Handle large file uploads (#62)

Previously the upload function only handled files of less than 128 KB.

deps/tools/rpiConfigServer_src/Application.cpp

@@ -7,6 +7,7 @@
 
 #include "Application.h"
 
+#include <stdlib.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
@@ -80,31 +81,62 @@ void Application::Set(wpi::StringRef appType,
     os << "exec " << appCommand << '\n';
   }
 
-  m_appType = appType;
-
   // terminate vision process so it reloads
   VisionStatus::GetInstance()->Terminate(onFail);
 
   UpdateStatus();
 }
 
-void Application::Upload(wpi::ArrayRef<uint8_t> contents,
-                         std::function<void(wpi::StringRef)> onFail) {
+int Application::StartUpload(wpi::StringRef appType, char* filename,
+                             std::function<void(wpi::StringRef)> onFail) {
+  int fd = mkstemp(filename);
+  if (fd < 0) {
+    wpi::SmallString<64> msg;
+    msg = "could not open temporary file: ";
+    msg += std::strerror(errno);
+    onFail(msg);
+  }
+  return fd;
+}
+
+void Application::Upload(int fd, bool text, wpi::ArrayRef<uint8_t> contents) {
+  // write contents
+  wpi::raw_fd_ostream out(fd, false);
+  if (text) {
+    wpi::StringRef str(reinterpret_cast<const char*>(contents.data()),
+                       contents.size());
+    // convert any Windows EOL to Unix
+    for (;;) {
+      size_t idx = str.find("\r\n");
+      if (idx == wpi::StringRef::npos) break;
+      out << str.slice(0, idx) << '\n';
+      str = str.slice(idx + 2, wpi::StringRef::npos);
+    }
+    out << str;
+    // ensure file ends with EOL
+    if (!str.empty() && str.back() != '\n') out << '\n';
+  } else {
+    out << contents;
+  }
+}
+
+void Application::FinishUpload(wpi::StringRef appType, int fd,
+                               const char* tmpFilename,
+                               std::function<void(wpi::StringRef)> onFail) {
   wpi::StringRef filename;
-  bool text = false;
-  if (m_appType == "upload-java") {
+  if (appType == "upload-java") {
     filename = "/uploaded.jar";
-  } else if (m_appType == "upload-cpp") {
+  } else if (appType == "upload-cpp") {
     filename = "/uploaded";
-  } else if (m_appType == "upload-python") {
+  } else if (appType == "upload-python") {
     filename = "/uploaded.py";
-    text = true;
   } else {
     wpi::SmallString<64> msg;
     msg = "cannot upload application type '";
-    msg += m_appType;
+    msg += appType;
     msg += "'";
     onFail(msg);
+    ::close(fd);
     return;
   }
 
@@ -112,54 +144,31 @@ void Application::Upload(wpi::ArrayRef<uint8_t> contents,
   pathname = EXEC_HOME;
   pathname += filename;
 
+  // change ownership
+  if (fchown(fd, APP_UID, APP_GID) == -1) {
+    wpi::errs() << "could not change app ownership: " << std::strerror(errno)
+                << '\n';
+  }
+
+  // set file to be executable
+  if (fchmod(fd, 0775) == -1) {
+    wpi::errs() << "could not change app permissions: " << std::strerror(errno)
+                << '\n';
+  }
+
+  // close temporary file
+  ::close(fd);
+
   // remove old file (need to do this as we can't overwrite a running exe)
   if (unlink(pathname.c_str()) == -1) {
     wpi::errs() << "could not remove app executable: " << std::strerror(errno)
                 << '\n';
   }
 
-  {
-    // open file for writing
-    std::error_code ec;
-    int fd;
-    if (wpi::sys::fs::openFileForWrite(pathname, fd, wpi::sys::fs::F_None)) {
-      wpi::SmallString<64> msg;
-      msg = "could not write ";
-      msg += pathname;
-      onFail(msg);
-      return;
-    }
-
-    // change ownership
-    if (fchown(fd, APP_UID, APP_GID) == -1) {
-      wpi::errs() << "could not change app ownership: " << std::strerror(errno)
-                  << '\n';
-    }
-
-    // set file to be executable
-    if (fchmod(fd, 0775) == -1) {
-      wpi::errs() << "could not change app permissions: "
-                  << std::strerror(errno) << '\n';
-    }
-
-    // write contents and close file
-    wpi::raw_fd_ostream out(fd, true);
-    if (text) {
-      wpi::StringRef str(reinterpret_cast<const char*>(contents.data()),
-                         contents.size());
-      // convert any Windows EOL to Unix
-      for (;;) {
-        size_t idx = str.find("\r\n");
-        if (idx == wpi::StringRef::npos) break;
-        out << str.slice(0, idx) << '\n';
-        str = str.slice(idx + 2, wpi::StringRef::npos);
-      }
-      out << str;
-      // ensure file ends with EOL
-      if (!str.empty() && str.back() != '\n') out << '\n';
-    } else {
-      out << contents;
-    }
+  // rename temporary file to new file
+  if (rename(tmpFilename, pathname.c_str()) == -1) {
+    wpi::errs() << "could not rename to app executable: "
+                << std::strerror(errno) << '\n';
   }
 
   // terminate vision process so it reloads

deps/tools/rpiConfigServer_src/Application.h

@@ -30,8 +30,11 @@ class Application {
 
   void Set(wpi::StringRef appType, std::function<void(wpi::StringRef)> onFail);
 
-  void Upload(wpi::ArrayRef<uint8_t> contents,
-              std::function<void(wpi::StringRef)> onFail);
+  int StartUpload(wpi::StringRef appType, char* filename,
+                  std::function<void(wpi::StringRef)> onFail);
+  void Upload(int fd, bool text, wpi::ArrayRef<uint8_t> contents);
+  void FinishUpload(wpi::StringRef appType, int fd, const char* tmpFilename,
+                    std::function<void(wpi::StringRef)> onFail);
 
   void UpdateStatus();
 
@@ -40,9 +43,6 @@ class Application {
   wpi::sig::Signal<const wpi::json&> status;
 
   static std::shared_ptr<Application> GetInstance();
-
- private:
-  std::string m_appType;
 };
 
 #endif  // RPICONFIGSERVER_APPLICATION_H_

deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp

@@ -7,6 +7,9 @@
 
 #include "WebSocketHandlers.h"
 
+#include <unistd.h>
+
+#include <cstring>
 #include <memory>
 
 #include <wpi/SmallVector.h>
@@ -29,7 +32,14 @@ namespace uv = wpi::uv;
 #define SERVICE "/service/camera"
 
 struct WebSocketData {
+  ~WebSocketData() {
+    if (uploadFd != -1) ::close(uploadFd);
+  }
+
   bool visionLogEnabled = false;
+  int uploadFd = -1;
+  bool uploadText = false;
+  char uploadFilename[128];
 
   wpi::sig::ScopedConnection sysStatusConn;
   wpi::sig::ScopedConnection sysWritableConn;
@@ -260,25 +270,43 @@ void ProcessWsText(wpi::WebSocket& ws, wpi::StringRef msg) {
       wpi::errs() << "could not read networkSave value: " << e.what() << '\n';
       return;
     }
-  } else if (t == "applicationSave") {
+  } else if (t.startswith("application")) {
+    wpi::StringRef subType = t.substr(11);
+
     auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) {
       SendWsText(*s, {{"type", "status"}, {"message", msg}});
     };
+
+    std::string appType;
     try {
-      Application::GetInstance()->Set(
-          j.at("applicationType").get_ref<const std::string&>(), statusFunc);
+      appType = j.at("applicationType").get<std::string>();
     } catch (const wpi::json::exception& e) {
       wpi::errs() << "could not read applicationSave value: " << e.what()
                   << '\n';
       return;
     }
+
+    if (subType == "Save") {
+      Application::GetInstance()->Set(appType, statusFunc);
+    } else if (subType == "StartUpload") {
+      auto d = ws.GetData<WebSocketData>();
+      std::strcpy(d->uploadFilename, EXEC_HOME "/appUploadXXXXXX");
+      d->uploadFd = Application::GetInstance()->StartUpload(
+          appType, d->uploadFilename, statusFunc);
+      d->uploadText = wpi::StringRef(appType).endswith("python");
+    } else if (subType == "FinishUpload") {
+      auto d = ws.GetData<WebSocketData>();
+      if (d->uploadFd != -1)
+        Application::GetInstance()->FinishUpload(appType, d->uploadFd,
+                                                 d->uploadFilename, statusFunc);
+      d->uploadFd = -1;
+      SendWsText(ws, {{"type", "applicationSaveComplete"}});
+    }
   }
 }
 
 void ProcessWsBinary(wpi::WebSocket& ws, wpi::ArrayRef<uint8_t> msg) {
-  auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) {
-    SendWsText(*s, {{"type", "status"}, {"message", msg}});
-  };
-  Application::GetInstance()->Upload(msg, statusFunc);
-  SendWsText(ws, {{"type", "applicationSaveComplete"}});
+  auto d = ws.GetData<WebSocketData>();
+  if (d->uploadFd != -1)
+    Application::GetInstance()->Upload(d->uploadFd, d->uploadText, msg);
 }

deps/tools/rpiConfigServer_src/resources/frcvision.js

@@ -588,12 +588,40 @@ $('#applicationSave').click(function() {
   if (applicationFiles.length <= 0) {
     return;
   }
+
   $('#applicationSave').button('loading');
-  var fr = new FileReader();
-  fr.onload = function(e) {
-    connection.send(e.target.result);
+
+  var msg = {
+    type: 'applicationStartUpload',
+    applicationType: $('#applicationType').val()
   };
-  fr.readAsArrayBuffer(applicationFiles.item(0));
+  connection.send(JSON.stringify(msg));
+
+  var reader = new FileReader();
+  var file = applicationFiles.item(0);
+
+  function uploadFile(start) {
+    var nextSlice = start + (64 * 1024) + 1;
+    reader.onloadend = function(e) {
+      if (e.target.readyState !== FileReader.DONE) {
+        return;
+      }
+      connection.send(e.target.result);
+      if (nextSlice < file.size) {
+        // more to go
+        uploadFile(nextSlice);
+      } else {
+        // done
+        var msg = {
+          type: 'applicationFinishUpload',
+          applicationType: $('#applicationType').val()
+        };
+        connection.send(JSON.stringify(msg));
+      }
+    }
+    reader.readAsArrayBuffer(file.slice(start, nextSlice));
+  }
+  uploadFile(0);
 });
 
 // Start with display disconnected and start initial connection attempt