From b0ecb03407ceb123083889777c2317a428602a4c Mon Sep 17 00:00:00 2001 From: Peter Johnson Date: Fri, 11 Jan 2019 23:14:41 -0800 Subject: [PATCH] Handle large file uploads (#62) Previously the upload function only handled files of less than 128 KB. --- .../tools/rpiConfigServer_src/Application.cpp | 113 ++++++++++-------- deps/tools/rpiConfigServer_src/Application.h | 10 +- .../rpiConfigServer_src/WebSocketHandlers.cpp | 44 +++++-- .../resources/frcvision.js | 36 +++++- 4 files changed, 134 insertions(+), 69 deletions(-) diff --git a/deps/tools/rpiConfigServer_src/Application.cpp b/deps/tools/rpiConfigServer_src/Application.cpp index 56a3b87..3145c97 100644 --- a/deps/tools/rpiConfigServer_src/Application.cpp +++ b/deps/tools/rpiConfigServer_src/Application.cpp @@ -7,6 +7,7 @@ #include "Application.h" +#include #include #include @@ -80,31 +81,62 @@ void Application::Set(wpi::StringRef appType, os << "exec " << appCommand << '\n'; } - m_appType = appType; - // terminate vision process so it reloads VisionStatus::GetInstance()->Terminate(onFail); UpdateStatus(); } -void Application::Upload(wpi::ArrayRef contents, - std::function onFail) { +int Application::StartUpload(wpi::StringRef appType, char* filename, + std::function onFail) { + int fd = mkstemp(filename); + if (fd < 0) { + wpi::SmallString<64> msg; + msg = "could not open temporary file: "; + msg += std::strerror(errno); + onFail(msg); + } + return fd; +} + +void Application::Upload(int fd, bool text, wpi::ArrayRef contents) { + // write contents + wpi::raw_fd_ostream out(fd, false); + if (text) { + wpi::StringRef str(reinterpret_cast(contents.data()), + contents.size()); + // convert any Windows EOL to Unix + for (;;) { + size_t idx = str.find("\r\n"); + if (idx == wpi::StringRef::npos) break; + out << str.slice(0, idx) << '\n'; + str = str.slice(idx + 2, wpi::StringRef::npos); + } + out << str; + // ensure file ends with EOL + if (!str.empty() && str.back() != '\n') out << '\n'; + } else { + out << contents; + } +} + +void Application::FinishUpload(wpi::StringRef appType, int fd, + const char* tmpFilename, + std::function onFail) { wpi::StringRef filename; - bool text = false; - if (m_appType == "upload-java") { + if (appType == "upload-java") { filename = "/uploaded.jar"; - } else if (m_appType == "upload-cpp") { + } else if (appType == "upload-cpp") { filename = "/uploaded"; - } else if (m_appType == "upload-python") { + } else if (appType == "upload-python") { filename = "/uploaded.py"; - text = true; } else { wpi::SmallString<64> msg; msg = "cannot upload application type '"; - msg += m_appType; + msg += appType; msg += "'"; onFail(msg); + ::close(fd); return; } @@ -112,54 +144,31 @@ void Application::Upload(wpi::ArrayRef contents, pathname = EXEC_HOME; pathname += filename; + // change ownership + if (fchown(fd, APP_UID, APP_GID) == -1) { + wpi::errs() << "could not change app ownership: " << std::strerror(errno) + << '\n'; + } + + // set file to be executable + if (fchmod(fd, 0775) == -1) { + wpi::errs() << "could not change app permissions: " << std::strerror(errno) + << '\n'; + } + + // close temporary file + ::close(fd); + // remove old file (need to do this as we can't overwrite a running exe) if (unlink(pathname.c_str()) == -1) { wpi::errs() << "could not remove app executable: " << std::strerror(errno) << '\n'; } - { - // open file for writing - std::error_code ec; - int fd; - if (wpi::sys::fs::openFileForWrite(pathname, fd, wpi::sys::fs::F_None)) { - wpi::SmallString<64> msg; - msg = "could not write "; - msg += pathname; - onFail(msg); - return; - } - - // change ownership - if (fchown(fd, APP_UID, APP_GID) == -1) { - wpi::errs() << "could not change app ownership: " << std::strerror(errno) - << '\n'; - } - - // set file to be executable - if (fchmod(fd, 0775) == -1) { - wpi::errs() << "could not change app permissions: " - << std::strerror(errno) << '\n'; - } - - // write contents and close file - wpi::raw_fd_ostream out(fd, true); - if (text) { - wpi::StringRef str(reinterpret_cast(contents.data()), - contents.size()); - // convert any Windows EOL to Unix - for (;;) { - size_t idx = str.find("\r\n"); - if (idx == wpi::StringRef::npos) break; - out << str.slice(0, idx) << '\n'; - str = str.slice(idx + 2, wpi::StringRef::npos); - } - out << str; - // ensure file ends with EOL - if (!str.empty() && str.back() != '\n') out << '\n'; - } else { - out << contents; - } + // rename temporary file to new file + if (rename(tmpFilename, pathname.c_str()) == -1) { + wpi::errs() << "could not rename to app executable: " + << std::strerror(errno) << '\n'; } // terminate vision process so it reloads diff --git a/deps/tools/rpiConfigServer_src/Application.h b/deps/tools/rpiConfigServer_src/Application.h index 12d0651..e1ec055 100644 --- a/deps/tools/rpiConfigServer_src/Application.h +++ b/deps/tools/rpiConfigServer_src/Application.h @@ -30,8 +30,11 @@ class Application { void Set(wpi::StringRef appType, std::function onFail); - void Upload(wpi::ArrayRef contents, - std::function onFail); + int StartUpload(wpi::StringRef appType, char* filename, + std::function onFail); + void Upload(int fd, bool text, wpi::ArrayRef contents); + void FinishUpload(wpi::StringRef appType, int fd, const char* tmpFilename, + std::function onFail); void UpdateStatus(); @@ -40,9 +43,6 @@ class Application { wpi::sig::Signal status; static std::shared_ptr GetInstance(); - - private: - std::string m_appType; }; #endif // RPICONFIGSERVER_APPLICATION_H_ diff --git a/deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp b/deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp index 4410742..4b9995f 100644 --- a/deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp +++ b/deps/tools/rpiConfigServer_src/WebSocketHandlers.cpp @@ -7,6 +7,9 @@ #include "WebSocketHandlers.h" +#include + +#include #include #include @@ -29,7 +32,14 @@ namespace uv = wpi::uv; #define SERVICE "/service/camera" struct WebSocketData { + ~WebSocketData() { + if (uploadFd != -1) ::close(uploadFd); + } + bool visionLogEnabled = false; + int uploadFd = -1; + bool uploadText = false; + char uploadFilename[128]; wpi::sig::ScopedConnection sysStatusConn; wpi::sig::ScopedConnection sysWritableConn; @@ -260,25 +270,43 @@ void ProcessWsText(wpi::WebSocket& ws, wpi::StringRef msg) { wpi::errs() << "could not read networkSave value: " << e.what() << '\n'; return; } - } else if (t == "applicationSave") { + } else if (t.startswith("application")) { + wpi::StringRef subType = t.substr(11); + auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) { SendWsText(*s, {{"type", "status"}, {"message", msg}}); }; + + std::string appType; try { - Application::GetInstance()->Set( - j.at("applicationType").get_ref(), statusFunc); + appType = j.at("applicationType").get(); } catch (const wpi::json::exception& e) { wpi::errs() << "could not read applicationSave value: " << e.what() << '\n'; return; } + + if (subType == "Save") { + Application::GetInstance()->Set(appType, statusFunc); + } else if (subType == "StartUpload") { + auto d = ws.GetData(); + std::strcpy(d->uploadFilename, EXEC_HOME "/appUploadXXXXXX"); + d->uploadFd = Application::GetInstance()->StartUpload( + appType, d->uploadFilename, statusFunc); + d->uploadText = wpi::StringRef(appType).endswith("python"); + } else if (subType == "FinishUpload") { + auto d = ws.GetData(); + if (d->uploadFd != -1) + Application::GetInstance()->FinishUpload(appType, d->uploadFd, + d->uploadFilename, statusFunc); + d->uploadFd = -1; + SendWsText(ws, {{"type", "applicationSaveComplete"}}); + } } } void ProcessWsBinary(wpi::WebSocket& ws, wpi::ArrayRef msg) { - auto statusFunc = [s = ws.shared_from_this()](wpi::StringRef msg) { - SendWsText(*s, {{"type", "status"}, {"message", msg}}); - }; - Application::GetInstance()->Upload(msg, statusFunc); - SendWsText(ws, {{"type", "applicationSaveComplete"}}); + auto d = ws.GetData(); + if (d->uploadFd != -1) + Application::GetInstance()->Upload(d->uploadFd, d->uploadText, msg); } diff --git a/deps/tools/rpiConfigServer_src/resources/frcvision.js b/deps/tools/rpiConfigServer_src/resources/frcvision.js index bbe52ff..4c038dc 100644 --- a/deps/tools/rpiConfigServer_src/resources/frcvision.js +++ b/deps/tools/rpiConfigServer_src/resources/frcvision.js @@ -588,12 +588,40 @@ $('#applicationSave').click(function() { if (applicationFiles.length <= 0) { return; } + $('#applicationSave').button('loading'); - var fr = new FileReader(); - fr.onload = function(e) { - connection.send(e.target.result); + + var msg = { + type: 'applicationStartUpload', + applicationType: $('#applicationType').val() }; - fr.readAsArrayBuffer(applicationFiles.item(0)); + connection.send(JSON.stringify(msg)); + + var reader = new FileReader(); + var file = applicationFiles.item(0); + + function uploadFile(start) { + var nextSlice = start + (64 * 1024) + 1; + reader.onloadend = function(e) { + if (e.target.readyState !== FileReader.DONE) { + return; + } + connection.send(e.target.result); + if (nextSlice < file.size) { + // more to go + uploadFile(nextSlice); + } else { + // done + var msg = { + type: 'applicationFinishUpload', + applicationType: $('#applicationType').val() + }; + connection.send(JSON.stringify(msg)); + } + } + reader.readAsArrayBuffer(file.slice(start, nextSlice)); + } + uploadFile(0); }); // Start with display disconnected and start initial connection attempt