Optimize upload/download via remote web browser

Makefile

@@ -13,7 +13,7 @@ ifeq ($(MAKECMDGOALS),)
   endif
 endif
 
-VERSION_TAG := v1.4
+VERSION_TAG := v1.5
 TITLE_ID    := FMGR88888
 PYTHON      ?= python3
 STRIP       ?= $(PS5_PAYLOAD_SDK)/bin/prospero-strip

README.md

@@ -123,6 +123,7 @@ This project was built with reference to these projects:
 
 - **[ps5-payload-dev/websrv](https://github.com/ps5-payload-dev/websrv):** HTTP server structure, static asset embedding ideas, PS5 browser/websrv behavior and PKG install function. License: GPLv3+.
 - **[ps5-payload-dev/ftpsrv](https://github.com/ps5-payload-dev/ftpsrv):** PS5 payload conventions, home screen launcher/install flow reference, process handling style and startup installation reference. License: GPLv3+.
+- **[seregonwar/zftpd](https://github.com/seregonwar/zftpd):** PS5 TCP socket buffer tuning and high-throughput transfer behavior reference. License: MIT.
 - **[itsPLK/ps5-payload-manager](https://github.com/itsPLK/ps5-payload-manager):** Payload building behavior. License: GPLv3.
 - **[libmicrohttpd](https://ftp.gnu.org/gnu/libmicrohttpd/):** Used as the embedded HTTP server library. It is licensed by GNU under the LGPL; this payload links it as the SDK-provided static library.
 - **[ps5-payload-dev/sdk](https://github.com/ps5-payload-dev/sdk):** Payload building foundation. License: GPLv3+.

assets/main.css

@@ -110,20 +110,55 @@ body {
   text-align: right;
 }
 
-.space-item {
+.space-info .space-item {
   flex: 0 0 auto;
+  display: inline-flex;
+  height: auto;
+  min-height: 0;
+  min-width: 0;
   margin-left: 14px;
+  padding: 0 2px;
+  border: 0;
+  border-radius: 3px;
+  background: transparent;
+  color: inherit;
+  font: inherit;
+  line-height: inherit;
+  text-align: right;
+  cursor: pointer;
   max-width: 100%;
   overflow: hidden;
   text-overflow: ellipsis;
 }
 
+.space-info .space-item:hover:not(:disabled),
+.space-info .space-item:focus:not(:disabled) {
+  color: #fff;
+  background: rgba(255, 255, 255, 0.1);
+  outline: none;
+}
+
+.space-info .space-item:focus-visible {
+  box-shadow: 0 0 0 2px #87bfff;
+}
+
+.space-info .space-item:active {
+  background: rgba(255, 255, 255, 0.18);
+}
+
+.space-info .space-item:disabled {
+  opacity: 1;
+  color: inherit;
+  cursor: default;
+  pointer-events: none;
+}
+
 .space-prefix {
   color: #8f9aa5;
   flex: 0 0 auto;
 }
 
-.space-item.current {
+.space-info .space-item.current {
   color: #f2f5f7;
   font-weight: 700;
 }

assets/main.js

@@ -29,7 +29,7 @@ let downloadFrame = null;
 let uploadXhr = null;
 let L = {};
 
-const APP_VERSION = "v1.4";
+const APP_VERSION = "v1.5";
 const LAST_PATH_KEY = "ps5-web-file-mgr:last-path";
 const SORT_KEY = "ps5-web-file-mgr:list-sort";
 const LOADING_DISPLAY_DELAY = 250;
@@ -604,13 +604,20 @@ async function refreshSpaces() {
       spaceInfoEl.appendChild(prefix);
     }
     for (const item of data.spaces || []) {
-      const span = document.createElement("span");
-      span.className = "space-item" + (item.current ? " current" : "");
+      const button = document.createElement("button");
+      button.type = "button";
+      button.className = "space-item" + (item.current ? " current" : "");
       const label = item.label_key ? t(item.label_key) : item.label;
-      span.textContent = label + ": " +
+      button.textContent = label + ": " +
         formatBytes(item.free, false) + "/" + formatBytes(item.total, false);
-      span.title = item.path + " " + span.textContent;
-      spaceInfoEl.appendChild(span);
+      button.title = item.path + " " + button.textContent;
+      if (item.current) button.setAttribute("aria-current", "location");
+      button.disabled = spaceNavigationLocked();
+      bindPress(button, () => {
+        if (!item.path || spaceNavigationLocked()) return;
+        load(item.path, undefined, false, true, "push");
+      });
+      spaceInfoEl.appendChild(button);
     }
     renderAddressPath();
   } catch (err) {
@@ -619,6 +626,11 @@ async function refreshSpaces() {
   }
 }
 
+function spaceNavigationLocked() {
+  return Boolean(busy || loadingPath || !contentLoadingEl.hidden ||
+    contentEl.classList.contains("loading"));
+}
+
 function updateSpecialStoragePaths(spaces) {
   const next = {};
   let hasMnt = false;
@@ -985,6 +997,9 @@ function singleSelected() {
 function updateButtons() {
   const items = selectedEntries();
   const locked = busy || contentEl.classList.contains("loading");
+  for (const button of spaceInfoEl.querySelectorAll(".space-item")) {
+    button.disabled = spaceNavigationLocked();
+  }
   document.getElementById("copyBtn").disabled = locked || items.length === 0;
   document.getElementById("moveBtn").disabled = locked || items.length === 0;
   document.getElementById("renameBtn").disabled = locked || items.length !== 1;
@@ -1234,6 +1249,7 @@ async function load(path, scrollTop, force, alertOnError, historyMode) {
   const shouldScrollTop = scrollTop !== false;
   if ((busy && !force) || loadingPath) return;
   loadingPath = path;
+  updateButtons();
   startContentLoadingTimer();
   try {
     setStatus(t("readDir"));

src/download.c

@@ -16,6 +16,7 @@
 #include "websrv.h"
 
 #define DOWNLOAD_ARCHIVE_NAME_LIMIT 20
+#define DOWNLOAD_BUFFER_SIZE (2 * 1024 * 1024)
 
 typedef struct tar_frame {
   char path[PATH_MAX];
@@ -47,7 +48,7 @@ typedef struct tar_stream {
 
 typedef struct download_file_stream {
   file_task_t *task;
-  FILE *file;
+  int fd;
   unsigned long long size;
   unsigned long long sent;
   int done;
@@ -295,6 +296,8 @@ static ssize_t
 tar_read(void *cls, uint64_t pos, char *buf, size_t max) {
   tar_stream_t *s = cls;
   size_t out = 0;
+  unsigned long long progress = 0;
+  char progress_current[PATH_MAX] = {0};
   (void)pos;
 
   while(out < max && !s->done) {
@@ -340,10 +343,10 @@ tar_read(void *cls, uint64_t pos, char *buf, size_t max) {
       }
       out += (size_t)n;
       s->file_remaining -= (unsigned long long)n;
-      if(s->task) {
-        task_update(s->task, TASK_RUNNING,
-                    s->current_file[0] ? s->current_file : NULL,
-                    (unsigned long long)n, NULL);
+      progress += (unsigned long long)n;
+      if(s->current_file[0]) {
+        snprintf(progress_current, sizeof(progress_current), "%s",
+                 s->current_file);
       }
       if(!s->file_remaining) {
         close(s->fd);
@@ -354,6 +357,11 @@ tar_read(void *cls, uint64_t pos, char *buf, size_t max) {
       }
     }
   }
+  if(s->task && progress) {
+    task_update(s->task, TASK_RUNNING,
+                progress_current[0] ? progress_current : NULL,
+                progress, NULL);
+  }
   return out ? (ssize_t)out : MHD_CONTENT_READER_END_OF_STREAM;
 }
 
@@ -589,22 +597,18 @@ api_download_prepare(struct MHD_Connection *conn, const char *body,
 static ssize_t
 download_file_read(void *cls, uint64_t pos, char *buf, size_t max) {
   download_file_stream_t *s = cls;
-  size_t len;
-  (void)pos;
+  ssize_t len;
 
   if(task_cancel_requested(s->task)) {
     s->error = ECANCELED;
     return MHD_CONTENT_READER_END_WITH_ERROR;
   }
-  if(fseek(s->file, (long)pos, SEEK_SET)) {
+  len = pread(s->fd, buf, max, (off_t)pos);
+  if(len < 0) {
     s->error = errno ? errno : EIO;
     return MHD_CONTENT_READER_END_WITH_ERROR;
   }
-  if(!(len = fread(buf, 1, max, s->file))) {
-    if(ferror(s->file)) {
-      s->error = errno ? errno : EIO;
-      return MHD_CONTENT_READER_END_WITH_ERROR;
-    }
+  if(!len) {
     s->done = 1;
     return MHD_CONTENT_READER_END_OF_STREAM;
   }
@@ -619,7 +623,7 @@ download_file_read(void *cls, uint64_t pos, char *buf, size_t max) {
     }
     task_update(s->task, TASK_RUNNING, s->task->src, add, NULL);
   }
-  return (ssize_t)len;
+  return len;
 }
 
 static void
@@ -630,8 +634,8 @@ download_file_close(void *cls) {
   if(!s) {
     return;
   }
-  if(s->file) {
-    fclose(s->file);
+  if(s->fd >= 0) {
+    close(s->fd);
   }
   canceled = task_cancel_requested(s->task);
   if(canceled) {
@@ -692,15 +696,17 @@ api_download(struct MHD_Connection *conn) {
       return send_json_error(conn, MHD_HTTP_INTERNAL_SERVER_ERROR, "out of memory");
     }
     file_stream->task = task;
+    file_stream->fd = -1;
     file_stream->size = (unsigned long long)st.st_size;
     file_stream->done = file_stream->size == 0;
-    file_stream->file = fopen(paths[0], "rb");
-    if(!file_stream->file) {
+    file_stream->fd = open(paths[0], O_RDONLY);
+    if(file_stream->fd < 0) {
       free(file_stream);
       task_update(task, TASK_FAILED, task->src, 0, strerror(errno));
       return send_json_error(conn, MHD_HTTP_NOT_FOUND, "file not found");
     }
-    resp = MHD_create_response_from_callback((uint64_t)st.st_size, 32 * 0x4000,
+    resp = MHD_create_response_from_callback((uint64_t)st.st_size,
+                                             DOWNLOAD_BUFFER_SIZE,
                                              download_file_read, file_stream,
                                              download_file_close);
     if(!resp) {
@@ -738,7 +744,8 @@ api_download(struct MHD_Connection *conn) {
       return send_json_error(conn, MHD_HTTP_INTERNAL_SERVER_ERROR, "out of memory");
     }
   }
-  resp = MHD_create_response_from_callback(MHD_SIZE_UNKNOWN, 32 * 0x4000,
+  resp = MHD_create_response_from_callback(MHD_SIZE_UNKNOWN,
+                                           DOWNLOAD_BUFFER_SIZE,
                                            tar_read, stream, tar_close);
   if(!resp) {
     tar_close(stream);

src/upload.c

@@ -14,9 +14,13 @@
 #include "json_util.h"
 #include "path_util.h"
 
+#define UPLOAD_BUFFER_SIZE (1024 * 1024)
+
 typedef struct upload_context {
   file_task_t *task;
   int fd;
+  char *buffer;
+  size_t buffered;
   char temp[PATH_MAX];
   char target[PATH_MAX];
   unsigned long long expected;
@@ -26,6 +30,35 @@ typedef struct upload_context {
   int task_done;
 } upload_context_t;
 
+static int
+upload_flush(upload_context_t *ctx) {
+  size_t written = 0;
+
+  while(written < ctx->buffered) {
+    ssize_t n;
+
+    if(ctx->task && task_cancel_requested(ctx->task)) {
+      ctx->failed = 1;
+      ctx->error = ECANCELED;
+      return -1;
+    }
+    n = write(ctx->fd, ctx->buffer + written, ctx->buffered - written);
+    if(n <= 0) {
+      ctx->failed = 1;
+      ctx->error = errno ? errno : EIO;
+      return -1;
+    }
+    written += (size_t)n;
+    ctx->written += (unsigned long long)n;
+  }
+  if(ctx->task && written) {
+    task_update(ctx->task, TASK_RUNNING, ctx->target,
+                (unsigned long long)written, NULL);
+  }
+  ctx->buffered = 0;
+  return 0;
+}
+
 static void
 finish_upload_task(file_task_t *task, task_state_t state, const char *current,
                    const char *error) {
@@ -331,6 +364,11 @@ filemgr_upload_begin(struct MHD_Connection *conn, void **upload_ctx) {
     ctx->error = errno;
     goto fail;
   }
+  if(!(ctx->buffer = malloc(UPLOAD_BUFFER_SIZE))) {
+    ctx->failed = 1;
+    ctx->error = ENOMEM;
+    goto fail;
+  }
 
 fail:
   free_paths(checked_dirs, checked_dir_count);
@@ -359,7 +397,6 @@ fail:
 int
 filemgr_upload_data(void *upload_ctx, const char *data, size_t size) {
   upload_context_t *ctx = upload_ctx;
-  size_t written = 0;
 
   if(!ctx || ctx->failed) {
     return -1;
@@ -369,19 +406,17 @@ filemgr_upload_data(void *upload_ctx, const char *data, size_t size) {
     ctx->error = ECANCELED;
     return -1;
   }
-  while(written < size) {
-    ssize_t n = write(ctx->fd, data + written, size - written);
-    if(n <= 0) {
-      ctx->failed = 1;
-      ctx->error = errno ? errno : EIO;
+  while(size) {
+    size_t space = UPLOAD_BUFFER_SIZE - ctx->buffered;
+    size_t take = size < space ? size : space;
+
+    memcpy(ctx->buffer + ctx->buffered, data, take);
+    ctx->buffered += take;
+    data += take;
+    size -= take;
+    if(ctx->buffered == UPLOAD_BUFFER_SIZE && upload_flush(ctx)) {
       return -1;
     }
-    written += (size_t)n;
-    ctx->written += (unsigned long long)n;
-    if(ctx->task) {
-      task_update(ctx->task, TASK_RUNNING, ctx->target,
-                  (unsigned long long)n, NULL);
-    }
   }
   return 0;
 }
@@ -409,6 +444,10 @@ filemgr_upload_finish(struct MHD_Connection *conn, void *upload_ctx) {
                            errno == ENOSPC ? MHD_HTTP_INSUFFICIENT_STORAGE :
                            MHD_HTTP_INTERNAL_SERVER_ERROR, NULL);
   }
+  if(ctx->buffered && upload_flush(ctx)) {
+    ctx->error = ctx->error ? ctx->error : EIO;
+    goto done;
+  }
   if(ctx->expected && ctx->written != ctx->expected) {
     ctx->error = EIO;
     goto done;
@@ -476,5 +515,6 @@ filemgr_upload_free(void *upload_ctx) {
       ctx->task_done = 1;
     }
   }
+  free(ctx->buffer);
   free(ctx);
 }

src/websrv.c

@@ -6,6 +6,7 @@
 #include <arpa/inet.h>
 #include <microhttpd.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <sys/socket.h>
 #include <unistd.h>
 
@@ -14,10 +15,24 @@
 #include "websrv.h"
 
 #define REQUEST_BODY_MAX (4 * 1024 * 1024)
+#define HTTP_CONNECTION_MEMORY_LIMIT (8 * 1024 * 1024)
+#define HTTP_CONNECTION_MEMORY_INCREMENT (2 * 1024 * 1024)
+#define HTTP_SOCKET_RCVBUF_SIZE (4 * 1024 * 1024)
+#define HTTP_SOCKET_SNDBUF_SIZE (4 * 1024 * 1024)
 
 static volatile sig_atomic_t g_stop_requested;
 static int g_listen_fd = -1;
 
+static void
+websrv_tune_connection_socket(int fd) {
+  const int sndbuf = HTTP_SOCKET_SNDBUF_SIZE;
+  const int nodelay = 1;
+
+  /* OrbisOS HTTP sockets need an explicit send buffer to fill a GbE link. */
+  (void)setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf));
+  (void)setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof(nodelay));
+}
+
 typedef struct request_context {
   char *body;
   size_t size;
@@ -190,6 +205,12 @@ websrv_listen(unsigned short port) {
     close(srvfd);
     return -1;
   }
+  {
+    const int rcvbuf = HTTP_SOCKET_RCVBUF_SIZE;
+
+    /* Set before listen so accepted PS5 sockets inherit the larger window. */
+    (void)setsockopt(srvfd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf));
+  }
 
   memset(&server_addr, 0, sizeof(server_addr));
   server_addr.sin_family = AF_INET;
@@ -211,8 +232,12 @@ websrv_listen(unsigned short port) {
 
   if(!(httpd = MHD_start_daemon(MHD_USE_THREAD_PER_CONNECTION | MHD_USE_ITC |
                                 MHD_USE_NO_LISTEN_SOCKET | MHD_USE_DEBUG |
-                                MHD_USE_INTERNAL_POLLING_THREAD,
+                                MHD_USE_INTERNAL_POLLING_THREAD | MHD_USE_TURBO,
                                 0, NULL, NULL, &websrv_on_request, NULL,
+                                MHD_OPTION_CONNECTION_MEMORY_LIMIT,
+                                (size_t)HTTP_CONNECTION_MEMORY_LIMIT,
+                                MHD_OPTION_CONNECTION_MEMORY_INCREMENT,
+                                (size_t)HTTP_CONNECTION_MEMORY_INCREMENT,
                                 MHD_OPTION_NOTIFY_COMPLETED,
                                 &websrv_on_completed, NULL, MHD_OPTION_END))) {
     perror("MHD_start_daemon");
@@ -226,6 +251,7 @@ websrv_listen(unsigned short port) {
       if(!g_stop_requested) perror("accept");
       break;
     }
+    websrv_tune_connection_socket(connfd);
     if(MHD_add_connection(httpd, connfd, (struct sockaddr *)&client_addr,
                           addr_len) != MHD_YES) {
       perror("MHD_add_connection");