mirror of
https://github.com/knutwurst/patchdl.git
synced 2026-10-06 06:00:21 +02:00
Download patches over a connection pool with a queue and resume
Replace the single sequential transfer with a pool of N worker threads that pull pieces of one manifest in parallel, lifting the per-connection ~7 MB/s ceiling. One job runs at a time; the rest queue. The connection count is configurable (1-16, default 4) and applies on the next start. Resume is tracked per piece in a sidecar bitmap that survives a reboot, and a one-time migration recognises a partial written by the old sequential build (a piece-aligned contiguous prefix on disk) and marks those pieces done so an in-progress download is not restarted from zero. Pause keeps the partial; Cancel deletes it. Both, plus Resume, are available at any point in a download's life. Concurrency review fixes folded in: - a job is published as the active (claimable) job only after its manifest/state/fd are attached, so a half-built job can no longer be settled to "done" before any bytes are fetched - cancel/pause during the admit I/O window only flag the job; admit_next is the sole finalizer, closing a use-after-free and a lost-pause race - resuming a paused job frees the stale per-job buffers and zeroes the committed counters before re-seeding, fixing a leak and a double-count - the background version.xml thread is joined on shutdown before the title list is freed - verify_downloads is snapshotted under its own lock before the pool lock - the web UI keeps Resume/Cancel after a failed transfer and bounds the local "downloading" bridge flag so a card cannot wedge
This commit is contained in:
1 parent
01eaef79f2
commit
21f6bd61ad
5 files changed
+1147
-367
No files matched your search
+205
-7
@@ -198,14 +198,14 @@ dns_lookup(const char *host, char *ip_out, size_t ip_sz) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pthread_mutex_unlock(&dns_cache_mtx);
|
/* Cold miss: resolve while HOLDING the cache lock (single-flight). N pool
|
||||||
|
workers needing the same CDN host would otherwise each blast Sony's
|
||||||
|
rate-limited resolver; this way one resolves and the rest get the cache.
|
||||||
|
It also serializes dns_resolve so its diagnostic globals can't be raced.
|
||||||
|
(This is the DNS lock, independent of the pool lock.) */
|
||||||
for (int attempt = 0; attempt < 4 && rc; attempt++)
|
for (int attempt = 0; attempt < 4 && rc; attempt++)
|
||||||
rc = dns_resolve(host, ip_out, ip_sz);
|
rc = dns_resolve(host, ip_out, ip_sz);
|
||||||
if (rc) return -1;
|
if (!rc && dns_cache_n < (int)(sizeof(dns_cache) / sizeof(dns_cache[0]))) {
|
||||||
|
|
||||||
pthread_mutex_lock(&dns_cache_mtx);
|
|
||||||
if (dns_cache_n < (int)(sizeof(dns_cache) / sizeof(dns_cache[0]))) {
|
|
||||||
strncpy(dns_cache[dns_cache_n].host, host,
|
strncpy(dns_cache[dns_cache_n].host, host,
|
||||||
sizeof(dns_cache[0].host) - 1);
|
sizeof(dns_cache[0].host) - 1);
|
||||||
strncpy(dns_cache[dns_cache_n].ip, ip_out,
|
strncpy(dns_cache[dns_cache_n].ip, ip_out,
|
||||||
@@ -213,7 +213,7 @@ dns_lookup(const char *host, char *ip_out, size_t ip_sz) {
|
|||||||
dns_cache_n++;
|
dns_cache_n++;
|
||||||
}
|
}
|
||||||
pthread_mutex_unlock(&dns_cache_mtx);
|
pthread_mutex_unlock(&dns_cache_mtx);
|
||||||
return 0;
|
return rc;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* ---------- HTTP GET via curl ------------------------------------------- */
|
/* ---------- HTTP GET via curl ------------------------------------------- */
|
||||||
@@ -678,6 +678,204 @@ patchdl_http_download_manifest(const char *manifest_url, const char *dest_path,
|
|||||||
bytes_out, NULL, NULL, 0, 0);
|
bytes_out, NULL, NULL, 0, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ---- global init + parallel piece download (connection pool) ----------- */
|
||||||
|
|
||||||
|
void patchdl_net_global_init(void) { curl_global_init(CURL_GLOBAL_ALL); }
|
||||||
|
void patchdl_net_global_cleanup(void) { curl_global_cleanup(); }
|
||||||
|
|
||||||
|
/* Write sink for one piece: pwrite at a fixed base offset (concurrent
|
||||||
|
non-overlapping pieces of the same fd are safe), tee into SHA-256 if asked,
|
||||||
|
and publish bytes-so-far for live progress. */
|
||||||
|
typedef struct {
|
||||||
|
int fd;
|
||||||
|
long long base;
|
||||||
|
long long written;
|
||||||
|
EVP_MD_CTX *md;
|
||||||
|
volatile long long *bytes_slot;
|
||||||
|
} piece_sink_t;
|
||||||
|
|
||||||
|
static size_t
|
||||||
|
piece_write_cb(void *ptr, size_t size, size_t nmemb, void *ud) {
|
||||||
|
piece_sink_t *s = (piece_sink_t *)ud;
|
||||||
|
size_t n = size * nmemb;
|
||||||
|
ssize_t w;
|
||||||
|
if (n == 0) return 0;
|
||||||
|
w = pwrite(s->fd, ptr, n, (off_t)(s->base + s->written));
|
||||||
|
if (w < 0 || (size_t)w != n) return 0; /* short write -> curl errors out */
|
||||||
|
if (s->md) EVP_DigestUpdate(s->md, ptr, n);
|
||||||
|
s->written += (long long)n;
|
||||||
|
if (s->bytes_slot) *s->bytes_slot = s->written;
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
piece_xfer_cb(void *clientp, curl_off_t dltotal, curl_off_t dlnow,
|
||||||
|
curl_off_t ultotal, curl_off_t ulnow) {
|
||||||
|
volatile int *abort_flag = (volatile int *)clientp;
|
||||||
|
(void)dltotal; (void)dlnow; (void)ultotal; (void)ulnow;
|
||||||
|
return (abort_flag && *abort_flag) ? 1 : 0; /* non-zero aborts the transfer */
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
patchdl_http_download_piece(const char *url, int fd,
|
||||||
|
long long file_offset, long long file_size,
|
||||||
|
const char *expected_sha256_or_null,
|
||||||
|
patchdl_piece_ctx_t *ctx) {
|
||||||
|
CURL *curl;
|
||||||
|
CURLcode res;
|
||||||
|
long http_code = 0;
|
||||||
|
char host[256], ip[INET_ADDRSTRLEN], rs443[512], rs80[512];
|
||||||
|
struct curl_slist *rl = NULL;
|
||||||
|
struct curl_blob ca_blob;
|
||||||
|
piece_sink_t sink;
|
||||||
|
int verify = (expected_sha256_or_null && expected_sha256_or_null[0]);
|
||||||
|
|
||||||
|
if (url_host(url, host, sizeof(host))) return -1;
|
||||||
|
if (!host_allowed(host)) return -1;
|
||||||
|
if (dns_lookup(host, ip, sizeof(ip))) return -1;
|
||||||
|
|
||||||
|
memset(&sink, 0, sizeof(sink));
|
||||||
|
sink.fd = fd;
|
||||||
|
sink.base = file_offset;
|
||||||
|
sink.bytes_slot = ctx ? ctx->bytes_slot : NULL;
|
||||||
|
if (verify) {
|
||||||
|
sink.md = EVP_MD_CTX_new();
|
||||||
|
if (sink.md) EVP_DigestInit_ex(sink.md, EVP_sha256(), NULL);
|
||||||
|
}
|
||||||
|
|
||||||
|
snprintf(rs443, sizeof(rs443), "%s:443:%s", host, ip);
|
||||||
|
rl = curl_slist_append(NULL, rs443);
|
||||||
|
snprintf(rs80, sizeof(rs80), "%s:80:%s", host, ip);
|
||||||
|
rl = curl_slist_append(rl, rs80);
|
||||||
|
|
||||||
|
ca_blob.data = (void *)PATCHDL_SCEI_DNAS_ROOT_PEM;
|
||||||
|
ca_blob.len = strlen(PATCHDL_SCEI_DNAS_ROOT_PEM);
|
||||||
|
ca_blob.flags = CURL_BLOB_COPY;
|
||||||
|
|
||||||
|
curl = curl_easy_init();
|
||||||
|
if (!curl) {
|
||||||
|
curl_slist_free_all(rl);
|
||||||
|
if (sink.md) EVP_MD_CTX_free(sink.md);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
curl_easy_setopt(curl, CURLOPT_URL, url);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_RESOLVE, rl);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, piece_write_cb);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &sink);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_CAINFO_BLOB, &ca_blob);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 2L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, "DEFAULT@SECLEVEL=0");
|
||||||
|
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx -> error, no body written */
|
||||||
|
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, 1024L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, 30L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_USERAGENT, "patchdl/1.0");
|
||||||
|
if (ctx && ctx->abort) {
|
||||||
|
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, piece_xfer_cb);
|
||||||
|
curl_easy_setopt(curl, CURLOPT_XFERINFODATA, (void *)ctx->abort);
|
||||||
|
}
|
||||||
|
|
||||||
|
res = curl_easy_perform(curl);
|
||||||
|
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
|
||||||
|
curl_easy_cleanup(curl);
|
||||||
|
curl_slist_free_all(rl);
|
||||||
|
|
||||||
|
if (res != CURLE_OK) {
|
||||||
|
if (sink.md) EVP_MD_CTX_free(sink.md);
|
||||||
|
return -1; /* network error / abort */
|
||||||
|
}
|
||||||
|
if (file_size > 0 && sink.written != file_size) {
|
||||||
|
if (sink.md) EVP_MD_CTX_free(sink.md);
|
||||||
|
return -1; /* short or over-long -> failed */
|
||||||
|
}
|
||||||
|
if (sink.md) {
|
||||||
|
unsigned char dig[EVP_MAX_MD_SIZE];
|
||||||
|
unsigned int dl = 0;
|
||||||
|
char hex[2 * EVP_MAX_MD_SIZE + 1];
|
||||||
|
EVP_DigestFinal_ex(sink.md, dig, &dl);
|
||||||
|
EVP_MD_CTX_free(sink.md);
|
||||||
|
hex_encode(dig, dl, hex, sizeof(hex));
|
||||||
|
if (strcasecmp(hex, expected_sha256_or_null) != 0)
|
||||||
|
return -2; /* integrity mismatch */
|
||||||
|
}
|
||||||
|
fdatasync(fd); /* durable before the caller sets the done bit */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void
|
||||||
|
patchdl_manifest_free(patchdl_manifest_t *m) {
|
||||||
|
if (!m || !m->pieces) return;
|
||||||
|
for (int i = 0; i < m->count; i++) free(m->pieces[i].url);
|
||||||
|
free(m->pieces);
|
||||||
|
m->pieces = NULL;
|
||||||
|
m->count = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
patchdl_fetch_manifest(const char *manifest_url, patchdl_manifest_t *out) {
|
||||||
|
patchdl_buf_t buf;
|
||||||
|
const char *pieces, *pieces_end, *p;
|
||||||
|
int cap = 0, n = 0;
|
||||||
|
long long running = 0;
|
||||||
|
|
||||||
|
memset(out, 0, sizeof(*out));
|
||||||
|
if (patchdl_http_get(manifest_url, &buf)) return -1;
|
||||||
|
if (!buf.data || !buf.size) { free(buf.data); return -1; }
|
||||||
|
|
||||||
|
pieces = strstr(buf.data, "\"pieces\"");
|
||||||
|
if (!pieces || !(pieces = strchr(pieces, '['))) { free(buf.data); return -1; }
|
||||||
|
pieces_end = strchr(pieces, ']');
|
||||||
|
|
||||||
|
for (p = pieces; (p = strstr(p, "\"url\"")) && (!pieces_end || p < pieces_end); p += 5)
|
||||||
|
cap++;
|
||||||
|
if (cap <= 0) { free(buf.data); return -1; }
|
||||||
|
out->pieces = calloc((size_t)cap, sizeof(patchdl_piece_t));
|
||||||
|
if (!out->pieces) { free(buf.data); return -1; }
|
||||||
|
|
||||||
|
p = pieces;
|
||||||
|
while ((p = strstr(p, "\"url\"")) && (!pieces_end || p < pieces_end) && n < cap) {
|
||||||
|
char url[768] = {0};
|
||||||
|
unsigned long long sz = 0, off = 0;
|
||||||
|
const char *obj_end = strchr(p, '}');
|
||||||
|
|
||||||
|
if (json_string_after(p, "url", url, sizeof(url)))
|
||||||
|
break;
|
||||||
|
json_u64_after(p, "fileSize", &sz);
|
||||||
|
if (json_u64_after(p, "fileOffset", &off) != 0)
|
||||||
|
off = (unsigned long long)running; /* no offset -> assume contiguous */
|
||||||
|
|
||||||
|
/* Validate tiling: pieces must be in order, contiguous, non-empty. */
|
||||||
|
if ((long long)off != running || sz == 0) {
|
||||||
|
patchdl_manifest_free(out);
|
||||||
|
free(buf.data);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
out->pieces[n].url = strdup(url);
|
||||||
|
out->pieces[n].offset = (long long)off;
|
||||||
|
out->pieces[n].size = (long long)sz;
|
||||||
|
json_string_after(p, "hashValue", out->pieces[n].hash,
|
||||||
|
sizeof(out->pieces[n].hash));
|
||||||
|
if (!out->pieces[n].url) {
|
||||||
|
patchdl_manifest_free(out);
|
||||||
|
free(buf.data);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
running += (long long)sz;
|
||||||
|
n++;
|
||||||
|
out->count = n; /* keep current so manifest_free frees exactly n */
|
||||||
|
p = obj_end ? obj_end + 1 : p + 5;
|
||||||
|
}
|
||||||
|
free(buf.data);
|
||||||
|
if (n == 0) { patchdl_manifest_free(out); return -1; }
|
||||||
|
out->total = running; /* authoritative assembled size */
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
void
|
void
|
||||||
patchdl_net_diag(const char *url, char *out_json, size_t sz) {
|
patchdl_net_diag(const char *url, char *out_json, size_t sz) {
|
||||||
char host[256] = {0}, ip[INET_ADDRSTRLEN] = {0};
|
char host[256] = {0}, ip[INET_ADDRSTRLEN] = {0};
|
||||||
|
|||||||
@@ -11,8 +11,52 @@ typedef struct {
|
|||||||
patchdl_buf_t *patchdl_buf_new(void);
|
patchdl_buf_t *patchdl_buf_new(void);
|
||||||
void patchdl_buf_free(patchdl_buf_t *b);
|
void patchdl_buf_free(patchdl_buf_t *b);
|
||||||
|
|
||||||
|
/* Call once, single-threaded, before any concurrent download worker starts /
|
||||||
|
after they have all joined. curl's global/OpenSSL init is otherwise lazy and
|
||||||
|
races across threads. */
|
||||||
|
void patchdl_net_global_init(void);
|
||||||
|
void patchdl_net_global_cleanup(void);
|
||||||
|
|
||||||
int patchdl_http_get(const char *url, patchdl_buf_t *out);
|
int patchdl_http_get(const char *url, patchdl_buf_t *out);
|
||||||
|
|
||||||
|
/* ---- parallel piece download (used by the connection pool) ------------- */
|
||||||
|
|
||||||
|
/* One piece of a split manifest package. `url` is heap-allocated. */
|
||||||
|
typedef struct {
|
||||||
|
char *url;
|
||||||
|
long long offset; /* byte offset of this piece in the assembled file */
|
||||||
|
long long size; /* exact length of this piece */
|
||||||
|
char hash[80]; /* manifest SHA-256 hex, or "" */
|
||||||
|
} patchdl_piece_t;
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
patchdl_piece_t *pieces;
|
||||||
|
int count;
|
||||||
|
long long total; /* assembled file size = sum of piece sizes */
|
||||||
|
} patchdl_manifest_t;
|
||||||
|
|
||||||
|
/* Fetch + parse a Sony JSON manifest into a validated, contiguously-tiled
|
||||||
|
piece list. Returns 0 on success (caller frees with patchdl_manifest_free),
|
||||||
|
-1 on fetch/parse/tiling failure. */
|
||||||
|
int patchdl_fetch_manifest(const char *manifest_url, patchdl_manifest_t *out);
|
||||||
|
void patchdl_manifest_free(patchdl_manifest_t *m);
|
||||||
|
|
||||||
|
/* Live state shared with one in-flight piece download. The worker owns these;
|
||||||
|
the curl callbacks read `abort` (set elsewhere) and publish progress into
|
||||||
|
`bytes_slot` (single-writer per worker slot). */
|
||||||
|
typedef struct {
|
||||||
|
volatile long long *bytes_slot; /* bytes written so far for this piece */
|
||||||
|
volatile int *abort; /* non-zero -> stop this transfer */
|
||||||
|
} patchdl_piece_ctx_t;
|
||||||
|
|
||||||
|
/* Download one whole piece and pwrite it into `fd` at `file_offset`. Concurrent
|
||||||
|
non-overlapping pieces of the same fd are safe. Returns 0 on success (and
|
||||||
|
fdatasyncs fd), -1 on network/IO/abort, -2 on a SHA-256 mismatch. */
|
||||||
|
int patchdl_http_download_piece(const char *url, int fd,
|
||||||
|
long long file_offset, long long file_size,
|
||||||
|
const char *expected_sha256_or_null,
|
||||||
|
patchdl_piece_ctx_t *ctx);
|
||||||
|
|
||||||
/* Progress callback. Return non-zero to ABORT the in-flight download (used to
|
/* Progress callback. Return non-zero to ABORT the in-flight download (used to
|
||||||
cancel large patch downloads); return 0 to continue. */
|
cancel large patch downloads); return 0 to continue. */
|
||||||
typedef int (*patchdl_download_progress_cb)(void *ctx,
|
typedef int (*patchdl_download_progress_cb)(void *ctx,
|
||||||
|
|||||||
+783
-286
File diff suppressed because it is too large.
Load diff
+110
-74
@@ -29,6 +29,7 @@ const fallback = {
|
|||||||
delete_pkg_after_install: true,
|
delete_pkg_after_install: true,
|
||||||
verify_downloads: false,
|
verify_downloads: false,
|
||||||
home_shortcut: true,
|
home_shortcut: true,
|
||||||
|
max_connections: 4,
|
||||||
source_policy: {
|
source_policy: {
|
||||||
official: { allow_check: true, allow_download: true, allow_install: true },
|
official: { allow_check: true, allow_download: true, allow_install: true },
|
||||||
external: { allow_check: true, allow_download: true, allow_install: true },
|
external: { allow_check: true, allow_download: true, allow_install: true },
|
||||||
@@ -116,6 +117,7 @@ function bindElements() {
|
|||||||
deleteAfterInstall: document.getElementById("deleteAfterInstall"),
|
deleteAfterInstall: document.getElementById("deleteAfterInstall"),
|
||||||
verifyDownloads: document.getElementById("verifyDownloads"),
|
verifyDownloads: document.getElementById("verifyDownloads"),
|
||||||
homeShortcut: document.getElementById("homeShortcut"),
|
homeShortcut: document.getElementById("homeShortcut"),
|
||||||
|
maxConnections: document.getElementById("maxConnections"),
|
||||||
refreshBtn: document.getElementById("refreshBtn"),
|
refreshBtn: document.getElementById("refreshBtn"),
|
||||||
saveBtn: document.getElementById("saveBtn"),
|
saveBtn: document.getElementById("saveBtn"),
|
||||||
clearLogBtn: document.getElementById("clearLogBtn"),
|
clearLogBtn: document.getElementById("clearLogBtn"),
|
||||||
@@ -173,27 +175,26 @@ async function loadInitialData() {
|
|||||||
// /api/titles carries no client-only progress flags, so preserve them across a
|
// /api/titles carries no client-only progress flags, so preserve them across a
|
||||||
// refresh — otherwise an in-flight download/install flips back to a clickable
|
// refresh — otherwise an in-flight download/install flips back to a clickable
|
||||||
// button mid-operation.
|
// button mid-operation.
|
||||||
|
// Carry client-only flags across a refresh; the pool's job list is the source
|
||||||
|
// of truth for download state and is reconciled right after.
|
||||||
const prev = new Map(state.titles.map((g) => [g.title_id, g]));
|
const prev = new Map(state.titles.map((g) => [g.title_id, g]));
|
||||||
titles.forEach((g) => {
|
titles.forEach((g) => {
|
||||||
const old = prev.get(g.title_id);
|
const old = prev.get(g.title_id);
|
||||||
// Carry client-only progress flags across a refresh — unless the server now
|
|
||||||
// reports the title up to date (the patch applied), in which case drop them.
|
|
||||||
if (old && g.status !== "up_to_date") {
|
if (old && g.status !== "up_to_date") {
|
||||||
if (old.downloading) g.downloading = true;
|
|
||||||
if (old.downloaded) g.downloaded = true;
|
if (old.downloaded) g.downloaded = true;
|
||||||
if (old.installing) g.installing = true;
|
if (old.installing) g.installing = true;
|
||||||
|
if (old._autoInstalled) g._autoInstalled = true;
|
||||||
if (old._localDownloading) g._localDownloading = true;
|
if (old._localDownloading) g._localDownloading = true;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
// Reconcile active downloads the server reports, so progress + Cancel show even
|
|
||||||
// after a hard reload or a download started from another session/device.
|
|
||||||
const activeDl = new Set(downloads.map((d) => d.title_id));
|
|
||||||
titles.forEach((g) => { if (activeDl.has(g.title_id)) g.downloading = true; });
|
|
||||||
|
|
||||||
state = { ...state, status, config, titles, downloads };
|
state = { ...state, status, config, titles, downloads };
|
||||||
|
reconcileFromJobs(downloads);
|
||||||
|
|
||||||
render();
|
render();
|
||||||
if (state.downloads.length) startDownloadPolling();
|
if (downloads.some((j) => j.state === "active" || j.state === "queued") ||
|
||||||
|
state.titles.some((g) => g._localDownloading))
|
||||||
|
startDownloadPolling();
|
||||||
showToast(state.usingFallback ? "Demo data loaded. API is not reachable yet." : "Data refreshed.");
|
showToast(state.usingFallback ? "Demo data loaded. API is not reachable yet." : "Data refreshed.");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -234,6 +235,7 @@ function renderSettings() {
|
|||||||
els.deleteAfterInstall.checked = Boolean(state.config.delete_pkg_after_install);
|
els.deleteAfterInstall.checked = Boolean(state.config.delete_pkg_after_install);
|
||||||
if (els.verifyDownloads) els.verifyDownloads.checked = Boolean(state.config.verify_downloads);
|
if (els.verifyDownloads) els.verifyDownloads.checked = Boolean(state.config.verify_downloads);
|
||||||
if (els.homeShortcut) els.homeShortcut.checked = state.config.home_shortcut !== false;
|
if (els.homeShortcut) els.homeShortcut.checked = state.config.home_shortcut !== false;
|
||||||
|
if (els.maxConnections) els.maxConnections.value = state.config.max_connections || 4;
|
||||||
els.allowlistHosts.replaceChildren(...(state.config.cdn_allowlist || []).map((host) => {
|
els.allowlistHosts.replaceChildren(...(state.config.cdn_allowlist || []).map((host) => {
|
||||||
const chip = document.createElement("span");
|
const chip = document.createElement("span");
|
||||||
chip.className = "host-chip";
|
chip.className = "host-chip";
|
||||||
@@ -441,6 +443,7 @@ function progressMetaHtml(d) {
|
|||||||
const done = Number(d.bytes) || 0;
|
const done = Number(d.bytes) || 0;
|
||||||
const total = Number(d.total_bytes) || 0;
|
const total = Number(d.total_bytes) || 0;
|
||||||
const speed = Number(d._speed) || 0;
|
const speed = Number(d._speed) || 0;
|
||||||
|
if (d.state === "queued") return `<span>Queued — waiting for a free slot</span>`;
|
||||||
const parts = [];
|
const parts = [];
|
||||||
parts.push(`<span><b>${formatBytes(done)}</b>${total > 0 ? ` / ${formatBytes(total)}` : ""}</span>`);
|
parts.push(`<span><b>${formatBytes(done)}</b>${total > 0 ? ` / ${formatBytes(total)}` : ""}</span>`);
|
||||||
if (speed > 0) parts.push(`<span><b>${formatBytes(speed)}/s</b></span>`);
|
if (speed > 0) parts.push(`<span><b>${formatBytes(speed)}/s</b></span>`);
|
||||||
@@ -463,55 +466,99 @@ function stopDownloadPolling() {
|
|||||||
downloadPollTimer = null;
|
downloadPollTimer = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Map the pool's job list onto per-title flags, and auto-install once a job
|
||||||
|
// finishes if "install after download" is on.
|
||||||
|
function reconcileFromJobs(jobs) {
|
||||||
|
const byId = new Map((jobs || []).map((j) => [j.title_id, j]));
|
||||||
|
state.titles.forEach((g) => {
|
||||||
|
const j = byId.get(g.title_id);
|
||||||
|
if (!j) {
|
||||||
|
// The pool never produced a job for our local intent: time it out so the
|
||||||
|
// card can't wedge in "Downloading" forever (server restart between POST
|
||||||
|
// and poll, or an unexpected response shape).
|
||||||
|
if (g._localDownloading && g._localSince &&
|
||||||
|
Date.now() - g._localSince > 12000) {
|
||||||
|
g._localDownloading = false;
|
||||||
|
}
|
||||||
|
if (g.downloading && !g._localDownloading) g.downloading = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
g._localDownloading = false; // the pool now tracks it
|
||||||
|
if (j.state === "active" || j.state === "queued") {
|
||||||
|
g.downloading = true;
|
||||||
|
g.resumable = false;
|
||||||
|
g._wasActive = true;
|
||||||
|
} else if (j.state === "paused") {
|
||||||
|
g.downloading = false;
|
||||||
|
g.resumable = true;
|
||||||
|
g.partial_bytes = Number(j.bytes) || g.partial_bytes || 0;
|
||||||
|
g._wasActive = false;
|
||||||
|
} else if (j.state === "done") {
|
||||||
|
g.downloading = false;
|
||||||
|
g.resumable = false;
|
||||||
|
g.downloaded = true;
|
||||||
|
g._wasActive = false;
|
||||||
|
if (state.config.install_after_download && !g.installing && !g._autoInstalled) {
|
||||||
|
g._autoInstalled = true;
|
||||||
|
doInstall(g);
|
||||||
|
}
|
||||||
|
} else if (j.state === "error") {
|
||||||
|
// The server keeps the partial + sidecar (resumable) on a post-retry
|
||||||
|
// network failure. Reflect that so primaryButton shows "Resume" and the
|
||||||
|
// red Cancel stays available to delete the kept partial.
|
||||||
|
const bytes = Number(j.bytes) || 0;
|
||||||
|
if (g._wasActive) {
|
||||||
|
showToast(`${g.name}: download failed${bytes > 0 ? " — partial kept, press Resume to continue" : "."}`);
|
||||||
|
}
|
||||||
|
g._wasActive = false;
|
||||||
|
g.downloading = false;
|
||||||
|
g.resumable = bytes > 0;
|
||||||
|
g.partial_bytes = bytes || g.partial_bytes || 0;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function refreshDownloads() {
|
async function refreshDownloads() {
|
||||||
let downloads;
|
let jobs;
|
||||||
try {
|
try {
|
||||||
const response = await fetch(API.downloads, { cache: "no-store" });
|
const response = await fetch(API.downloads, { cache: "no-store" });
|
||||||
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
||||||
downloads = await response.json();
|
jobs = await response.json();
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return; // keep last state on a transient failure
|
return; // keep last state on a transient failure
|
||||||
}
|
}
|
||||||
|
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
const activeIds = new Set();
|
const ids = new Set();
|
||||||
downloads.forEach((d) => {
|
jobs.forEach((j) => {
|
||||||
activeIds.add(d.title_id);
|
ids.add(j.title_id);
|
||||||
const done = Number(d.bytes) || 0;
|
const done = Number(j.bytes) || 0;
|
||||||
const prev = dlMeta[d.title_id];
|
const prev = dlMeta[j.title_id];
|
||||||
if (prev && now > prev.t) {
|
if (prev && now > prev.t) {
|
||||||
if (done >= prev.bytes) {
|
if (done >= prev.bytes) {
|
||||||
const inst = ((done - prev.bytes) * 1000) / (now - prev.t); // bytes/s
|
const inst = ((done - prev.bytes) * 1000) / (now - prev.t); // bytes/s
|
||||||
prev.speed = prev.speed ? prev.speed * 0.5 + inst * 0.5 : inst; // smoothed
|
prev.speed = prev.speed ? prev.speed * 0.5 + inst * 0.5 : inst; // smoothed
|
||||||
} else {
|
} else {
|
||||||
prev.speed = 0; // counter went backwards -> re-baseline, no stale speed
|
prev.speed = 0; // counter went backwards -> re-baseline
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const meta = prev || (dlMeta[d.title_id] = { speed: 0 });
|
const meta = prev || (dlMeta[j.title_id] = { speed: 0 });
|
||||||
meta.bytes = done;
|
meta.bytes = done;
|
||||||
meta.t = now;
|
meta.t = now;
|
||||||
d._speed = meta.speed || 0;
|
j._speed = meta.speed || 0;
|
||||||
});
|
});
|
||||||
Object.keys(dlMeta).forEach((id) => { if (!activeIds.has(id)) delete dlMeta[id]; });
|
Object.keys(dlMeta).forEach((id) => { if (!ids.has(id)) delete dlMeta[id]; });
|
||||||
|
|
||||||
state.downloads = downloads;
|
state.downloads = jobs;
|
||||||
|
|
||||||
// Reconcile downloading flags with the server. A structural change (a download
|
|
||||||
// appeared or finished) needs a full re-render to add/remove the progress block
|
|
||||||
// and morph the button; otherwise update the bar in place.
|
|
||||||
const before = downloadingIds();
|
const before = downloadingIds();
|
||||||
state.titles.forEach((g) => {
|
reconcileFromJobs(jobs);
|
||||||
if (activeIds.has(g.title_id)) g.downloading = true;
|
|
||||||
else if (g.downloading && !g._localDownloading) g.downloading = false;
|
|
||||||
});
|
|
||||||
if (downloadingIds() !== before) renderGames();
|
if (downloadingIds() !== before) renderGames();
|
||||||
else applyDownloadProgress();
|
else applyDownloadProgress();
|
||||||
|
|
||||||
if (!downloads.length && !state.titles.some((g) => g.downloading)) {
|
const busy = jobs.some((j) => j.state === "active" || j.state === "queued") ||
|
||||||
if (++emptyPolls >= 3) stopDownloadPolling();
|
state.titles.some((g) => g._localDownloading);
|
||||||
} else {
|
if (!busy) { if (++emptyPolls >= 3) stopDownloadPolling(); }
|
||||||
emptyPolls = 0;
|
else emptyPolls = 0;
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function downloadingIds() {
|
function downloadingIds() {
|
||||||
@@ -522,6 +569,11 @@ function downloadingIds() {
|
|||||||
function applyDownloadProgress() {
|
function applyDownloadProgress() {
|
||||||
let needRender = false;
|
let needRender = false;
|
||||||
state.downloads.forEach((d) => {
|
state.downloads.forEach((d) => {
|
||||||
|
// Only titles currently downloading render a progress tile; paused/done/error
|
||||||
|
// jobs linger in the pool list but have no .progress bar — skip them so a
|
||||||
|
// missing tile for a non-downloading title doesn't force a full rebuild.
|
||||||
|
const g = state.titles.find((t) => t.title_id === d.title_id);
|
||||||
|
if (!g || !g.downloading) return;
|
||||||
const card = els.gameGrid.querySelector(`[data-title-id="${d.title_id}"]`);
|
const card = els.gameGrid.querySelector(`[data-title-id="${d.title_id}"]`);
|
||||||
const bar = card && card.querySelector(".card-progress .progress > i");
|
const bar = card && card.querySelector(".card-progress .progress > i");
|
||||||
const meta = card && card.querySelector(".card-progress .progress-meta");
|
const meta = card && card.querySelector(".card-progress .progress-meta");
|
||||||
@@ -563,6 +615,9 @@ async function saveConfig() {
|
|||||||
delete_pkg_after_install: els.deleteAfterInstall.checked,
|
delete_pkg_after_install: els.deleteAfterInstall.checked,
|
||||||
verify_downloads: els.verifyDownloads ? els.verifyDownloads.checked : Boolean(state.config.verify_downloads),
|
verify_downloads: els.verifyDownloads ? els.verifyDownloads.checked : Boolean(state.config.verify_downloads),
|
||||||
home_shortcut: els.homeShortcut ? els.homeShortcut.checked : state.config.home_shortcut !== false,
|
home_shortcut: els.homeShortcut ? els.homeShortcut.checked : state.config.home_shortcut !== false,
|
||||||
|
max_connections: els.maxConnections
|
||||||
|
? Math.max(1, Math.min(16, parseInt(els.maxConnections.value, 10) || 4))
|
||||||
|
: (state.config.max_connections || 4),
|
||||||
};
|
};
|
||||||
try {
|
try {
|
||||||
await postJson(API.config, config);
|
await postJson(API.config, config);
|
||||||
@@ -578,19 +633,31 @@ async function saveConfig() {
|
|||||||
|
|
||||||
/* ---------------- actions (data layer) ---------------- */
|
/* ---------------- actions (data layer) ---------------- */
|
||||||
|
|
||||||
|
// Enqueue a download. The pool returns immediately (202); progress, completion
|
||||||
|
// and (if configured) auto-install are driven by reconcileFromJobs() on poll.
|
||||||
async function doDownload(game) {
|
async function doDownload(game) {
|
||||||
game.downloading = true;
|
game.downloading = true;
|
||||||
game._localDownloading = true; // this client owns it; don't let a poll clear it
|
game._localDownloading = true; // until the pool reports a job for this title
|
||||||
|
game._localSince = Date.now(); // bounded in reconcileFromJobs if no job appears
|
||||||
|
game._autoInstalled = false;
|
||||||
state.downloads = state.downloads.filter((i) => i.title_id !== game.title_id);
|
state.downloads = state.downloads.filter((i) => i.title_id !== game.title_id);
|
||||||
state.downloads.push({ title_id: game.title_id, name: game.name, version: game.compatible_version || "", progress: 0, bytes: 0, total_bytes: 0 });
|
state.downloads.push({ title_id: game.title_id, name: game.name,
|
||||||
state.logs.push(`[${timeNow()}] Download started: ${game.title_id} ${game.compatible_version}`);
|
version: game.compatible_version || "",
|
||||||
|
state: "queued", progress: 0, bytes: 0, total_bytes: 0 });
|
||||||
renderGames();
|
renderGames();
|
||||||
renderLogs();
|
|
||||||
startDownloadPolling();
|
startDownloadPolling();
|
||||||
|
|
||||||
let r;
|
|
||||||
try {
|
try {
|
||||||
r = await postJson(API.action(game.title_id, "download"), {});
|
const r = await postJson(API.action(game.title_id, "download"), {});
|
||||||
|
if (r && r.downloaded && r.already) {
|
||||||
|
game.downloading = false;
|
||||||
|
game._localDownloading = false;
|
||||||
|
game.downloaded = true;
|
||||||
|
renderGames();
|
||||||
|
} else {
|
||||||
|
state.logs.push(`[${timeNow()}] Download queued: ${game.title_id} ${game.compatible_version || ""}`);
|
||||||
|
renderLogs();
|
||||||
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
game.downloading = false;
|
game.downloading = false;
|
||||||
game._localDownloading = false;
|
game._localDownloading = false;
|
||||||
@@ -598,40 +665,8 @@ async function doDownload(game) {
|
|||||||
const why = reasonText(error);
|
const why = reasonText(error);
|
||||||
state.logs.push(`[${timeNow()}] download ${game.title_id} blocked: ${why}`);
|
state.logs.push(`[${timeNow()}] download ${game.title_id} blocked: ${why}`);
|
||||||
showToast(`${game.name}: ${why}`);
|
showToast(`${game.name}: ${why}`);
|
||||||
renderGames(); renderLogs(); stopDownloadPolling();
|
renderGames(); renderLogs();
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
game.downloading = false;
|
|
||||||
game._localDownloading = false;
|
|
||||||
state.downloads = state.downloads.filter((i) => i.title_id !== game.title_id);
|
|
||||||
|
|
||||||
// Cancel / pause / soft failure returns HTTP 200 with ok:false (not thrown).
|
|
||||||
if (!r || r.ok === false) {
|
|
||||||
game.downloaded = false;
|
|
||||||
if (r && r.paused) {
|
|
||||||
// Paused: keep the partial, mark resumable so the tile shows Resume.
|
|
||||||
game.resumable = true;
|
|
||||||
game.partial_bytes = r.bytes || game.partial_bytes || 0;
|
|
||||||
state.logs.push(`[${timeNow()}] Download paused: ${game.title_id} (${formatBytes(game.partial_bytes)})`);
|
|
||||||
showToast(`${game.name}: paused at ${formatBytes(game.partial_bytes)}.`);
|
|
||||||
} else {
|
|
||||||
const what = r && r.cancelled ? "cancelled" : "failed";
|
|
||||||
state.logs.push(`[${timeNow()}] Download ${what}: ${game.title_id}`);
|
|
||||||
showToast(`${game.name}: download ${what}.`);
|
|
||||||
}
|
|
||||||
renderGames(); renderLogs(); stopDownloadPolling();
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
game.downloaded = true;
|
|
||||||
game.resumable = false;
|
|
||||||
game.partial_bytes = 0;
|
|
||||||
const sz = r && r.bytes ? formatBytes(r.bytes) : "?";
|
|
||||||
state.logs.push(`[${timeNow()}] Downloaded ${game.title_id} ${game.compatible_version} (${sz}, internal)`);
|
|
||||||
showToast(`${game.name}: downloaded ${sz}.`);
|
|
||||||
renderGames(); renderLogs(); stopDownloadPolling();
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function doInstall(game) {
|
async function doInstall(game) {
|
||||||
@@ -680,11 +715,12 @@ async function cancelDownload(titleId) {
|
|||||||
async function runTitleAction(titleId, action) {
|
async function runTitleAction(titleId, action) {
|
||||||
const game = state.titles.find((i) => i.title_id === titleId);
|
const game = state.titles.find((i) => i.title_id === titleId);
|
||||||
if (!game) return;
|
if (!game) return;
|
||||||
if (action === "download") await doDownload(game);
|
// "update" and "download" both just enqueue; for "update" (install-after-
|
||||||
|
// download on) the reconciler auto-installs once the pool reports it done.
|
||||||
|
if (action === "download" || action === "update") await doDownload(game);
|
||||||
else if (action === "install") await doInstall(game);
|
else if (action === "install") await doInstall(game);
|
||||||
else if (action === "pause") await doPause(game);
|
else if (action === "pause") await doPause(game);
|
||||||
else if (action === "cancel") await cancelDownload(titleId);
|
else if (action === "cancel") await cancelDownload(titleId);
|
||||||
else if (action === "update") { if (await doDownload(game)) await doInstall(game); }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pause only sends the signal; the in-flight doDownload() request returns its
|
// Pause only sends the signal; the in-flight doDownload() request returns its
|
||||||
|
|||||||
@@ -186,6 +186,11 @@
|
|||||||
<span class="track"></span>
|
<span class="track"></span>
|
||||||
</span>
|
</span>
|
||||||
</label>
|
</label>
|
||||||
|
<label class="field">
|
||||||
|
<span>Parallel download connections</span>
|
||||||
|
<input id="maxConnections" type="number" min="1" max="16" step="1"
|
||||||
|
title="Connections used to download a patch in parallel — applies after a restart" />
|
||||||
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="allowlist">
|
<div class="allowlist">
|
||||||
|
|||||||
Reference in new issue
Block a user