From 66e4912485b17b325f6309a067f5ab93dc50d666 Mon Sep 17 00:00:00 2001 From: Knutwurst <36196269+knutwurst@users.noreply.github.com> Date: Wed, 24 Jun 2026 08:58:32 +0200 Subject: [PATCH] Resume interrupted downloads across a reboot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An interrupted download (cancel excepted) now keeps its partial package on disk instead of deleting it, and records the manifest it belongs to in a sidecar (state.json). On the next start the title is flagged resumable and the UI shows a "Paused — X downloaded" note with a Resume button. Resume refetches the manifest, skips every piece already fully on disk, and re-fetches only the one partially-written piece (piece-granular, no HTTP range needed), appending the rest. The fresh-download path is unchanged. A partial belonging to a different/older manifest is dropped and the download starts clean; a corrupt (failed SHA-256) download is not kept. Survives a reboot: a killed payload runs no cleanup, so the partial and its sidecar persist under /data/patchdl until resumed, completed, or deleted. --- src/patchdl_net.c | 85 +++++++++++++++-------- src/patchdl_net.h | 7 +- src/patchdl_scan.h | 2 + src/patchdl_websrv.c | 156 ++++++++++++++++++++++++++++++++++++++++--- web/app.js | 30 +++++++-- 5 files changed, 234 insertions(+), 46 deletions(-) diff --git a/src/patchdl_net.c b/src/patchdl_net.c index e22bcfe..560441e 100644 --- a/src/patchdl_net.c +++ b/src/patchdl_net.c @@ -504,15 +504,13 @@ patchdl_http_download_manifest_progress(const char *manifest_url, const char *dest_path, long long *bytes_out, patchdl_download_progress_cb cb, - void *ctx, int verify) { + void *ctx, int verify, int resume) { patchdl_buf_t manifest; - const char *pieces; - const char *p; - FILE *fp; - long long total = 0; + const char *pieces, *pieces_end, *p; + FILE *fp = NULL; + long long total = 0, have = 0; unsigned long long manifest_total = 0; - int count = 0; - int rc = -1; + int count = 0, started, rc = -1; if (bytes_out) *bytes_out = 0; if (patchdl_http_get(manifest_url, &manifest)) @@ -529,14 +527,20 @@ patchdl_http_download_manifest_progress(const char *manifest_url, } /* Bound the scan to the pieces array; otherwise a later "url" key in the manifest (e.g. playgoChunkCrcUrl) could be appended as a bogus piece. */ - const char *pieces_end = strchr(pieces, ']'); + pieces_end = strchr(pieces, ']'); json_u64_after(manifest.data, "originalFileSize", &manifest_total); - fp = fopen(dest_path, "wb"); - if (!fp) { - free(manifest.data); - return -1; + /* Resume: reopen the existing partial and keep its bytes; else start clean. + Pieces already fully on disk are skipped, and the one piece that was only + partially written is dropped and re-fetched whole (piece-granular resume, + no HTTP range needed). */ + if (resume) { + fp = fopen(dest_path, "r+b"); + if (fp) { fseek(fp, 0, SEEK_END); have = ftell(fp); if (have < 0) have = 0; } } + if (!fp) { fp = fopen(dest_path, "wb"); have = 0; } + if (!fp) { free(manifest.data); return -1; } + started = (have <= 0); p = pieces; while ((p = strstr(p, "\"url\"")) && (!pieces_end || p < pieces_end)) { @@ -547,29 +551,51 @@ patchdl_http_download_manifest_progress(const char *manifest_url, unsigned long long offset = 0; int have_offset, drc; const char *obj_end = strchr(p, '}'); - progress_state_t progress = { - cb, - ctx, - total, - manifest_total ? (long long)manifest_total : 0 - }; if (json_string_after(p, "url", url, sizeof(url))) break; json_u64_after(p, "fileSize", &expected); have_offset = (json_u64_after(p, "fileOffset", &offset) == 0); - if (verify) - json_string_after(p, "hashValue", hash, sizeof(hash)); + + /* Piece already fully present from a previous run: skip the download. */ + if (!started && have_offset && expected && + have >= (long long)(offset + expected)) { + total = (long long)(offset + expected); + count++; + if (cb && cb(ctx, total, manifest_total ? (long long)manifest_total : total)) + goto done; + p = obj_end ? obj_end + 1 : p + 5; + continue; + } + + /* First piece we must (re)download: drop any partial bytes of it so the + appended data lines up exactly, then append from here on. */ + if (!started) { + long long start_at = have_offset ? (long long)offset : 0; + fflush(fp); + if (ftruncate(fileno(fp), (off_t)start_at) != 0) + goto done; /* can't resume cleanly; keep partial */ + fseek(fp, 0, SEEK_END); + total = start_at; + started = 1; + } /* Pieces are concatenated in array order; each one's fileOffset must - equal the bytes written so far. A manifest that lists them out of - order would otherwise silently produce a corrupt package. */ + equal the bytes written so far, or the package would be corrupt. */ if (have_offset && offset != (unsigned long long)total) goto done; - /* drc: 0 ok, -1 network/cancel, -2 SHA-256 mismatch (propagated out). */ - drc = http_download_to_file_progress(url, fp, &got, &progress, - hash[0] ? hash : NULL); + if (verify) + json_string_after(p, "hashValue", hash, sizeof(hash)); + + { + progress_state_t progress = { + cb, ctx, total, manifest_total ? (long long)manifest_total : 0 + }; + /* drc: 0 ok, -1 network/cancel, -2 SHA-256 mismatch. */ + drc = http_download_to_file_progress(url, fp, &got, &progress, + hash[0] ? hash : NULL); + } if (drc) { if (drc == -2) rc = -2; goto done; @@ -593,7 +619,8 @@ patchdl_http_download_manifest_progress(const char *manifest_url, done: fclose(fp); free(manifest.data); - if (rc) unlink(dest_path); + /* Keep the partial on failure so it can be resumed; the caller deletes it + on cancel or on a corrupt-verify (-2). */ return rc; } @@ -601,7 +628,7 @@ int patchdl_http_download_manifest(const char *manifest_url, const char *dest_path, long long *bytes_out) { return patchdl_http_download_manifest_progress(manifest_url, dest_path, - bytes_out, NULL, NULL, 0); + bytes_out, NULL, NULL, 0, 0); } void @@ -696,8 +723,8 @@ patchdl_http_download_manifest_progress(const char *manifest_url, const char *dest_path, long long *bytes_out, patchdl_download_progress_cb cb, - void *ctx, int verify) { - (void)cb; (void)ctx; (void)verify; + void *ctx, int verify, int resume) { + (void)cb; (void)ctx; (void)verify; (void)resume; return patchdl_http_download_manifest(manifest_url, dest_path, bytes_out); } diff --git a/src/patchdl_net.h b/src/patchdl_net.h index 54af4de..94698fd 100644 --- a/src/patchdl_net.h +++ b/src/patchdl_net.h @@ -29,14 +29,17 @@ int patchdl_http_download_progress(const char *url, const char *dest_path, /* Download a Sony JSON package manifest by concatenating every entry in "pieces" into one installable PKG. When `verify` is non-zero each piece is - checked against its manifest SHA-256 (a mismatch returns -2). */ + checked against its manifest SHA-256 (a mismatch returns -2). When `resume` + is non-zero an existing partial at dest_path is kept: fully-downloaded pieces + are skipped and only the remainder is fetched (survives a reboot). On any + failure the partial is left in place for a later resume. */ int patchdl_http_download_manifest(const char *manifest_url, const char *dest_path, long long *bytes_out); int patchdl_http_download_manifest_progress(const char *manifest_url, const char *dest_path, long long *bytes_out, patchdl_download_progress_cb cb, - void *ctx, int verify); + void *ctx, int verify, int resume); /* Diagnostic: run the GET pipeline for `url` and write a JSON report (dns result/ip, curl code, http status, bytes) into `out_json`. */ diff --git a/src/patchdl_scan.h b/src/patchdl_scan.h index de9de97..80ef6a7 100644 --- a/src/patchdl_scan.h +++ b/src/patchdl_scan.h @@ -28,6 +28,8 @@ typedef struct { char patch_storage_title_id[16]; /* title id embedded in delta_url */ int verxml_done; int enabled; /* user policy, persisted in config.json */ + int resumable; /* a partial download is on disk */ + long long partial_bytes; /* size of that partial, for the UI */ } patchdl_title_t; int patchdl_scan(patchdl_title_t **titles_out, size_t *count_out); diff --git a/src/patchdl_websrv.c b/src/patchdl_websrv.c index d6659d4..96220ac 100644 --- a/src/patchdl_websrv.c +++ b/src/patchdl_websrv.c @@ -589,6 +589,8 @@ build_titles_json(void) { (!t->patch_title_id[0] || !strncmp(t->patch_title_id, t->title_id, 9)) ? "true" : "false"); jbuf_appendf(&j, ",\"enabled\":%s", t->enabled ? "true" : "false"); + jbuf_appendf(&j, ",\"resumable\":%s", t->resumable ? "true" : "false"); + jbuf_appendf(&j, ",\"partial_bytes\":%lld", t->partial_bytes); jbuf_append(&j, ",\"mode\":"); jbuf_append_str(&j, title_mode_str(t->source_type)); jbuf_append(&j, ",\"queued\":false"); @@ -828,6 +830,8 @@ remove_title_dir(const char *title_id) { rmdir(dir); } +static void set_title_resumable(const char *title_id, int resumable, long long bytes); + /* Cancel an in-progress download for this title (the worker aborts and removes the partial), or delete an already-downloaded package if nothing is running. */ static enum MHD_Result @@ -842,8 +846,10 @@ do_cancel(struct MHD_Connection *conn, const char *title_id) { } pthread_mutex_unlock(&g_mutex); - if (!was_active) + if (!was_active) { remove_title_dir(title_id); + set_title_resumable(title_id, 0, 0); + } snprintf(resp, sizeof(resp), "{\"ok\":true,\"cancelled\":%s,\"deleted\":true}", @@ -851,13 +857,78 @@ do_cancel(struct MHD_Connection *conn, const char *title_id) { return queue_json_owned(conn, MHD_HTTP_OK, strdup(resp)); } +/* ---------- resume sidecar (/data/patchdl//state.json) ----------- */ + +static long long +file_size(const char *path) { + struct stat st; + if (stat(path, &st) == 0 && S_ISREG(st.st_mode)) return (long long)st.st_size; + return -1; +} + +static void +title_state_path(const char *title_id, char *out, size_t sz) { + snprintf(out, sz, "%s/%s/state.json", PATCHDL_DL_DIR, title_id); +} + +/* Record which manifest a partial download belongs to, so a resume only + continues a partial that matches the current patch. Sony CDN URLs contain no + characters that need JSON escaping, so the value is embedded verbatim. */ +static void +write_dl_state(const char *title_id, const char *manifest_url) { + char path[320]; + FILE *f; + title_state_path(title_id, path, sizeof(path)); + f = fopen(path, "w"); + if (!f) return; + fprintf(f, "{\"manifest_url\":\"%s\"}\n", manifest_url); + fclose(f); +} + +static int +read_dl_state_url(const char *title_id, char *out, size_t sz) { + char path[320], buf[1280]; + FILE *f; + size_t n; + out[0] = '\0'; + title_state_path(title_id, path, sizeof(path)); + f = fopen(path, "r"); + if (!f) return -1; + n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + buf[n] = '\0'; + json_get_str(buf, "manifest_url", out, sz); + return out[0] ? 0 : -1; +} + +static void +remove_dl_state(const char *title_id) { + char path[320]; + title_state_path(title_id, path, sizeof(path)); + unlink(path); +} + +/* Keep the resumable flag (set at startup) in sync after a download finishes or + its partial is deleted, so /api/titles stops reporting a stale partial. */ +static void +set_title_resumable(const char *title_id, int resumable, long long bytes) { + pthread_mutex_lock(&g_mutex); + for (size_t i = 0; i < g_title_count; i++) + if (!strcmp(g_titles[i].title_id, title_id)) { + g_titles[i].resumable = resumable; + g_titles[i].partial_bytes = bytes; + break; + } + pthread_mutex_unlock(&g_mutex); +} + static enum MHD_Result do_download(struct MHD_Connection *conn, const char *title_id, patchdl_source_t src, const char *patch_url, const char *name, const char *version, int enabled) { char dir[256], dest[320], resp[640]; long long bytes = 0; - int verify = 0, dlrc; + int verify = 0, dlrc, is_manifest, resume = 0; if (!enabled) return queue_json(conn, MHD_HTTP_FORBIDDEN, @@ -877,6 +948,22 @@ do_download(struct MHD_Connection *conn, const char *title_id, mkdir(dir, 0777); title_pkg_path(title_id, patch_url, dest, sizeof(dest)); + /* Resume: keep an existing partial only if it belongs to THIS manifest + (recorded in the sidecar); a partial from a different/older patch is + dropped. The partial survives a reboot because a killed process runs no + cleanup. Only manifest downloads resume. */ + is_manifest = url_is_manifest(patch_url); + if (is_manifest && file_size(dest) > 0) { + char prev_url[1024] = {0}; + if (read_dl_state_url(title_id, prev_url, sizeof(prev_url)) == 0 && + !strcmp(prev_url, patch_url)) + resume = 1; + else + unlink(dest); /* stale partial from a different patch */ + } + if (is_manifest) + write_dl_state(title_id, patch_url); + pthread_mutex_lock(&g_mutex); if (g_dl.active) { pthread_mutex_unlock(&g_mutex); @@ -892,9 +979,9 @@ do_download(struct MHD_Connection *conn, const char *title_id, verify = g_cfg.verify_downloads; pthread_mutex_unlock(&g_mutex); - dlrc = url_is_manifest(patch_url) + dlrc = is_manifest ? patchdl_http_download_manifest_progress(patch_url, dest, &bytes, - download_progress_cb, NULL, verify) + download_progress_cb, NULL, verify, resume) : patchdl_http_download_progress(patch_url, dest, &bytes, download_progress_cb, NULL); if (dlrc) { @@ -905,16 +992,27 @@ do_download(struct MHD_Connection *conn, const char *title_id, g_dl.cancel = 0; pthread_mutex_unlock(&g_mutex); - /* The download function already unlinked the partial file on failure; - drop the now-empty title dir too. */ - unlink(dest); - rmdir(dir); - - if (was_cancel) + if (was_cancel) { + /* user cancelled: drop the partial + its resume sidecar */ + unlink(dest); + remove_dl_state(title_id); + rmdir(dir); + set_title_resumable(title_id, 0, 0); return queue_json(conn, MHD_HTTP_OK, "{\"ok\":false,\"cancelled\":true," "\"reason\":\"download_cancelled\"}"); - /* -2 = a piece failed its SHA-256 (only possible when verify is on). */ + } + if (dlrc == -2) { + /* corrupt data (failed SHA-256): don't keep it for resume */ + unlink(dest); + remove_dl_state(title_id); + rmdir(dir); + set_title_resumable(title_id, 0, 0); + } else { + /* network failure: the partial + sidecar are kept, and the title is + now resumable (also survives a reboot). */ + set_title_resumable(title_id, 1, file_size(dest)); + } snprintf(resp, sizeof(resp), "{\"ok\":false,\"reason\":\"%s\"}", dlrc == -2 ? "piece_verify_failed" : "download_failed"); return queue_json_owned(conn, MHD_HTTP_BAD_GATEWAY, strdup(resp)); @@ -926,6 +1024,9 @@ do_download(struct MHD_Connection *conn, const char *title_id, g_dl.active = 0; pthread_mutex_unlock(&g_mutex); + remove_dl_state(title_id); /* complete: drop sidecar, keep the pkg */ + set_title_resumable(title_id, 0, 0); /* no longer a partial */ + /* shadowmount: download allowed, install is not (per source policy). */ snprintf(resp, sizeof(resp), "{\"ok\":true,\"downloaded\":true,\"bytes\":%lld,\"path\":\"%s\"," @@ -1204,6 +1305,35 @@ on_request(void *cls, struct MHD_Connection *conn, const char *url, return queue_asset(conn, url); } +/* A title is resumable when its download dir holds both a resume sidecar and a + partial .pkg. Records the partial size for the UI. Single-threaded startup. */ +static void +detect_resumable_partials(void) { + char dir[288], state[320], pkg[576]; + DIR *d; + struct dirent *e; + + for (size_t i = 0; i < g_title_count; i++) { + patchdl_title_t *t = &g_titles[i]; + title_state_path(t->title_id, state, sizeof(state)); + if (file_size(state) < 0) continue; /* no sidecar -> not resumable */ + snprintf(dir, sizeof(dir), "%s/%s", PATCHDL_DL_DIR, t->title_id); + d = opendir(dir); + if (!d) continue; + while ((e = readdir(d))) { + size_t nl = strlen(e->d_name); + if (nl > 4 && !strcmp(e->d_name + nl - 4, ".pkg")) { + long long sz; + snprintf(pkg, sizeof(pkg), "%s/%s", dir, e->d_name); + sz = file_size(pkg); + if (sz > 0) { t->resumable = 1; t->partial_bytes = sz; } + break; + } + } + closedir(d); + } +} + /* ---------- lifecycle -------------------------------------------------- */ int @@ -1226,6 +1356,10 @@ patchdl_websrv_start(unsigned short port) { g_titles[i].enabled = (g_titles[i].source_type != PATCHDL_SOURCE_UNKNOWN); load_config(); + /* Flag titles that have a partial download on disk so the UI can offer + Resume after a reboot. */ + detect_resumable_partials(); + /* Build the diagnostic dump now, while single-threaded — the root-vnode swap it performs is unsafe once MHD worker threads are running. */ g_debug_json = patchdl_scan_debug_json(); diff --git a/web/app.js b/web/app.js index 45b3f31..8d746e8 100644 --- a/web/app.js +++ b/web/app.js @@ -312,6 +312,7 @@ function createGameCard(game) { </div> <div class="pills"> ${game.downloading ? `<span class="pill live">Downloading</span>` : ""} + ${game.resumable && !game.downloading ? `<span class="pill warn">Paused</span>` : ""} ${statusPill(game)} ${sourcePill(game)} </div> @@ -334,12 +335,12 @@ function createGameCard(game) { else btn.addEventListener("click", () => runTitleAction(game.title_id, act.action)); actions.appendChild(btn); } - // Delete a finished (not-yet-installed) download. - if (game.downloaded && !game.installing && !game.downloading) { + // Delete a finished or paused (partial) download. + if ((game.downloaded || game.resumable) && !game.installing && !game.downloading) { const del = document.createElement("button"); del.className = "row-button is-ghost"; del.textContent = "Delete"; - del.title = "Delete the downloaded package"; + del.title = game.resumable ? "Delete the partial download" : "Delete the downloaded package"; del.addEventListener("click", () => cancelDownload(game.title_id)); actions.appendChild(del); } @@ -357,6 +358,17 @@ function createGameCard(game) { <div class="progress-meta">${progressMetaHtml(d)}</div> `; card.appendChild(prog); + } else if (game.resumable && game.partial_bytes > 0) { + // ---- paused partial (survived a reboot) ---- + const note = document.createElement("div"); + note.className = "card-progress"; + note.innerHTML = ` + <div class="progress-meta"> + <span>Paused — <b>${formatBytes(game.partial_bytes)}</b> downloaded</span> + <span>Resume to continue</span> + </div> + `; + card.appendChild(note); } return card; @@ -391,6 +403,8 @@ function tileButton(game) { if (!isInstallAllowed(game)) return null; if (game.downloaded && game.status === "available") return { label: "Install", action: "install", variant: "update", hint: "Install the downloaded patch (modifies the game)." }; + if (game.resumable) + return { label: "Resume", action: "download", variant: "update", hint: "Resume the interrupted download where it stopped." }; if (game.status !== "available") return null; return state.config.install_after_download ? { label: "Update", action: "update", variant: "update", hint: "Download and install the update." } @@ -603,6 +617,8 @@ async function doDownload(game) { } game.downloaded = true; + game.resumable = false; + game.partial_bytes = 0; const sz = r && r.bytes ? formatBytes(r.bytes) : "?"; state.logs.push(`[${timeNow()}] Downloaded ${game.title_id} ${game.compatible_version} (${sz}, internal)`); showToast(`${game.name}: downloaded ${sz}.`); @@ -640,7 +656,13 @@ async function cancelDownload(titleId) { } catch (error) { showToast(`${game ? game.name : titleId}: ${reasonText(error)}`); } - if (game) { game.downloading = false; game._localDownloading = false; game.downloaded = false; } + if (game) { + game.downloading = false; + game._localDownloading = false; + game.downloaded = false; + game.resumable = false; + game.partial_bytes = 0; + } state.downloads = state.downloads.filter((i) => i.title_id !== titleId); state.logs.push(`[${timeNow()}] Download cancelled / deleted: ${titleId}`); renderGames(); renderLogs();