Net: drop HTTPS pin on the streaming download paths

Real regression from c9d721f: pinning CURLOPT_PROTOCOLS_STR /
REDIR_PROTOCOLS_STR to "https" on the piece downloader (and the simple
file streamer) broke real-world Sony patch downloads. Banishers reliably
aborted after ~73 MB and Last of Us Part I after ~127 MB — the Sony CDN
appears to 302 the piece URL to an http:// signed edge inside its own
infrastructure, and refusing those redirects killed the transfer mid-piece.

The smaller patchdl_http_get path (version.xml + manifest JSON) keeps the
HTTPS pin since those payloads always come back from the public https
endpoints. The defence still holds elsewhere: host_allowed gates every
URL to the Sony CDN allowlist, TLS verifies against the pinned SCEI root
even on a 302, and NOSIGNAL stays so a connection RST can't smuggle a
SIGPIPE back into the worker thread.

For future regressions of this class /api/downloads now exposes the last
CURLcode + HTTP status per job (last_curl_rc, last_http_code) so we don't
have to instrument the binary again to find out what curl returned.
This commit is contained in:
Knutwurst committed 2026-06-24 22:13:16 +02:00
1 parent 2bc15cbaa2
commit a371a67521
3 files changed
+34 -8

No files matched your search

+16 -5
View File
@@ -452,8 +452,12 @@ http_download_to_file_progress(const char *url, FILE *fp, long long *bytes_out,
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, "DEFAULT@SECLEVEL=0");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "https");
/* HTTPS pin removed on the streaming download path: the Sony CDN sometimes
302s a piece URL to a signed http:// edge inside its own infrastructure,
and refusing those redirects was breaking real-world downloads.
host_allowed + TLS-against-pinned-root on every leg already gate the
hosts we'll talk to. NOSIGNAL stays — it protects the worker from a
SIGPIPE on connection RST. */
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
/* No total timeout (patches can be large); abort only on a long stall. */
@@ -822,7 +826,8 @@ int
patchdl_http_download_piece(const char *url, int fd,
long long file_offset, long long file_size,
const char *expected_sha256_or_null,
patchdl_piece_ctx_t *ctx) {
patchdl_piece_ctx_t *ctx,
int *curl_rc_out, long *http_code_out) {
CURL *curl;
CURLcode res;
long http_code = 0;
@@ -832,6 +837,9 @@ patchdl_http_download_piece(const char *url, int fd,
piece_sink_t sink;
int verify = (expected_sha256_or_null && expected_sha256_or_null[0]);
if (curl_rc_out) *curl_rc_out = 0;
if (http_code_out) *http_code_out = 0;
if (url_host(url, host, sizeof(host))) return -1;
if (!host_allowed(host)) return -1;
if (dns_lookup(host, ip, sizeof(ip))) return -1;
@@ -871,8 +879,8 @@ patchdl_http_download_piece(const char *url, int fd,
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, "DEFAULT@SECLEVEL=0");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "https");
/* HTTPS pin removed on the streaming piece path — see the same change in
http_download_to_file_progress for the why. */
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx -> error, no body written */
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
@@ -890,6 +898,9 @@ patchdl_http_download_piece(const char *url, int fd,
curl_easy_cleanup(curl);
curl_slist_free_all(rl);
if (curl_rc_out) *curl_rc_out = (int)res;
if (http_code_out) *http_code_out = http_code;
if (res != CURLE_OK) {
if (sink.md) EVP_MD_CTX_free(sink.md);
return -1; /* network error / abort */
+5 -2
View File
@@ -52,11 +52,14 @@ typedef struct {
/* Download one whole piece and pwrite it into `fd` at `file_offset`. Concurrent
non-overlapping pieces of the same fd are safe. Returns 0 on success (and
fdatasyncs fd), -1 on network/IO/abort, -2 on a SHA-256 mismatch. */
fdatasyncs fd), -1 on network/IO/abort, -2 on a SHA-256 mismatch.
`curl_rc_out`/`http_code_out` (either may be NULL) receive the last libcurl
CURLcode + HTTP status for failure diagnosis. */
int patchdl_http_download_piece(const char *url, int fd,
long long file_offset, long long file_size,
const char *expected_sha256_or_null,
patchdl_piece_ctx_t *ctx);
patchdl_piece_ctx_t *ctx,
int *curl_rc_out, long *http_code_out);
/* Read-only: SHA-256 a [offset, offset+size) region of fd into out_hex
(caller provides >= 65 bytes). Returns 0 on success. */
+13 -1
View File
@@ -96,6 +96,8 @@ typedef struct dl_job {
int inflight;
int fd; /* O_RDWR dest fd, -1 until admit */
int rc; /* 0 ok, -1 net/io, -2 verify */
int last_curl_rc; /* CURLcode from the most recent piece */
long last_http_code; /* HTTP status from the most recent piece */
struct dl_job *next;
} dl_job_t;
@@ -827,6 +829,8 @@ build_downloads_json(void) {
jbuf_append(&j, ",\"state\":"); jbuf_append_str(&j, job_state_str(job->state));
jbuf_appendf(&j, ",\"progress\":%d", progress);
jbuf_appendf(&j, ",\"bytes\":%lld,\"total_bytes\":%lld", bytes, total);
jbuf_appendf(&j, ",\"rc\":%d,\"last_curl_rc\":%d,\"last_http_code\":%ld",
job->rc, job->last_curl_rc, job->last_http_code);
jbuf_append(&j, "}");
}
jbuf_append(&j, "]");
@@ -1340,6 +1344,7 @@ admit_next(void) {
free(q->ps); q->ps = NULL;
free(q->bitmap); q->bitmap = NULL;
q->done_bytes = 0; q->pieces_failed = 0; q->rc = 0;
q->last_curl_rc = 0; q->last_http_code = 0;
q->mf = mf; q->ps = ps; q->bitmap = bitmap; q->fd = fd; q->total = total;
for (int i = 0; i < mf.count; i++) q->ps[i].slot = -1;
@@ -1447,15 +1452,22 @@ dl_worker(void *arg) {
pthread_mutex_unlock(&g_pool.mtx);
/* ---- download the piece, NO lock ---- */
int last_curl_rc = 0;
long last_http_code = 0;
{
patchdl_piece_ctx_t ctx = { &g_pool.inflight_bytes[slot], abort_ptr };
rc = patchdl_http_download_piece(url, fd, off, sz,
verify && hash[0] ? hash : NULL, &ctx);
verify && hash[0] ? hash : NULL, &ctx,
&last_curl_rc, &last_http_code);
}
pthread_mutex_lock(&g_pool.mtx);
g_pool.inflight_bytes[slot] = 0;
job->inflight--;
if (rc != 0) {
job->last_curl_rc = last_curl_rc;
job->last_http_code = last_http_code;
}
job->ps[pidx].slot = -1;
if (my_seq != job->seq) {
/* job cancelled/torn down under us: discard result (do not touch