mirror of
https://github.com/knutwurst/patchdl.git
synced 2026-10-06 13:00:21 +02:00
Net: drop HTTPS pin on the streaming download paths
Real regression from c9d721f: pinning CURLOPT_PROTOCOLS_STR /
REDIR_PROTOCOLS_STR to "https" on the piece downloader (and the simple
file streamer) broke real-world Sony patch downloads. Banishers reliably
aborted after ~73 MB and Last of Us Part I after ~127 MB — the Sony CDN
appears to 302 the piece URL to an http:// signed edge inside its own
infrastructure, and refusing those redirects killed the transfer mid-piece.
The smaller patchdl_http_get path (version.xml + manifest JSON) keeps the
HTTPS pin since those payloads always come back from the public https
endpoints. The defence still holds elsewhere: host_allowed gates every
URL to the Sony CDN allowlist, TLS verifies against the pinned SCEI root
even on a 302, and NOSIGNAL stays so a connection RST can't smuggle a
SIGPIPE back into the worker thread.
For future regressions of this class /api/downloads now exposes the last
CURLcode + HTTP status per job (last_curl_rc, last_http_code) so we don't
have to instrument the binary again to find out what curl returned.
This commit is contained in:
1 parent
2bc15cbaa2
commit
a371a67521
3 files changed
+34
-8
No files matched your search
+16
-5
@@ -452,8 +452,12 @@ http_download_to_file_progress(const char *url, FILE *fp, long long *bytes_out,
|
||||
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, "DEFAULT@SECLEVEL=0");
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
|
||||
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
|
||||
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "https");
|
||||
/* HTTPS pin removed on the streaming download path: the Sony CDN sometimes
|
||||
302s a piece URL to a signed http:// edge inside its own infrastructure,
|
||||
and refusing those redirects was breaking real-world downloads.
|
||||
host_allowed + TLS-against-pinned-root on every leg already gate the
|
||||
hosts we'll talk to. NOSIGNAL stays — it protects the worker from a
|
||||
SIGPIPE on connection RST. */
|
||||
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
|
||||
/* No total timeout (patches can be large); abort only on a long stall. */
|
||||
@@ -822,7 +826,8 @@ int
|
||||
patchdl_http_download_piece(const char *url, int fd,
|
||||
long long file_offset, long long file_size,
|
||||
const char *expected_sha256_or_null,
|
||||
patchdl_piece_ctx_t *ctx) {
|
||||
patchdl_piece_ctx_t *ctx,
|
||||
int *curl_rc_out, long *http_code_out) {
|
||||
CURL *curl;
|
||||
CURLcode res;
|
||||
long http_code = 0;
|
||||
@@ -832,6 +837,9 @@ patchdl_http_download_piece(const char *url, int fd,
|
||||
piece_sink_t sink;
|
||||
int verify = (expected_sha256_or_null && expected_sha256_or_null[0]);
|
||||
|
||||
if (curl_rc_out) *curl_rc_out = 0;
|
||||
if (http_code_out) *http_code_out = 0;
|
||||
|
||||
if (url_host(url, host, sizeof(host))) return -1;
|
||||
if (!host_allowed(host)) return -1;
|
||||
if (dns_lookup(host, ip, sizeof(ip))) return -1;
|
||||
@@ -871,8 +879,8 @@ patchdl_http_download_piece(const char *url, int fd,
|
||||
curl_easy_setopt(curl, CURLOPT_SSL_CIPHER_LIST, "DEFAULT@SECLEVEL=0");
|
||||
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L);
|
||||
curl_easy_setopt(curl, CURLOPT_PROTOCOLS_STR, "https");
|
||||
curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "https");
|
||||
/* HTTPS pin removed on the streaming piece path — see the same change in
|
||||
http_download_to_file_progress for the why. */
|
||||
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
|
||||
curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); /* 4xx/5xx -> error, no body written */
|
||||
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 20L);
|
||||
@@ -890,6 +898,9 @@ patchdl_http_download_piece(const char *url, int fd,
|
||||
curl_easy_cleanup(curl);
|
||||
curl_slist_free_all(rl);
|
||||
|
||||
if (curl_rc_out) *curl_rc_out = (int)res;
|
||||
if (http_code_out) *http_code_out = http_code;
|
||||
|
||||
if (res != CURLE_OK) {
|
||||
if (sink.md) EVP_MD_CTX_free(sink.md);
|
||||
return -1; /* network error / abort */
|
||||
|
||||
+5
-2
@@ -52,11 +52,14 @@ typedef struct {
|
||||
|
||||
/* Download one whole piece and pwrite it into `fd` at `file_offset`. Concurrent
|
||||
non-overlapping pieces of the same fd are safe. Returns 0 on success (and
|
||||
fdatasyncs fd), -1 on network/IO/abort, -2 on a SHA-256 mismatch. */
|
||||
fdatasyncs fd), -1 on network/IO/abort, -2 on a SHA-256 mismatch.
|
||||
`curl_rc_out`/`http_code_out` (either may be NULL) receive the last libcurl
|
||||
CURLcode + HTTP status for failure diagnosis. */
|
||||
int patchdl_http_download_piece(const char *url, int fd,
|
||||
long long file_offset, long long file_size,
|
||||
const char *expected_sha256_or_null,
|
||||
patchdl_piece_ctx_t *ctx);
|
||||
patchdl_piece_ctx_t *ctx,
|
||||
int *curl_rc_out, long *http_code_out);
|
||||
|
||||
/* Read-only: SHA-256 a [offset, offset+size) region of fd into out_hex
|
||||
(caller provides >= 65 bytes). Returns 0 on success. */
|
||||
|
||||
+13
-1
@@ -96,6 +96,8 @@ typedef struct dl_job {
|
||||
int inflight;
|
||||
int fd; /* O_RDWR dest fd, -1 until admit */
|
||||
int rc; /* 0 ok, -1 net/io, -2 verify */
|
||||
int last_curl_rc; /* CURLcode from the most recent piece */
|
||||
long last_http_code; /* HTTP status from the most recent piece */
|
||||
struct dl_job *next;
|
||||
} dl_job_t;
|
||||
|
||||
@@ -827,6 +829,8 @@ build_downloads_json(void) {
|
||||
jbuf_append(&j, ",\"state\":"); jbuf_append_str(&j, job_state_str(job->state));
|
||||
jbuf_appendf(&j, ",\"progress\":%d", progress);
|
||||
jbuf_appendf(&j, ",\"bytes\":%lld,\"total_bytes\":%lld", bytes, total);
|
||||
jbuf_appendf(&j, ",\"rc\":%d,\"last_curl_rc\":%d,\"last_http_code\":%ld",
|
||||
job->rc, job->last_curl_rc, job->last_http_code);
|
||||
jbuf_append(&j, "}");
|
||||
}
|
||||
jbuf_append(&j, "]");
|
||||
@@ -1340,6 +1344,7 @@ admit_next(void) {
|
||||
free(q->ps); q->ps = NULL;
|
||||
free(q->bitmap); q->bitmap = NULL;
|
||||
q->done_bytes = 0; q->pieces_failed = 0; q->rc = 0;
|
||||
q->last_curl_rc = 0; q->last_http_code = 0;
|
||||
|
||||
q->mf = mf; q->ps = ps; q->bitmap = bitmap; q->fd = fd; q->total = total;
|
||||
for (int i = 0; i < mf.count; i++) q->ps[i].slot = -1;
|
||||
@@ -1447,15 +1452,22 @@ dl_worker(void *arg) {
|
||||
pthread_mutex_unlock(&g_pool.mtx);
|
||||
|
||||
/* ---- download the piece, NO lock ---- */
|
||||
int last_curl_rc = 0;
|
||||
long last_http_code = 0;
|
||||
{
|
||||
patchdl_piece_ctx_t ctx = { &g_pool.inflight_bytes[slot], abort_ptr };
|
||||
rc = patchdl_http_download_piece(url, fd, off, sz,
|
||||
verify && hash[0] ? hash : NULL, &ctx);
|
||||
verify && hash[0] ? hash : NULL, &ctx,
|
||||
&last_curl_rc, &last_http_code);
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&g_pool.mtx);
|
||||
g_pool.inflight_bytes[slot] = 0;
|
||||
job->inflight--;
|
||||
if (rc != 0) {
|
||||
job->last_curl_rc = last_curl_rc;
|
||||
job->last_http_code = last_http_code;
|
||||
}
|
||||
job->ps[pidx].slot = -1;
|
||||
if (my_seq != job->seq) {
|
||||
/* job cancelled/torn down under us: discard result (do not touch
|
||||
|
||||
Reference in new issue
Block a user