diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 00ced36..ca3846d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -16,7 +16,7 @@ the end. | `src/doom/` | id's original game code with the 64-bit and portability fixes listed below | | `src/port/` | Doom's `i_*` layer (main, system, video, sound, network, controller mapping) on top of the platform API, and the button glyphs shared with the launcher | | `src/launcher/` | The game list shown before the engine starts: IWAD discovery, settings, Doom-style drawing from the bundled WAD, HTML folder listings, the importer (HTTP or local file, WAD or ZIP/7Z/RAR), and the send screen's HTTP server and web page (`upload.c`, `upload.html`) | -| `src/platform/platform.h` | The platform API: time, log, files and folder listing, video present, pad, rumble, audio output, HTTP, text input | +| `src/platform/` | The platform API (`platform.h`: time, log, files and folder listing, video present, pad, rumble, audio output, HTTP, text input), the test plan reader, and the URL helpers (`url.c`) shared by the launcher and the PS5 HTTP backend | | `src/audio/` | Sound engine: SFX mixer, MUS sequencer with DMX-style voice allocation, OPL FM synth | | `src/ps5/` | PS5 backend: startup (`crt0.c`), system, VideoOut, AGC compute presenter, pad, AudioOut, `sceHttp`, the IME keyboard | | `src/ps5/present.cl` | GPU kernel: palette lookup, sharp-bilinear scaling to 1080p, tiled scanout writes (gfx1010, wave64) | @@ -83,7 +83,9 @@ and maps the chosen game to Doom's game mode, mission and language. The launcher file's header and directory, and tells The Ultimate DOOM from DOOM by the presence of `E4M1`; - imports any ZIP, 7Z or RAR found there that it has not imported before (recorded in `/download0/launcher.cfg` by size and path), extracting only known IWAD names; -- downloads from a typed link with `plat_http_get`: an HTML answer is parsed as a folder listing +- downloads from a typed link with `plat_http_get`, which follows redirects and reports where it + ended up (the PS5 backend follows `Location` itself, as `sceHttp`'s automatic redirects do not + say where they land), so a short link to a folder works: an HTML answer is parsed as a folder listing (links under the folder that end in `.wad`, `.zip`, `.7z`, `.rar` or `/`), a WAD is copied, an archive goes through libarchive. libarchive seeks with HTTP range requests; a server without them is read through or re-read from the start instead; diff --git a/TODO.md b/TODO.md index 93cf2e1..2a2583f 100644 --- a/TODO.md +++ b/TODO.md @@ -13,11 +13,3 @@ The launcher only starts the main games; add-on WADs found in archives are skipp 6, which need episode, intermission and sky support beyond episode 4. - [ ] *Legacy of Rust* (`id1.wad` with `id1-*.wad` and `id24res.wad`): needs the id24 extensions (new weapons, monsters and the rest of the spec). Largest of these. - -## Importer - -- [ ] Folder links behind a redirect (for example a shorturl.at link to a folder): the listing's - relative links are resolved against the typed address, not the address the redirect lands - on, so the files cannot be opened. Learn the final address (`sceHttpSetRedirectCallback`, - `CURLINFO_EFFECTIVE_URL` on the PC) and resolve against it. Short links straight to a file - already work. diff --git a/src/host/http.c b/src/host/http.c index 2d746cc..3db6e8a 100644 --- a/src/host/http.c +++ b/src/host/http.c @@ -79,7 +79,7 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i char range[32]; long status = 0; curl_off_t length = -1; - const char *type = NULL; + const char *type = NULL, *final = NULL; pthread_once(&once, start_curl); if (!http) @@ -119,9 +119,11 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i } curl_easy_getinfo(http->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length); curl_easy_getinfo(http->easy, CURLINFO_CONTENT_TYPE, &type); + curl_easy_getinfo(http->easy, CURLINFO_EFFECTIVE_URL, &final); info->status = (int)status; info->length = length; snprintf(info->type, sizeof(info->type), "%s", type ? type : ""); + snprintf(info->url, sizeof(info->url), "%s", final ? final : url); return http; } diff --git a/src/launcher/import.c b/src/launcher/import.c index 46bfe71..96b5896 100644 --- a/src/launcher/import.c +++ b/src/launcher/import.c @@ -15,6 +15,7 @@ #include "games.h" #include "import.h" #include "platform.h" +#include "url.h" #define CHUNK (256 * 1024) #define PEEK (64 * 1024) @@ -25,6 +26,7 @@ typedef struct { int remote; const char *url; + char final_url[LINK_URL]; // where url led after redirects plat_http_t *http; int fd; long long position, size; @@ -96,12 +98,14 @@ static int open_remote(source_t *s, long long offset) char error[128]; plat_http_close(s->http); - s->http = plat_http_get(s->url, (uint64_t)offset, &info, error, sizeof(error)); + s->http = plat_http_get(s->final_url[0] ? s->final_url : s->url, (uint64_t)offset, &info, error, + sizeof(error)); if (!s->http) { fail("%s", error); return -1; } + snprintf(s->final_url, sizeof(s->final_url), "%s", info.url); if (info.status == 206) { s->ranges = 1; @@ -335,7 +339,7 @@ static int read_listing(source_t *s) while (used < LISTING_LIMIT && (got = raw_read(s, html + used, LISTING_LIMIT - used)) > 0) used += got; html[used] = 0; - url_directory(base, sizeof(base), s->url); + url_directory(base, sizeof(base), s->final_url); listing_parse(&listing, base, html, (size_t)used); free(html); pthread_mutex_lock(&lock); diff --git a/src/launcher/launcher.c b/src/launcher/launcher.c index e30c558..fe1aaa3 100644 --- a/src/launcher/launcher.c +++ b/src/launcher/launcher.c @@ -14,6 +14,7 @@ #include "settings.h" #include "test_plan.h" #include "upload.h" +#include "url.h" #define FRAME_US 16667 #define REPEAT_DELAY 18 diff --git a/src/launcher/listing.c b/src/launcher/listing.c index 8b1bf3c..d0604f8 100644 --- a/src/launcher/listing.c +++ b/src/launcher/listing.c @@ -5,102 +5,7 @@ #include "games.h" #include "listing.h" - -static const char *path_start(const char *url) -{ - const char *scheme = strstr(url, "://"); - const char *slash; - - if (!scheme) - return url; - slash = strchr(scheme + 3, '/'); - return slash ? slash : url + strlen(url); -} - -void url_clean(char *out, size_t size, const char *typed) -{ - size_t used = 0; - const char *end; - - while (*typed == ' ') - typed++; - end = typed + strlen(typed); - while (end > typed && end[-1] == ' ') - end--; - if (!strstr(typed, "://")) - used = (size_t)snprintf(out, size, "http://"); - for (; typed < end && used + 4 < size; typed++) - { - if (*typed == ' ') - used += (size_t)snprintf(out + used, size - used, "%%20"); - else - out[used++] = *typed; - } - out[used < size ? used : size - 1] = 0; -} - -void url_directory(char *out, size_t size, const char *url) -{ - const char *path = path_start(url); - const char *last = strrchr(path, '/'); - - snprintf(out, size, "%s", url); - if (!*path || (last && last[1] && !strchr(last + 1, '.'))) - strncat(out, "/", size - strlen(out) - 1); -} - -void url_resolve(char *out, size_t size, const char *base, const char *href) -{ - const char *path = path_start(base); - const char *last; - - if (strstr(href, "://")) - snprintf(out, size, "%s", href); - else if (href[0] == '/' && href[1] == '/') - snprintf(out, size, "%.*s%s", (int)(strstr(base, "//") - base), base, href); - else if (href[0] == '/') - snprintf(out, size, "%.*s%s", (int)(path - base), base, href); - else - { - last = strrchr(path, '/'); - if (last) - snprintf(out, size, "%.*s%s", (int)(last + 1 - base), base, href); - else - snprintf(out, size, "%s/%s", base, href); - } -} - -static int hex(int c) -{ - if (c >= '0' && c <= '9') - return c - '0'; - c = tolower(c); - return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; -} - -void url_file_name(char *out, size_t size, const char *url) -{ - const char *path = path_start(url); - const char *end = path + strcspn(path, "?#"); - const char *start; - size_t used = 0; - - if (end > path && end[-1] == '/') - end--; - for (start = end; start > path && start[-1] != '/'; start--) - ; - while (start < end && used + 1 < size) - { - if (*start == '%' && end - start >= 3 && hex(start[1]) >= 0 && hex(start[2]) >= 0) - { - out[used++] = (char)(hex(start[1]) * 16 + hex(start[2])); - start += 3; - } - else - out[used++] = *start++; - } - out[used] = 0; -} +#include "url.h" static void decode_entities(char *text) { diff --git a/src/launcher/listing.h b/src/launcher/listing.h index 7132c77..c8a8bfb 100644 --- a/src/launcher/listing.h +++ b/src/launcher/listing.h @@ -21,10 +21,6 @@ typedef struct int count; } listing_t; -void url_clean(char *out, size_t size, const char *typed); -void url_directory(char *out, size_t size, const char *url); -void url_resolve(char *out, size_t size, const char *base, const char *href); -void url_file_name(char *out, size_t size, const char *url); void listing_parse(listing_t *listing, const char *base, const char *html, size_t length); #endif diff --git a/src/launcher/upload.c b/src/launcher/upload.c index a927e25..d0df485 100644 --- a/src/launcher/upload.c +++ b/src/launcher/upload.c @@ -15,6 +15,7 @@ #include "platform.h" #include "upload.h" #include "upload_page.h" +#include "url.h" #define HEAD_LIMIT 8192 #define CHUNK (256 * 1024) @@ -155,38 +156,18 @@ static const char *header_value(const char *name) return NULL; } -static int hex(int c) -{ - return c >= '0' && c <= '9' ? c - '0' - : c >= 'a' && c <= 'f' ? c - 'a' + 10 - : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1; -} - -// Turns the request path after /upload/ into a safe file name: decoded, without folders, lower -// case, and one of the kinds the importer reads. +// Turns the request path after /upload/ into a safe file name: decoded, lower case, with anything +// that could name a folder replaced, and one of the kinds the importer reads. static int clean_name(char *out, size_t size, const char *path) { char decoded[UPLOAD_NAME * 3]; - const char *base, *dot; - size_t n = 0; + const char *dot; + size_t n; - while (*path && *path != '?' && n < sizeof(decoded) - 1) + url_file_name(decoded, sizeof(decoded), path); + for (n = 0; decoded[n] && n < size - 1; n++) { - if (*path == '%' && hex(path[1]) >= 0 && hex(path[2]) >= 0) - { - decoded[n++] = (char)(hex(path[1]) * 16 + hex(path[2])); - path += 3; - } - else - decoded[n++] = *path++; - } - decoded[n] = 0; - base = decoded + n; - while (base > decoded && base[-1] != '/' && base[-1] != '\\') - base--; - for (n = 0; base[n] && n < size - 1; n++) - { - unsigned char c = (unsigned char)base[n]; + unsigned char c = (unsigned char)decoded[n]; out[n] = isalnum(c) || strchr(" .-_()[]+", c) ? (char)tolower(c) : '_'; } diff --git a/src/platform/platform.h b/src/platform/platform.h index c62c521..127083f 100644 --- a/src/platform/platform.h +++ b/src/platform/platform.h @@ -47,6 +47,7 @@ typedef struct int status; int64_t length; char type[64]; + char url[768]; // where the request ended up after redirects } plat_http_info_t; typedef struct plat_http plat_http_t; diff --git a/src/platform/url.c b/src/platform/url.c new file mode 100644 index 0000000..4cec011 --- /dev/null +++ b/src/platform/url.c @@ -0,0 +1,101 @@ +#include +#include +#include + +#include "url.h" + +static const char *path_start(const char *url) +{ + const char *scheme = strstr(url, "://"); + const char *slash; + + if (!scheme) + return url; + slash = strchr(scheme + 3, '/'); + return slash ? slash : url + strlen(url); +} + +void url_clean(char *out, size_t size, const char *typed) +{ + size_t used = 0; + const char *end; + + while (*typed == ' ') + typed++; + end = typed + strlen(typed); + while (end > typed && end[-1] == ' ') + end--; + if (!strstr(typed, "://")) + used = (size_t)snprintf(out, size, "http://"); + for (; typed < end && used + 4 < size; typed++) + { + if (*typed == ' ') + used += (size_t)snprintf(out + used, size - used, "%%20"); + else + out[used++] = *typed; + } + out[used < size ? used : size - 1] = 0; +} + +void url_directory(char *out, size_t size, const char *url) +{ + const char *path = path_start(url); + const char *last = strrchr(path, '/'); + + snprintf(out, size, "%s", url); + if (!*path || (last && last[1] && !strchr(last + 1, '.'))) + strncat(out, "/", size - strlen(out) - 1); +} + +void url_resolve(char *out, size_t size, const char *base, const char *href) +{ + const char *path = path_start(base); + const char *last; + + if (strstr(href, "://")) + snprintf(out, size, "%s", href); + else if (href[0] == '/' && href[1] == '/') + snprintf(out, size, "%.*s%s", (int)(strstr(base, "//") - base), base, href); + else if (href[0] == '/') + snprintf(out, size, "%.*s%s", (int)(path - base), base, href); + else + { + last = strrchr(path, '/'); + if (last) + snprintf(out, size, "%.*s%s", (int)(last + 1 - base), base, href); + else + snprintf(out, size, "%s/%s", base, href); + } +} + +static int hex(int c) +{ + if (c >= '0' && c <= '9') + return c - '0'; + c = tolower(c); + return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; +} + +void url_file_name(char *out, size_t size, const char *url) +{ + const char *path = path_start(url); + const char *end = path + strcspn(path, "?#"); + const char *start; + size_t used = 0; + + if (end > path && end[-1] == '/') + end--; + for (start = end; start > path && start[-1] != '/'; start--) + ; + while (start < end && used + 1 < size) + { + if (*start == '%' && end - start >= 3 && hex(start[1]) >= 0 && hex(start[2]) >= 0) + { + out[used++] = (char)(hex(start[1]) * 16 + hex(start[2])); + start += 3; + } + else + out[used++] = *start++; + } + out[used] = 0; +} diff --git a/src/platform/url.h b/src/platform/url.h new file mode 100644 index 0000000..b6fea64 --- /dev/null +++ b/src/platform/url.h @@ -0,0 +1,15 @@ +#ifndef URL_H +#define URL_H + +#include + +// Adds http:// when no scheme was typed and encodes spaces. +void url_clean(char *out, size_t size, const char *typed); +// The folder a link points at, ending in a slash. +void url_directory(char *out, size_t size, const char *url); +// A link from a page or a Location header, made absolute against the address it came from. +void url_resolve(char *out, size_t size, const char *base, const char *href); +// The last part of the path, percent-decoded. +void url_file_name(char *out, size_t size, const char *url); + +#endif diff --git a/src/ps5/http.c b/src/ps5/http.c index deca9ee..09712b5 100644 --- a/src/ps5/http.c +++ b/src/ps5/http.c @@ -5,6 +5,7 @@ #include "platform.h" #include "sce.h" +#include "url.h" #define NET_POOL (1024 * 1024) #define SSL_POOL (304 * 1024) @@ -13,6 +14,7 @@ #define METHOD_GET 0 #define HEADER_OVERWRITE 0 #define TIMEOUT_US 15000000 +#define MAX_REDIRECTS 8 #define VERIFY_FLAGS (0x01 | 0x04 | 0x08 | 0x10 | 0x20 | 0x80) struct plat_http @@ -39,7 +41,7 @@ static int start_http(void) template_id = sceHttpCreateTemplate(http, "DOOM-PS5", HTTP_1_1, 0); if (template_id < 0) return template_id; - if ((result = sceHttpSetAutoRedirect(template_id, 1)) < 0 || + if ((result = sceHttpSetAutoRedirect(template_id, 0)) < 0 || (result = sceHttpSetResolveTimeOut(template_id, TIMEOUT_US)) < 0 || (result = sceHttpSetConnectTimeOut(template_id, TIMEOUT_US)) < 0 || (result = sceHttpSetSendTimeOut(template_id, TIMEOUT_US)) < 0 || @@ -65,13 +67,47 @@ static void header_value(int request, const char *name, char *out, size_t size) out[value_size] = 0; } +static int send_get(plat_http_t *http, const char *url, uint64_t offset, int *status) +{ + char range[40]; + int result; + + http->connection = result = sceHttpCreateConnectionWithURL(template_id, url, 1); + if (result >= 0) + http->request = result = sceHttpCreateRequestWithURL(http->connection, METHOD_GET, url, 0); + if (result >= 0 && offset) + { + snprintf(range, sizeof(range), "bytes=%llu-", (unsigned long long)offset); + result = sceHttpAddRequestHeader(http->request, "Range", range, HEADER_OVERWRITE); + } + if (result >= 0) + result = sceHttpSendRequest(http->request, NULL, 0); + if (result >= 0) + result = sceHttpGetStatusCode(http->request, status); + return result; +} + +static void drop_request(plat_http_t *http) +{ + if (http->request >= 0) + sceHttpDeleteRequest(http->request); + if (http->connection >= 0) + sceHttpDeleteConnection(http->connection); + http->request = http->connection = -1; +} + +static int is_redirect(int status) +{ + return status == 301 || status == 302 || status == 303 || status == 307 || status == 308; +} + plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *info, char *error, int error_size) { plat_http_t *http; - char range[40]; + char location[sizeof(info->url)], next[sizeof(info->url)]; uint64_t length = 0; - int status = 0, has_length = -1, result; + int status = 0, has_length = -1, redirects, result; pthread_mutex_lock(&lock); result = template_id < 0 ? start_http() : 0; @@ -85,19 +121,23 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i http = calloc(1, sizeof(*http)); if (!http) return NULL; - http->request = -1; - http->connection = result = sceHttpCreateConnectionWithURL(template_id, url, 1); - if (result >= 0) - http->request = result = sceHttpCreateRequestWithURL(http->connection, METHOD_GET, url, 0); - if (result >= 0 && offset) + http->request = http->connection = -1; + + // Redirects are followed here rather than by sceHttp, to learn the address the request ends + // up at: a folder listing's relative links are relative to that one. + snprintf(info->url, sizeof(info->url), "%s", url); + for (redirects = 0;; redirects++) { - snprintf(range, sizeof(range), "bytes=%llu-", (unsigned long long)offset); - result = sceHttpAddRequestHeader(http->request, "Range", range, HEADER_OVERWRITE); + result = send_get(http, info->url, offset, &status); + if (result < 0 || !is_redirect(status) || redirects == MAX_REDIRECTS) + break; + header_value(http->request, "Location", location, sizeof(location)); + if (!location[0]) + break; + url_resolve(next, sizeof(next), info->url, location); + snprintf(info->url, sizeof(info->url), "%s", next); + drop_request(http); } - if (result >= 0) - result = sceHttpSendRequest(http->request, NULL, 0); - if (result >= 0) - result = sceHttpGetStatusCode(http->request, &status); if (result < 0) { snprintf(error, error_size, "connection failed (0x%08x)", (unsigned)result); @@ -130,9 +170,6 @@ void plat_http_close(plat_http_t *http) { if (!http) return; - if (http->request >= 0) - sceHttpDeleteRequest(http->request); - if (http->connection >= 0) - sceHttpDeleteConnection(http->connection); + drop_request(http); free(http); }