Follow redirects to folder links and resolve the listing where they land

A short link to a folder could not be imported: the listing's relative links were resolved against
the typed address instead of the one the redirect ended at. plat_http_get now reports the final
address. The PS5 backend follows Location headers itself, since sceHttp's automatic redirects do
not say where they land; the PC backend asks libcurl. The importer resolves listings against that
address and sends later range requests straight to it.

The URL helpers move from the launcher's listing code to src/platform/url.c so the PS5 backend can
resolve Location headers with them, and the upload server reuses url_file_name for file names.

DEVELOPMENT.md

@@ -16,7 +16,7 @@ the end.
 | `src/doom/` | id's original game code with the 64-bit and portability fixes listed below |
 | `src/port/` | Doom's `i_*` layer (main, system, video, sound, network, controller mapping) on top of the platform API, and the button glyphs shared with the launcher |
 | `src/launcher/` | The game list shown before the engine starts: IWAD discovery, settings, Doom-style drawing from the bundled WAD, HTML folder listings, the importer (HTTP or local file, WAD or ZIP/7Z/RAR), and the send screen's HTTP server and web page (`upload.c`, `upload.html`) |
-| `src/platform/platform.h` | The platform API: time, log, files and folder listing, video present, pad, rumble, audio output, HTTP, text input |
+| `src/platform/` | The platform API (`platform.h`: time, log, files and folder listing, video present, pad, rumble, audio output, HTTP, text input), the test plan reader, and the URL helpers (`url.c`) shared by the launcher and the PS5 HTTP backend |
 | `src/audio/` | Sound engine: SFX mixer, MUS sequencer with DMX-style voice allocation, OPL FM synth |
 | `src/ps5/` | PS5 backend: startup (`crt0.c`), system, VideoOut, AGC compute presenter, pad, AudioOut, `sceHttp`, the IME keyboard |
 | `src/ps5/present.cl` | GPU kernel: palette lookup, sharp-bilinear scaling to 1080p, tiled scanout writes (gfx1010, wave64) |
@@ -83,7 +83,9 @@ and maps the chosen game to Doom's game mode, mission and language. The launcher
   file's header and directory, and tells The Ultimate DOOM from DOOM by the presence of `E4M1`;
 - imports any ZIP, 7Z or RAR found there that it has not imported before (recorded in
   `/download0/launcher.cfg` by size and path), extracting only known IWAD names;
-- downloads from a typed link with `plat_http_get`: an HTML answer is parsed as a folder listing
+- downloads from a typed link with `plat_http_get`, which follows redirects and reports where it
+  ended up (the PS5 backend follows `Location` itself, as `sceHttp`'s automatic redirects do not
+  say where they land), so a short link to a folder works: an HTML answer is parsed as a folder listing
   (links under the folder that end in `.wad`, `.zip`, `.7z`, `.rar` or `/`), a WAD is copied, an
   archive goes through libarchive. libarchive seeks with HTTP range requests; a server without them
   is read through or re-read from the start instead;

TODO.md

@@ -13,11 +13,3 @@ The launcher only starts the main games; add-on WADs found in archives are skipp
       6, which need episode, intermission and sky support beyond episode 4.
 - [ ] *Legacy of Rust* (`id1.wad` with `id1-*.wad` and `id24res.wad`): needs the id24 extensions
       (new weapons, monsters and the rest of the spec). Largest of these.
-
-## Importer
-
-- [ ] Folder links behind a redirect (for example a shorturl.at link to a folder): the listing's
-      relative links are resolved against the typed address, not the address the redirect lands
-      on, so the files cannot be opened. Learn the final address (`sceHttpSetRedirectCallback`,
-      `CURLINFO_EFFECTIVE_URL` on the PC) and resolve against it. Short links straight to a file
-      already work.

src/host/http.c

@@ -79,7 +79,7 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i
     char range[32];
     long status = 0;
     curl_off_t length = -1;
-    const char *type = NULL;
+    const char *type = NULL, *final = NULL;
 
     pthread_once(&once, start_curl);
     if (!http)
@@ -119,9 +119,11 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i
     }
     curl_easy_getinfo(http->easy, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length);
     curl_easy_getinfo(http->easy, CURLINFO_CONTENT_TYPE, &type);
+    curl_easy_getinfo(http->easy, CURLINFO_EFFECTIVE_URL, &final);
     info->status = (int)status;
     info->length = length;
     snprintf(info->type, sizeof(info->type), "%s", type ? type : "");
+    snprintf(info->url, sizeof(info->url), "%s", final ? final : url);
     return http;
 }
 

src/launcher/import.c

@@ -15,6 +15,7 @@
 #include "games.h"
 #include "import.h"
 #include "platform.h"
+#include "url.h"
 
 #define CHUNK (256 * 1024)
 #define PEEK (64 * 1024)
@@ -25,6 +26,7 @@ typedef struct
 {
     int remote;
     const char *url;
+    char final_url[LINK_URL]; // where url led after redirects
     plat_http_t *http;
     int fd;
     long long position, size;
@@ -96,12 +98,14 @@ static int open_remote(source_t *s, long long offset)
     char error[128];
 
     plat_http_close(s->http);
-    s->http = plat_http_get(s->url, (uint64_t)offset, &info, error, sizeof(error));
+    s->http = plat_http_get(s->final_url[0] ? s->final_url : s->url, (uint64_t)offset, &info, error,
+                            sizeof(error));
     if (!s->http)
     {
         fail("%s", error);
         return -1;
     }
+    snprintf(s->final_url, sizeof(s->final_url), "%s", info.url);
     if (info.status == 206)
     {
         s->ranges = 1;
@@ -335,7 +339,7 @@ static int read_listing(source_t *s)
     while (used < LISTING_LIMIT && (got = raw_read(s, html + used, LISTING_LIMIT - used)) > 0)
         used += got;
     html[used] = 0;
-    url_directory(base, sizeof(base), s->url);
+    url_directory(base, sizeof(base), s->final_url);
     listing_parse(&listing, base, html, (size_t)used);
     free(html);
     pthread_mutex_lock(&lock);

src/launcher/launcher.c

@@ -14,6 +14,7 @@
 #include "settings.h"
 #include "test_plan.h"
 #include "upload.h"
+#include "url.h"
 
 #define FRAME_US 16667
 #define REPEAT_DELAY 18

src/launcher/listing.c

@@ -5,102 +5,7 @@
 
 #include "games.h"
 #include "listing.h"
-
-static const char *path_start(const char *url)
-{
-    const char *scheme = strstr(url, "://");
-    const char *slash;
-
-    if (!scheme)
-        return url;
-    slash = strchr(scheme + 3, '/');
-    return slash ? slash : url + strlen(url);
-}
-
-void url_clean(char *out, size_t size, const char *typed)
-{
-    size_t used = 0;
-    const char *end;
-
-    while (*typed == ' ')
-        typed++;
-    end = typed + strlen(typed);
-    while (end > typed && end[-1] == ' ')
-        end--;
-    if (!strstr(typed, "://"))
-        used = (size_t)snprintf(out, size, "http://");
-    for (; typed < end && used + 4 < size; typed++)
-    {
-        if (*typed == ' ')
-            used += (size_t)snprintf(out + used, size - used, "%%20");
-        else
-            out[used++] = *typed;
-    }
-    out[used < size ? used : size - 1] = 0;
-}
-
-void url_directory(char *out, size_t size, const char *url)
-{
-    const char *path = path_start(url);
-    const char *last = strrchr(path, '/');
-
-    snprintf(out, size, "%s", url);
-    if (!*path || (last && last[1] && !strchr(last + 1, '.')))
-        strncat(out, "/", size - strlen(out) - 1);
-}
-
-void url_resolve(char *out, size_t size, const char *base, const char *href)
-{
-    const char *path = path_start(base);
-    const char *last;
-
-    if (strstr(href, "://"))
-        snprintf(out, size, "%s", href);
-    else if (href[0] == '/' && href[1] == '/')
-        snprintf(out, size, "%.*s%s", (int)(strstr(base, "//") - base), base, href);
-    else if (href[0] == '/')
-        snprintf(out, size, "%.*s%s", (int)(path - base), base, href);
-    else
-    {
-        last = strrchr(path, '/');
-        if (last)
-            snprintf(out, size, "%.*s%s", (int)(last + 1 - base), base, href);
-        else
-            snprintf(out, size, "%s/%s", base, href);
-    }
-}
-
-static int hex(int c)
-{
-    if (c >= '0' && c <= '9')
-        return c - '0';
-    c = tolower(c);
-    return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1;
-}
-
-void url_file_name(char *out, size_t size, const char *url)
-{
-    const char *path = path_start(url);
-    const char *end = path + strcspn(path, "?#");
-    const char *start;
-    size_t used = 0;
-
-    if (end > path && end[-1] == '/')
-        end--;
-    for (start = end; start > path && start[-1] != '/'; start--)
-        ;
-    while (start < end && used + 1 < size)
-    {
-        if (*start == '%' && end - start >= 3 && hex(start[1]) >= 0 && hex(start[2]) >= 0)
-        {
-            out[used++] = (char)(hex(start[1]) * 16 + hex(start[2]));
-            start += 3;
-        }
-        else
-            out[used++] = *start++;
-    }
-    out[used] = 0;
-}
+#include "url.h"
 
 static void decode_entities(char *text)
 {

src/launcher/listing.h

@@ -21,10 +21,6 @@ typedef struct
     int count;
 } listing_t;
 
-void url_clean(char *out, size_t size, const char *typed);
-void url_directory(char *out, size_t size, const char *url);
-void url_resolve(char *out, size_t size, const char *base, const char *href);
-void url_file_name(char *out, size_t size, const char *url);
 void listing_parse(listing_t *listing, const char *base, const char *html, size_t length);
 
 #endif

src/launcher/upload.c

@@ -15,6 +15,7 @@
 #include "platform.h"
 #include "upload.h"
 #include "upload_page.h"
+#include "url.h"
 
 #define HEAD_LIMIT 8192
 #define CHUNK (256 * 1024)
@@ -155,38 +156,18 @@ static const char *header_value(const char *name)
     return NULL;
 }
 
-static int hex(int c)
-{
-    return c >= '0' && c <= '9' ? c - '0'
-         : c >= 'a' && c <= 'f' ? c - 'a' + 10
-         : c >= 'A' && c <= 'F' ? c - 'A' + 10 : -1;
-}
-
-// Turns the request path after /upload/ into a safe file name: decoded, without folders, lower
-// case, and one of the kinds the importer reads.
+// Turns the request path after /upload/ into a safe file name: decoded, lower case, with anything
+// that could name a folder replaced, and one of the kinds the importer reads.
 static int clean_name(char *out, size_t size, const char *path)
 {
     char decoded[UPLOAD_NAME * 3];
-    const char *base, *dot;
-    size_t n = 0;
+    const char *dot;
+    size_t n;
 
-    while (*path && *path != '?' && n < sizeof(decoded) - 1)
+    url_file_name(decoded, sizeof(decoded), path);
+    for (n = 0; decoded[n] && n < size - 1; n++)
     {
-        if (*path == '%' && hex(path[1]) >= 0 && hex(path[2]) >= 0)
-        {
-            decoded[n++] = (char)(hex(path[1]) * 16 + hex(path[2]));
-            path += 3;
-        }
-        else
-            decoded[n++] = *path++;
-    }
-    decoded[n] = 0;
-    base = decoded + n;
-    while (base > decoded && base[-1] != '/' && base[-1] != '\\')
-        base--;
-    for (n = 0; base[n] && n < size - 1; n++)
-    {
-        unsigned char c = (unsigned char)base[n];
+        unsigned char c = (unsigned char)decoded[n];
 
         out[n] = isalnum(c) || strchr(" .-_()[]+", c) ? (char)tolower(c) : '_';
     }

src/platform/platform.h

@@ -47,6 +47,7 @@ typedef struct
     int status;
     int64_t length;
     char type[64];
+    char url[768]; // where the request ended up after redirects
 } plat_http_info_t;
 
 typedef struct plat_http plat_http_t;

src/platform/url.c

@@ -0,0 +1,101 @@
+#include <ctype.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "url.h"
+
+static const char *path_start(const char *url)
+{
+    const char *scheme = strstr(url, "://");
+    const char *slash;
+
+    if (!scheme)
+        return url;
+    slash = strchr(scheme + 3, '/');
+    return slash ? slash : url + strlen(url);
+}
+
+void url_clean(char *out, size_t size, const char *typed)
+{
+    size_t used = 0;
+    const char *end;
+
+    while (*typed == ' ')
+        typed++;
+    end = typed + strlen(typed);
+    while (end > typed && end[-1] == ' ')
+        end--;
+    if (!strstr(typed, "://"))
+        used = (size_t)snprintf(out, size, "http://");
+    for (; typed < end && used + 4 < size; typed++)
+    {
+        if (*typed == ' ')
+            used += (size_t)snprintf(out + used, size - used, "%%20");
+        else
+            out[used++] = *typed;
+    }
+    out[used < size ? used : size - 1] = 0;
+}
+
+void url_directory(char *out, size_t size, const char *url)
+{
+    const char *path = path_start(url);
+    const char *last = strrchr(path, '/');
+
+    snprintf(out, size, "%s", url);
+    if (!*path || (last && last[1] && !strchr(last + 1, '.')))
+        strncat(out, "/", size - strlen(out) - 1);
+}
+
+void url_resolve(char *out, size_t size, const char *base, const char *href)
+{
+    const char *path = path_start(base);
+    const char *last;
+
+    if (strstr(href, "://"))
+        snprintf(out, size, "%s", href);
+    else if (href[0] == '/' && href[1] == '/')
+        snprintf(out, size, "%.*s%s", (int)(strstr(base, "//") - base), base, href);
+    else if (href[0] == '/')
+        snprintf(out, size, "%.*s%s", (int)(path - base), base, href);
+    else
+    {
+        last = strrchr(path, '/');
+        if (last)
+            snprintf(out, size, "%.*s%s", (int)(last + 1 - base), base, href);
+        else
+            snprintf(out, size, "%s/%s", base, href);
+    }
+}
+
+static int hex(int c)
+{
+    if (c >= '0' && c <= '9')
+        return c - '0';
+    c = tolower(c);
+    return c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1;
+}
+
+void url_file_name(char *out, size_t size, const char *url)
+{
+    const char *path = path_start(url);
+    const char *end = path + strcspn(path, "?#");
+    const char *start;
+    size_t used = 0;
+
+    if (end > path && end[-1] == '/')
+        end--;
+    for (start = end; start > path && start[-1] != '/'; start--)
+        ;
+    while (start < end && used + 1 < size)
+    {
+        if (*start == '%' && end - start >= 3 && hex(start[1]) >= 0 && hex(start[2]) >= 0)
+        {
+            out[used++] = (char)(hex(start[1]) * 16 + hex(start[2]));
+            start += 3;
+        }
+        else
+            out[used++] = *start++;
+    }
+    out[used] = 0;
+}

src/platform/url.h

@@ -0,0 +1,15 @@
+#ifndef URL_H
+#define URL_H
+
+#include <stddef.h>
+
+// Adds http:// when no scheme was typed and encodes spaces.
+void url_clean(char *out, size_t size, const char *typed);
+// The folder a link points at, ending in a slash.
+void url_directory(char *out, size_t size, const char *url);
+// A link from a page or a Location header, made absolute against the address it came from.
+void url_resolve(char *out, size_t size, const char *base, const char *href);
+// The last part of the path, percent-decoded.
+void url_file_name(char *out, size_t size, const char *url);
+
+#endif

src/ps5/http.c

@@ -5,6 +5,7 @@
 
 #include "platform.h"
 #include "sce.h"
+#include "url.h"
 
 #define NET_POOL (1024 * 1024)
 #define SSL_POOL (304 * 1024)
@@ -13,6 +14,7 @@
 #define METHOD_GET 0
 #define HEADER_OVERWRITE 0
 #define TIMEOUT_US 15000000
+#define MAX_REDIRECTS 8
 #define VERIFY_FLAGS (0x01 | 0x04 | 0x08 | 0x10 | 0x20 | 0x80)
 
 struct plat_http
@@ -39,7 +41,7 @@ static int start_http(void)
     template_id = sceHttpCreateTemplate(http, "DOOM-PS5", HTTP_1_1, 0);
     if (template_id < 0)
         return template_id;
-    if ((result = sceHttpSetAutoRedirect(template_id, 1)) < 0 ||
+    if ((result = sceHttpSetAutoRedirect(template_id, 0)) < 0 ||
         (result = sceHttpSetResolveTimeOut(template_id, TIMEOUT_US)) < 0 ||
         (result = sceHttpSetConnectTimeOut(template_id, TIMEOUT_US)) < 0 ||
         (result = sceHttpSetSendTimeOut(template_id, TIMEOUT_US)) < 0 ||
@@ -65,13 +67,47 @@ static void header_value(int request, const char *name, char *out, size_t size)
     out[value_size] = 0;
 }
 
+static int send_get(plat_http_t *http, const char *url, uint64_t offset, int *status)
+{
+    char range[40];
+    int result;
+
+    http->connection = result = sceHttpCreateConnectionWithURL(template_id, url, 1);
+    if (result >= 0)
+        http->request = result = sceHttpCreateRequestWithURL(http->connection, METHOD_GET, url, 0);
+    if (result >= 0 && offset)
+    {
+        snprintf(range, sizeof(range), "bytes=%llu-", (unsigned long long)offset);
+        result = sceHttpAddRequestHeader(http->request, "Range", range, HEADER_OVERWRITE);
+    }
+    if (result >= 0)
+        result = sceHttpSendRequest(http->request, NULL, 0);
+    if (result >= 0)
+        result = sceHttpGetStatusCode(http->request, status);
+    return result;
+}
+
+static void drop_request(plat_http_t *http)
+{
+    if (http->request >= 0)
+        sceHttpDeleteRequest(http->request);
+    if (http->connection >= 0)
+        sceHttpDeleteConnection(http->connection);
+    http->request = http->connection = -1;
+}
+
+static int is_redirect(int status)
+{
+    return status == 301 || status == 302 || status == 303 || status == 307 || status == 308;
+}
+
 plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *info,
                            char *error, int error_size)
 {
     plat_http_t *http;
-    char range[40];
+    char location[sizeof(info->url)], next[sizeof(info->url)];
     uint64_t length = 0;
-    int status = 0, has_length = -1, result;
+    int status = 0, has_length = -1, redirects, result;
 
     pthread_mutex_lock(&lock);
     result = template_id < 0 ? start_http() : 0;
@@ -85,19 +121,23 @@ plat_http_t *plat_http_get(const char *url, uint64_t offset, plat_http_info_t *i
     http = calloc(1, sizeof(*http));
     if (!http)
         return NULL;
-    http->request = -1;
-    http->connection = result = sceHttpCreateConnectionWithURL(template_id, url, 1);
-    if (result >= 0)
-        http->request = result = sceHttpCreateRequestWithURL(http->connection, METHOD_GET, url, 0);
-    if (result >= 0 && offset)
+    http->request = http->connection = -1;
+
+    // Redirects are followed here rather than by sceHttp, to learn the address the request ends
+    // up at: a folder listing's relative links are relative to that one.
+    snprintf(info->url, sizeof(info->url), "%s", url);
+    for (redirects = 0;; redirects++)
     {
-        snprintf(range, sizeof(range), "bytes=%llu-", (unsigned long long)offset);
-        result = sceHttpAddRequestHeader(http->request, "Range", range, HEADER_OVERWRITE);
+        result = send_get(http, info->url, offset, &status);
+        if (result < 0 || !is_redirect(status) || redirects == MAX_REDIRECTS)
+            break;
+        header_value(http->request, "Location", location, sizeof(location));
+        if (!location[0])
+            break;
+        url_resolve(next, sizeof(next), info->url, location);
+        snprintf(info->url, sizeof(info->url), "%s", next);
+        drop_request(http);
     }
-    if (result >= 0)
-        result = sceHttpSendRequest(http->request, NULL, 0);
-    if (result >= 0)
-        result = sceHttpGetStatusCode(http->request, &status);
     if (result < 0)
     {
         snprintf(error, error_size, "connection failed (0x%08x)", (unsigned)result);
@@ -130,9 +170,6 @@ void plat_http_close(plat_http_t *http)
 {
     if (!http)
         return;
-    if (http->request >= 0)
-        sceHttpDeleteRequest(http->request);
-    if (http->connection >= 0)
-        sceHttpDeleteConnection(http->connection);
+    drop_request(http);
     free(http);
 }