Store: real-source fixes found by running search against live repos

- F-Droid: percent-encode repo file names (a '#' in one screenshot name broke
  the whole main repo); a bad image name drops that image, not the app.
- Search: sources still fetching report 'loading' (UI says so and refreshes
  quietly); indexes warm up at server start; page-only SideQuest is not
  searched and appears as a 'Browse SideQuest' link instead of an error.
- Browse (empty query) ranks VR, artwork and recent updates first; the F-Droid
  archive is off by default (old versions only).
- Throttled sources fall back to their last cached copy; per-host message.
- Curated GitHub list gains Open Saber Plus (MIT) with icon and screenshots.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
saphidandClaude Opus 5.5 committed 2026-09-28 22:02:38 +10:00
1 parent f10b5fe159
commit 69f790b83a
12 files changed
+200 -111

No files matched your search

+119 -89
View File
@@ -1,90 +1,120 @@
[
{
"url": "https://avatars.githubusercontent.com/u/2757344?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://avatars.githubusercontent.com/u/784805?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://avatars.githubusercontent.com/u/94376830?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/KhronosGroup/OpenXR-SDK-Source",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/LWJGL/lwjgl3",
"error": "HTTP Error 429: Too Many Requests"
},
{
"url": "https://opengraph.githubassets.com/1/bjornbytes/lovr",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/sahibzada-allahyar/YC-Killer",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/KhronosGroup/OpenXR-SDK-Source/3ed64d0f9bb680f24b80a085091e5c8fab38f7b7/src/tests/hello_xr/android_resources/vulkan/mipmap-xxxhdpi/ic_helloxr_launcher.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/Assets/Resources/DefaultImages/OpenBrushLogo.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/open-brush.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_0a9c208e26a43cf34879c2ca361d4c8f18af8cba.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_19b25b86ef55c0d8769a65135d60eaae8fa40553.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_785ea37d63378146dfe0f0ffa3f1d5c155ca978f.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://www.supertux.org/images/0_7_0/github_preview.png",
"status": 200,
"content_type": "image/png",
"image": true
}
]
{
"url": "https://avatars.githubusercontent.com/u/2757344?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://avatars.githubusercontent.com/u/784805?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://avatars.githubusercontent.com/u/94376830?v=4",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/KhronosGroup/OpenXR-SDK-Source",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/LWJGL/lwjgl3",
"error": "HTTP Error 429: Too Many Requests"
},
{
"url": "https://opengraph.githubassets.com/1/bjornbytes/lovr",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/sahibzada-allahyar/YC-Killer",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/KhronosGroup/OpenXR-SDK-Source/3ed64d0f9bb680f24b80a085091e5c8fab38f7b7/src/tests/hello_xr/android_resources/vulkan/mipmap-xxxhdpi/ic_helloxr_launcher.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/Assets/Resources/DefaultImages/OpenBrushLogo.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/open-brush.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_0a9c208e26a43cf34879c2ca361d4c8f18af8cba.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_19b25b86ef55c0d8769a65135d60eaae8fa40553.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_785ea37d63378146dfe0f0ffa3f1d5c155ca978f.1920x1080.jpg",
"status": 200,
"content_type": "image/jpeg",
"image": true
},
{
"url": "https://www.supertux.org/images/0_7_0/github_preview.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://opengraph.githubassets.com/1/arpruss/OpenSaberPlus",
"status": 200,
"content_type": "image/png",
"image": true
},
{
"url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_1.gif",
"status": 200,
"content_type": "image/gif",
"image": true
},
{
"url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_2.gif",
"status": 200,
"content_type": "image/gif",
"image": true
},
{
"url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_3.gif",
"status": 200,
"content_type": "image/gif",
"image": true
}
]
+4 -2
View File
@@ -26,7 +26,7 @@ class PublisherSources(unittest.TestCase):
def test_curated_artwork_has_recorded_image_evidence(self):
evidence = {r['url']: r for r in json.loads((FIX / 'artwork-check.json').read_text())}
entries = github.search(github.sources()[0], '')
self.assertEqual(len(entries), 3)
self.assertEqual(len(entries), 4)
for entry in entries:
self.assertTrue(entry['summary'])
images = entry['images']
@@ -138,8 +138,10 @@ class PublisherSources(unittest.TestCase):
self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index')
self.assertEqual(op.call_count, 1)
error = urllib.error.HTTPError('https://api.github.com/x', 403, 'limited', {}, None)
with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'rate limit'):
with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'FRAME_GITHUB_TOKEN'):
_web.read('https://api.github.com/x', ('api.github.com',))
with patch.object(_web, 'open_url', side_effect=error), patch.object(_web.time, 'time', return_value=1e12):
self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') # throttled: stale copy
with patch.object(_web, 'read', return_value=b'<html>'), self.assertRaises(SourceError):
github._api('/x')
+1 -1
View File
@@ -71,7 +71,7 @@ class SearchTests(SettingsTest):
result = search.search(timeout=.03)
self.assertLess(time.monotonic() - started, .3)
self.assertTrue(result['apps'])
self.assertEqual([s['status'] for s in result['sources']], ['ok', 'timed out', 'error'])
self.assertEqual([s['status'] for s in result['sources']], ['ok', 'loading', 'error'])
search.search('other', timeout=.03)
self.assertEqual(calls, [''])
finally:
+2
View File
@@ -127,6 +127,8 @@ class Repositories(unittest.TestCase):
fdroid._url(url)
with self.assertRaisesRegex(SourceError, 'conflicting'):
fdroid._url(URL + '?fingerprint=' + PIN, '0' * 64)
self.assertEqual(fdroid._child(URL, '/app/en-US/phoneScreenshots/#0 a.png'),
URL + 'app/en-US/phoneScreenshots/%230%20a.png') # real F-Droid screenshot name
for name in ['../x.apk', '%2e%2e/x.apk', 'https://evil.org/a.apk', '//evil.org/../x', 'x?token=y', 'x\\y']:
with self.subTest(name=name), self.assertRaises(SourceError):
fdroid._child(URL, name)
+1 -2
View File
@@ -26,8 +26,7 @@ class SideQuestTests(unittest.TestCase):
self.assertEqual(entry['images']['screenshots'], [])
with self.assertRaisesRegex(SourceError, 'page-only'):
sidequest.download(source, '123')
with self.assertRaisesRegex(SourceError, 'page-only'):
sidequest.search(source, 'open saber')
self.assertEqual(sidequest.search(source, 'open saber'), [])
self.assertEqual(sidequest.search(source, 'open saber', 0), [])
def test_invalid_ids(self):
+6 -1
View File
@@ -63,7 +63,12 @@ def read(url, hosts, headers=None, ttl=3600):
return data
except urllib.error.HTTPError as e:
if e.code in (403, 429):
raise SourceError('Source refused access or reached its rate limit; try later (GitHub accepts FRAME_GITHUB_TOKEN)') from e
if os.path.isfile(path): # throttled: an older copy beats no results
with open(path, 'rb') as f:
return f.read()
host = urllib.parse.urlsplit(url).hostname or 'The source'
hint = ' (set FRAME_GITHUB_TOKEN to raise the limit)' if host.endswith('github.com') else ''
raise SourceError(host + ' is limiting requests right now; try again later' + hint) from e
raise SourceError('Source HTTP error: ' + str(e.code)) from e
except (OSError, ValueError) as e:
raise SourceError('Could not read source: ' + str(e)) from e
+9 -3
View File
@@ -68,7 +68,9 @@ def _child(base, name):
decoded = urllib.parse.unquote(name)
if not name or '\\' in decoded or any(x in ('.', '..') for x in decoded.split('/')):
raise SourceError('unsafe repository file name')
url = urllib.parse.urljoin(base, name)
if '?' in decoded or urllib.parse.urlsplit(decoded).scheme:
raise SourceError('repository file is outside its repository')
url = urllib.parse.urljoin(base, urllib.parse.quote(decoded, safe='/')) # names may contain '#' or spaces
if not url.startswith(base) or urllib.parse.urlsplit(url).query or urllib.parse.urlsplit(url).fragment:
raise SourceError('repository file is outside its repository')
return url
@@ -246,7 +248,8 @@ def sources():
('izzyondroid', 'IzzyOnDroid', 'https://apt.izzysoft.de/fdroid/repo/', IZZY_PIN)]
settings = _read()
return [dict(id=i, kind=KIND, name=n, url=u, fingerprint=p, builtin=True,
enabled=settings['enabled'].get(i, True), trust='community')
# The archive only holds superseded versions; it clutters search unless asked for.
enabled=settings['enabled'].get(i, i != 'fdroid-archive'), trust='community')
for i, n, u, p in builtins] + settings['repos']
@@ -288,7 +291,10 @@ def _summary(value):
def _images(meta, base):
def url(file):
name = file.get('name') if isinstance(file, dict) else file
return _child(base, name) if isinstance(name, str) and name else None
try:
return _child(base, name) if isinstance(name, str) and name else None
except SourceError:
return None # one odd image name mustn't hide the app
icon = url(_text(meta.get('icon')))
banner = url(_text(meta.get('featureGraphic')))
+20 -1
View File
@@ -36,7 +36,7 @@
{
"repo": "SgtBilko76/SuperTux-3D",
"name": "SuperTux 3D",
"summary": "Run and jump through Tux’s platform adventure on a layered 3D screen in VR.",
"summary": "Run and jump through Tux\u2019s platform adventure on a layered 3D screen in VR.",
"license": "GPL-3.0",
"vr": true,
"asset_pattern": "*Quest*.apk",
@@ -49,5 +49,24 @@
]
},
"icon": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png"
},
{
"repo": "arpruss/OpenSaberPlus",
"name": "Open Saber Plus",
"summary": "Slash glowing blocks to the beat with two lightsabers, with songs from BeatSaver.",
"license": "MIT",
"vr": true,
"asset_pattern": "OpenSaberPlus.apk",
"allow_prerelease": false,
"images": {
"icon": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png",
"banner": "https://opengraph.githubassets.com/1/arpruss/OpenSaberPlus",
"screenshots": [
"https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_1.gif",
"https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_2.gif",
"https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_3.gif"
]
},
"icon": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png"
}
]
+25 -4
View File
@@ -189,9 +189,19 @@ def group(entries, query='', vr=None, installable=False):
result.append({'name': best.get('name'), 'package': best.get('package'),
'summary': best.get('summary'), 'offers': offers})
q = normalise(query)
def browse(a): # a headset store: VR first, then apps with artwork, newest first
o = a['offers'][0]
art = o.get('images') or {}
return (o.get('vr') is not True, not art.get('banner'), not art.get('screenshots'),
''.join(chr(0x10ffff - ord(c)) for c in str(o.get('updated') or '')))
if not q:
result.sort(key=lambda a: (not any(o['fit']['installable'] is True for o in a['offers']),) + browse(a))
return result
result.sort(key=lambda a: (not any(normalise(o.get('name')) == q for o in a['offers']),
not any(o['fit']['installable'] is True for o in a['offers']),
not any(normalise(o.get('name')).startswith(q) for o in a['offers']),
a['offers'][0].get('vr') is not True,
normalise(a['name'])))
return result
@@ -220,13 +230,16 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l
items, errors = registry()
if source and source not in [s['id'] for _, s in items]:
raise SourceError('Unknown source')
tasks = [(s, _launch(m, s, query, limit)) for m, s in items
if s['enabled'] and (not source or s['id'] == source)]
chosen = [(m, s) for m, s in items if s['enabled'] and (not source or s['id'] == source)]
# Page-only sources (SideQuest) can't be searched; offer a link to browse them instead.
elsewhere = [{'name': s['name'], 'url': s['url']} for m, s in chosen if s.get('page_only')]
tasks = [(s, _launch(m, s, query, limit)) for m, s in chosen if not s.get('page_only')]
entries, statuses = [], list(errors)
for s, task in tasks:
status = {'id': s['id'], 'name': s['name']}
if task is None or not task['event'].wait(max(0, task['started'] + timeout - time.monotonic())):
status.update(status='timed out')
# Still working (e.g. first download of a large index); it keeps going and fills the cache.
status.update(status='loading')
elif 'error' in task:
status.update(status='error', error=task['error'])
else:
@@ -235,7 +248,15 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l
statuses.append(status)
with _lock:
_status[s['id']] = {k: v for k, v in status.items() if k not in ('id', 'name')}
return {'apps': group(entries, query, vr, installable), 'sources': statuses}
return {'apps': group(entries, query, vr, installable), 'sources': statuses, 'elsewhere': elsewhere}
def warm():
"""Start every enabled source's index download in the background (server start, new repo)."""
items, _ = registry()
for m, s in items:
if s['enabled'] and not s.get('page_only'):
_launch(m, s, '', 1)
def install(source_id, entry_id, version_code=None, progress=None):
+1 -3
View File
@@ -18,9 +18,7 @@ def sources():
def search(source, query, limit=50):
# Do not invent catalogue results or interpret a query as a verified free app.
if limit <= 0:
return []
raise SourceError(REASON + ' ' + URL + '/apps')
return []
def details(source, entry_id):
+11 -5
View File
@@ -1995,18 +1995,24 @@ function renderStore() {
$("searchGrid").innerHTML=`<button class="store-hero" data-app="${apps.indexOf(featured)}" aria-label="Explore ${esc(featured.name)}">${storeArt(o,"",true)}<span class="hero-copy"><span class="store-eyebrow">Step into something new</span><strong>${esc(featured.name)}</strong><span>${esc(storeText(featured.summary))}</span><span class="hero-cta">Explore ${esc(featured.name)} <span aria-hidden="true">&nbsp; ↗</span></span></span></button>` + storeRow(apps.some(a=>a.offers[0].popularity) ? "Popular apps" : "Explore apps","More ways to make your Frame yours",popular) + storeRow(tested.length ? "VR that works on the Frame" : "Step into VR",tested.length ? "Experiences checked on a Frame" : "A new perspective, a whole new playground",vr) + storeRow("Recently updated","Fresh from their creators",recent);
}
function storeChips() { $("searchFilters").querySelectorAll("button").forEach(b=>b.setAttribute("aria-pressed",b.dataset.filter==='all' ? sourceState.vr===null && !sourceState.installable : b.dataset.filter==='installable' ? sourceState.installable : sourceState.vr===(b.dataset.filter==='vr'))); }
async function searchSources() {
async function searchSources(quiet) {
const request=++sourceState.request;
$("searchGrid").setAttribute("aria-busy","true");
$("searchGrid").innerHTML=`<div class="store-grid" role="status" aria-label="Finding apps">${Array.from({length:8},()=>'<div class="store-skeleton" aria-hidden="true"></div>').join("")}</div>`;
if(!quiet)$("searchGrid").innerHTML=`<div class="store-grid" role="status" aria-label="Finding apps">${Array.from({length:8},()=>'<div class="store-skeleton" aria-hidden="true"></div>').join("")}</div>`;
const params=new URLSearchParams({q:$("sourceQ").value,source:sourceState.source,installable:String(sourceState.installable)});
if(sourceState.vr!==null)params.set("vr",String(sourceState.vr));
try {
const result=await api("/api/search?"+params);if(request!==sourceState.request)return;
sourceState.apps=result.apps;
const unavailable=result.sources.filter(s=>s.status==='error'||s.status==='timed out').map(s=>storeName(s.name));
$("sourceStatus").textContent=unavailable.length ? `${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.` : "";
$("sourceStatus").hidden=!unavailable.length;
const loading=result.sources.filter(s=>s.status==='loading').map(s=>storeName(s.name));
const notes=[];
if(loading.length)notes.push(`Still fetching ${loading.join(" and ")}; more results will appear in a moment.`);
if(unavailable.length)notes.push(`${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.`);
$("sourceStatus").innerHTML=notes.map(esc).join(" ")+(result.elsewhere||[]).map(x=>` <a href="${esc(x.url)}" target="_blank" rel="noopener">Browse ${esc(storeName(x.name))} ↗</a>`).join("");
$("sourceStatus").hidden=!notes.length && !(result.elsewhere||[]).length;
clearTimeout(sourceState.retry);
if(loading.length)sourceState.retry=setTimeout(()=>{ if(request===sourceState.request)searchSources(true); },6000);
renderStore();await loadSources();
} catch(e) { if(request===sourceState.request) { $("searchGrid").innerHTML=storeEmpty();$("sourceStatus").hidden=false;$("sourceStatus").textContent="We couldn't reach the app shelves. Try again in a moment."; } }
finally { if(request===sourceState.request)$("searchGrid").setAttribute("aria-busy","false"); }
@@ -2075,7 +2081,7 @@ $("addSource").onsubmit=e=>{e.preventDefault();const f=new FormData(e.target);ch
document.querySelectorAll('[data-close]').forEach(b=>b.onclick=()=>$(b.dataset.close).close());
$('appDetail').addEventListener('close',()=>{sourceState.detailRequest++;});
document.querySelectorAll('[data-store-tab]').forEach(b=>b.onclick=()=>{const browse=b.dataset.storeTab==='browse';$('appStore').hidden=!browse;$('androidLibrary').hidden=browse;$('repNew').hidden=browse;$('sourcesOpen').hidden=!browse;document.querySelectorAll('[data-store-tab]').forEach(t=>{t.classList.toggle('on',t===b);t.setAttribute('aria-pressed',t===b);});if(!browse)loadAndroid();});
loadSources().then(searchSources);
loadSources().then(()=>searchSources());
// ---- Android apps (own Lepton instance each) + catalogue ----
const VLABEL = { works: "Works on Frame", likely: "Should work", maybe: "Might work", unlikely: "Probably crashes", no: "Won't work" };
+1
View File
@@ -1623,6 +1623,7 @@ def main():
args = ap.parse_args()
httpd = ThreadingHTTPServer(("127.0.0.1", args.port), Handler)
sweep_tmp()
threading.Thread(target=apk_search.warm, daemon=True).start() # big indexes download before the first search
if not frame_host.WINDOWS:
signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt))
if args.exit_on_eof: