diff --git a/tests/fixtures/more_sources/artwork-check.json b/tests/fixtures/more_sources/artwork-check.json index 8457ca4..d16be22 100644 --- a/tests/fixtures/more_sources/artwork-check.json +++ b/tests/fixtures/more_sources/artwork-check.json @@ -1,90 +1,120 @@ [ - { - "url": "https://avatars.githubusercontent.com/u/2757344?v=4", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://avatars.githubusercontent.com/u/784805?v=4", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://avatars.githubusercontent.com/u/94376830?v=4", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://opengraph.githubassets.com/1/KhronosGroup/OpenXR-SDK-Source", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://opengraph.githubassets.com/1/LWJGL/lwjgl3", - "error": "HTTP Error 429: Too Many Requests" - }, - { - "url": "https://opengraph.githubassets.com/1/bjornbytes/lovr", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://opengraph.githubassets.com/1/sahibzada-allahyar/YC-Killer", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://raw.githubusercontent.com/KhronosGroup/OpenXR-SDK-Source/3ed64d0f9bb680f24b80a085091e5c8fab38f7b7/src/tests/hello_xr/android_resources/vulkan/mipmap-xxxhdpi/ic_helloxr_launcher.png", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/Assets/Resources/DefaultImages/OpenBrushLogo.png", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/open-brush.png", - "status": 200, - "content_type": "image/png", - "image": true - }, - { - "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_0a9c208e26a43cf34879c2ca361d4c8f18af8cba.1920x1080.jpg", - "status": 200, - "content_type": "image/jpeg", - "image": true - }, - { - "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_19b25b86ef55c0d8769a65135d60eaae8fa40553.1920x1080.jpg", - "status": 200, - "content_type": "image/jpeg", - "image": true - }, - { - "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_785ea37d63378146dfe0f0ffa3f1d5c155ca978f.1920x1080.jpg", - "status": 200, - "content_type": "image/jpeg", - "image": true - }, - { - "url": "https://www.supertux.org/images/0_7_0/github_preview.png", - "status": 200, - "content_type": "image/png", - "image": true - } -] + { + "url": "https://avatars.githubusercontent.com/u/2757344?v=4", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://avatars.githubusercontent.com/u/784805?v=4", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://avatars.githubusercontent.com/u/94376830?v=4", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://opengraph.githubassets.com/1/KhronosGroup/OpenXR-SDK-Source", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://opengraph.githubassets.com/1/LWJGL/lwjgl3", + "error": "HTTP Error 429: Too Many Requests" + }, + { + "url": "https://opengraph.githubassets.com/1/bjornbytes/lovr", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://opengraph.githubassets.com/1/sahibzada-allahyar/YC-Killer", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/KhronosGroup/OpenXR-SDK-Source/3ed64d0f9bb680f24b80a085091e5c8fab38f7b7/src/tests/hello_xr/android_resources/vulkan/mipmap-xxxhdpi/ic_helloxr_launcher.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/Assets/Resources/DefaultImages/OpenBrushLogo.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/icosa-foundation/open-brush/56acbce831c7e9f257bfee9e21853da99773787b/open-brush.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_0a9c208e26a43cf34879c2ca361d4c8f18af8cba.1920x1080.jpg", + "status": 200, + "content_type": "image/jpeg", + "image": true + }, + { + "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_19b25b86ef55c0d8769a65135d60eaae8fa40553.1920x1080.jpg", + "status": 200, + "content_type": "image/jpeg", + "image": true + }, + { + "url": "https://shared.akamai.steamstatic.com/store_item_assets/steam/apps/1634870/ss_785ea37d63378146dfe0f0ffa3f1d5c155ca978f.1920x1080.jpg", + "status": 200, + "content_type": "image/jpeg", + "image": true + }, + { + "url": "https://www.supertux.org/images/0_7_0/github_preview.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://opengraph.githubassets.com/1/arpruss/OpenSaberPlus", + "status": 200, + "content_type": "image/png", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_1.gif", + "status": 200, + "content_type": "image/gif", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_2.gif", + "status": 200, + "content_type": "image/gif", + "image": true + }, + { + "url": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_3.gif", + "status": 200, + "content_type": "image/gif", + "image": true + } +] \ No newline at end of file diff --git a/tests/test_apk_more_sources.py b/tests/test_apk_more_sources.py index 1709e2d..4c788f1 100644 --- a/tests/test_apk_more_sources.py +++ b/tests/test_apk_more_sources.py @@ -26,7 +26,7 @@ class PublisherSources(unittest.TestCase): def test_curated_artwork_has_recorded_image_evidence(self): evidence = {r['url']: r for r in json.loads((FIX / 'artwork-check.json').read_text())} entries = github.search(github.sources()[0], '') - self.assertEqual(len(entries), 3) + self.assertEqual(len(entries), 4) for entry in entries: self.assertTrue(entry['summary']) images = entry['images'] @@ -138,8 +138,10 @@ class PublisherSources(unittest.TestCase): self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') self.assertEqual(op.call_count, 1) error = urllib.error.HTTPError('https://api.github.com/x', 403, 'limited', {}, None) - with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'rate limit'): + with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'FRAME_GITHUB_TOKEN'): _web.read('https://api.github.com/x', ('api.github.com',)) + with patch.object(_web, 'open_url', side_effect=error), patch.object(_web.time, 'time', return_value=1e12): + self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') # throttled: stale copy with patch.object(_web, 'read', return_value=b''), self.assertRaises(SourceError): github._api('/x') diff --git a/tests/test_apk_search.py b/tests/test_apk_search.py index d5c1389..3e249da 100644 --- a/tests/test_apk_search.py +++ b/tests/test_apk_search.py @@ -71,7 +71,7 @@ class SearchTests(SettingsTest): result = search.search(timeout=.03) self.assertLess(time.monotonic() - started, .3) self.assertTrue(result['apps']) - self.assertEqual([s['status'] for s in result['sources']], ['ok', 'timed out', 'error']) + self.assertEqual([s['status'] for s in result['sources']], ['ok', 'loading', 'error']) search.search('other', timeout=.03) self.assertEqual(calls, ['']) finally: diff --git a/tests/test_fdroid_sources.py b/tests/test_fdroid_sources.py index 461b329..50cab33 100644 --- a/tests/test_fdroid_sources.py +++ b/tests/test_fdroid_sources.py @@ -127,6 +127,8 @@ class Repositories(unittest.TestCase): fdroid._url(url) with self.assertRaisesRegex(SourceError, 'conflicting'): fdroid._url(URL + '?fingerprint=' + PIN, '0' * 64) + self.assertEqual(fdroid._child(URL, '/app/en-US/phoneScreenshots/#0 a.png'), + URL + 'app/en-US/phoneScreenshots/%230%20a.png') # real F-Droid screenshot name for name in ['../x.apk', '%2e%2e/x.apk', 'https://evil.org/a.apk', '//evil.org/../x', 'x?token=y', 'x\\y']: with self.subTest(name=name), self.assertRaises(SourceError): fdroid._child(URL, name) diff --git a/tests/test_sidequest.py b/tests/test_sidequest.py index 495d685..11d8d19 100644 --- a/tests/test_sidequest.py +++ b/tests/test_sidequest.py @@ -26,8 +26,7 @@ class SideQuestTests(unittest.TestCase): self.assertEqual(entry['images']['screenshots'], []) with self.assertRaisesRegex(SourceError, 'page-only'): sidequest.download(source, '123') - with self.assertRaisesRegex(SourceError, 'page-only'): - sidequest.search(source, 'open saber') + self.assertEqual(sidequest.search(source, 'open saber'), []) self.assertEqual(sidequest.search(source, 'open saber', 0), []) def test_invalid_ids(self): diff --git a/ui/apk_sources/_web.py b/ui/apk_sources/_web.py index 1ed8d4a..d86e843 100644 --- a/ui/apk_sources/_web.py +++ b/ui/apk_sources/_web.py @@ -63,7 +63,12 @@ def read(url, hosts, headers=None, ttl=3600): return data except urllib.error.HTTPError as e: if e.code in (403, 429): - raise SourceError('Source refused access or reached its rate limit; try later (GitHub accepts FRAME_GITHUB_TOKEN)') from e + if os.path.isfile(path): # throttled: an older copy beats no results + with open(path, 'rb') as f: + return f.read() + host = urllib.parse.urlsplit(url).hostname or 'The source' + hint = ' (set FRAME_GITHUB_TOKEN to raise the limit)' if host.endswith('github.com') else '' + raise SourceError(host + ' is limiting requests right now; try again later' + hint) from e raise SourceError('Source HTTP error: ' + str(e.code)) from e except (OSError, ValueError) as e: raise SourceError('Could not read source: ' + str(e)) from e diff --git a/ui/apk_sources/fdroid.py b/ui/apk_sources/fdroid.py index df62699..7063a7f 100644 --- a/ui/apk_sources/fdroid.py +++ b/ui/apk_sources/fdroid.py @@ -68,7 +68,9 @@ def _child(base, name): decoded = urllib.parse.unquote(name) if not name or '\\' in decoded or any(x in ('.', '..') for x in decoded.split('/')): raise SourceError('unsafe repository file name') - url = urllib.parse.urljoin(base, name) + if '?' in decoded or urllib.parse.urlsplit(decoded).scheme: + raise SourceError('repository file is outside its repository') + url = urllib.parse.urljoin(base, urllib.parse.quote(decoded, safe='/')) # names may contain '#' or spaces if not url.startswith(base) or urllib.parse.urlsplit(url).query or urllib.parse.urlsplit(url).fragment: raise SourceError('repository file is outside its repository') return url @@ -246,7 +248,8 @@ def sources(): ('izzyondroid', 'IzzyOnDroid', 'https://apt.izzysoft.de/fdroid/repo/', IZZY_PIN)] settings = _read() return [dict(id=i, kind=KIND, name=n, url=u, fingerprint=p, builtin=True, - enabled=settings['enabled'].get(i, True), trust='community') + # The archive only holds superseded versions; it clutters search unless asked for. + enabled=settings['enabled'].get(i, i != 'fdroid-archive'), trust='community') for i, n, u, p in builtins] + settings['repos'] @@ -288,7 +291,10 @@ def _summary(value): def _images(meta, base): def url(file): name = file.get('name') if isinstance(file, dict) else file - return _child(base, name) if isinstance(name, str) and name else None + try: + return _child(base, name) if isinstance(name, str) and name else None + except SourceError: + return None # one odd image name mustn't hide the app icon = url(_text(meta.get('icon'))) banner = url(_text(meta.get('featureGraphic'))) diff --git a/ui/apk_sources/github_curated.json b/ui/apk_sources/github_curated.json index 113515e..90cdf21 100644 --- a/ui/apk_sources/github_curated.json +++ b/ui/apk_sources/github_curated.json @@ -36,7 +36,7 @@ { "repo": "SgtBilko76/SuperTux-3D", "name": "SuperTux 3D", - "summary": "Run and jump through Tux’s platform adventure on a layered 3D screen in VR.", + "summary": "Run and jump through Tux\u2019s platform adventure on a layered 3D screen in VR.", "license": "GPL-3.0", "vr": true, "asset_pattern": "*Quest*.apk", @@ -49,5 +49,24 @@ ] }, "icon": "https://raw.githubusercontent.com/SgtBilko76/SuperTux-3D/1955493ee6f1000e048c58db40d4904df827210e/data/images/engine/icons/supertux-256x256.png" + }, + { + "repo": "arpruss/OpenSaberPlus", + "name": "Open Saber Plus", + "summary": "Slash glowing blocks to the beat with two lightsabers, with songs from BeatSaver.", + "license": "MIT", + "vr": true, + "asset_pattern": "OpenSaberPlus.apk", + "allow_prerelease": false, + "images": { + "icon": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png", + "banner": "https://opengraph.githubassets.com/1/arpruss/OpenSaberPlus", + "screenshots": [ + "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_1.gif", + "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_2.gif", + "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/doc/images/OS0.4.0_3.gif" + ] + }, + "icon": "https://raw.githubusercontent.com/arpruss/OpenSaberPlus/8c5295cdaadf02ea7418b0c6cbea20240ba913af/icon.png" } ] diff --git a/ui/apk_sources/search.py b/ui/apk_sources/search.py index d1f3d18..8af14d1 100644 --- a/ui/apk_sources/search.py +++ b/ui/apk_sources/search.py @@ -189,9 +189,19 @@ def group(entries, query='', vr=None, installable=False): result.append({'name': best.get('name'), 'package': best.get('package'), 'summary': best.get('summary'), 'offers': offers}) q = normalise(query) + + def browse(a): # a headset store: VR first, then apps with artwork, newest first + o = a['offers'][0] + art = o.get('images') or {} + return (o.get('vr') is not True, not art.get('banner'), not art.get('screenshots'), + ''.join(chr(0x10ffff - ord(c)) for c in str(o.get('updated') or ''))) + if not q: + result.sort(key=lambda a: (not any(o['fit']['installable'] is True for o in a['offers']),) + browse(a)) + return result result.sort(key=lambda a: (not any(normalise(o.get('name')) == q for o in a['offers']), not any(o['fit']['installable'] is True for o in a['offers']), not any(normalise(o.get('name')).startswith(q) for o in a['offers']), + a['offers'][0].get('vr') is not True, normalise(a['name']))) return result @@ -220,13 +230,16 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l items, errors = registry() if source and source not in [s['id'] for _, s in items]: raise SourceError('Unknown source') - tasks = [(s, _launch(m, s, query, limit)) for m, s in items - if s['enabled'] and (not source or s['id'] == source)] + chosen = [(m, s) for m, s in items if s['enabled'] and (not source or s['id'] == source)] + # Page-only sources (SideQuest) can't be searched; offer a link to browse them instead. + elsewhere = [{'name': s['name'], 'url': s['url']} for m, s in chosen if s.get('page_only')] + tasks = [(s, _launch(m, s, query, limit)) for m, s in chosen if not s.get('page_only')] entries, statuses = [], list(errors) for s, task in tasks: status = {'id': s['id'], 'name': s['name']} if task is None or not task['event'].wait(max(0, task['started'] + timeout - time.monotonic())): - status.update(status='timed out') + # Still working (e.g. first download of a large index); it keeps going and fills the cache. + status.update(status='loading') elif 'error' in task: status.update(status='error', error=task['error']) else: @@ -235,7 +248,15 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l statuses.append(status) with _lock: _status[s['id']] = {k: v for k, v in status.items() if k not in ('id', 'name')} - return {'apps': group(entries, query, vr, installable), 'sources': statuses} + return {'apps': group(entries, query, vr, installable), 'sources': statuses, 'elsewhere': elsewhere} + + +def warm(): + """Start every enabled source's index download in the background (server start, new repo).""" + items, _ = registry() + for m, s in items: + if s['enabled'] and not s.get('page_only'): + _launch(m, s, '', 1) def install(source_id, entry_id, version_code=None, progress=None): diff --git a/ui/apk_sources/sidequest.py b/ui/apk_sources/sidequest.py index dfed2ee..ed6fe08 100644 --- a/ui/apk_sources/sidequest.py +++ b/ui/apk_sources/sidequest.py @@ -18,9 +18,7 @@ def sources(): def search(source, query, limit=50): # Do not invent catalogue results or interpret a query as a verified free app. - if limit <= 0: - return [] - raise SourceError(REASON + ' ' + URL + '/apps') + return [] def details(source, entry_id): diff --git a/ui/index.html b/ui/index.html index 60796f6..1e6a933 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1995,18 +1995,24 @@ function renderStore() { $("searchGrid").innerHTML=`` + storeRow(apps.some(a=>a.offers[0].popularity) ? "Popular apps" : "Explore apps","More ways to make your Frame yours",popular) + storeRow(tested.length ? "VR that works on the Frame" : "Step into VR",tested.length ? "Experiences checked on a Frame" : "A new perspective, a whole new playground",vr) + storeRow("Recently updated","Fresh from their creators",recent); } function storeChips() { $("searchFilters").querySelectorAll("button").forEach(b=>b.setAttribute("aria-pressed",b.dataset.filter==='all' ? sourceState.vr===null && !sourceState.installable : b.dataset.filter==='installable' ? sourceState.installable : sourceState.vr===(b.dataset.filter==='vr'))); } -async function searchSources() { +async function searchSources(quiet) { const request=++sourceState.request; $("searchGrid").setAttribute("aria-busy","true"); - $("searchGrid").innerHTML=`
${Array.from({length:8},()=>'').join("")}
`; + if(!quiet)$("searchGrid").innerHTML=`
${Array.from({length:8},()=>'').join("")}
`; const params=new URLSearchParams({q:$("sourceQ").value,source:sourceState.source,installable:String(sourceState.installable)}); if(sourceState.vr!==null)params.set("vr",String(sourceState.vr)); try { const result=await api("/api/search?"+params);if(request!==sourceState.request)return; sourceState.apps=result.apps; const unavailable=result.sources.filter(s=>s.status==='error'||s.status==='timed out').map(s=>storeName(s.name)); - $("sourceStatus").textContent=unavailable.length ? `${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.` : ""; - $("sourceStatus").hidden=!unavailable.length; + const loading=result.sources.filter(s=>s.status==='loading').map(s=>storeName(s.name)); + const notes=[]; + if(loading.length)notes.push(`Still fetching ${loading.join(" and ")}; more results will appear in a moment.`); + if(unavailable.length)notes.push(`${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.`); + $("sourceStatus").innerHTML=notes.map(esc).join(" ")+(result.elsewhere||[]).map(x=>` Browse ${esc(storeName(x.name))} ↗`).join(""); + $("sourceStatus").hidden=!notes.length && !(result.elsewhere||[]).length; + clearTimeout(sourceState.retry); + if(loading.length)sourceState.retry=setTimeout(()=>{ if(request===sourceState.request)searchSources(true); },6000); renderStore();await loadSources(); } catch(e) { if(request===sourceState.request) { $("searchGrid").innerHTML=storeEmpty();$("sourceStatus").hidden=false;$("sourceStatus").textContent="We couldn't reach the app shelves. Try again in a moment."; } } finally { if(request===sourceState.request)$("searchGrid").setAttribute("aria-busy","false"); } @@ -2075,7 +2081,7 @@ $("addSource").onsubmit=e=>{e.preventDefault();const f=new FormData(e.target);ch document.querySelectorAll('[data-close]').forEach(b=>b.onclick=()=>$(b.dataset.close).close()); $('appDetail').addEventListener('close',()=>{sourceState.detailRequest++;}); document.querySelectorAll('[data-store-tab]').forEach(b=>b.onclick=()=>{const browse=b.dataset.storeTab==='browse';$('appStore').hidden=!browse;$('androidLibrary').hidden=browse;$('repNew').hidden=browse;$('sourcesOpen').hidden=!browse;document.querySelectorAll('[data-store-tab]').forEach(t=>{t.classList.toggle('on',t===b);t.setAttribute('aria-pressed',t===b);});if(!browse)loadAndroid();}); -loadSources().then(searchSources); +loadSources().then(()=>searchSources()); // ---- Android apps (own Lepton instance each) + catalogue ---- const VLABEL = { works: "Works on Frame", likely: "Should work", maybe: "Might work", unlikely: "Probably crashes", no: "Won't work" }; diff --git a/ui/server.py b/ui/server.py index ab635f6..dbc8ad2 100755 --- a/ui/server.py +++ b/ui/server.py @@ -1623,6 +1623,7 @@ def main(): args = ap.parse_args() httpd = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) sweep_tmp() + threading.Thread(target=apk_search.warm, daemon=True).start() # big indexes download before the first search if not frame_host.WINDOWS: signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt)) if args.exit_on_eof: