diff --git a/docs/apk-versions-measurement.txt b/docs/apk-versions-measurement.txt new file mode 100644 index 0000000..6976f2d --- /dev/null +++ b/docs/apk-versions-measurement.txt @@ -0,0 +1,24 @@ +cold_seconds 33.303869292 +fdroid {"package": "org.fdroid.fdroid", "versions": [{"version": "2.0.0", "version_code": 2000050, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000050.apk", "sha256": "94938d324b03755fb324f143229240c75640e809cdd3a90cc0a545b5d0a61931", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000050.apk", "source": "F-Droid"}, {"version": "2.0-rc1", "version_code": 2000041, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000041.apk", "sha256": "36b343514ea2cdd353b7554c14b3da3f4309653b67be5ceb2e912b25597e3d85", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000041.apk", "source": "F-Droid"}, {"version": "2.0-rc0", "version_code": 2000040, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000040.apk", "sha256": "cd492ba2d5a425ad80aa22f4b13d9815b2ad487e9b2b6df6b67c31b957b19f2b", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000040.apk", "source": "F-Droid"}, {"version": "2.0-alpha11", "version_code": 2000011, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000011.apk", "sha256": "730f81f2cf860b2671410e3bb7a9f00f99ebd8020aea941c316db1f55cd030e3", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000011.apk", "source": "F-Droid"}, {"version": "2.0-alpha10", "version_code": 2000010, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000010.apk", "sha256": "0b6c65becb0712828625976c6baa76fc40153f77871487b5bc231c3433856b50", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000010.apk", "source": "F-Droid"}, {"version": "2.0-alpha9", "version_code": 2000009, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000009.apk", "sha256": "d6e5eaea63107aed8e46a1c2de3db4c878dcafccb832ec7048798c1275b0a1b7", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000009.apk", "source": "F-Droid"}, {"version": "2.0-alpha8", "version_code": 2000008, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000008.apk", "sha256": "72e139a04fcd0f587a85de5b118245f1cbcf2939874a4f3e298ae6eaea98653c", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000008.apk", "source": "F-Droid"}, {"version": "2.0-alpha7", "version_code": 2000007, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000007.apk", "sha256": "2acb72be96ae91342d52822c1b100edc60c5240eea630e4fc59cb7c4e18d7fbf", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000007.apk", "source": "F-Droid"}], "total": 250, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=org.fdroid.fdroid"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=org.fdroid.fdroid"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/org.fdroid.fdroid"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=org.fdroid.fdroid"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=org.fdroid.fdroid"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []} +termux_warm_seconds 0.00022699999999531428 +termux {"package": "com.termux", "versions": [{"version": "0.119.0-beta.3", "version_code": 1022, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1022.apk", "sha256": "fdd476982cd74f2f00aac12d3683b1fa260a0b2d146411b94e09d773be3a7b56", "url": "https://f-droid.org/repo/com.termux_1022.apk", "source": "F-Droid"}, {"version": "0.119.0-beta.2", "version_code": 1021, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1021.apk", "sha256": "54f472928acf98dd58ba4a1444cac41a4f144e015696ffe1fd81f13a23b997d0", "url": "https://f-droid.org/repo/com.termux_1021.apk", "source": "F-Droid"}, {"version": "0.118.3", "version_code": 1002, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1002.apk", "sha256": "e6265a57eb5ca363808488e3b01955958bed93bc0c8a0d281849b363b11027ec", "url": "https://f-droid.org/repo/com.termux_1002.apk", "source": "F-Droid"}, {"version": "0.118.2", "version_code": 1001, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1001.apk", "sha256": "75a8dd74d6fe69681af450869d872cc6a7ce13dc6ef5f0d9c1b94ed9cc734d50", "url": "https://f-droid.org/archive/com.termux_1001.apk", "source": "F-Droid archive"}, {"version": "0.118.1", "version_code": 1000, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1000.apk", "sha256": "f137958392a800fca583bfc00f191b8edb29b77c705fddf27dffb6c26ca5d413", "url": "https://f-droid.org/archive/com.termux_1000.apk", "source": "F-Droid archive"}, {"version": "0.118.0", "version_code": 118, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_118.apk", "sha256": "822ac152bd7c2d9770b87c1feea03f22f2349a91b94481b268c739493a260f0b", "url": "https://f-droid.org/archive/com.termux_118.apk", "source": "F-Droid archive"}, {"version": "0.117", "version_code": 117, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_117.apk", "sha256": "956f5945dd616206b1f226c54b1908110a1e8e2e759ba71c6f888577fa739993", "url": "https://f-droid.org/archive/com.termux_117.apk", "source": "F-Droid archive"}, {"version": "0.116", "version_code": 116, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_116.apk", "sha256": "fccc12892e3cfda140d23aaefb668e20934231ee5364d7ca8810b8e4ca0d178e", "url": "https://f-droid.org/archive/com.termux_116.apk", "source": "F-Droid archive"}], "total": 71, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=com.termux"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=com.termux"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/com.termux"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=com.termux"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=com.termux"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []} +warm_mean_seconds 0.000251964583000003 +cache_bytes 16460680 + 33.97 real 5.68 user 0.64 sys + 105906176 maximum resident set size + 0 average shared memory size + 0 average unshared data size + 0 average unshared stack size + 25234 page reclaims + 24 page faults + 0 swaps + 0 block input operations + 0 block output operations + 31 messages sent + 24566 messages received + 0 signals received + 259 voluntary context switches + 22262 involuntary context switches + 52738979493 instructions retired + 18814737630 cycles elapsed + 117670368 peak memory footprint diff --git a/docs/apks.md b/docs/apks.md index 4a7428d..4cb17b5 100644 --- a/docs/apks.md +++ b/docs/apks.md @@ -50,6 +50,10 @@ needs to be confirmed or started in the headset. Lepton is Android 11 (API 30), with arm64-v8a only. If Frame Control refuses an APK, it shows compatible versions from F-Droid's main and archive repos. +The refusal appears immediately; a separate request checks F-Droid while the +dialog shows “Checking F-Droid for older versions…”. It shows at most eight +distinct version names, newest first, preferring an arm64-only build when +available. The count includes all compatible builds before release deduplication. Choose **Install** to download a listed version, verify its SHA-256 against the index, and install it as its own app. @@ -320,3 +324,51 @@ because gamescope scales Lepton's surface to fit the same panel. Also unverified whether the settings survive the app or its Lepton instance relaunching. Lepton Development rebuilds its Android data on exit, so there they probably don't. + +## Version lookup performance + +Measured on 2026-09-28 with Python 3.9.6 against live f-droid.org, using an +empty temporary cache. Both repositories download sequentially under one lock. +The reducer decodes one package at a time with the standard library; the +`.installable-v1` files contain only compatible build records grouped by package. +Raw downloads are removed after reduction. The compact files refresh after one +day and are retained in memory by file mtime. Install uses the cached records +without refreshing the indexes or rerunning release selection. + +- Cold lookup: 33.304 s (process wall time 33.97 s). +- Maximum RSS: 105,906,176 bytes (101 MiB). +- Warm F-Droid lookup: 0.252 ms mean over 1,000 calls. +- Warm Termux lookup: 0.227 ms. +- Combined reduced cache: 16,460,680 bytes (15.7 MiB). +- F-Droid: 8 of 250 compatible builds, newest 2.0.0 (code 2000050). +- Termux: 8 of 71 compatible builds, newest 0.119.0-beta.3 (code 1022). +- Both repository lookups returned no errors. + +Full results and `/usr/bin/time -l` output: [measurement](apk-versions-measurement.txt). +Network timings depend on the connection and repository state. Reproduce with: + +```sh +/usr/bin/time -l python3 -c ' +import json, sys, tempfile, time +sys.path.insert(0, "ui") +import frame_catalog as c, frame_apk_versions as v +with tempfile.TemporaryDirectory() as directory: + c.CATALOG = directory + start = time.perf_counter() + print(json.dumps(v.alternatives("org.fdroid.fdroid"))) + print("cold_seconds", time.perf_counter() - start) + print(json.dumps(v.alternatives("com.termux"))) + start = time.perf_counter() + for _ in range(1000): + v.alternatives("org.fdroid.fdroid") + print("warm_mean_seconds", (time.perf_counter() - start) / 1000) +' +``` + +Verification: `python3 -m unittest discover -s tests` passed 138 tests; +`python3 -m py_compile ui/*.py`, inline JavaScript `node --check`, and +`git diff --check` passed. Tests include concurrent cache use, failed refresh, +mtime invalidation, streaming boundaries, cap/deduplication, endpoint validation, +nonblocking refusal, and cached installation identity checks. No browser visual +verification or physical headset installation was performed. Independent model +review was omitted to honor the task's explicit no-delegation instruction. diff --git a/tests/test_frame_apk_versions.py b/tests/test_frame_apk_versions.py index 6c29382..8e3c050 100644 --- a/tests/test_frame_apk_versions.py +++ b/tests/test_frame_apk_versions.py @@ -5,6 +5,7 @@ import os import sys import tempfile import unittest +from concurrent.futures import ThreadPoolExecutor from unittest.mock import patch sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ui')) @@ -66,12 +67,78 @@ class VersionsTest(unittest.TestCase): self.assertEqual(len(result['links']), 5) def test_no_compatible_versions(self): - with patch.object(frame_catalog, 'load_index', return_value={ - 'packages': {'org.example.app': {'versions': {'x': build(10, 33)}}}}): + with patch.object(frame_catalog, 'load_index', return_value={}): result = versions.alternatives('org.example.app') self.assertEqual(result['versions'], []) self.assertEqual(len(result['links']), 5) + def test_reduction_memory_cache_and_refresh(self): + repo = versions.REPOS[0][1] + index = frame_catalog.load_index(repo) + self.assertEqual([v['version_code'] for v in index['org.example.app']], [3, 2]) + self.assertEqual(set(index['org.example.app'][0]), + {'version', 'version_code', 'min_sdk', 'abis', 'name', 'sha256'}) + raw = os.path.join(self.tmp.name, 'data', 'index-v2.json') + self.assertFalse(os.path.exists(raw)) + with patch.object(frame_catalog.json, 'load', side_effect=AssertionError('reparsed')): + self.assertIs(frame_catalog.load_index(repo), index) + path = raw + '.installable-v1' + with open(path, 'w') as f: + json.dump({}, f) + os.utime(path, ns=(1, 1)) + self.assertEqual(frame_catalog.load_index(repo, cached_only=True), {}) + payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(9)}}}}).encode() + with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(payload)) as fetch: + self.assertEqual(frame_catalog.load_index(repo)['org.example.app'][0]['version_code'], 9) + fetch.assert_called_once() + self.assertFalse(os.path.exists(raw)) + + def test_concurrent_requests_share_download(self): + raw = os.path.join(self.tmp.name, 'data', 'index-v2.json') + os.remove(raw) + payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(7)}}}}).encode() + with patch.object(frame_catalog.urllib.request, 'urlopen', side_effect=lambda *a, **k: io.BytesIO(payload)) as fetch: + with ThreadPoolExecutor(max_workers=4) as pool: + indexes = list(pool.map(frame_catalog.load_index, [versions.REPOS[0][1]] * 4)) + fetch.assert_called_once() + self.assertTrue(all(index is indexes[0] for index in indexes)) + + def test_failed_refresh_preserves_cache(self): + repo = versions.REPOS[0][1] + index = frame_catalog.load_index(repo) + path = os.path.join(self.tmp.name, 'data', 'index-v2.json.installable-v1') + os.utime(path, ns=(1, 1)) + with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(b'{')): + with self.assertRaises(ValueError): + frame_catalog.load_index(repo) + self.assertEqual(frame_catalog.load_index(repo, cached_only=True), index) + self.assertFalse(any(name.endswith('.part') for name in os.listdir(os.path.dirname(path)))) + + def test_stream_boundaries_and_invalid_index(self): + raw = os.path.join(self.tmp.name, 'stream.json') + with open(raw, 'w') as f: + json.dump({'repo': {'description': 'é' * 70000}, 'packages': { + 'org.example.app': {'metadata': {'text': 'escaped " packages { }' * 6000}, + 'versions': {'x': build(7)}}}, 'tail': {}}, f) + self.assertEqual(frame_catalog._reduce_index(raw)['org.example.app'][0]['version_code'], 7) + for invalid in ('{}', '{"packages": []}', '{"packages": {', '{"packages": {}} trailing'): + with open(raw, 'w') as f: + f.write(invalid) + with self.assertRaises(ValueError): + frame_catalog._reduce_index(raw) + + def test_cap_and_preferred_build(self): + records = [dict(version=str(i), version_code=i, min_sdk=21, abis=[], + name='/app_%s.apk' % i, sha256=str(i)) for i in range(20)] + records += [dict(records[-1], version_code=21, abis=['arm64-v8a'], name='/arm.apk'), + dict(records[-1], version_code=22, abis=['arm64-v8a', 'x86_64'], name='/all.apk')] + with patch.object(frame_catalog, 'load_index', return_value={'org.example.app': records}): + result = versions.alternatives('org.example.app') + self.assertEqual(result['total'], 22) + self.assertEqual(len(result['versions']), 8) + self.assertEqual(len({v['version'] for v in result['versions']}), 8) + self.assertEqual([v['version_code'] for v in result['versions']], [21, 18, 17, 16, 15, 14, 13, 12]) + def test_android_names_and_verdict(self): for sdk, name in [(23, 'Android 6.0'), (30, 'Android 11'), (32, 'Android 12L'), (33, 'Android 13'), (99, 'Android API 99')]: self.assertEqual(versions.android_name(sdk), name) @@ -86,7 +153,9 @@ class VersionsTest(unittest.TestCase): self.assertIn('no arm64-v8a build', versions.describe(info)) def test_install_resolves_index_hash(self): - with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \ + versions.alternatives('org.example.app') + with patch.object(versions, 'alternatives', side_effect=AssertionError('recomputed')), \ + patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \ patch.object(frame_android, 'apk_info', return_value={'package': 'org.example.app', 'version_code': 1}), \ patch.object(frame_android, 'install', return_value={'label': 'Example'}) as install: versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk') @@ -95,30 +164,64 @@ class VersionsTest(unittest.TestCase): with self.assertRaises(frame_android.FrameError): versions.install('org.example.app', 'https://evil.example/app.apk') + def test_install_checks_identity_without_network_refresh(self): + versions.alternatives('org.example.app') + for name in ('index-v2.json', 'index-v2.archive.json'): + os.utime(os.path.join(self.tmp.name, 'data', name + '.installable-v1'), ns=(1, 1)) + for info in ({'package': 'wrong.package', 'version_code': 1}, + {'package': 'org.example.app', 'version_code': 99}): + with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk'), \ + patch.object(frame_android, 'apk_info', return_value=info), \ + patch.object(frame_android, 'install') as install: + with self.assertRaises(frame_android.FrameError): + versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk') + install.assert_not_called() + self.network.assert_not_called() + class UploadVersionsTest(unittest.TestCase): - def test_blocked_uploads_return_alternatives_before_ssh(self): + def test_endpoint_validation(self): + import server + for query in ('', 'package=', 'package=foo', 'package=a..b', 'package=a.1b', + 'package=a.b/path', 'package=a.b&package=c.d', 'package=a.b&code=-1', + 'package=a.b&code=x', 'package=a.b&code=', 'package=a.b&code=1&code=2'): + handler = object.__new__(server.Handler) + handler.path = '/api/apk-versions?' + query + with patch.object(handler, 'local_request', return_value=True), \ + patch.object(handler, 'send_json') as reply, \ + patch.object(versions, 'alternatives') as lookup: + handler.do_GET() + self.assertEqual(reply.call_args[0][1], 400, query) + lookup.assert_not_called() + handler.path = '/api/apk-versions?package=org.example_app.demo&code=123' + with patch.object(handler, 'local_request', return_value=True), \ + patch.object(handler, 'send_json') as reply, \ + patch.object(versions, 'alternatives', return_value={'total': 0}) as lookup: + handler.do_GET() + lookup.assert_called_once_with('org.example_app.demo', 123) + reply.assert_called_once_with({'total': 0}) + + def test_blocked_uploads_do_not_lookup_before_reply(self): import server info = {'package': 'org.example.app', 'label': 'Example', 'version': '5', 'version_code': 5, 'min_sdk': 33, 'abis': [], 'icon_png': None} - result = {'package': info['package'], 'versions': [], 'links': versions.search_links(info['package'])} for mode in ('apkinfo', 'apk'): handler = object.__new__(server.Handler) handler.headers = {'X-Filename': 'app.apk', 'X-Mode': mode, 'Content-Length': '1'} handler.rfile = io.BytesIO(b'x') with patch.object(frame_android, 'apk_info', return_value=dict(info)), \ - patch.object(versions, 'alternatives', return_value=result) as lookup, \ + patch.object(versions, 'alternatives', side_effect=AssertionError('lookup during upload')) as lookup, \ patch.object(server, 'ensure_master') as ssh: if mode == 'apkinfo': reply = handler.upload() - self.assertEqual(reply['apk']['alternatives'], result) + self.assertNotIn('alternatives', reply['apk']) self.assertIn('API 33', reply['apk']['blocker']) else: with self.assertRaises(server.Failure) as error: handler.upload() self.assertEqual(error.exception.status, 400) - self.assertEqual(error.exception.alternatives, result) - lookup.assert_called_once_with('org.example.app', 5) + self.assertEqual(error.exception.apk['package'], info['package']) + lookup.assert_not_called() ssh.assert_not_called() diff --git a/ui/frame_apk_versions.py b/ui/frame_apk_versions.py index 8a8559f..e69d3e5 100644 --- a/ui/frame_apk_versions.py +++ b/ui/frame_apk_versions.py @@ -3,7 +3,6 @@ from urllib.parse import quote, urlencode import frame_android import frame_catalog -from pick import installable ANDROID = dict(enumerate([ '1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3', @@ -45,38 +44,42 @@ def search_links(package): )] -def alternatives(package, current_version_code=None): - """All compatible builds, ordered by version code; keep per-ABI builds distinct.""" +def _versions(package, cached_only=False): versions, errors, seen = [], [], set() for source, repo in REPOS: try: - index = frame_catalog.load_index(repo) + index = frame_catalog.load_index(repo, cached_only=cached_only) except (OSError, ValueError) as e: errors.append(f'Could not check {source}: {e}') continue - for v in index.get('packages', {}).get(package, {}).get('versions', {}).values(): - m, file = v['manifest'], v['file'] - code = m.get('versionCode', 0) - if not installable(v): - continue - url = repo + file['name'].lstrip('/') - key = (code, file.get('sha256') or url) - if code == current_version_code or key in seen: + for v in index.get(package, []): + url = repo + v['name'].lstrip('/') + key = (v['version_code'], v.get('sha256') or url) + if key in seen: continue seen.add(key) - versions.append({'version': m.get('versionName', ''), 'version_code': code, - 'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1), - 'abis': m.get('nativecode') or [], 'url': url, - 'source': source, 'sha256': file.get('sha256')}) - versions.sort(key=lambda v: v['version_code'], reverse=True) - return {'package': package, 'versions': versions, 'links': search_links(package), - 'note': NOTE, 'errors': errors} + versions.append(dict(v, url=url, source=source)) + return versions, errors + + +def alternatives(package, current_version_code=None): + """At most eight releases, preferring arm64-only builds over universal builds.""" + versions, errors = _versions(package) + versions = [v for v in versions if v['version_code'] != current_version_code] + total = len(versions) + versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True) + releases = {} + for v in versions: + releases.setdefault(v['version'], v) + versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8] + return {'package': package, 'versions': versions, 'total': total, + 'links': search_links(package), 'note': NOTE, 'errors': errors} def install(package, url): # Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL. - result = alternatives(package) - version = next((v for v in result['versions'] if v['url'] == url), None) + records, _ = _versions(package, cached_only=True) + version = next((v for v in records if v['url'] == url), None) if not version: raise frame_android.FrameError('That version is no longer available; check the APK again') apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package}) diff --git a/ui/frame_catalog.py b/ui/frame_catalog.py index 3a3db39..17ddd83 100644 --- a/ui/frame_catalog.py +++ b/ui/frame_catalog.py @@ -24,29 +24,127 @@ _env = {} _index_lock = threading.Lock() +_indexes = {} -def load_index(repo): - """Share the catalogue's index files; installed apps use its writable cache.""" +class _IndexReader: + """Decode one object member at a time; never retain the whole raw index.""" + def __init__(self, stream): + self.stream, self.buffer = stream, '' + self.decoder = json.JSONDecoder() + + def fill(self): + chunk = self.stream.read(1 << 16) + if not chunk: + raise ValueError('incomplete F-Droid index') + self.buffer += chunk + + def peek(self): + self.buffer = self.buffer.lstrip() + while not self.buffer: + self.fill() + self.buffer = self.buffer.lstrip() + return self.buffer[0] + + def expect(self, char): + if self.peek() != char: + raise ValueError('invalid F-Droid index') + self.buffer = self.buffer[1:] + + def value(self): + self.peek() + while True: + try: + value, end = self.decoder.raw_decode(self.buffer) + self.buffer = self.buffer[end:] + return value + except json.JSONDecodeError: + self.fill() + + def members(self): + self.expect('{') + if self.peek() != '}': + while True: + key = self.value() + if not isinstance(key, str): + raise ValueError('invalid F-Droid index key') + self.expect(':') + yield key + if self.peek() == '}': + break + self.expect(',') + self.expect('}') + + +def _reduce_index(path): + from pick import installable + packages = {} + found = False + with open(path, encoding='utf-8') as f: + reader = _IndexReader(f) + for key in reader.members(): + if key != 'packages': + reader.value() + continue + found = True + for package in reader.members(): + records = [] + for v in reader.value().get('versions', {}).values(): + if not installable(v): + continue + m, file = v['manifest'], v['file'] + records.append({'version': m.get('versionName', ''), + 'version_code': m.get('versionCode', 0), + 'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1), + 'abis': m.get('nativecode') or [], + 'name': file['name'], 'sha256': file.get('sha256')}) + if records: + packages[package] = records + if reader.buffer.strip() or f.read().strip(): + raise ValueError('trailing data in F-Droid index') + if not found: + raise ValueError('invalid F-Droid index') + return packages + + +def load_index(repo, cached_only=False): + """Compact installable records by package, cached on disk and by mtime in memory.""" if repo not in APK_HOSTS: raise ValueError('unexpected index URL') directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data') filename = 'index-v2.json' if repo == APK_HOSTS[0] else 'index-v2.archive.json' - path = os.path.join(directory, filename) + raw = os.path.join(directory, filename) + path = raw + '.installable-v1' with _index_lock: - if os.path.exists(path) and time.time() - os.path.getmtime(path) < 86400: - with open(path) as f: - return json.load(f) + mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None + if mtime is not None and (cached_only or time.time() - mtime / 1e9 < 86400): + cached = _indexes.get(path) + if cached is None or cached[0] != mtime: + with open(path) as f: + cached = (mtime, json.load(f)) + _indexes[path] = cached + return cached[1] + if cached_only: + return {} os.makedirs(directory, exist_ok=True) fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory) + os.close(fd) try: - with os.fdopen(fd, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r: - shutil.copyfileobj(r, f, 1 << 20) - with open(tmp) as f: - index = json.load(f) - if not isinstance(index.get('packages'), dict): - raise ValueError('invalid F-Droid index') + if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400: + index = _reduce_index(raw) + refreshed = os.stat(raw).st_mtime_ns + else: + with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r: + shutil.copyfileobj(r, f, 1 << 20) + index = _reduce_index(tmp) + refreshed = time.time_ns() + with open(tmp, 'w') as f: + json.dump(index, f, separators=(',', ':')) + os.utime(tmp, ns=(refreshed, refreshed)) os.replace(tmp, path) + _indexes[path] = (os.stat(path).st_mtime_ns, index) + if os.path.exists(raw): + os.remove(raw) return index finally: if os.path.exists(tmp): diff --git a/ui/index.html b/ui/index.html index 4f8e733..3ebf51d 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1154,16 +1154,33 @@ function upload(file, mode) { xhr.onload = () => { $("prog").style.display = "none"; let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; } - if (data.alternatives) showApkAlternatives(data.error, data.alternatives); + if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk); xhr.status < 300 ? resolve(data) : reject(new Error(data.error)); }; xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); }; xhr.send(file); }); } +let apkLookup = 0; +async function checkApkAlternatives(apk) { + const lookup = ++apkLookup; + $("apkAltReason").textContent = apk.blocker; + $("apkAltVersions").textContent = "Checking F-Droid for older versions…"; + $("apkAltVersions").onclick = null; + for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = ""; + if (!$("apkAltDlg").open) $("apkAltDlg").showModal(); + const query = new URLSearchParams({package: apk.package}); + if (apk.version_code != null) query.set("code", apk.version_code); + try { + const result = await api(`/api/apk-versions?${query}`); + if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result); + } catch (e) { + if (lookup === apkLookup) $("apkAltVersions").textContent = e.message; + } +} function showApkAlternatives(reason, result) { $("apkAltReason").textContent = reason; - $("apkAltNote").textContent = result.note; + $("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`; $("apkAltErrors").textContent = result.errors.join(" · "); $("apkAltLinks").innerHTML = result.links.map(l => `${esc(l.source)}`).join(" · "); $("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) => @@ -1740,7 +1757,7 @@ $("repFile").onchange = async () => { const { apk } = await upload(file, "apkinfo"); $("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label; if (!$("repSrc").value) $("repSrc").value = file.name; - if (apk.alternatives) showApkAlternatives(apk.blocker, apk.alternatives); + if (apk.blocker) checkApkAlternatives(apk); $("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`; } catch (e) { $("repFileNote").textContent = e.message; } $("repFile").value = ""; diff --git a/ui/server.py b/ui/server.py index be078a3..6ecdfa1 100755 --- a/ui/server.py +++ b/ui/server.py @@ -81,10 +81,10 @@ exit 1 class Failure(Exception): - def __init__(self, message, status=502, alternatives=None): + def __init__(self, message, status=502, apk=None): super().__init__(message) self.status = status - self.alternatives = alternatives + self.apk = apk _master_lock = threading.Lock() @@ -445,6 +445,16 @@ def open_thing(body): raise Failure("unknown target", 400) +def apk_versions(query): + args = parse_qs(query, keep_blank_values=True) + packages, codes = args.get('package', []), args.get('code', []) + if len(packages) != 1 or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)+', packages[0]): + raise Failure('invalid Android package id', 400) + if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])): + raise Failure('invalid version code', 400) + return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None) + + def android(body): """Android apps, each in its own persistent Lepton instance (frame_android.py).""" action, pkg = body.get("action"), str(body.get("package", "")) @@ -1170,6 +1180,8 @@ class Handler(BaseHTTPRequestHandler): elif path == "/api/host": self.send_json({"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER, "computer": "Mac" if frame_host.MAC else "PC"}) + elif path == "/api/apk-versions": + self.send_json(apk_versions(url.query)) elif path == "/api/android": ensure_master() self.send_json({"apps": frame_android.list_apps()}) @@ -1207,7 +1219,7 @@ class Handler(BaseHTTPRequestHandler): else: self.send_json({"error": "not found"}, 404) except Failure as e: - self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status) + self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status) except frame_android.FrameError as e: self.send_json({"error": str(e)}, 502) except Exception as e: @@ -1233,7 +1245,7 @@ class Handler(BaseHTTPRequestHandler): raise Failure("request body must be a JSON object", 400) self.send_json(handler(body)) except Failure as e: - self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status) + self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status) except (ValueError, TypeError) as e: self.send_json({"error": f"bad request: {e}"}, 400) except frame_android.FrameError as e: @@ -1333,7 +1345,6 @@ class Handler(BaseHTTPRequestHandler): info["blocker"] = None except frame_android.FrameError as e: info["blocker"] = str(e) - info["alternatives"] = frame_apk_versions.alternatives(info["package"], info.get("version_code")) return {"message": f"Read {info['label']} {info['version']}", "apk": info} if mode == "title": keep = True # stage_title owns tmp now, and removes it on failure @@ -1346,7 +1357,7 @@ class Handler(BaseHTTPRequestHandler): try: frame_android.check_installable(info) except frame_android.FrameError as e: - raise Failure(str(e), 400, frame_apk_versions.alternatives(info["package"], info.get("version_code"))) + raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)}) ensure_master() try: m = frame_android.install(str(dest), source=name)