Reduce F-Droid indexes and fetch APK alternatives asynchronously

docs/apk-versions-measurement.txt

@@ -0,0 +1,24 @@
+cold_seconds 33.303869292
+fdroid {"package": "org.fdroid.fdroid", "versions": [{"version": "2.0.0", "version_code": 2000050, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000050.apk", "sha256": "94938d324b03755fb324f143229240c75640e809cdd3a90cc0a545b5d0a61931", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000050.apk", "source": "F-Droid"}, {"version": "2.0-rc1", "version_code": 2000041, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000041.apk", "sha256": "36b343514ea2cdd353b7554c14b3da3f4309653b67be5ceb2e912b25597e3d85", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000041.apk", "source": "F-Droid"}, {"version": "2.0-rc0", "version_code": 2000040, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000040.apk", "sha256": "cd492ba2d5a425ad80aa22f4b13d9815b2ad487e9b2b6df6b67c31b957b19f2b", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000040.apk", "source": "F-Droid"}, {"version": "2.0-alpha11", "version_code": 2000011, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000011.apk", "sha256": "730f81f2cf860b2671410e3bb7a9f00f99ebd8020aea941c316db1f55cd030e3", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000011.apk", "source": "F-Droid"}, {"version": "2.0-alpha10", "version_code": 2000010, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000010.apk", "sha256": "0b6c65becb0712828625976c6baa76fc40153f77871487b5bc231c3433856b50", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000010.apk", "source": "F-Droid"}, {"version": "2.0-alpha9", "version_code": 2000009, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000009.apk", "sha256": "d6e5eaea63107aed8e46a1c2de3db4c878dcafccb832ec7048798c1275b0a1b7", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000009.apk", "source": "F-Droid"}, {"version": "2.0-alpha8", "version_code": 2000008, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000008.apk", "sha256": "72e139a04fcd0f587a85de5b118245f1cbcf2939874a4f3e298ae6eaea98653c", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000008.apk", "source": "F-Droid"}, {"version": "2.0-alpha7", "version_code": 2000007, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000007.apk", "sha256": "2acb72be96ae91342d52822c1b100edc60c5240eea630e4fc59cb7c4e18d7fbf", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000007.apk", "source": "F-Droid"}], "total": 250, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=org.fdroid.fdroid"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=org.fdroid.fdroid"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/org.fdroid.fdroid"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=org.fdroid.fdroid"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=org.fdroid.fdroid"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []}
+termux_warm_seconds 0.00022699999999531428
+termux {"package": "com.termux", "versions": [{"version": "0.119.0-beta.3", "version_code": 1022, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1022.apk", "sha256": "fdd476982cd74f2f00aac12d3683b1fa260a0b2d146411b94e09d773be3a7b56", "url": "https://f-droid.org/repo/com.termux_1022.apk", "source": "F-Droid"}, {"version": "0.119.0-beta.2", "version_code": 1021, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1021.apk", "sha256": "54f472928acf98dd58ba4a1444cac41a4f144e015696ffe1fd81f13a23b997d0", "url": "https://f-droid.org/repo/com.termux_1021.apk", "source": "F-Droid"}, {"version": "0.118.3", "version_code": 1002, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1002.apk", "sha256": "e6265a57eb5ca363808488e3b01955958bed93bc0c8a0d281849b363b11027ec", "url": "https://f-droid.org/repo/com.termux_1002.apk", "source": "F-Droid"}, {"version": "0.118.2", "version_code": 1001, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1001.apk", "sha256": "75a8dd74d6fe69681af450869d872cc6a7ce13dc6ef5f0d9c1b94ed9cc734d50", "url": "https://f-droid.org/archive/com.termux_1001.apk", "source": "F-Droid archive"}, {"version": "0.118.1", "version_code": 1000, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1000.apk", "sha256": "f137958392a800fca583bfc00f191b8edb29b77c705fddf27dffb6c26ca5d413", "url": "https://f-droid.org/archive/com.termux_1000.apk", "source": "F-Droid archive"}, {"version": "0.118.0", "version_code": 118, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_118.apk", "sha256": "822ac152bd7c2d9770b87c1feea03f22f2349a91b94481b268c739493a260f0b", "url": "https://f-droid.org/archive/com.termux_118.apk", "source": "F-Droid archive"}, {"version": "0.117", "version_code": 117, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_117.apk", "sha256": "956f5945dd616206b1f226c54b1908110a1e8e2e759ba71c6f888577fa739993", "url": "https://f-droid.org/archive/com.termux_117.apk", "source": "F-Droid archive"}, {"version": "0.116", "version_code": 116, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_116.apk", "sha256": "fccc12892e3cfda140d23aaefb668e20934231ee5364d7ca8810b8e4ca0d178e", "url": "https://f-droid.org/archive/com.termux_116.apk", "source": "F-Droid archive"}], "total": 71, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=com.termux"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=com.termux"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/com.termux"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=com.termux"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=com.termux"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []}
+warm_mean_seconds 0.000251964583000003
+cache_bytes 16460680
+       33.97 real         5.68 user         0.64 sys
+           105906176  maximum resident set size
+                   0  average shared memory size
+                   0  average unshared data size
+                   0  average unshared stack size
+               25234  page reclaims
+                  24  page faults
+                   0  swaps
+                   0  block input operations
+                   0  block output operations
+                  31  messages sent
+               24566  messages received
+                   0  signals received
+                 259  voluntary context switches
+               22262  involuntary context switches
+         52738979493  instructions retired
+         18814737630  cycles elapsed
+           117670368  peak memory footprint

docs/apks.md

@@ -50,6 +50,10 @@ needs to be confirmed or started in the headset.
 
 Lepton is Android 11 (API 30), with arm64-v8a only. If Frame Control refuses
 an APK, it shows compatible versions from F-Droid's main and archive repos.
+The refusal appears immediately; a separate request checks F-Droid while the
+dialog shows “Checking F-Droid for older versions…”. It shows at most eight
+distinct version names, newest first, preferring an arm64-only build when
+available. The count includes all compatible builds before release deduplication.
 Choose **Install** to download a listed version, verify its SHA-256 against
 the index, and install it as its own app.
 
@@ -320,3 +324,51 @@ because gamescope scales Lepton's surface to fit the same panel. Also unverified
 whether the settings survive the app or its Lepton instance relaunching.
 Lepton Development rebuilds its Android data on exit, so there they probably
 don't.
+
+## Version lookup performance
+
+Measured on 2026-09-28 with Python 3.9.6 against live f-droid.org, using an
+empty temporary cache. Both repositories download sequentially under one lock.
+The reducer decodes one package at a time with the standard library; the
+`.installable-v1` files contain only compatible build records grouped by package.
+Raw downloads are removed after reduction. The compact files refresh after one
+day and are retained in memory by file mtime. Install uses the cached records
+without refreshing the indexes or rerunning release selection.
+
+- Cold lookup: 33.304 s (process wall time 33.97 s).
+- Maximum RSS: 105,906,176 bytes (101 MiB).
+- Warm F-Droid lookup: 0.252 ms mean over 1,000 calls.
+- Warm Termux lookup: 0.227 ms.
+- Combined reduced cache: 16,460,680 bytes (15.7 MiB).
+- F-Droid: 8 of 250 compatible builds, newest 2.0.0 (code 2000050).
+- Termux: 8 of 71 compatible builds, newest 0.119.0-beta.3 (code 1022).
+- Both repository lookups returned no errors.
+
+Full results and `/usr/bin/time -l` output: [measurement](apk-versions-measurement.txt).
+Network timings depend on the connection and repository state. Reproduce with:
+
+```sh
+/usr/bin/time -l python3 -c '
+import json, sys, tempfile, time
+sys.path.insert(0, "ui")
+import frame_catalog as c, frame_apk_versions as v
+with tempfile.TemporaryDirectory() as directory:
+    c.CATALOG = directory
+    start = time.perf_counter()
+    print(json.dumps(v.alternatives("org.fdroid.fdroid")))
+    print("cold_seconds", time.perf_counter() - start)
+    print(json.dumps(v.alternatives("com.termux")))
+    start = time.perf_counter()
+    for _ in range(1000):
+        v.alternatives("org.fdroid.fdroid")
+    print("warm_mean_seconds", (time.perf_counter() - start) / 1000)
+'
+```
+
+Verification: `python3 -m unittest discover -s tests` passed 138 tests;
+`python3 -m py_compile ui/*.py`, inline JavaScript `node --check`, and
+`git diff --check` passed. Tests include concurrent cache use, failed refresh,
+mtime invalidation, streaming boundaries, cap/deduplication, endpoint validation,
+nonblocking refusal, and cached installation identity checks. No browser visual
+verification or physical headset installation was performed. Independent model
+review was omitted to honor the task's explicit no-delegation instruction.

tests/test_frame_apk_versions.py

@@ -5,6 +5,7 @@ import os
 import sys
 import tempfile
 import unittest
+from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import patch
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ui'))
@@ -66,12 +67,78 @@ class VersionsTest(unittest.TestCase):
         self.assertEqual(len(result['links']), 5)
 
     def test_no_compatible_versions(self):
-        with patch.object(frame_catalog, 'load_index', return_value={
+        with patch.object(frame_catalog, 'load_index', return_value={}):
-                'packages': {'org.example.app': {'versions': {'x': build(10, 33)}}}}):
             result = versions.alternatives('org.example.app')
         self.assertEqual(result['versions'], [])
         self.assertEqual(len(result['links']), 5)
 
+    def test_reduction_memory_cache_and_refresh(self):
+        repo = versions.REPOS[0][1]
+        index = frame_catalog.load_index(repo)
+        self.assertEqual([v['version_code'] for v in index['org.example.app']], [3, 2])
+        self.assertEqual(set(index['org.example.app'][0]),
+                         {'version', 'version_code', 'min_sdk', 'abis', 'name', 'sha256'})
+        raw = os.path.join(self.tmp.name, 'data', 'index-v2.json')
+        self.assertFalse(os.path.exists(raw))
+        with patch.object(frame_catalog.json, 'load', side_effect=AssertionError('reparsed')):
+            self.assertIs(frame_catalog.load_index(repo), index)
+        path = raw + '.installable-v1'
+        with open(path, 'w') as f:
+            json.dump({}, f)
+        os.utime(path, ns=(1, 1))
+        self.assertEqual(frame_catalog.load_index(repo, cached_only=True), {})
+        payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(9)}}}}).encode()
+        with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(payload)) as fetch:
+            self.assertEqual(frame_catalog.load_index(repo)['org.example.app'][0]['version_code'], 9)
+            fetch.assert_called_once()
+        self.assertFalse(os.path.exists(raw))
+
+    def test_concurrent_requests_share_download(self):
+        raw = os.path.join(self.tmp.name, 'data', 'index-v2.json')
+        os.remove(raw)
+        payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(7)}}}}).encode()
+        with patch.object(frame_catalog.urllib.request, 'urlopen', side_effect=lambda *a, **k: io.BytesIO(payload)) as fetch:
+            with ThreadPoolExecutor(max_workers=4) as pool:
+                indexes = list(pool.map(frame_catalog.load_index, [versions.REPOS[0][1]] * 4))
+            fetch.assert_called_once()
+        self.assertTrue(all(index is indexes[0] for index in indexes))
+
+    def test_failed_refresh_preserves_cache(self):
+        repo = versions.REPOS[0][1]
+        index = frame_catalog.load_index(repo)
+        path = os.path.join(self.tmp.name, 'data', 'index-v2.json.installable-v1')
+        os.utime(path, ns=(1, 1))
+        with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(b'{')):
+            with self.assertRaises(ValueError):
+                frame_catalog.load_index(repo)
+        self.assertEqual(frame_catalog.load_index(repo, cached_only=True), index)
+        self.assertFalse(any(name.endswith('.part') for name in os.listdir(os.path.dirname(path))))
+
+    def test_stream_boundaries_and_invalid_index(self):
+        raw = os.path.join(self.tmp.name, 'stream.json')
+        with open(raw, 'w') as f:
+            json.dump({'repo': {'description': 'é' * 70000}, 'packages': {
+                'org.example.app': {'metadata': {'text': 'escaped " packages { }' * 6000},
+                                    'versions': {'x': build(7)}}}, 'tail': {}}, f)
+        self.assertEqual(frame_catalog._reduce_index(raw)['org.example.app'][0]['version_code'], 7)
+        for invalid in ('{}', '{"packages": []}', '{"packages": {', '{"packages": {}} trailing'):
+            with open(raw, 'w') as f:
+                f.write(invalid)
+            with self.assertRaises(ValueError):
+                frame_catalog._reduce_index(raw)
+
+    def test_cap_and_preferred_build(self):
+        records = [dict(version=str(i), version_code=i, min_sdk=21, abis=[],
+                        name='/app_%s.apk' % i, sha256=str(i)) for i in range(20)]
+        records += [dict(records[-1], version_code=21, abis=['arm64-v8a'], name='/arm.apk'),
+                    dict(records[-1], version_code=22, abis=['arm64-v8a', 'x86_64'], name='/all.apk')]
+        with patch.object(frame_catalog, 'load_index', return_value={'org.example.app': records}):
+            result = versions.alternatives('org.example.app')
+        self.assertEqual(result['total'], 22)
+        self.assertEqual(len(result['versions']), 8)
+        self.assertEqual(len({v['version'] for v in result['versions']}), 8)
+        self.assertEqual([v['version_code'] for v in result['versions']], [21, 18, 17, 16, 15, 14, 13, 12])
+
     def test_android_names_and_verdict(self):
         for sdk, name in [(23, 'Android 6.0'), (30, 'Android 11'), (32, 'Android 12L'), (33, 'Android 13'), (99, 'Android API 99')]:
             self.assertEqual(versions.android_name(sdk), name)
@@ -86,7 +153,9 @@ class VersionsTest(unittest.TestCase):
         self.assertIn('no arm64-v8a build', versions.describe(info))
 
     def test_install_resolves_index_hash(self):
-        with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \
+        versions.alternatives('org.example.app')
+        with patch.object(versions, 'alternatives', side_effect=AssertionError('recomputed')), \
+                patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \
                 patch.object(frame_android, 'apk_info', return_value={'package': 'org.example.app', 'version_code': 1}), \
                 patch.object(frame_android, 'install', return_value={'label': 'Example'}) as install:
             versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk')
@@ -95,30 +164,64 @@ class VersionsTest(unittest.TestCase):
         with self.assertRaises(frame_android.FrameError):
             versions.install('org.example.app', 'https://evil.example/app.apk')
 
+    def test_install_checks_identity_without_network_refresh(self):
+        versions.alternatives('org.example.app')
+        for name in ('index-v2.json', 'index-v2.archive.json'):
+            os.utime(os.path.join(self.tmp.name, 'data', name + '.installable-v1'), ns=(1, 1))
+        for info in ({'package': 'wrong.package', 'version_code': 1},
+                     {'package': 'org.example.app', 'version_code': 99}):
+            with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk'), \
+                    patch.object(frame_android, 'apk_info', return_value=info), \
+                    patch.object(frame_android, 'install') as install:
+                with self.assertRaises(frame_android.FrameError):
+                    versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk')
+                install.assert_not_called()
+        self.network.assert_not_called()
+
 
 class UploadVersionsTest(unittest.TestCase):
-    def test_blocked_uploads_return_alternatives_before_ssh(self):
+    def test_endpoint_validation(self):
+        import server
+        for query in ('', 'package=', 'package=foo', 'package=a..b', 'package=a.1b',
+                      'package=a.b/path', 'package=a.b&package=c.d', 'package=a.b&code=-1',
+                      'package=a.b&code=x', 'package=a.b&code=', 'package=a.b&code=1&code=2'):
+            handler = object.__new__(server.Handler)
+            handler.path = '/api/apk-versions?' + query
+            with patch.object(handler, 'local_request', return_value=True), \
+                    patch.object(handler, 'send_json') as reply, \
+                    patch.object(versions, 'alternatives') as lookup:
+                handler.do_GET()
+                self.assertEqual(reply.call_args[0][1], 400, query)
+                lookup.assert_not_called()
+        handler.path = '/api/apk-versions?package=org.example_app.demo&code=123'
+        with patch.object(handler, 'local_request', return_value=True), \
+                patch.object(handler, 'send_json') as reply, \
+                patch.object(versions, 'alternatives', return_value={'total': 0}) as lookup:
+            handler.do_GET()
+            lookup.assert_called_once_with('org.example_app.demo', 123)
+            reply.assert_called_once_with({'total': 0})
+
+    def test_blocked_uploads_do_not_lookup_before_reply(self):
         import server
         info = {'package': 'org.example.app', 'label': 'Example', 'version': '5',
                 'version_code': 5, 'min_sdk': 33, 'abis': [], 'icon_png': None}
-        result = {'package': info['package'], 'versions': [], 'links': versions.search_links(info['package'])}
         for mode in ('apkinfo', 'apk'):
             handler = object.__new__(server.Handler)
             handler.headers = {'X-Filename': 'app.apk', 'X-Mode': mode, 'Content-Length': '1'}
             handler.rfile = io.BytesIO(b'x')
             with patch.object(frame_android, 'apk_info', return_value=dict(info)), \
-                    patch.object(versions, 'alternatives', return_value=result) as lookup, \
+                    patch.object(versions, 'alternatives', side_effect=AssertionError('lookup during upload')) as lookup, \
                     patch.object(server, 'ensure_master') as ssh:
                 if mode == 'apkinfo':
                     reply = handler.upload()
-                    self.assertEqual(reply['apk']['alternatives'], result)
+                    self.assertNotIn('alternatives', reply['apk'])
                     self.assertIn('API 33', reply['apk']['blocker'])
                 else:
                     with self.assertRaises(server.Failure) as error:
                         handler.upload()
                     self.assertEqual(error.exception.status, 400)
-                    self.assertEqual(error.exception.alternatives, result)
+                    self.assertEqual(error.exception.apk['package'], info['package'])
-                lookup.assert_called_once_with('org.example.app', 5)
+                lookup.assert_not_called()
                 ssh.assert_not_called()
 
 

ui/frame_apk_versions.py

@@ -3,7 +3,6 @@ from urllib.parse import quote, urlencode
 
 import frame_android
 import frame_catalog
-from pick import installable
 
 ANDROID = dict(enumerate([
     '1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3',
@@ -45,38 +44,42 @@ def search_links(package):
     )]
 
 
-def alternatives(package, current_version_code=None):
+def _versions(package, cached_only=False):
-    """All compatible builds, ordered by version code; keep per-ABI builds distinct."""
     versions, errors, seen = [], [], set()
     for source, repo in REPOS:
         try:
-            index = frame_catalog.load_index(repo)
+            index = frame_catalog.load_index(repo, cached_only=cached_only)
         except (OSError, ValueError) as e:
             errors.append(f'Could not check {source}: {e}')
             continue
-        for v in index.get('packages', {}).get(package, {}).get('versions', {}).values():
+        for v in index.get(package, []):
-            m, file = v['manifest'], v['file']
+            url = repo + v['name'].lstrip('/')
-            code = m.get('versionCode', 0)
+            key = (v['version_code'], v.get('sha256') or url)
-            if not installable(v):
+            if key in seen:
-                continue
-            url = repo + file['name'].lstrip('/')
-            key = (code, file.get('sha256') or url)
-            if code == current_version_code or key in seen:
                 continue
             seen.add(key)
-            versions.append({'version': m.get('versionName', ''), 'version_code': code,
+            versions.append(dict(v, url=url, source=source))
-                             'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
+    return versions, errors
-                             'abis': m.get('nativecode') or [], 'url': url,
+
-                             'source': source, 'sha256': file.get('sha256')})
+
-    versions.sort(key=lambda v: v['version_code'], reverse=True)
+def alternatives(package, current_version_code=None):
-    return {'package': package, 'versions': versions, 'links': search_links(package),
+    """At most eight releases, preferring arm64-only builds over universal builds."""
-            'note': NOTE, 'errors': errors}
+    versions, errors = _versions(package)
+    versions = [v for v in versions if v['version_code'] != current_version_code]
+    total = len(versions)
+    versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True)
+    releases = {}
+    for v in versions:
+        releases.setdefault(v['version'], v)
+    versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8]
+    return {'package': package, 'versions': versions, 'total': total,
+            'links': search_links(package), 'note': NOTE, 'errors': errors}
 
 
 def install(package, url):
     # Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL.
-    result = alternatives(package)
+    records, _ = _versions(package, cached_only=True)
-    version = next((v for v in result['versions'] if v['url'] == url), None)
+    version = next((v for v in records if v['url'] == url), None)
     if not version:
         raise frame_android.FrameError('That version is no longer available; check the APK again')
     apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package})

ui/frame_catalog.py

@@ -24,29 +24,127 @@ _env = {}
 
 
 _index_lock = threading.Lock()
+_indexes = {}
 
 
-def load_index(repo):
+class _IndexReader:
-    """Share the catalogue's index files; installed apps use its writable cache."""
+    """Decode one object member at a time; never retain the whole raw index."""
+    def __init__(self, stream):
+        self.stream, self.buffer = stream, ''
+        self.decoder = json.JSONDecoder()
+
+    def fill(self):
+        chunk = self.stream.read(1 << 16)
+        if not chunk:
+            raise ValueError('incomplete F-Droid index')
+        self.buffer += chunk
+
+    def peek(self):
+        self.buffer = self.buffer.lstrip()
+        while not self.buffer:
+            self.fill()
+            self.buffer = self.buffer.lstrip()
+        return self.buffer[0]
+
+    def expect(self, char):
+        if self.peek() != char:
+            raise ValueError('invalid F-Droid index')
+        self.buffer = self.buffer[1:]
+
+    def value(self):
+        self.peek()
+        while True:
+            try:
+                value, end = self.decoder.raw_decode(self.buffer)
+                self.buffer = self.buffer[end:]
+                return value
+            except json.JSONDecodeError:
+                self.fill()
+
+    def members(self):
+        self.expect('{')
+        if self.peek() != '}':
+            while True:
+                key = self.value()
+                if not isinstance(key, str):
+                    raise ValueError('invalid F-Droid index key')
+                self.expect(':')
+                yield key
+                if self.peek() == '}':
+                    break
+                self.expect(',')
+        self.expect('}')
+
+
+def _reduce_index(path):
+    from pick import installable
+    packages = {}
+    found = False
+    with open(path, encoding='utf-8') as f:
+        reader = _IndexReader(f)
+        for key in reader.members():
+            if key != 'packages':
+                reader.value()
+                continue
+            found = True
+            for package in reader.members():
+                records = []
+                for v in reader.value().get('versions', {}).values():
+                    if not installable(v):
+                        continue
+                    m, file = v['manifest'], v['file']
+                    records.append({'version': m.get('versionName', ''),
+                                    'version_code': m.get('versionCode', 0),
+                                    'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
+                                    'abis': m.get('nativecode') or [],
+                                    'name': file['name'], 'sha256': file.get('sha256')})
+                if records:
+                    packages[package] = records
+        if reader.buffer.strip() or f.read().strip():
+            raise ValueError('trailing data in F-Droid index')
+    if not found:
+        raise ValueError('invalid F-Droid index')
+    return packages
+
+
+def load_index(repo, cached_only=False):
+    """Compact installable records by package, cached on disk and by mtime in memory."""
     if repo not in APK_HOSTS:
         raise ValueError('unexpected index URL')
     directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data')
     filename = 'index-v2.json' if repo == APK_HOSTS[0] else 'index-v2.archive.json'
-    path = os.path.join(directory, filename)
+    raw = os.path.join(directory, filename)
+    path = raw + '.installable-v1'
     with _index_lock:
-        if os.path.exists(path) and time.time() - os.path.getmtime(path) < 86400:
+        mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None
+        if mtime is not None and (cached_only or time.time() - mtime / 1e9 < 86400):
+            cached = _indexes.get(path)
+            if cached is None or cached[0] != mtime:
                 with open(path) as f:
-                return json.load(f)
+                    cached = (mtime, json.load(f))
+                _indexes[path] = cached
+            return cached[1]
+        if cached_only:
+            return {}
         os.makedirs(directory, exist_ok=True)
         fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory)
+        os.close(fd)
         try:
-            with os.fdopen(fd, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
+            if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400:
+                index = _reduce_index(raw)
+                refreshed = os.stat(raw).st_mtime_ns
+            else:
+                with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
                     shutil.copyfileobj(r, f, 1 << 20)
-            with open(tmp) as f:
+                index = _reduce_index(tmp)
-                index = json.load(f)
+                refreshed = time.time_ns()
-            if not isinstance(index.get('packages'), dict):
+            with open(tmp, 'w') as f:
-                raise ValueError('invalid F-Droid index')
+                json.dump(index, f, separators=(',', ':'))
+            os.utime(tmp, ns=(refreshed, refreshed))
             os.replace(tmp, path)
+            _indexes[path] = (os.stat(path).st_mtime_ns, index)
+            if os.path.exists(raw):
+                os.remove(raw)
             return index
         finally:
             if os.path.exists(tmp):

ui/index.html

@@ -1154,16 +1154,33 @@ function upload(file, mode) {
     xhr.onload = () => {
       $("prog").style.display = "none";
       let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; }
-      if (data.alternatives) showApkAlternatives(data.error, data.alternatives);
+      if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk);
       xhr.status < 300 ? resolve(data) : reject(new Error(data.error));
     };
     xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); };
     xhr.send(file);
   });
 }
+let apkLookup = 0;
+async function checkApkAlternatives(apk) {
+  const lookup = ++apkLookup;
+  $("apkAltReason").textContent = apk.blocker;
+  $("apkAltVersions").textContent = "Checking F-Droid for older versions…";
+  $("apkAltVersions").onclick = null;
+  for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = "";
+  if (!$("apkAltDlg").open) $("apkAltDlg").showModal();
+  const query = new URLSearchParams({package: apk.package});
+  if (apk.version_code != null) query.set("code", apk.version_code);
+  try {
+    const result = await api(`/api/apk-versions?${query}`);
+    if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result);
+  } catch (e) {
+    if (lookup === apkLookup) $("apkAltVersions").textContent = e.message;
+  }
+}
 function showApkAlternatives(reason, result) {
   $("apkAltReason").textContent = reason;
-  $("apkAltNote").textContent = result.note;
+  $("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`;
   $("apkAltErrors").textContent = result.errors.join(" · ");
   $("apkAltLinks").innerHTML = result.links.map(l => `<a href="${esc(l.url)}" target="_blank" rel="noopener noreferrer">${esc(l.source)}</a>`).join(" · ");
   $("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) =>
@@ -1740,7 +1757,7 @@ $("repFile").onchange = async () => {
     const { apk } = await upload(file, "apkinfo");
     $("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label;
     if (!$("repSrc").value) $("repSrc").value = file.name;
-    if (apk.alternatives) showApkAlternatives(apk.blocker, apk.alternatives);
+    if (apk.blocker) checkApkAlternatives(apk);
     $("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`;
   } catch (e) { $("repFileNote").textContent = e.message; }
   $("repFile").value = "";

ui/server.py

@@ -81,10 +81,10 @@ exit 1
 
 
 class Failure(Exception):
-    def __init__(self, message, status=502, alternatives=None):
+    def __init__(self, message, status=502, apk=None):
         super().__init__(message)
         self.status = status
-        self.alternatives = alternatives
+        self.apk = apk
 
 
 _master_lock = threading.Lock()
@@ -445,6 +445,16 @@ def open_thing(body):
     raise Failure("unknown target", 400)
 
 
+def apk_versions(query):
+    args = parse_qs(query, keep_blank_values=True)
+    packages, codes = args.get('package', []), args.get('code', [])
+    if len(packages) != 1 or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)+', packages[0]):
+        raise Failure('invalid Android package id', 400)
+    if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])):
+        raise Failure('invalid version code', 400)
+    return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None)
+
+
 def android(body):
     """Android apps, each in its own persistent Lepton instance (frame_android.py)."""
     action, pkg = body.get("action"), str(body.get("package", ""))
@@ -1170,6 +1180,8 @@ class Handler(BaseHTTPRequestHandler):
             elif path == "/api/host":
                 self.send_json({"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER,
                                 "computer": "Mac" if frame_host.MAC else "PC"})
+            elif path == "/api/apk-versions":
+                self.send_json(apk_versions(url.query))
             elif path == "/api/android":
                 ensure_master()
                 self.send_json({"apps": frame_android.list_apps()})
@@ -1207,7 +1219,7 @@ class Handler(BaseHTTPRequestHandler):
             else:
                 self.send_json({"error": "not found"}, 404)
         except Failure as e:
-            self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
+            self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
         except frame_android.FrameError as e:
             self.send_json({"error": str(e)}, 502)
         except Exception as e:
@@ -1233,7 +1245,7 @@ class Handler(BaseHTTPRequestHandler):
                 raise Failure("request body must be a JSON object", 400)
             self.send_json(handler(body))
         except Failure as e:
-            self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
+            self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
         except (ValueError, TypeError) as e:
             self.send_json({"error": f"bad request: {e}"}, 400)
         except frame_android.FrameError as e:
@@ -1333,7 +1345,6 @@ class Handler(BaseHTTPRequestHandler):
                     info["blocker"] = None
                 except frame_android.FrameError as e:
                     info["blocker"] = str(e)
-                    info["alternatives"] = frame_apk_versions.alternatives(info["package"], info.get("version_code"))
                 return {"message": f"Read {info['label']} {info['version']}", "apk": info}
             if mode == "title":
                 keep = True  # stage_title owns tmp now, and removes it on failure
@@ -1346,7 +1357,7 @@ class Handler(BaseHTTPRequestHandler):
                 try:
                     frame_android.check_installable(info)
                 except frame_android.FrameError as e:
-                    raise Failure(str(e), 400, frame_apk_versions.alternatives(info["package"], info.get("version_code")))
+                    raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)})
                 ensure_master()
                 try:
                     m = frame_android.install(str(dest), source=name)