mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-06 06:00:33 +02:00
Reduce F-Droid indexes and fetch APK alternatives asynchronously
This commit is contained in:
saphid committed
1 parent
fbe7ba9575
commit
32196b4260
7 files changed
+357
-49
No files matched your search
@@ -0,0 +1,24 @@
|
||||
cold_seconds 33.303869292
|
||||
fdroid {"package": "org.fdroid.fdroid", "versions": [{"version": "2.0.0", "version_code": 2000050, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000050.apk", "sha256": "94938d324b03755fb324f143229240c75640e809cdd3a90cc0a545b5d0a61931", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000050.apk", "source": "F-Droid"}, {"version": "2.0-rc1", "version_code": 2000041, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000041.apk", "sha256": "36b343514ea2cdd353b7554c14b3da3f4309653b67be5ceb2e912b25597e3d85", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000041.apk", "source": "F-Droid"}, {"version": "2.0-rc0", "version_code": 2000040, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000040.apk", "sha256": "cd492ba2d5a425ad80aa22f4b13d9815b2ad487e9b2b6df6b67c31b957b19f2b", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000040.apk", "source": "F-Droid"}, {"version": "2.0-alpha11", "version_code": 2000011, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000011.apk", "sha256": "730f81f2cf860b2671410e3bb7a9f00f99ebd8020aea941c316db1f55cd030e3", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000011.apk", "source": "F-Droid"}, {"version": "2.0-alpha10", "version_code": 2000010, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000010.apk", "sha256": "0b6c65becb0712828625976c6baa76fc40153f77871487b5bc231c3433856b50", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000010.apk", "source": "F-Droid"}, {"version": "2.0-alpha9", "version_code": 2000009, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000009.apk", "sha256": "d6e5eaea63107aed8e46a1c2de3db4c878dcafccb832ec7048798c1275b0a1b7", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000009.apk", "source": "F-Droid"}, {"version": "2.0-alpha8", "version_code": 2000008, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000008.apk", "sha256": "72e139a04fcd0f587a85de5b118245f1cbcf2939874a4f3e298ae6eaea98653c", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000008.apk", "source": "F-Droid"}, {"version": "2.0-alpha7", "version_code": 2000007, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/org.fdroid.fdroid_2000007.apk", "sha256": "2acb72be96ae91342d52822c1b100edc60c5240eea630e4fc59cb7c4e18d7fbf", "url": "https://f-droid.org/repo/org.fdroid.fdroid_2000007.apk", "source": "F-Droid"}], "total": 250, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=org.fdroid.fdroid"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=org.fdroid.fdroid"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/org.fdroid.fdroid"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=org.fdroid.fdroid"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=org.fdroid.fdroid"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []}
|
||||
termux_warm_seconds 0.00022699999999531428
|
||||
termux {"package": "com.termux", "versions": [{"version": "0.119.0-beta.3", "version_code": 1022, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1022.apk", "sha256": "fdd476982cd74f2f00aac12d3683b1fa260a0b2d146411b94e09d773be3a7b56", "url": "https://f-droid.org/repo/com.termux_1022.apk", "source": "F-Droid"}, {"version": "0.119.0-beta.2", "version_code": 1021, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1021.apk", "sha256": "54f472928acf98dd58ba4a1444cac41a4f144e015696ffe1fd81f13a23b997d0", "url": "https://f-droid.org/repo/com.termux_1021.apk", "source": "F-Droid"}, {"version": "0.118.3", "version_code": 1002, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1002.apk", "sha256": "e6265a57eb5ca363808488e3b01955958bed93bc0c8a0d281849b363b11027ec", "url": "https://f-droid.org/repo/com.termux_1002.apk", "source": "F-Droid"}, {"version": "0.118.2", "version_code": 1001, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1001.apk", "sha256": "75a8dd74d6fe69681af450869d872cc6a7ce13dc6ef5f0d9c1b94ed9cc734d50", "url": "https://f-droid.org/archive/com.termux_1001.apk", "source": "F-Droid archive"}, {"version": "0.118.1", "version_code": 1000, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_1000.apk", "sha256": "f137958392a800fca583bfc00f191b8edb29b77c705fddf27dffb6c26ca5d413", "url": "https://f-droid.org/archive/com.termux_1000.apk", "source": "F-Droid archive"}, {"version": "0.118.0", "version_code": 118, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_118.apk", "sha256": "822ac152bd7c2d9770b87c1feea03f22f2349a91b94481b268c739493a260f0b", "url": "https://f-droid.org/archive/com.termux_118.apk", "source": "F-Droid archive"}, {"version": "0.117", "version_code": 117, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_117.apk", "sha256": "956f5945dd616206b1f226c54b1908110a1e8e2e759ba71c6f888577fa739993", "url": "https://f-droid.org/archive/com.termux_117.apk", "source": "F-Droid archive"}, {"version": "0.116", "version_code": 116, "min_sdk": 24, "abis": ["arm64-v8a", "armeabi-v7a", "x86", "x86_64"], "name": "/com.termux_116.apk", "sha256": "fccc12892e3cfda140d23aaefb668e20934231ee5364d7ca8810b8e4ca0d178e", "url": "https://f-droid.org/archive/com.termux_116.apk", "source": "F-Droid archive"}], "total": 71, "links": [{"source": "APKMirror", "url": "https://www.apkmirror.com/?post_type=app_release&s=com.termux"}, {"source": "APKPure", "url": "https://apkpure.com/search?q=com.termux"}, {"source": "Uptodown", "url": "https://en.uptodown.com/android/search/com.termux"}, {"source": "F-Droid", "url": "https://search.f-droid.org/?q=com.termux"}, {"source": "GitHub", "url": "https://github.com/search?type=repositories&q=com.termux"}], "note": "Pick a version whose minimum is Android 11 or lower and that has an arm64-v8a build (or no native code). Installable does not mean every feature works.", "errors": []}
|
||||
warm_mean_seconds 0.000251964583000003
|
||||
cache_bytes 16460680
|
||||
33.97 real 5.68 user 0.64 sys
|
||||
105906176 maximum resident set size
|
||||
0 average shared memory size
|
||||
0 average unshared data size
|
||||
0 average unshared stack size
|
||||
25234 page reclaims
|
||||
24 page faults
|
||||
0 swaps
|
||||
0 block input operations
|
||||
0 block output operations
|
||||
31 messages sent
|
||||
24566 messages received
|
||||
0 signals received
|
||||
259 voluntary context switches
|
||||
22262 involuntary context switches
|
||||
52738979493 instructions retired
|
||||
18814737630 cycles elapsed
|
||||
117670368 peak memory footprint
|
||||
@@ -50,6 +50,10 @@ needs to be confirmed or started in the headset.
|
||||
|
||||
Lepton is Android 11 (API 30), with arm64-v8a only. If Frame Control refuses
|
||||
an APK, it shows compatible versions from F-Droid's main and archive repos.
|
||||
The refusal appears immediately; a separate request checks F-Droid while the
|
||||
dialog shows “Checking F-Droid for older versions…”. It shows at most eight
|
||||
distinct version names, newest first, preferring an arm64-only build when
|
||||
available. The count includes all compatible builds before release deduplication.
|
||||
Choose **Install** to download a listed version, verify its SHA-256 against
|
||||
the index, and install it as its own app.
|
||||
|
||||
@@ -320,3 +324,51 @@ because gamescope scales Lepton's surface to fit the same panel. Also unverified
|
||||
whether the settings survive the app or its Lepton instance relaunching.
|
||||
Lepton Development rebuilds its Android data on exit, so there they probably
|
||||
don't.
|
||||
|
||||
## Version lookup performance
|
||||
|
||||
Measured on 2026-09-28 with Python 3.9.6 against live f-droid.org, using an
|
||||
empty temporary cache. Both repositories download sequentially under one lock.
|
||||
The reducer decodes one package at a time with the standard library; the
|
||||
`.installable-v1` files contain only compatible build records grouped by package.
|
||||
Raw downloads are removed after reduction. The compact files refresh after one
|
||||
day and are retained in memory by file mtime. Install uses the cached records
|
||||
without refreshing the indexes or rerunning release selection.
|
||||
|
||||
- Cold lookup: 33.304 s (process wall time 33.97 s).
|
||||
- Maximum RSS: 105,906,176 bytes (101 MiB).
|
||||
- Warm F-Droid lookup: 0.252 ms mean over 1,000 calls.
|
||||
- Warm Termux lookup: 0.227 ms.
|
||||
- Combined reduced cache: 16,460,680 bytes (15.7 MiB).
|
||||
- F-Droid: 8 of 250 compatible builds, newest 2.0.0 (code 2000050).
|
||||
- Termux: 8 of 71 compatible builds, newest 0.119.0-beta.3 (code 1022).
|
||||
- Both repository lookups returned no errors.
|
||||
|
||||
Full results and `/usr/bin/time -l` output: [measurement](apk-versions-measurement.txt).
|
||||
Network timings depend on the connection and repository state. Reproduce with:
|
||||
|
||||
```sh
|
||||
/usr/bin/time -l python3 -c '
|
||||
import json, sys, tempfile, time
|
||||
sys.path.insert(0, "ui")
|
||||
import frame_catalog as c, frame_apk_versions as v
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
c.CATALOG = directory
|
||||
start = time.perf_counter()
|
||||
print(json.dumps(v.alternatives("org.fdroid.fdroid")))
|
||||
print("cold_seconds", time.perf_counter() - start)
|
||||
print(json.dumps(v.alternatives("com.termux")))
|
||||
start = time.perf_counter()
|
||||
for _ in range(1000):
|
||||
v.alternatives("org.fdroid.fdroid")
|
||||
print("warm_mean_seconds", (time.perf_counter() - start) / 1000)
|
||||
'
|
||||
```
|
||||
|
||||
Verification: `python3 -m unittest discover -s tests` passed 138 tests;
|
||||
`python3 -m py_compile ui/*.py`, inline JavaScript `node --check`, and
|
||||
`git diff --check` passed. Tests include concurrent cache use, failed refresh,
|
||||
mtime invalidation, streaming boundaries, cap/deduplication, endpoint validation,
|
||||
nonblocking refusal, and cached installation identity checks. No browser visual
|
||||
verification or physical headset installation was performed. Independent model
|
||||
review was omitted to honor the task's explicit no-delegation instruction.
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'ui'))
|
||||
@@ -66,12 +67,78 @@ class VersionsTest(unittest.TestCase):
|
||||
self.assertEqual(len(result['links']), 5)
|
||||
|
||||
def test_no_compatible_versions(self):
|
||||
with patch.object(frame_catalog, 'load_index', return_value={
|
||||
'packages': {'org.example.app': {'versions': {'x': build(10, 33)}}}}):
|
||||
with patch.object(frame_catalog, 'load_index', return_value={}):
|
||||
result = versions.alternatives('org.example.app')
|
||||
self.assertEqual(result['versions'], [])
|
||||
self.assertEqual(len(result['links']), 5)
|
||||
|
||||
def test_reduction_memory_cache_and_refresh(self):
|
||||
repo = versions.REPOS[0][1]
|
||||
index = frame_catalog.load_index(repo)
|
||||
self.assertEqual([v['version_code'] for v in index['org.example.app']], [3, 2])
|
||||
self.assertEqual(set(index['org.example.app'][0]),
|
||||
{'version', 'version_code', 'min_sdk', 'abis', 'name', 'sha256'})
|
||||
raw = os.path.join(self.tmp.name, 'data', 'index-v2.json')
|
||||
self.assertFalse(os.path.exists(raw))
|
||||
with patch.object(frame_catalog.json, 'load', side_effect=AssertionError('reparsed')):
|
||||
self.assertIs(frame_catalog.load_index(repo), index)
|
||||
path = raw + '.installable-v1'
|
||||
with open(path, 'w') as f:
|
||||
json.dump({}, f)
|
||||
os.utime(path, ns=(1, 1))
|
||||
self.assertEqual(frame_catalog.load_index(repo, cached_only=True), {})
|
||||
payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(9)}}}}).encode()
|
||||
with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(payload)) as fetch:
|
||||
self.assertEqual(frame_catalog.load_index(repo)['org.example.app'][0]['version_code'], 9)
|
||||
fetch.assert_called_once()
|
||||
self.assertFalse(os.path.exists(raw))
|
||||
|
||||
def test_concurrent_requests_share_download(self):
|
||||
raw = os.path.join(self.tmp.name, 'data', 'index-v2.json')
|
||||
os.remove(raw)
|
||||
payload = json.dumps({'packages': {'org.example.app': {'versions': {'x': build(7)}}}}).encode()
|
||||
with patch.object(frame_catalog.urllib.request, 'urlopen', side_effect=lambda *a, **k: io.BytesIO(payload)) as fetch:
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
indexes = list(pool.map(frame_catalog.load_index, [versions.REPOS[0][1]] * 4))
|
||||
fetch.assert_called_once()
|
||||
self.assertTrue(all(index is indexes[0] for index in indexes))
|
||||
|
||||
def test_failed_refresh_preserves_cache(self):
|
||||
repo = versions.REPOS[0][1]
|
||||
index = frame_catalog.load_index(repo)
|
||||
path = os.path.join(self.tmp.name, 'data', 'index-v2.json.installable-v1')
|
||||
os.utime(path, ns=(1, 1))
|
||||
with patch.object(frame_catalog.urllib.request, 'urlopen', return_value=io.BytesIO(b'{')):
|
||||
with self.assertRaises(ValueError):
|
||||
frame_catalog.load_index(repo)
|
||||
self.assertEqual(frame_catalog.load_index(repo, cached_only=True), index)
|
||||
self.assertFalse(any(name.endswith('.part') for name in os.listdir(os.path.dirname(path))))
|
||||
|
||||
def test_stream_boundaries_and_invalid_index(self):
|
||||
raw = os.path.join(self.tmp.name, 'stream.json')
|
||||
with open(raw, 'w') as f:
|
||||
json.dump({'repo': {'description': 'é' * 70000}, 'packages': {
|
||||
'org.example.app': {'metadata': {'text': 'escaped " packages { }' * 6000},
|
||||
'versions': {'x': build(7)}}}, 'tail': {}}, f)
|
||||
self.assertEqual(frame_catalog._reduce_index(raw)['org.example.app'][0]['version_code'], 7)
|
||||
for invalid in ('{}', '{"packages": []}', '{"packages": {', '{"packages": {}} trailing'):
|
||||
with open(raw, 'w') as f:
|
||||
f.write(invalid)
|
||||
with self.assertRaises(ValueError):
|
||||
frame_catalog._reduce_index(raw)
|
||||
|
||||
def test_cap_and_preferred_build(self):
|
||||
records = [dict(version=str(i), version_code=i, min_sdk=21, abis=[],
|
||||
name='/app_%s.apk' % i, sha256=str(i)) for i in range(20)]
|
||||
records += [dict(records[-1], version_code=21, abis=['arm64-v8a'], name='/arm.apk'),
|
||||
dict(records[-1], version_code=22, abis=['arm64-v8a', 'x86_64'], name='/all.apk')]
|
||||
with patch.object(frame_catalog, 'load_index', return_value={'org.example.app': records}):
|
||||
result = versions.alternatives('org.example.app')
|
||||
self.assertEqual(result['total'], 22)
|
||||
self.assertEqual(len(result['versions']), 8)
|
||||
self.assertEqual(len({v['version'] for v in result['versions']}), 8)
|
||||
self.assertEqual([v['version_code'] for v in result['versions']], [21, 18, 17, 16, 15, 14, 13, 12])
|
||||
|
||||
def test_android_names_and_verdict(self):
|
||||
for sdk, name in [(23, 'Android 6.0'), (30, 'Android 11'), (32, 'Android 12L'), (33, 'Android 13'), (99, 'Android API 99')]:
|
||||
self.assertEqual(versions.android_name(sdk), name)
|
||||
@@ -86,7 +153,9 @@ class VersionsTest(unittest.TestCase):
|
||||
self.assertIn('no arm64-v8a build', versions.describe(info))
|
||||
|
||||
def test_install_resolves_index_hash(self):
|
||||
with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \
|
||||
versions.alternatives('org.example.app')
|
||||
with patch.object(versions, 'alternatives', side_effect=AssertionError('recomputed')), \
|
||||
patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk') as fetch, \
|
||||
patch.object(frame_android, 'apk_info', return_value={'package': 'org.example.app', 'version_code': 1}), \
|
||||
patch.object(frame_android, 'install', return_value={'label': 'Example'}) as install:
|
||||
versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk')
|
||||
@@ -95,30 +164,64 @@ class VersionsTest(unittest.TestCase):
|
||||
with self.assertRaises(frame_android.FrameError):
|
||||
versions.install('org.example.app', 'https://evil.example/app.apk')
|
||||
|
||||
def test_install_checks_identity_without_network_refresh(self):
|
||||
versions.alternatives('org.example.app')
|
||||
for name in ('index-v2.json', 'index-v2.archive.json'):
|
||||
os.utime(os.path.join(self.tmp.name, 'data', name + '.installable-v1'), ns=(1, 1))
|
||||
for info in ({'package': 'wrong.package', 'version_code': 1},
|
||||
{'package': 'org.example.app', 'version_code': 99}):
|
||||
with patch.object(frame_catalog, 'fetch_apk', return_value='/tmp/example.apk'), \
|
||||
patch.object(frame_android, 'apk_info', return_value=info), \
|
||||
patch.object(frame_android, 'install') as install:
|
||||
with self.assertRaises(frame_android.FrameError):
|
||||
versions.install('org.example.app', 'https://f-droid.org/archive/example_1.apk')
|
||||
install.assert_not_called()
|
||||
self.network.assert_not_called()
|
||||
|
||||
|
||||
class UploadVersionsTest(unittest.TestCase):
|
||||
def test_blocked_uploads_return_alternatives_before_ssh(self):
|
||||
def test_endpoint_validation(self):
|
||||
import server
|
||||
for query in ('', 'package=', 'package=foo', 'package=a..b', 'package=a.1b',
|
||||
'package=a.b/path', 'package=a.b&package=c.d', 'package=a.b&code=-1',
|
||||
'package=a.b&code=x', 'package=a.b&code=', 'package=a.b&code=1&code=2'):
|
||||
handler = object.__new__(server.Handler)
|
||||
handler.path = '/api/apk-versions?' + query
|
||||
with patch.object(handler, 'local_request', return_value=True), \
|
||||
patch.object(handler, 'send_json') as reply, \
|
||||
patch.object(versions, 'alternatives') as lookup:
|
||||
handler.do_GET()
|
||||
self.assertEqual(reply.call_args[0][1], 400, query)
|
||||
lookup.assert_not_called()
|
||||
handler.path = '/api/apk-versions?package=org.example_app.demo&code=123'
|
||||
with patch.object(handler, 'local_request', return_value=True), \
|
||||
patch.object(handler, 'send_json') as reply, \
|
||||
patch.object(versions, 'alternatives', return_value={'total': 0}) as lookup:
|
||||
handler.do_GET()
|
||||
lookup.assert_called_once_with('org.example_app.demo', 123)
|
||||
reply.assert_called_once_with({'total': 0})
|
||||
|
||||
def test_blocked_uploads_do_not_lookup_before_reply(self):
|
||||
import server
|
||||
info = {'package': 'org.example.app', 'label': 'Example', 'version': '5',
|
||||
'version_code': 5, 'min_sdk': 33, 'abis': [], 'icon_png': None}
|
||||
result = {'package': info['package'], 'versions': [], 'links': versions.search_links(info['package'])}
|
||||
for mode in ('apkinfo', 'apk'):
|
||||
handler = object.__new__(server.Handler)
|
||||
handler.headers = {'X-Filename': 'app.apk', 'X-Mode': mode, 'Content-Length': '1'}
|
||||
handler.rfile = io.BytesIO(b'x')
|
||||
with patch.object(frame_android, 'apk_info', return_value=dict(info)), \
|
||||
patch.object(versions, 'alternatives', return_value=result) as lookup, \
|
||||
patch.object(versions, 'alternatives', side_effect=AssertionError('lookup during upload')) as lookup, \
|
||||
patch.object(server, 'ensure_master') as ssh:
|
||||
if mode == 'apkinfo':
|
||||
reply = handler.upload()
|
||||
self.assertEqual(reply['apk']['alternatives'], result)
|
||||
self.assertNotIn('alternatives', reply['apk'])
|
||||
self.assertIn('API 33', reply['apk']['blocker'])
|
||||
else:
|
||||
with self.assertRaises(server.Failure) as error:
|
||||
handler.upload()
|
||||
self.assertEqual(error.exception.status, 400)
|
||||
self.assertEqual(error.exception.alternatives, result)
|
||||
lookup.assert_called_once_with('org.example.app', 5)
|
||||
self.assertEqual(error.exception.apk['package'], info['package'])
|
||||
lookup.assert_not_called()
|
||||
ssh.assert_not_called()
|
||||
|
||||
|
||||
|
||||
+24
-21
@@ -3,7 +3,6 @@ from urllib.parse import quote, urlencode
|
||||
|
||||
import frame_android
|
||||
import frame_catalog
|
||||
from pick import installable
|
||||
|
||||
ANDROID = dict(enumerate([
|
||||
'1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3',
|
||||
@@ -45,38 +44,42 @@ def search_links(package):
|
||||
)]
|
||||
|
||||
|
||||
def alternatives(package, current_version_code=None):
|
||||
"""All compatible builds, ordered by version code; keep per-ABI builds distinct."""
|
||||
def _versions(package, cached_only=False):
|
||||
versions, errors, seen = [], [], set()
|
||||
for source, repo in REPOS:
|
||||
try:
|
||||
index = frame_catalog.load_index(repo)
|
||||
index = frame_catalog.load_index(repo, cached_only=cached_only)
|
||||
except (OSError, ValueError) as e:
|
||||
errors.append(f'Could not check {source}: {e}')
|
||||
continue
|
||||
for v in index.get('packages', {}).get(package, {}).get('versions', {}).values():
|
||||
m, file = v['manifest'], v['file']
|
||||
code = m.get('versionCode', 0)
|
||||
if not installable(v):
|
||||
continue
|
||||
url = repo + file['name'].lstrip('/')
|
||||
key = (code, file.get('sha256') or url)
|
||||
if code == current_version_code or key in seen:
|
||||
for v in index.get(package, []):
|
||||
url = repo + v['name'].lstrip('/')
|
||||
key = (v['version_code'], v.get('sha256') or url)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
versions.append({'version': m.get('versionName', ''), 'version_code': code,
|
||||
'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
|
||||
'abis': m.get('nativecode') or [], 'url': url,
|
||||
'source': source, 'sha256': file.get('sha256')})
|
||||
versions.sort(key=lambda v: v['version_code'], reverse=True)
|
||||
return {'package': package, 'versions': versions, 'links': search_links(package),
|
||||
'note': NOTE, 'errors': errors}
|
||||
versions.append(dict(v, url=url, source=source))
|
||||
return versions, errors
|
||||
|
||||
|
||||
def alternatives(package, current_version_code=None):
|
||||
"""At most eight releases, preferring arm64-only builds over universal builds."""
|
||||
versions, errors = _versions(package)
|
||||
versions = [v for v in versions if v['version_code'] != current_version_code]
|
||||
total = len(versions)
|
||||
versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True)
|
||||
releases = {}
|
||||
for v in versions:
|
||||
releases.setdefault(v['version'], v)
|
||||
versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8]
|
||||
return {'package': package, 'versions': versions, 'total': total,
|
||||
'links': search_links(package), 'note': NOTE, 'errors': errors}
|
||||
|
||||
|
||||
def install(package, url):
|
||||
# Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL.
|
||||
result = alternatives(package)
|
||||
version = next((v for v in result['versions'] if v['url'] == url), None)
|
||||
records, _ = _versions(package, cached_only=True)
|
||||
version = next((v for v in records if v['url'] == url), None)
|
||||
if not version:
|
||||
raise frame_android.FrameError('That version is no longer available; check the APK again')
|
||||
apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package})
|
||||
|
||||
+108
-10
@@ -24,29 +24,127 @@ _env = {}
|
||||
|
||||
|
||||
_index_lock = threading.Lock()
|
||||
_indexes = {}
|
||||
|
||||
|
||||
def load_index(repo):
|
||||
"""Share the catalogue's index files; installed apps use its writable cache."""
|
||||
class _IndexReader:
|
||||
"""Decode one object member at a time; never retain the whole raw index."""
|
||||
def __init__(self, stream):
|
||||
self.stream, self.buffer = stream, ''
|
||||
self.decoder = json.JSONDecoder()
|
||||
|
||||
def fill(self):
|
||||
chunk = self.stream.read(1 << 16)
|
||||
if not chunk:
|
||||
raise ValueError('incomplete F-Droid index')
|
||||
self.buffer += chunk
|
||||
|
||||
def peek(self):
|
||||
self.buffer = self.buffer.lstrip()
|
||||
while not self.buffer:
|
||||
self.fill()
|
||||
self.buffer = self.buffer.lstrip()
|
||||
return self.buffer[0]
|
||||
|
||||
def expect(self, char):
|
||||
if self.peek() != char:
|
||||
raise ValueError('invalid F-Droid index')
|
||||
self.buffer = self.buffer[1:]
|
||||
|
||||
def value(self):
|
||||
self.peek()
|
||||
while True:
|
||||
try:
|
||||
value, end = self.decoder.raw_decode(self.buffer)
|
||||
self.buffer = self.buffer[end:]
|
||||
return value
|
||||
except json.JSONDecodeError:
|
||||
self.fill()
|
||||
|
||||
def members(self):
|
||||
self.expect('{')
|
||||
if self.peek() != '}':
|
||||
while True:
|
||||
key = self.value()
|
||||
if not isinstance(key, str):
|
||||
raise ValueError('invalid F-Droid index key')
|
||||
self.expect(':')
|
||||
yield key
|
||||
if self.peek() == '}':
|
||||
break
|
||||
self.expect(',')
|
||||
self.expect('}')
|
||||
|
||||
|
||||
def _reduce_index(path):
|
||||
from pick import installable
|
||||
packages = {}
|
||||
found = False
|
||||
with open(path, encoding='utf-8') as f:
|
||||
reader = _IndexReader(f)
|
||||
for key in reader.members():
|
||||
if key != 'packages':
|
||||
reader.value()
|
||||
continue
|
||||
found = True
|
||||
for package in reader.members():
|
||||
records = []
|
||||
for v in reader.value().get('versions', {}).values():
|
||||
if not installable(v):
|
||||
continue
|
||||
m, file = v['manifest'], v['file']
|
||||
records.append({'version': m.get('versionName', ''),
|
||||
'version_code': m.get('versionCode', 0),
|
||||
'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
|
||||
'abis': m.get('nativecode') or [],
|
||||
'name': file['name'], 'sha256': file.get('sha256')})
|
||||
if records:
|
||||
packages[package] = records
|
||||
if reader.buffer.strip() or f.read().strip():
|
||||
raise ValueError('trailing data in F-Droid index')
|
||||
if not found:
|
||||
raise ValueError('invalid F-Droid index')
|
||||
return packages
|
||||
|
||||
|
||||
def load_index(repo, cached_only=False):
|
||||
"""Compact installable records by package, cached on disk and by mtime in memory."""
|
||||
if repo not in APK_HOSTS:
|
||||
raise ValueError('unexpected index URL')
|
||||
directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data')
|
||||
filename = 'index-v2.json' if repo == APK_HOSTS[0] else 'index-v2.archive.json'
|
||||
path = os.path.join(directory, filename)
|
||||
raw = os.path.join(directory, filename)
|
||||
path = raw + '.installable-v1'
|
||||
with _index_lock:
|
||||
if os.path.exists(path) and time.time() - os.path.getmtime(path) < 86400:
|
||||
mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None
|
||||
if mtime is not None and (cached_only or time.time() - mtime / 1e9 < 86400):
|
||||
cached = _indexes.get(path)
|
||||
if cached is None or cached[0] != mtime:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
cached = (mtime, json.load(f))
|
||||
_indexes[path] = cached
|
||||
return cached[1]
|
||||
if cached_only:
|
||||
return {}
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory)
|
||||
os.close(fd)
|
||||
try:
|
||||
with os.fdopen(fd, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
|
||||
if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400:
|
||||
index = _reduce_index(raw)
|
||||
refreshed = os.stat(raw).st_mtime_ns
|
||||
else:
|
||||
with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
|
||||
shutil.copyfileobj(r, f, 1 << 20)
|
||||
with open(tmp) as f:
|
||||
index = json.load(f)
|
||||
if not isinstance(index.get('packages'), dict):
|
||||
raise ValueError('invalid F-Droid index')
|
||||
index = _reduce_index(tmp)
|
||||
refreshed = time.time_ns()
|
||||
with open(tmp, 'w') as f:
|
||||
json.dump(index, f, separators=(',', ':'))
|
||||
os.utime(tmp, ns=(refreshed, refreshed))
|
||||
os.replace(tmp, path)
|
||||
_indexes[path] = (os.stat(path).st_mtime_ns, index)
|
||||
if os.path.exists(raw):
|
||||
os.remove(raw)
|
||||
return index
|
||||
finally:
|
||||
if os.path.exists(tmp):
|
||||
|
||||
+20
-3
@@ -1154,16 +1154,33 @@ function upload(file, mode) {
|
||||
xhr.onload = () => {
|
||||
$("prog").style.display = "none";
|
||||
let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; }
|
||||
if (data.alternatives) showApkAlternatives(data.error, data.alternatives);
|
||||
if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk);
|
||||
xhr.status < 300 ? resolve(data) : reject(new Error(data.error));
|
||||
};
|
||||
xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); };
|
||||
xhr.send(file);
|
||||
});
|
||||
}
|
||||
let apkLookup = 0;
|
||||
async function checkApkAlternatives(apk) {
|
||||
const lookup = ++apkLookup;
|
||||
$("apkAltReason").textContent = apk.blocker;
|
||||
$("apkAltVersions").textContent = "Checking F-Droid for older versions…";
|
||||
$("apkAltVersions").onclick = null;
|
||||
for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = "";
|
||||
if (!$("apkAltDlg").open) $("apkAltDlg").showModal();
|
||||
const query = new URLSearchParams({package: apk.package});
|
||||
if (apk.version_code != null) query.set("code", apk.version_code);
|
||||
try {
|
||||
const result = await api(`/api/apk-versions?${query}`);
|
||||
if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result);
|
||||
} catch (e) {
|
||||
if (lookup === apkLookup) $("apkAltVersions").textContent = e.message;
|
||||
}
|
||||
}
|
||||
function showApkAlternatives(reason, result) {
|
||||
$("apkAltReason").textContent = reason;
|
||||
$("apkAltNote").textContent = result.note;
|
||||
$("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`;
|
||||
$("apkAltErrors").textContent = result.errors.join(" · ");
|
||||
$("apkAltLinks").innerHTML = result.links.map(l => `<a href="${esc(l.url)}" target="_blank" rel="noopener noreferrer">${esc(l.source)}</a>`).join(" · ");
|
||||
$("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) =>
|
||||
@@ -1740,7 +1757,7 @@ $("repFile").onchange = async () => {
|
||||
const { apk } = await upload(file, "apkinfo");
|
||||
$("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label;
|
||||
if (!$("repSrc").value) $("repSrc").value = file.name;
|
||||
if (apk.alternatives) showApkAlternatives(apk.blocker, apk.alternatives);
|
||||
if (apk.blocker) checkApkAlternatives(apk);
|
||||
$("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`;
|
||||
} catch (e) { $("repFileNote").textContent = e.message; }
|
||||
$("repFile").value = "";
|
||||
|
||||
+17
-6
@@ -81,10 +81,10 @@ exit 1
|
||||
|
||||
|
||||
class Failure(Exception):
|
||||
def __init__(self, message, status=502, alternatives=None):
|
||||
def __init__(self, message, status=502, apk=None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.alternatives = alternatives
|
||||
self.apk = apk
|
||||
|
||||
|
||||
_master_lock = threading.Lock()
|
||||
@@ -445,6 +445,16 @@ def open_thing(body):
|
||||
raise Failure("unknown target", 400)
|
||||
|
||||
|
||||
def apk_versions(query):
|
||||
args = parse_qs(query, keep_blank_values=True)
|
||||
packages, codes = args.get('package', []), args.get('code', [])
|
||||
if len(packages) != 1 or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)+', packages[0]):
|
||||
raise Failure('invalid Android package id', 400)
|
||||
if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])):
|
||||
raise Failure('invalid version code', 400)
|
||||
return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None)
|
||||
|
||||
|
||||
def android(body):
|
||||
"""Android apps, each in its own persistent Lepton instance (frame_android.py)."""
|
||||
action, pkg = body.get("action"), str(body.get("package", ""))
|
||||
@@ -1170,6 +1180,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
elif path == "/api/host":
|
||||
self.send_json({"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER,
|
||||
"computer": "Mac" if frame_host.MAC else "PC"})
|
||||
elif path == "/api/apk-versions":
|
||||
self.send_json(apk_versions(url.query))
|
||||
elif path == "/api/android":
|
||||
ensure_master()
|
||||
self.send_json({"apps": frame_android.list_apps()})
|
||||
@@ -1207,7 +1219,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
self.send_json({"error": "not found"}, 404)
|
||||
except Failure as e:
|
||||
self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
|
||||
self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
|
||||
except frame_android.FrameError as e:
|
||||
self.send_json({"error": str(e)}, 502)
|
||||
except Exception as e:
|
||||
@@ -1233,7 +1245,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
raise Failure("request body must be a JSON object", 400)
|
||||
self.send_json(handler(body))
|
||||
except Failure as e:
|
||||
self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
|
||||
self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
|
||||
except (ValueError, TypeError) as e:
|
||||
self.send_json({"error": f"bad request: {e}"}, 400)
|
||||
except frame_android.FrameError as e:
|
||||
@@ -1333,7 +1345,6 @@ class Handler(BaseHTTPRequestHandler):
|
||||
info["blocker"] = None
|
||||
except frame_android.FrameError as e:
|
||||
info["blocker"] = str(e)
|
||||
info["alternatives"] = frame_apk_versions.alternatives(info["package"], info.get("version_code"))
|
||||
return {"message": f"Read {info['label']} {info['version']}", "apk": info}
|
||||
if mode == "title":
|
||||
keep = True # stage_title owns tmp now, and removes it on failure
|
||||
@@ -1346,7 +1357,7 @@ class Handler(BaseHTTPRequestHandler):
|
||||
try:
|
||||
frame_android.check_installable(info)
|
||||
except frame_android.FrameError as e:
|
||||
raise Failure(str(e), 400, frame_apk_versions.alternatives(info["package"], info.get("version_code")))
|
||||
raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)})
|
||||
ensure_master()
|
||||
try:
|
||||
m = frame_android.install(str(dest), source=name)
|
||||
|
||||
Reference in new issue
Block a user