diff --git a/.gitignore b/.gitignore index d89ecc2..230bfe5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .DS_Store __pycache__/ apk-catalog/data/cache/ -apk-catalog/data/index-v2.json* +apk-catalog/data/index-v2*.json* compat-db/.env.lakebed.server compat-db/.lakebed/ tests/smoke/results/ diff --git a/docs/apks.md b/docs/apks.md index 050e58f..8bca60a 100644 --- a/docs/apks.md +++ b/docs/apks.md @@ -46,6 +46,33 @@ Lepton Development must be installed once. Over SSH, `ssh frame 'steam steam://install/3056000'` queues it, but the install still needs to be confirmed or started in the headset. +## When an app needs a newer Android + +Lepton is Android 11 (API 30), with arm64-v8a only. If Frame Control refuses +an APK, it shows compatible versions from F-Droid's main and archive repos and +IzzyOnDroid. It shows at most eight version names, newest first, preferring an +arm64-only build, and says how many compatible builds it found in total. +Each index is reduced to its compatible builds once a day and cached (about +16 MB). The first lookup takes about 30 s and 100 MB of memory; later ones are +instant. +Choose **Install** to download a listed version, verify its SHA-256 against +the index, and install it as its own app. + +You can also inspect a file or look up a package from the command line: + +```sh +python3 ui/frame_android.py info some-app.apk +python3 ui/frame_android.py versions some-app.apk +python3 ui/frame_android.py versions org.example.app +``` + +The search links open APKMirror, APKPure, Uptodown, F-Droid and GitHub. Pick a +version whose minimum is Android 11 or lower and that has an arm64-v8a build +(or no native code). Frame Control does not fetch APKs from those search sites. +Older versions may lack fixes, and being installable does not guarantee an +app will run: see the missing services below. Android may refuse a downgrade +or an update signed by a different publisher; removing the app deletes its data. + ## Installed apps disappear when Lepton Development closes (verified 2026-09-25) Lepton Development runs in a throwaway "dev" context. When it exits for any diff --git a/tests/test_frame_apk.py b/tests/test_frame_apk.py index 40119d4..1af14f9 100644 --- a/tests/test_frame_apk.py +++ b/tests/test_frame_apk.py @@ -35,7 +35,7 @@ def manifest(package, label_ref, version_ref, min_sdk, package_raw=True, foreign foreign_label adds a non-android `label` attribute after android:label. """ strings = ['label', 'icon', 'versionName', 'minSdkVersion', 'package', 'manifest', 'uses-sdk', - 'application', package, 'junk', 'label'] # the second 'label' has no android id + 'application', package, 'junk', 'label', 'versionCode'] # the second 'label' has no android id resmap = struct.pack('<4I', 0x01010001, 0x01010002, 0x0101021c, 0x0101020c) resmap = struct.pack('= 2}), diff --git a/ui/frame_apk_versions.py b/ui/frame_apk_versions.py new file mode 100644 index 0000000..e5964f4 --- /dev/null +++ b/ui/frame_apk_versions.py @@ -0,0 +1,90 @@ +"""Explain APK requirements and find installable versions in F-Droid's indexes.""" +from urllib.parse import quote, urlencode + +import frame_android +import frame_catalog + +ANDROID = dict(enumerate([ + '1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3', + '3.0', '3.1', '3.2', '4.0', '4.0.3', '4.1', '4.2', '4.3', '4.4', '4.4W', + '5.0', '5.1', '6.0', '7.0', '7.1', '8.0', '8.1', '9', '10', '11', '12', + '12L', '13', '14', '15', '16', +], 1)) +REPOS = (('F-Droid', 'https://f-droid.org/repo/'), + ('F-Droid archive', 'https://f-droid.org/archive/'), + ('IzzyOnDroid', 'https://apt.izzysoft.de/fdroid/repo/')) +NOTE = ('Pick a version whose minimum is Android 11 or lower and that has an ' + 'arm64-v8a build (or no native code). Installable does not mean every feature works.') + + +def android_name(sdk): + return 'Android ' + ANDROID[sdk] if sdk in ANDROID else f'Android API {sdk}' + + +def describe(info): + sdk = info.get('min_sdk') + minimum = f'{android_name(sdk)} (API {sdk})' if sdk else 'not specified' + try: + frame_android.check_installable(info) + verdict = 'Lepton can install this APK. Features may still need services Lepton lacks.' + except frame_android.FrameError as e: + verdict = f'Lepton cannot install this APK: {e}' + return (f"{info['package']} · {info.get('version') or '?'} " + f"(code {info.get('version_code') if info.get('version_code') is not None else '?'})\n" + f"Minimum: {minimum}\nABIs: {', '.join(info['abis']) or 'no native code'}\n{verdict}") + + +def search_links(package): + q = quote(package, safe='') + return [{'source': name, 'url': url} for name, url in ( + ('APKMirror', 'https://www.apkmirror.com/?' + urlencode({'post_type': 'app_release', 's': package})), + ('APKPure', 'https://apkpure.com/search?q=' + q), + ('Uptodown', 'https://en.uptodown.com/android/search/' + q), + ('F-Droid', 'https://search.f-droid.org/?q=' + q), + ('GitHub', 'https://github.com/search?type=repositories&q=' + q), + )] + + +def _versions(package, cached_only=False): + versions, errors, seen = [], [], set() + for source, repo in REPOS: + try: + index = frame_catalog.load_index(repo, cached_only=cached_only) + except Exception as e: # one bad repo (dropped download, odd index) mustn't hide the others + errors.append(f'Could not check {source}: {e}') + continue + for v in index.get(package, []): + url = repo + v['name'].lstrip('/') + key = (v['version_code'], v.get('sha256') or url) + if key in seen: + continue + seen.add(key) + versions.append(dict(v, url=url, source=source)) + return versions, errors + + +def alternatives(package, current_version_code=None): + """At most eight releases, preferring arm64-only builds over universal builds.""" + versions, errors = _versions(package) + versions = [v for v in versions if v['version_code'] != current_version_code] + total = len(versions) + versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True) + releases = {} + for v in versions: + releases.setdefault(v['version'], v) + versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8] + return {'package': package, 'versions': versions, 'total': total, + 'links': search_links(package), 'note': NOTE, 'errors': errors} + + +def install(package, url): + # Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL. + records, _ = _versions(package, cached_only=True) + version = next((v for v in records if v['url'] == url), None) + if not version: + raise frame_android.FrameError('That version is no longer available; check the APK again') + apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package}) + info = frame_android.apk_info(apk) + if info['package'] != package or info.get('version_code') != version['version_code']: + raise frame_android.FrameError('The downloaded APK does not match the selected version') + return frame_android.install(apk, source=version['source']) diff --git a/ui/frame_catalog.py b/ui/frame_catalog.py index 28c0161..d5b808e 100644 --- a/ui/frame_catalog.py +++ b/ui/frame_catalog.py @@ -2,7 +2,7 @@ list, verified downloads, installs into per-app Lepton instances, and compatibility reports. Python stdlib only. """ -import hashlib, os, shutil, sys, tempfile, threading, time, urllib.error, urllib.request +import hashlib, json, os, shutil, sys, tempfile, threading, time, urllib.error, urllib.request ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) CATALOG = os.path.join(ROOT, 'apk-catalog') @@ -17,12 +17,149 @@ import frame_compat_db as compat_db # noqa: E402 # the per-user cache (FRAME_CONTROL_APP is set by app/main.js). CACHE = (str(frame_host.cache_dir('apk')) if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data', 'cache')) -APK_HOSTS = ('https://f-droid.org/repo/', 'https://f-droid.org/archive/') +# Repo base URL -> local name of its index; every APK download must come from one of these. +INDEX_FILES = {'https://f-droid.org/repo/': 'index-v2.json', + 'https://f-droid.org/archive/': 'index-v2.archive.json', + 'https://apt.izzysoft.de/fdroid/repo/': 'index-v2.izzy.json'} +APK_HOSTS = tuple(INDEX_FILES) _lock = threading.Lock() _cache = {'mtime': None, 'sig': None, 'apps': None, 'by_pkg': None} _env = {} +_index_lock = threading.Lock() +_indexes = {} + + +class _IndexReader: + """Decode one object member at a time; never retain the whole raw index.""" + def __init__(self, stream): + self.stream, self.buffer = stream, '' + self.decoder = json.JSONDecoder() + + def fill(self): + chunk = self.stream.read(1 << 16) + if not chunk: + raise ValueError('incomplete F-Droid index') + self.buffer += chunk + + def peek(self): + self.buffer = self.buffer.lstrip() + while not self.buffer: + self.fill() + self.buffer = self.buffer.lstrip() + return self.buffer[0] + + def expect(self, char): + if self.peek() != char: + raise ValueError('invalid F-Droid index') + self.buffer = self.buffer[1:] + + def value(self): + self.peek() + while True: + try: + value, end = self.decoder.raw_decode(self.buffer) + self.buffer = self.buffer[end:] + return value + except json.JSONDecodeError: + self.fill() + + def members(self): + self.expect('{') + if self.peek() != '}': + while True: + key = self.value() + if not isinstance(key, str): + raise ValueError('invalid F-Droid index key') + self.expect(':') + yield key + if self.peek() == '}': + break + self.expect(',') + self.expect('}') + + +def _reduce_index(path): + from pick import installable + packages = {} + found = False + with open(path, encoding='utf-8') as f: + reader = _IndexReader(f) + for key in reader.members(): + if key != 'packages': + reader.value() + continue + found = True + for package in reader.members(): + records, entry = [], reader.value() + versions = entry.get('versions') if isinstance(entry, dict) else None + for v in (versions.values() if isinstance(versions, dict) else ()): + # Skip malformed entries rather than losing the whole repo. + if not (isinstance(v, dict) and isinstance(v.get('manifest'), dict) + and isinstance(v.get('file'), dict) and v['file'].get('name')): + continue + if not installable(v): + continue + m, file = v['manifest'], v['file'] + records.append({'version': m.get('versionName', ''), + 'version_code': m.get('versionCode', 0), + 'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1), + 'abis': m.get('nativecode') or [], + 'name': file['name'], 'sha256': file.get('sha256')}) + if records: + packages[package] = records + if reader.buffer.strip() or f.read().strip(): + raise ValueError('trailing data in F-Droid index') + if not found: + raise ValueError('invalid F-Droid index') + return packages + + +def load_index(repo, cached_only=False): + """Compact installable records by package, cached on disk and by mtime in memory.""" + if repo not in APK_HOSTS: + raise ValueError('unexpected index URL') + directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data') + filename = INDEX_FILES[repo] + raw = os.path.join(directory, filename) + path = raw + '.installable-v1' + with _index_lock: + mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None + # A newer raw index (the catalogue script refreshed it) outdates the reduced copy. + newer_raw = mtime is not None and os.path.exists(raw) and os.stat(raw).st_mtime_ns > mtime + if mtime is not None and (cached_only or (time.time() - mtime / 1e9 < 86400 and not newer_raw)): + cached = _indexes.get(path) + if cached is None or cached[0] != mtime: + with open(path) as f: + cached = (mtime, json.load(f)) + _indexes[path] = cached + return cached[1] + if cached_only: + return {} + os.makedirs(directory, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory) + os.close(fd) + try: + if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400: + index = _reduce_index(raw) + refreshed = os.stat(raw).st_mtime_ns + else: + with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r: + shutil.copyfileobj(r, f, 1 << 20) + index = _reduce_index(tmp) + refreshed = time.time_ns() + with open(tmp, 'w') as f: + json.dump(index, f, separators=(',', ':')) + os.utime(tmp, ns=(refreshed, refreshed)) + os.replace(tmp, path) + _indexes[path] = (os.stat(path).st_mtime_ns, index) + return index + finally: + if os.path.exists(tmp): + os.remove(tmp) + + def catalog(): """Rated apps with the database's reports applied.""" path = os.path.join(CATALOG, 'site', 'apps.js') diff --git a/ui/index.html b/ui/index.html index 3845ef7..8ac1265 100644 --- a/ui/index.html +++ b/ui/index.html @@ -254,10 +254,10 @@ .and-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 2fr); gap: 22px; align-items: start; } .and-col { display: grid; gap: 22px; align-content: start; } .rep-item .s { white-space: normal; } - #repDlg, #titleDlg, #wiDlg, #pwDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px; + #repDlg, #titleDlg, #wiDlg, #pwDlg, #apkAltDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px; padding: 22px; width: min(560px, 92vw); box-shadow: 0 20px 60px rgba(0,0,0,.6); } - #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop { background: rgba(0,0,0,.55); } - #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); } + #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop, #apkAltDlg::backdrop { background: rgba(0,0,0,.55); } + #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2, #apkAltDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); } #repForm label, #titleForm label { display: block; font-size: 12.5px; color: var(--muted); margin-top: 10px; } #repForm label input[type=text], #repForm textarea, #titleForm label input, #titleForm label select { margin-top: 5px; } #titleForm select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; @@ -656,6 +656,16 @@ Ready Show ▴ + +

Try another APK version

+

+
+

+ +

+
+ +

Report an APK

@@ -1410,12 +1420,56 @@ function upload(file, mode) { xhr.onload = () => { $("prog").style.display = "none"; let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; } + if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk); xhr.status < 300 ? resolve(data) : reject(new Error(data.error)); }; xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); }; xhr.send(file); }); } +let apkLookup = 0; +async function checkApkAlternatives(apk) { + const lookup = ++apkLookup; + $("apkAltReason").textContent = apk.blocker; + $("apkAltVersions").textContent = "Checking F-Droid for older versions…"; + $("apkAltVersions").onclick = null; + for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = ""; + if (!$("apkAltDlg").open) $("apkAltDlg").showModal(); + const query = new URLSearchParams({package: apk.package}); + if (apk.version_code != null) query.set("code", apk.version_code); + try { + const result = await api(`/api/apk-versions?${query}`); + if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result); + } catch (e) { + if (lookup === apkLookup) $("apkAltVersions").textContent = e.message; + } +} +function showApkAlternatives(reason, result) { + $("apkAltReason").textContent = reason; + $("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`; + $("apkAltErrors").textContent = result.errors.join(" · "); + $("apkAltLinks").innerHTML = result.links.map(l => `${esc(l.source)}`).join(" · "); + $("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) => + `
${esc(v.version || "?")} code ${esc(v.version_code)}
+
${esc(v.source)} · minimum API ${esc(v.min_sdk)} · ${esc(v.abis.join(", ") || "no native code")}
+
`).join("") : + `

No compatible version found in F-Droid. Try the searches below.

`; + $("apkAltVersions").onclick = async e => { + const b = e.target.closest("[data-version]"); if (!b) return; + const v = result.versions[+b.dataset.version]; + if (installing.has(result.package)) return; + b.disabled = true; b.textContent = "Installing…"; + const res = await runJob(`Install ${result.package} ${v.version}`, result.package, () => api("/api/android", { + action: "install", package: result.package, url: v.url + })); + if (res) $("apkAltDlg").close(); else { b.disabled = false; b.textContent = "Install"; } + await loadAndroid(); + if (cat.apps) { const y = window.scrollY; filterCatalog(); window.scrollTo(0, y); } + }; + if (!$("apkAltDlg").open) $("apkAltDlg").showModal(); +} +$("apkAltClose").onclick = () => $("apkAltDlg").close(); + const TITLE_EXT = /\.(zip|exe)$/i; async function sendFiles(files, dirs = new Set()) { for (const [i, f] of files.entries()) { @@ -1996,6 +2050,7 @@ $("repFile").onchange = async () => { const { apk } = await upload(file, "apkinfo"); $("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label; if (!$("repSrc").value) $("repSrc").value = file.name; + if (apk.blocker) checkApkAlternatives(apk); $("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`; } catch (e) { $("repFileNote").textContent = e.message; } $("repFile").value = ""; diff --git a/ui/server.py b/ui/server.py index ef22af6..dd86ff6 100755 --- a/ui/server.py +++ b/ui/server.py @@ -37,6 +37,7 @@ from urllib.parse import parse_qs, unquote, urlparse sys.path.insert(0, str(Path(__file__).resolve().parent)) import frame_android # noqa: E402 +import frame_apk_versions # noqa: E402 import frame_catalog # noqa: E402 import frame_host # noqa: E402 import frame_store # noqa: E402 @@ -91,9 +92,10 @@ exit 1 class Failure(Exception): - def __init__(self, message, status=502): + def __init__(self, message, status=502, apk=None): super().__init__(message) self.status = status + self.apk = apk # What ssh prints when it never reached the Frame, and what to tell the user @@ -561,16 +563,28 @@ def open_thing(body): raise Failure("unknown target", 400) +def apk_versions(query): + args = parse_qs(query, keep_blank_values=True) + packages, codes = args.get('package', []), args.get('code', []) + if len(packages) != 1 or not frame_android.PKG_RE.match(packages[0]): + raise Failure('invalid Android package id', 400) + if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])): + raise Failure('invalid version code', 400) + return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None) + + def android(body): """Android apps, each in its own persistent Lepton instance (frame_android.py).""" action, pkg = body.get("action"), str(body.get("package", "")) ensure_master() try: if action == "install": - frame_catalog.app(pkg) # an unknown package fails now, not in the background + url = body.get("url") + if not url: + frame_catalog.app(pkg) # an unknown package fails now, not in the background def work(): - m = frame_catalog.install(pkg) + m = frame_apk_versions.install(pkg, url) if url else frame_catalog.install(pkg) return {"message": f"Installed {m['label']}. It's in the Steam library; launching it opens its own panel.", "app": m} return start_job(f"Install {pkg}", work) @@ -1303,8 +1317,10 @@ class Handler(BaseHTTPRequestHandler): def send_json(self, obj, status=200): self.send_bytes(json.dumps(obj).encode(), "application/json", status) - def send_error_json(self, message, status): + def send_error_json(self, message, status, apk=None): body, offline_status = error_body(message) + if apk is not None: + body["apk"] = apk self.send_json(body, offline_status or status) def do_GET(self): @@ -1319,6 +1335,8 @@ class Handler(BaseHTTPRequestHandler): self.send_json({"os": "SteamOS", "fileManager": None, "computer": DEVICE, "mobile": True} if LOCAL else {"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER, "computer": "Mac" if frame_host.MAC else "PC"}) + elif path == "/api/apk-versions": + self.send_json(apk_versions(url.query)) elif path == "/api/android": ensure_master() self.send_json({"apps": frame_android.list_apps()}) @@ -1358,7 +1376,7 @@ class Handler(BaseHTTPRequestHandler): else: self.send_json({"error": "not found"}, 404) except Failure as e: - self.send_error_json(str(e), e.status) + self.send_error_json(str(e), e.status, e.apk) except frame_android.FrameError as e: self.send_error_json(str(e), 502) except Exception as e: @@ -1384,7 +1402,7 @@ class Handler(BaseHTTPRequestHandler): raise Failure("request body must be a JSON object", 400) self.send_json(handler(body)) except Failure as e: - self.send_error_json(str(e), e.status) + self.send_error_json(str(e), e.status, e.apk) except (ValueError, TypeError) as e: self.send_json({"error": f"bad request: {e}"}, 400) except frame_android.FrameError as e: @@ -1489,6 +1507,14 @@ class Handler(BaseHTTPRequestHandler): keep = True # stage_title owns tmp now, and removes it on failure return stage_title(str(dest), temp_dir=str(tmp)) if mode == "apk": + try: + info = frame_android.apk_info(str(dest)) + except frame_android.FrameError as e: + raise Failure(str(e), 400) + try: + frame_android.check_installable(info) + except frame_android.FrameError as e: + raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)}) ensure_master() try: m = frame_android.install(str(dest), source=name)