Merge remote-tracking branch 'origin/main' into frame-input

ui/frame_android.py

@@ -6,7 +6,7 @@ apps, the lepton-show-flatscreen marker; plus a non-Steam shortcut, so it shows
 in the Steam library and gets its own SteamVR panel. Nothing goes through
 Lepton Development, which wipes its apps on exit. See docs/apks.md.
 
-Python stdlib only. CLI: python3 ui/frame_android.py {install APK|list|launch PKG|stop PKG|remove PKG|probe PKG}
+Python stdlib only. CLI: python3 ui/frame_android.py {info APK|versions APK-or-PKG|install APK|list|launch PKG|stop PKG|remove PKG|probe PKG}
 """
 import json, os, re, shlex, shutil, subprocess, sys, threading, time, zlib
 
@@ -276,7 +276,15 @@ def probe(pkg, wait=20):
 def main():
     cmd, *args = sys.argv[1:] or ['help']
     try:
-        if cmd == 'install':
+        if cmd in ('info', 'versions'):
+            import frame_apk_versions
+            if cmd == 'info':
+                print(frame_apk_versions.describe(apk_info(args[0])))
+                return
+            info = apk_info(args[0]) if os.path.isfile(args[0]) or args[0].lower().endswith('.apk') else None
+            r = frame_apk_versions.alternatives(
+                info['package'] if info else args[0], info.get('version_code') if info else None)
+        elif cmd == 'install':
             r = install(args[0], flatscreen='--vr' not in args)
         elif cmd == 'list':
             r = list_apps()

ui/frame_apk.py

@@ -229,6 +229,7 @@ def apk_info(path):
         min_sdk = sdk.get('minSdkVersion')
         info = {
             'package': package,
+            'version_code': manifest.get('versionCode', (None, None))[1],
             'version': _text(manifest.get('versionName'), res) or '',
             'label': _text(app.get('label'), res) or package,
             'abis': sorted({n.split('/')[1] for n in names if n.startswith('lib/') and n.count('/') >= 2}),

ui/frame_apk_versions.py

@@ -0,0 +1,90 @@
+"""Explain APK requirements and find installable versions in F-Droid's indexes."""
+from urllib.parse import quote, urlencode
+
+import frame_android
+import frame_catalog
+
+ANDROID = dict(enumerate([
+    '1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3',
+    '3.0', '3.1', '3.2', '4.0', '4.0.3', '4.1', '4.2', '4.3', '4.4', '4.4W',
+    '5.0', '5.1', '6.0', '7.0', '7.1', '8.0', '8.1', '9', '10', '11', '12',
+    '12L', '13', '14', '15', '16',
+], 1))
+REPOS = (('F-Droid', 'https://f-droid.org/repo/'),
+         ('F-Droid archive', 'https://f-droid.org/archive/'),
+         ('IzzyOnDroid', 'https://apt.izzysoft.de/fdroid/repo/'))
+NOTE = ('Pick a version whose minimum is Android 11 or lower and that has an '
+        'arm64-v8a build (or no native code). Installable does not mean every feature works.')
+
+
+def android_name(sdk):
+    return 'Android ' + ANDROID[sdk] if sdk in ANDROID else f'Android API {sdk}'
+
+
+def describe(info):
+    sdk = info.get('min_sdk')
+    minimum = f'{android_name(sdk)} (API {sdk})' if sdk else 'not specified'
+    try:
+        frame_android.check_installable(info)
+        verdict = 'Lepton can install this APK. Features may still need services Lepton lacks.'
+    except frame_android.FrameError as e:
+        verdict = f'Lepton cannot install this APK: {e}'
+    return (f"{info['package']} · {info.get('version') or '?'} "
+            f"(code {info.get('version_code') if info.get('version_code') is not None else '?'})\n"
+            f"Minimum: {minimum}\nABIs: {', '.join(info['abis']) or 'no native code'}\n{verdict}")
+
+
+def search_links(package):
+    q = quote(package, safe='')
+    return [{'source': name, 'url': url} for name, url in (
+        ('APKMirror', 'https://www.apkmirror.com/?' + urlencode({'post_type': 'app_release', 's': package})),
+        ('APKPure', 'https://apkpure.com/search?q=' + q),
+        ('Uptodown', 'https://en.uptodown.com/android/search/' + q),
+        ('F-Droid', 'https://search.f-droid.org/?q=' + q),
+        ('GitHub', 'https://github.com/search?type=repositories&q=' + q),
+    )]
+
+
+def _versions(package, cached_only=False):
+    versions, errors, seen = [], [], set()
+    for source, repo in REPOS:
+        try:
+            index = frame_catalog.load_index(repo, cached_only=cached_only)
+        except Exception as e:  # one bad repo (dropped download, odd index) mustn't hide the others
+            errors.append(f'Could not check {source}: {e}')
+            continue
+        for v in index.get(package, []):
+            url = repo + v['name'].lstrip('/')
+            key = (v['version_code'], v.get('sha256') or url)
+            if key in seen:
+                continue
+            seen.add(key)
+            versions.append(dict(v, url=url, source=source))
+    return versions, errors
+
+
+def alternatives(package, current_version_code=None):
+    """At most eight releases, preferring arm64-only builds over universal builds."""
+    versions, errors = _versions(package)
+    versions = [v for v in versions if v['version_code'] != current_version_code]
+    total = len(versions)
+    versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True)
+    releases = {}
+    for v in versions:
+        releases.setdefault(v['version'], v)
+    versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8]
+    return {'package': package, 'versions': versions, 'total': total,
+            'links': search_links(package), 'note': NOTE, 'errors': errors}
+
+
+def install(package, url):
+    # Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL.
+    records, _ = _versions(package, cached_only=True)
+    version = next((v for v in records if v['url'] == url), None)
+    if not version:
+        raise frame_android.FrameError('That version is no longer available; check the APK again')
+    apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package})
+    info = frame_android.apk_info(apk)
+    if info['package'] != package or info.get('version_code') != version['version_code']:
+        raise frame_android.FrameError('The downloaded APK does not match the selected version')
+    return frame_android.install(apk, source=version['source'])

ui/frame_catalog.py

@@ -2,7 +2,7 @@
 list, verified downloads, installs into per-app Lepton instances, and
 compatibility reports. Python stdlib only.
 """
-import hashlib, os, shutil, sys, tempfile, threading, time, urllib.error, urllib.request
+import hashlib, json, os, shutil, sys, tempfile, threading, time, urllib.error, urllib.request
 
 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 CATALOG = os.path.join(ROOT, 'apk-catalog')
@@ -17,12 +17,149 @@ import frame_compat_db as compat_db  # noqa: E402
 # the per-user cache (FRAME_CONTROL_APP is set by app/main.js).
 CACHE = (str(frame_host.cache_dir('apk')) if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG
          else os.path.join(CATALOG, 'data', 'cache'))
-APK_HOSTS = ('https://f-droid.org/repo/', 'https://f-droid.org/archive/')
+# Repo base URL -> local name of its index; every APK download must come from one of these.
+INDEX_FILES = {'https://f-droid.org/repo/': 'index-v2.json',
+               'https://f-droid.org/archive/': 'index-v2.archive.json',
+               'https://apt.izzysoft.de/fdroid/repo/': 'index-v2.izzy.json'}
+APK_HOSTS = tuple(INDEX_FILES)
 _lock = threading.Lock()
 _cache = {'mtime': None, 'sig': None, 'apps': None, 'by_pkg': None}
 _env = {}
 
 
+_index_lock = threading.Lock()
+_indexes = {}
+
+
+class _IndexReader:
+    """Decode one object member at a time; never retain the whole raw index."""
+    def __init__(self, stream):
+        self.stream, self.buffer = stream, ''
+        self.decoder = json.JSONDecoder()
+
+    def fill(self):
+        chunk = self.stream.read(1 << 16)
+        if not chunk:
+            raise ValueError('incomplete F-Droid index')
+        self.buffer += chunk
+
+    def peek(self):
+        self.buffer = self.buffer.lstrip()
+        while not self.buffer:
+            self.fill()
+            self.buffer = self.buffer.lstrip()
+        return self.buffer[0]
+
+    def expect(self, char):
+        if self.peek() != char:
+            raise ValueError('invalid F-Droid index')
+        self.buffer = self.buffer[1:]
+
+    def value(self):
+        self.peek()
+        while True:
+            try:
+                value, end = self.decoder.raw_decode(self.buffer)
+                self.buffer = self.buffer[end:]
+                return value
+            except json.JSONDecodeError:
+                self.fill()
+
+    def members(self):
+        self.expect('{')
+        if self.peek() != '}':
+            while True:
+                key = self.value()
+                if not isinstance(key, str):
+                    raise ValueError('invalid F-Droid index key')
+                self.expect(':')
+                yield key
+                if self.peek() == '}':
+                    break
+                self.expect(',')
+        self.expect('}')
+
+
+def _reduce_index(path):
+    from pick import installable
+    packages = {}
+    found = False
+    with open(path, encoding='utf-8') as f:
+        reader = _IndexReader(f)
+        for key in reader.members():
+            if key != 'packages':
+                reader.value()
+                continue
+            found = True
+            for package in reader.members():
+                records, entry = [], reader.value()
+                versions = entry.get('versions') if isinstance(entry, dict) else None
+                for v in (versions.values() if isinstance(versions, dict) else ()):
+                    # Skip malformed entries rather than losing the whole repo.
+                    if not (isinstance(v, dict) and isinstance(v.get('manifest'), dict)
+                            and isinstance(v.get('file'), dict) and v['file'].get('name')):
+                        continue
+                    if not installable(v):
+                        continue
+                    m, file = v['manifest'], v['file']
+                    records.append({'version': m.get('versionName', ''),
+                                    'version_code': m.get('versionCode', 0),
+                                    'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
+                                    'abis': m.get('nativecode') or [],
+                                    'name': file['name'], 'sha256': file.get('sha256')})
+                if records:
+                    packages[package] = records
+        if reader.buffer.strip() or f.read().strip():
+            raise ValueError('trailing data in F-Droid index')
+    if not found:
+        raise ValueError('invalid F-Droid index')
+    return packages
+
+
+def load_index(repo, cached_only=False):
+    """Compact installable records by package, cached on disk and by mtime in memory."""
+    if repo not in APK_HOSTS:
+        raise ValueError('unexpected index URL')
+    directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data')
+    filename = INDEX_FILES[repo]
+    raw = os.path.join(directory, filename)
+    path = raw + '.installable-v1'
+    with _index_lock:
+        mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None
+        # A newer raw index (the catalogue script refreshed it) outdates the reduced copy.
+        newer_raw = mtime is not None and os.path.exists(raw) and os.stat(raw).st_mtime_ns > mtime
+        if mtime is not None and (cached_only or (time.time() - mtime / 1e9 < 86400 and not newer_raw)):
+            cached = _indexes.get(path)
+            if cached is None or cached[0] != mtime:
+                with open(path) as f:
+                    cached = (mtime, json.load(f))
+                _indexes[path] = cached
+            return cached[1]
+        if cached_only:
+            return {}
+        os.makedirs(directory, exist_ok=True)
+        fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory)
+        os.close(fd)
+        try:
+            if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400:
+                index = _reduce_index(raw)
+                refreshed = os.stat(raw).st_mtime_ns
+            else:
+                with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
+                    shutil.copyfileobj(r, f, 1 << 20)
+                index = _reduce_index(tmp)
+                refreshed = time.time_ns()
+            with open(tmp, 'w') as f:
+                json.dump(index, f, separators=(',', ':'))
+            os.utime(tmp, ns=(refreshed, refreshed))
+            os.replace(tmp, path)
+            _indexes[path] = (os.stat(path).st_mtime_ns, index)
+            return index
+        finally:
+            if os.path.exists(tmp):
+                os.remove(tmp)
+
+
 def catalog():
     """Rated apps with the database's reports applied."""
     path = os.path.join(CATALOG, 'site', 'apps.js')

ui/index.html

@@ -254,10 +254,10 @@
   .and-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 2fr); gap: 22px; align-items: start; }
   .and-col { display: grid; gap: 22px; align-content: start; }
   .rep-item .s { white-space: normal; }
-  #repDlg, #titleDlg, #wiDlg, #pwDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px;
+  #repDlg, #titleDlg, #wiDlg, #pwDlg, #apkAltDlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px;
             padding: 22px; width: min(560px, 92vw); box-shadow: 0 20px 60px rgba(0,0,0,.6); }
-  #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop { background: rgba(0,0,0,.55); }
-  #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); }
+  #repDlg::backdrop, #titleDlg::backdrop, #wiDlg::backdrop, #pwDlg::backdrop, #apkAltDlg::backdrop { background: rgba(0,0,0,.55); }
+  #repDlg h2, #titleDlg h2, #wiDlg h2, #pwDlg h2, #apkAltDlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); }
   #repForm label, #titleForm label { display: block; font-size: 12.5px; color: var(--muted); margin-top: 10px; }
   #repForm label input[type=text], #repForm textarea, #titleForm label input, #titleForm label select { margin-top: 5px; }
   #titleForm select { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent;
@@ -703,6 +703,16 @@
   <span class="last" id="lastLog">Ready</span>
   <span class="sub" id="drawerHint">Show ▴</span>
 </div>
+<dialog id="apkAltDlg" aria-labelledby="apkAltTitle">
+  <h2 id="apkAltTitle">Try another APK version</h2>
+  <p id="apkAltReason"></p>
+  <div class="list" id="apkAltVersions"></div>
+  <p class="sub" id="apkAltNote"></p>
+  <div id="apkAltLinks"></div>
+  <p class="sub" id="apkAltErrors"></p>
+  <div class="actions"><button id="apkAltClose">Close</button></div>
+</dialog>
+
 <dialog id="repDlg" aria-labelledby="repTitle">
   <form method="dialog" id="repForm">
     <h2 id="repTitle">Report an APK</h2>
@@ -1457,12 +1467,56 @@ function upload(file, mode) {
     xhr.onload = () => {
       $("prog").style.display = "none";
       let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; }
+      if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk);
       xhr.status < 300 ? resolve(data) : reject(new Error(data.error));
     };
     xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); };
     xhr.send(file);
   });
 }
+let apkLookup = 0;
+async function checkApkAlternatives(apk) {
+  const lookup = ++apkLookup;
+  $("apkAltReason").textContent = apk.blocker;
+  $("apkAltVersions").textContent = "Checking F-Droid for older versions…";
+  $("apkAltVersions").onclick = null;
+  for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = "";
+  if (!$("apkAltDlg").open) $("apkAltDlg").showModal();
+  const query = new URLSearchParams({package: apk.package});
+  if (apk.version_code != null) query.set("code", apk.version_code);
+  try {
+    const result = await api(`/api/apk-versions?${query}`);
+    if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result);
+  } catch (e) {
+    if (lookup === apkLookup) $("apkAltVersions").textContent = e.message;
+  }
+}
+function showApkAlternatives(reason, result) {
+  $("apkAltReason").textContent = reason;
+  $("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`;
+  $("apkAltErrors").textContent = result.errors.join(" · ");
+  $("apkAltLinks").innerHTML = result.links.map(l => `<a href="${esc(l.url)}" target="_blank" rel="noopener noreferrer">${esc(l.source)}</a>`).join(" · ");
+  $("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) =>
+    `<div class="item"><div class="grow"><div class="t">${esc(v.version || "?")} <span class="sub">code ${esc(v.version_code)}</span></div>
+    <div class="s">${esc(v.source)} · minimum API ${esc(v.min_sdk)} · ${esc(v.abis.join(", ") || "no native code")}</div></div>
+    <button class="small" data-version="${i}" ${v.sha256 ? "" : "disabled"}>Install</button></div>`).join("") :
+    `<p>No compatible version found in F-Droid. Try the searches below.</p>`;
+  $("apkAltVersions").onclick = async e => {
+    const b = e.target.closest("[data-version]"); if (!b) return;
+    const v = result.versions[+b.dataset.version];
+    if (installing.has(result.package)) return;
+    b.disabled = true; b.textContent = "Installing…";
+    const res = await runJob(`Install ${result.package} ${v.version}`, result.package, () => api("/api/android", {
+      action: "install", package: result.package, url: v.url
+    }));
+    if (res) $("apkAltDlg").close(); else { b.disabled = false; b.textContent = "Install"; }
+    await loadAndroid();
+    if (cat.apps) { const y = window.scrollY; filterCatalog(); window.scrollTo(0, y); }
+  };
+  if (!$("apkAltDlg").open) $("apkAltDlg").showModal();
+}
+$("apkAltClose").onclick = () => $("apkAltDlg").close();
+
 const TITLE_EXT = /\.(zip|exe)$/i;
 async function sendFiles(files, dirs = new Set()) {
   for (const [i, f] of files.entries()) {
@@ -2043,6 +2097,7 @@ $("repFile").onchange = async () => {
     const { apk } = await upload(file, "apkinfo");
     $("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label;
     if (!$("repSrc").value) $("repSrc").value = file.name;
+    if (apk.blocker) checkApkAlternatives(apk);
     $("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`;
   } catch (e) { $("repFileNote").textContent = e.message; }
   $("repFile").value = "";

ui/server.py

@@ -38,6 +38,7 @@ from urllib.parse import parse_qs, unquote, urlparse
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 
 import frame_android  # noqa: E402
+import frame_apk_versions  # noqa: E402
 import frame_catalog  # noqa: E402
 import frame_host  # noqa: E402
 import frame_store  # noqa: E402
@@ -94,9 +95,10 @@ exit 1
 
 
 class Failure(Exception):
-    def __init__(self, message, status=502):
+    def __init__(self, message, status=502, apk=None):
         super().__init__(message)
         self.status = status
+        self.apk = apk
 
 
 # What ssh prints when it never reached the Frame, and what to tell the user
@@ -719,16 +721,28 @@ def open_thing(body):
     raise Failure("unknown target", 400)
 
 
+def apk_versions(query):
+    args = parse_qs(query, keep_blank_values=True)
+    packages, codes = args.get('package', []), args.get('code', [])
+    if len(packages) != 1 or not frame_android.PKG_RE.match(packages[0]):
+        raise Failure('invalid Android package id', 400)
+    if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])):
+        raise Failure('invalid version code', 400)
+    return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None)
+
+
 def android(body):
     """Android apps, each in its own persistent Lepton instance (frame_android.py)."""
     action, pkg = body.get("action"), str(body.get("package", ""))
     ensure_master()
     try:
         if action == "install":
-            frame_catalog.app(pkg)  # an unknown package fails now, not in the background
+            url = body.get("url")
+            if not url:
+                frame_catalog.app(pkg)  # an unknown package fails now, not in the background
 
             def work():
-                m = frame_catalog.install(pkg)
+                m = frame_apk_versions.install(pkg, url) if url else frame_catalog.install(pkg)
                 return {"message": f"Installed {m['label']}. It's in the Steam library; launching it opens its own panel.",
                         "app": m}
             return start_job(f"Install {pkg}", work)
@@ -1462,8 +1476,10 @@ class Handler(BaseHTTPRequestHandler):
     def send_json(self, obj, status=200):
         self.send_bytes(json.dumps(obj).encode(), "application/json", status)
 
-    def send_error_json(self, message, status):
+    def send_error_json(self, message, status, apk=None):
         body, offline_status = error_body(message)
+        if apk is not None:
+            body["apk"] = apk
         self.send_json(body, offline_status or status)
 
     def do_GET(self):
@@ -1478,6 +1494,8 @@ class Handler(BaseHTTPRequestHandler):
                 self.send_json({"os": "SteamOS", "fileManager": None, "computer": DEVICE, "mobile": True} if LOCAL else
                                {"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER,
                                 "computer": "Mac" if frame_host.MAC else "PC"})
+            elif path == "/api/apk-versions":
+                self.send_json(apk_versions(url.query))
             elif path == "/api/android":
                 ensure_master()
                 self.send_json({"apps": frame_android.list_apps()})
@@ -1519,7 +1537,7 @@ class Handler(BaseHTTPRequestHandler):
             else:
                 self.send_json({"error": "not found"}, 404)
         except Failure as e:
-            self.send_error_json(str(e), e.status)
+            self.send_error_json(str(e), e.status, e.apk)
         except frame_android.FrameError as e:
             self.send_error_json(str(e), 502)
         except Exception as e:
@@ -1545,7 +1563,7 @@ class Handler(BaseHTTPRequestHandler):
                 raise Failure("request body must be a JSON object", 400)
             self.send_json(handler(body))
         except Failure as e:
-            self.send_error_json(str(e), e.status)
+            self.send_error_json(str(e), e.status, e.apk)
         except (ValueError, TypeError) as e:
             self.send_json({"error": f"bad request: {e}"}, 400)
         except frame_android.FrameError as e:
@@ -1650,6 +1668,14 @@ class Handler(BaseHTTPRequestHandler):
                 keep = True  # stage_title owns tmp now, and removes it on failure
                 return stage_title(str(dest), temp_dir=str(tmp))
             if mode == "apk":
+                try:
+                    info = frame_android.apk_info(str(dest))
+                except frame_android.FrameError as e:
+                    raise Failure(str(e), 400)
+                try:
+                    frame_android.check_installable(info)
+                except frame_android.FrameError as e:
+                    raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)})
                 ensure_master()
                 try:
                     m = frame_android.install(str(dest), source=name)