Reduce F-Droid indexes and fetch APK alternatives asynchronously

ui/frame_apk_versions.py

@@ -3,7 +3,6 @@ from urllib.parse import quote, urlencode
 
 import frame_android
 import frame_catalog
-from pick import installable
 
 ANDROID = dict(enumerate([
     '1.0', '1.1', '1.5', '1.6', '2.0', '2.0.1', '2.1', '2.2', '2.3', '2.3.3',
@@ -45,38 +44,42 @@ def search_links(package):
     )]
 
 
-def alternatives(package, current_version_code=None):
-    """All compatible builds, ordered by version code; keep per-ABI builds distinct."""
+def _versions(package, cached_only=False):
     versions, errors, seen = [], [], set()
     for source, repo in REPOS:
         try:
-            index = frame_catalog.load_index(repo)
+            index = frame_catalog.load_index(repo, cached_only=cached_only)
         except (OSError, ValueError) as e:
             errors.append(f'Could not check {source}: {e}')
             continue
-        for v in index.get('packages', {}).get(package, {}).get('versions', {}).values():
-            m, file = v['manifest'], v['file']
-            code = m.get('versionCode', 0)
-            if not installable(v):
-                continue
-            url = repo + file['name'].lstrip('/')
-            key = (code, file.get('sha256') or url)
-            if code == current_version_code or key in seen:
+        for v in index.get(package, []):
+            url = repo + v['name'].lstrip('/')
+            key = (v['version_code'], v.get('sha256') or url)
+            if key in seen:
                 continue
             seen.add(key)
-            versions.append({'version': m.get('versionName', ''), 'version_code': code,
-                             'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
-                             'abis': m.get('nativecode') or [], 'url': url,
-                             'source': source, 'sha256': file.get('sha256')})
-    versions.sort(key=lambda v: v['version_code'], reverse=True)
-    return {'package': package, 'versions': versions, 'links': search_links(package),
-            'note': NOTE, 'errors': errors}
+            versions.append(dict(v, url=url, source=source))
+    return versions, errors
+
+
+def alternatives(package, current_version_code=None):
+    """At most eight releases, preferring arm64-only builds over universal builds."""
+    versions, errors = _versions(package)
+    versions = [v for v in versions if v['version_code'] != current_version_code]
+    total = len(versions)
+    versions.sort(key=lambda v: (v['abis'] == ['arm64-v8a'], v['version_code']), reverse=True)
+    releases = {}
+    for v in versions:
+        releases.setdefault(v['version'], v)
+    versions = sorted(releases.values(), key=lambda v: v['version_code'], reverse=True)[:8]
+    return {'package': package, 'versions': versions, 'total': total,
+            'links': search_links(package), 'note': NOTE, 'errors': errors}
 
 
 def install(package, url):
     # Resolve the selection again: the client cannot supply a trusted hash or arbitrary URL.
-    result = alternatives(package)
-    version = next((v for v in result['versions'] if v['url'] == url), None)
+    records, _ = _versions(package, cached_only=True)
+    version = next((v for v in records if v['url'] == url), None)
     if not version:
         raise frame_android.FrameError('That version is no longer available; check the APK again')
     apk = frame_catalog.fetch_apk({'a': version['url'], 'h': version['sha256'], 'n': package})

ui/frame_catalog.py

@@ -24,29 +24,127 @@ _env = {}
 
 
 _index_lock = threading.Lock()
+_indexes = {}
 
 
-def load_index(repo):
-    """Share the catalogue's index files; installed apps use its writable cache."""
+class _IndexReader:
+    """Decode one object member at a time; never retain the whole raw index."""
+    def __init__(self, stream):
+        self.stream, self.buffer = stream, ''
+        self.decoder = json.JSONDecoder()
+
+    def fill(self):
+        chunk = self.stream.read(1 << 16)
+        if not chunk:
+            raise ValueError('incomplete F-Droid index')
+        self.buffer += chunk
+
+    def peek(self):
+        self.buffer = self.buffer.lstrip()
+        while not self.buffer:
+            self.fill()
+            self.buffer = self.buffer.lstrip()
+        return self.buffer[0]
+
+    def expect(self, char):
+        if self.peek() != char:
+            raise ValueError('invalid F-Droid index')
+        self.buffer = self.buffer[1:]
+
+    def value(self):
+        self.peek()
+        while True:
+            try:
+                value, end = self.decoder.raw_decode(self.buffer)
+                self.buffer = self.buffer[end:]
+                return value
+            except json.JSONDecodeError:
+                self.fill()
+
+    def members(self):
+        self.expect('{')
+        if self.peek() != '}':
+            while True:
+                key = self.value()
+                if not isinstance(key, str):
+                    raise ValueError('invalid F-Droid index key')
+                self.expect(':')
+                yield key
+                if self.peek() == '}':
+                    break
+                self.expect(',')
+        self.expect('}')
+
+
+def _reduce_index(path):
+    from pick import installable
+    packages = {}
+    found = False
+    with open(path, encoding='utf-8') as f:
+        reader = _IndexReader(f)
+        for key in reader.members():
+            if key != 'packages':
+                reader.value()
+                continue
+            found = True
+            for package in reader.members():
+                records = []
+                for v in reader.value().get('versions', {}).values():
+                    if not installable(v):
+                        continue
+                    m, file = v['manifest'], v['file']
+                    records.append({'version': m.get('versionName', ''),
+                                    'version_code': m.get('versionCode', 0),
+                                    'min_sdk': m.get('usesSdk', {}).get('minSdkVersion', 1),
+                                    'abis': m.get('nativecode') or [],
+                                    'name': file['name'], 'sha256': file.get('sha256')})
+                if records:
+                    packages[package] = records
+        if reader.buffer.strip() or f.read().strip():
+            raise ValueError('trailing data in F-Droid index')
+    if not found:
+        raise ValueError('invalid F-Droid index')
+    return packages
+
+
+def load_index(repo, cached_only=False):
+    """Compact installable records by package, cached on disk and by mtime in memory."""
     if repo not in APK_HOSTS:
         raise ValueError('unexpected index URL')
     directory = CACHE if os.environ.get('FRAME_CONTROL_APP') or '.app/Contents/Resources' in CATALOG else os.path.join(CATALOG, 'data')
     filename = 'index-v2.json' if repo == APK_HOSTS[0] else 'index-v2.archive.json'
-    path = os.path.join(directory, filename)
+    raw = os.path.join(directory, filename)
+    path = raw + '.installable-v1'
     with _index_lock:
-        if os.path.exists(path) and time.time() - os.path.getmtime(path) < 86400:
-            with open(path) as f:
-                return json.load(f)
+        mtime = os.stat(path).st_mtime_ns if os.path.exists(path) else None
+        if mtime is not None and (cached_only or time.time() - mtime / 1e9 < 86400):
+            cached = _indexes.get(path)
+            if cached is None or cached[0] != mtime:
+                with open(path) as f:
+                    cached = (mtime, json.load(f))
+                _indexes[path] = cached
+            return cached[1]
+        if cached_only:
+            return {}
         os.makedirs(directory, exist_ok=True)
         fd, tmp = tempfile.mkstemp(prefix=filename, suffix='.part', dir=directory)
+        os.close(fd)
         try:
-            with os.fdopen(fd, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
-                shutil.copyfileobj(r, f, 1 << 20)
-            with open(tmp) as f:
-                index = json.load(f)
-            if not isinstance(index.get('packages'), dict):
-                raise ValueError('invalid F-Droid index')
+            if os.path.exists(raw) and time.time() - os.path.getmtime(raw) < 86400:
+                index = _reduce_index(raw)
+                refreshed = os.stat(raw).st_mtime_ns
+            else:
+                with open(tmp, 'wb') as f, urllib.request.urlopen(repo + 'index-v2.json', timeout=30) as r:
+                    shutil.copyfileobj(r, f, 1 << 20)
+                index = _reduce_index(tmp)
+                refreshed = time.time_ns()
+            with open(tmp, 'w') as f:
+                json.dump(index, f, separators=(',', ':'))
+            os.utime(tmp, ns=(refreshed, refreshed))
             os.replace(tmp, path)
+            _indexes[path] = (os.stat(path).st_mtime_ns, index)
+            if os.path.exists(raw):
+                os.remove(raw)
             return index
         finally:
             if os.path.exists(tmp):

ui/index.html

@@ -1154,16 +1154,33 @@ function upload(file, mode) {
     xhr.onload = () => {
       $("prog").style.display = "none";
       let data; try { data = JSON.parse(xhr.responseText); } catch { data = { error: `HTTP ${xhr.status}` }; }
-      if (data.alternatives) showApkAlternatives(data.error, data.alternatives);
+      if (data.apk?.blocker && xhr.status >= 300) checkApkAlternatives(data.apk);
       xhr.status < 300 ? resolve(data) : reject(new Error(data.error));
     };
     xhr.onerror = () => { $("prog").style.display = "none"; reject(new Error("network error")); };
     xhr.send(file);
   });
 }
+let apkLookup = 0;
+async function checkApkAlternatives(apk) {
+  const lookup = ++apkLookup;
+  $("apkAltReason").textContent = apk.blocker;
+  $("apkAltVersions").textContent = "Checking F-Droid for older versions…";
+  $("apkAltVersions").onclick = null;
+  for (const id of ["apkAltNote", "apkAltErrors", "apkAltLinks"]) $(id).textContent = "";
+  if (!$("apkAltDlg").open) $("apkAltDlg").showModal();
+  const query = new URLSearchParams({package: apk.package});
+  if (apk.version_code != null) query.set("code", apk.version_code);
+  try {
+    const result = await api(`/api/apk-versions?${query}`);
+    if (lookup === apkLookup && $("apkAltDlg").open) showApkAlternatives(apk.blocker, result);
+  } catch (e) {
+    if (lookup === apkLookup) $("apkAltVersions").textContent = e.message;
+  }
+}
 function showApkAlternatives(reason, result) {
   $("apkAltReason").textContent = reason;
-  $("apkAltNote").textContent = result.note;
+  $("apkAltNote").textContent = `${result.versions.length} of ${result.total} compatible versions. ${result.note}`;
   $("apkAltErrors").textContent = result.errors.join(" · ");
   $("apkAltLinks").innerHTML = result.links.map(l => `<a href="${esc(l.url)}" target="_blank" rel="noopener noreferrer">${esc(l.source)}</a>`).join(" · ");
   $("apkAltVersions").innerHTML = result.versions.length ? result.versions.map((v, i) =>
@@ -1740,7 +1757,7 @@ $("repFile").onchange = async () => {
     const { apk } = await upload(file, "apkinfo");
     $("repPkg").value = apk.package; $("repVer").value = apk.version; $("repLabel").value = apk.label;
     if (!$("repSrc").value) $("repSrc").value = file.name;
-    if (apk.alternatives) showApkAlternatives(apk.blocker, apk.alternatives);
+    if (apk.blocker) checkApkAlternatives(apk);
     $("repFileNote").textContent = apk.blocker ? `Note: ${apk.blocker}` : `${apk.package} ${apk.version}`;
   } catch (e) { $("repFileNote").textContent = e.message; }
   $("repFile").value = "";

ui/server.py

@@ -81,10 +81,10 @@ exit 1
 
 
 class Failure(Exception):
-    def __init__(self, message, status=502, alternatives=None):
+    def __init__(self, message, status=502, apk=None):
         super().__init__(message)
         self.status = status
-        self.alternatives = alternatives
+        self.apk = apk
 
 
 _master_lock = threading.Lock()
@@ -445,6 +445,16 @@ def open_thing(body):
     raise Failure("unknown target", 400)
 
 
+def apk_versions(query):
+    args = parse_qs(query, keep_blank_values=True)
+    packages, codes = args.get('package', []), args.get('code', [])
+    if len(packages) != 1 or not re.fullmatch(r'[A-Za-z][A-Za-z0-9_]*(?:\.[A-Za-z][A-Za-z0-9_]*)+', packages[0]):
+        raise Failure('invalid Android package id', 400)
+    if codes and (len(codes) != 1 or not re.fullmatch(r'[0-9]{1,19}', codes[0])):
+        raise Failure('invalid version code', 400)
+    return frame_apk_versions.alternatives(packages[0], int(codes[0]) if codes else None)
+
+
 def android(body):
     """Android apps, each in its own persistent Lepton instance (frame_android.py)."""
     action, pkg = body.get("action"), str(body.get("package", ""))
@@ -1170,6 +1180,8 @@ class Handler(BaseHTTPRequestHandler):
             elif path == "/api/host":
                 self.send_json({"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER,
                                 "computer": "Mac" if frame_host.MAC else "PC"})
+            elif path == "/api/apk-versions":
+                self.send_json(apk_versions(url.query))
             elif path == "/api/android":
                 ensure_master()
                 self.send_json({"apps": frame_android.list_apps()})
@@ -1207,7 +1219,7 @@ class Handler(BaseHTTPRequestHandler):
             else:
                 self.send_json({"error": "not found"}, 404)
         except Failure as e:
-            self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
+            self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
         except frame_android.FrameError as e:
             self.send_json({"error": str(e)}, 502)
         except Exception as e:
@@ -1233,7 +1245,7 @@ class Handler(BaseHTTPRequestHandler):
                 raise Failure("request body must be a JSON object", 400)
             self.send_json(handler(body))
         except Failure as e:
-            self.send_json({"error": str(e), **({"alternatives": e.alternatives} if e.alternatives is not None else {})}, e.status)
+            self.send_json({"error": str(e), **({"apk": e.apk} if e.apk is not None else {})}, e.status)
         except (ValueError, TypeError) as e:
             self.send_json({"error": f"bad request: {e}"}, 400)
         except frame_android.FrameError as e:
@@ -1333,7 +1345,6 @@ class Handler(BaseHTTPRequestHandler):
                     info["blocker"] = None
                 except frame_android.FrameError as e:
                     info["blocker"] = str(e)
-                    info["alternatives"] = frame_apk_versions.alternatives(info["package"], info.get("version_code"))
                 return {"message": f"Read {info['label']} {info['version']}", "apk": info}
             if mode == "title":
                 keep = True  # stage_title owns tmp now, and removes it on failure
@@ -1346,7 +1357,7 @@ class Handler(BaseHTTPRequestHandler):
                 try:
                     frame_android.check_installable(info)
                 except frame_android.FrameError as e:
-                    raise Failure(str(e), 400, frame_apk_versions.alternatives(info["package"], info.get("version_code")))
+                    raise Failure(str(e), 400, {"package": info["package"], "version_code": info.get("version_code"), "blocker": str(e)})
                 ensure_master()
                 try:
                     m = frame_android.install(str(dest), source=name)