Store: prune download caches (2 GB LRU for APKs, orphaned .part/temp, old listings)

Runs after each APK download and at server start. APKs used in the last hour
are kept; an F-Droid cache hit refreshes the APK's mtime.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/test_apk_more_sources.py

@@ -20,6 +20,10 @@ class PublisherSources(unittest.TestCase):
         self.addCleanup(self.network.stop)
         _web._limited.clear()
         self.addCleanup(_web._limited.clear)
+        self.root = Path(self.tmp.name) / 'caches' / 'apk-sources'
+        roots = patch.object(_web.frame_host, 'cache_dir', lambda *p: self.root.parent.joinpath(*p))
+        roots.start()
+        self.addCleanup(roots.stop)
 
     def test_curated_search_is_offline(self):
         self.assertEqual(github.search(github.sources()[0], 'hello')[0]['id'], 'KhronosGroup/OpenXR-SDK-Source')
@@ -148,6 +152,27 @@ class PublisherSources(unittest.TestCase):
         with patch.object(_web, 'read', return_value=b'<html>'), self.assertRaises(SourceError):
             github._api('/x')
 
+    def test_prune_caps_apks_by_age_and_removes_orphans(self):
+        now = 1e9
+        self.root.mkdir(parents=True)
+        def make(folder, name, size, age):
+            path = Path(folder) / name
+            path.mkdir() if size is None else path.write_bytes(b'x' * size)
+            os.utime(str(path), (now - age, now - age))
+            return path
+        pub = self.tmp.name
+        oldest = make(self.root, 'a.apk', 40, 9000)
+        old = make(pub, 'b.apk', 40, 8000)
+        kept = make(self.root, 'c.apk', 40, 7200)
+        recent = make(pub, 'd.apk', 40, 60)  # just downloaded: never pruned
+        orphan, busy = make(self.root, 'x.part', 5, 90000), make(pub, 'y.part', 5, 60)
+        listing, fresh = make(pub, 'l.data', 5, 8 * 86400), make(pub, 'm.data', 5, 3600)
+        tmpdir = make(self.root, 'tmpabc', None, 90000)
+        with patch.object(_web, 'APK_CAP', 100), patch.object(_web.time, 'time', return_value=now):
+            _web.prune()
+        self.assertEqual([p.exists() for p in (oldest, old, kept, recent)], [False, False, True, True])
+        self.assertEqual([p.exists() for p in (orphan, busy, listing, fresh, tmpdir)], [False, True, False, True, False])
+
     def test_backoff_honours_retry_after_per_host(self):
         from apk_sources import SourceLimited
         from email.utils import formatdate

tests/test_fdroid_sources.py

@@ -1,6 +1,7 @@
 """Offline authenticated repository fixtures; no tests contact a server."""
 import io
 import json
+import os
 from pathlib import Path
 import sys
 import tempfile
@@ -104,8 +105,10 @@ class Repositories(unittest.TestCase):
         self.assertTrue(downloaded['verified'])
         self.assertEqual(Path(downloaded['apk']).read_bytes(), (FIXTURES / 'example.apk').read_bytes())
         count = self.fetch_mock.call_count
+        os.utime(downloaded['apk'], (1, 1))
         fdroid.download(source, 'org.example.app', 1)
         self.assertEqual(self.fetch_mock.call_count, count)
+        self.assertGreater(Path(downloaded['apk']).stat().st_mtime, 1)  # reuse counts as recent use
 
     def test_wrong_pin_is_not_saved(self):
         with self.assertRaisesRegex(SourceError, 'fingerprint mismatch'):

ui/apk_sources/_web.py

@@ -1,11 +1,12 @@
 """Small HTTPS cache and APK downloader for public publisher sources."""
-import hashlib, os, tempfile, threading, time, urllib.error, urllib.parse, urllib.request, zipfile
+import hashlib, os, shutil, tempfile, threading, time, urllib.error, urllib.parse, urllib.request, zipfile
 from email.utils import parsedate_to_datetime
 
 import frame_host
 from . import SourceError, SourceLimited
 
 UA = 'FrameControl/0.1'
+APK_CAP = 2 * 1024 ** 3  # cached APKs across all sources, least recently used go first
 BACKOFF = 600  # seconds to leave a host alone after 403/429 without Retry-After
 _limited = {}  # host -> time.time() before which we don't contact it
 _limited_lock = threading.Lock()
@@ -53,6 +54,41 @@ def cache():
     return str(path)
 
 
+def prune():
+    """Trim the download caches: APKs to APK_CAP by mtime, orphaned .part/temp files, old listings."""
+    now, apks = time.time(), []
+    for folder in (str(frame_host.cache_dir('apk-sources')), cache()):
+        try:
+            names = os.listdir(folder)
+        except OSError:
+            continue
+        for name in names:
+            path = os.path.join(folder, name)
+            try:
+                st = os.lstat(path)
+                age = now - st.st_mtime
+                if os.path.isdir(path) and not os.path.islink(path):
+                    if name.startswith('tmp') and age > 86400:  # an interrupted F-Droid index download
+                        shutil.rmtree(path, ignore_errors=True)
+                elif (name.endswith('.part') and age > 86400) or (name.endswith('.data') and age > 7 * 86400):
+                    os.remove(path)
+                elif name.endswith('.apk'):
+                    apks.append((st.st_mtime, st.st_size, path))
+            except OSError:
+                pass
+    total = sum(size for _, size, _ in apks)
+    for mtime, size, path in sorted(apks):
+        if total <= APK_CAP:
+            break
+        if now - mtime < 3600:  # may be about to be installed
+            continue
+        try:
+            os.remove(path)
+            total -= size
+        except OSError:
+            pass
+
+
 def checked_url(url, hosts):
     try:
         p = urllib.parse.urlsplit(url)
@@ -153,6 +189,7 @@ def apk(url, hosts, digest=None, name=None):
                 raise SourceError('Download is not an APK')
         path = os.path.join(cache(), actual + '.apk')
         os.replace(tmp, path)
+        prune()
         return {'apk': path, 'obb': [], 'sha256': actual, 'verified': bool(digest)}
     except urllib.error.HTTPError as e:
         if e.code in (403, 429):

ui/apk_sources/fdroid.py

@@ -614,7 +614,9 @@ def download(source, entry_id, version_code=None):
     sha = version['sha256']
     path = frame_host.cache_dir('apk-sources', sha + '.apk')
     try:
-        if not path.exists() or _sha256(path) != sha:
+        if path.exists() and _sha256(path) == sha:
+            os.utime(str(path))  # most recently used, for cache pruning
+        else:
             path.parent.mkdir(parents=True, exist_ok=True)
             fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part')
             os.close(fd)
@@ -626,6 +628,7 @@ def download(source, entry_id, version_code=None):
             finally:
                 if os.path.exists(tmp):
                     os.unlink(tmp)
+            _web.prune()
         return {'apk': str(path), 'obb': [], 'sha256': sha, 'verified': True}
     except SourceLimited as e:
         raise _limited(source, e) from e

ui/apk_sources/search.py

@@ -279,6 +279,8 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l
 
 def warm():
     """Start every enabled source's index download in the background (server start, new repo)."""
+    from apk_sources import _web
+    _web.prune()
     items, _ = registry()
     for m, s in items:
         if s['enabled'] and not s.get('page_only'):