diff --git a/tests/test_apk_more_sources.py b/tests/test_apk_more_sources.py index ab3e540..2cbcf10 100644 --- a/tests/test_apk_more_sources.py +++ b/tests/test_apk_more_sources.py @@ -20,6 +20,10 @@ class PublisherSources(unittest.TestCase): self.addCleanup(self.network.stop) _web._limited.clear() self.addCleanup(_web._limited.clear) + self.root = Path(self.tmp.name) / 'caches' / 'apk-sources' + roots = patch.object(_web.frame_host, 'cache_dir', lambda *p: self.root.parent.joinpath(*p)) + roots.start() + self.addCleanup(roots.stop) def test_curated_search_is_offline(self): self.assertEqual(github.search(github.sources()[0], 'hello')[0]['id'], 'KhronosGroup/OpenXR-SDK-Source') @@ -148,6 +152,27 @@ class PublisherSources(unittest.TestCase): with patch.object(_web, 'read', return_value=b''), self.assertRaises(SourceError): github._api('/x') + def test_prune_caps_apks_by_age_and_removes_orphans(self): + now = 1e9 + self.root.mkdir(parents=True) + def make(folder, name, size, age): + path = Path(folder) / name + path.mkdir() if size is None else path.write_bytes(b'x' * size) + os.utime(str(path), (now - age, now - age)) + return path + pub = self.tmp.name + oldest = make(self.root, 'a.apk', 40, 9000) + old = make(pub, 'b.apk', 40, 8000) + kept = make(self.root, 'c.apk', 40, 7200) + recent = make(pub, 'd.apk', 40, 60) # just downloaded: never pruned + orphan, busy = make(self.root, 'x.part', 5, 90000), make(pub, 'y.part', 5, 60) + listing, fresh = make(pub, 'l.data', 5, 8 * 86400), make(pub, 'm.data', 5, 3600) + tmpdir = make(self.root, 'tmpabc', None, 90000) + with patch.object(_web, 'APK_CAP', 100), patch.object(_web.time, 'time', return_value=now): + _web.prune() + self.assertEqual([p.exists() for p in (oldest, old, kept, recent)], [False, False, True, True]) + self.assertEqual([p.exists() for p in (orphan, busy, listing, fresh, tmpdir)], [False, True, False, True, False]) + def test_backoff_honours_retry_after_per_host(self): from apk_sources import SourceLimited from email.utils import formatdate diff --git a/tests/test_fdroid_sources.py b/tests/test_fdroid_sources.py index a6b72e6..a2935c4 100644 --- a/tests/test_fdroid_sources.py +++ b/tests/test_fdroid_sources.py @@ -1,6 +1,7 @@ """Offline authenticated repository fixtures; no tests contact a server.""" import io import json +import os from pathlib import Path import sys import tempfile @@ -104,8 +105,10 @@ class Repositories(unittest.TestCase): self.assertTrue(downloaded['verified']) self.assertEqual(Path(downloaded['apk']).read_bytes(), (FIXTURES / 'example.apk').read_bytes()) count = self.fetch_mock.call_count + os.utime(downloaded['apk'], (1, 1)) fdroid.download(source, 'org.example.app', 1) self.assertEqual(self.fetch_mock.call_count, count) + self.assertGreater(Path(downloaded['apk']).stat().st_mtime, 1) # reuse counts as recent use def test_wrong_pin_is_not_saved(self): with self.assertRaisesRegex(SourceError, 'fingerprint mismatch'): diff --git a/ui/apk_sources/_web.py b/ui/apk_sources/_web.py index 1ed80f1..6f83838 100644 --- a/ui/apk_sources/_web.py +++ b/ui/apk_sources/_web.py @@ -1,11 +1,12 @@ """Small HTTPS cache and APK downloader for public publisher sources.""" -import hashlib, os, tempfile, threading, time, urllib.error, urllib.parse, urllib.request, zipfile +import hashlib, os, shutil, tempfile, threading, time, urllib.error, urllib.parse, urllib.request, zipfile from email.utils import parsedate_to_datetime import frame_host from . import SourceError, SourceLimited UA = 'FrameControl/0.1' +APK_CAP = 2 * 1024 ** 3 # cached APKs across all sources, least recently used go first BACKOFF = 600 # seconds to leave a host alone after 403/429 without Retry-After _limited = {} # host -> time.time() before which we don't contact it _limited_lock = threading.Lock() @@ -53,6 +54,41 @@ def cache(): return str(path) +def prune(): + """Trim the download caches: APKs to APK_CAP by mtime, orphaned .part/temp files, old listings.""" + now, apks = time.time(), [] + for folder in (str(frame_host.cache_dir('apk-sources')), cache()): + try: + names = os.listdir(folder) + except OSError: + continue + for name in names: + path = os.path.join(folder, name) + try: + st = os.lstat(path) + age = now - st.st_mtime + if os.path.isdir(path) and not os.path.islink(path): + if name.startswith('tmp') and age > 86400: # an interrupted F-Droid index download + shutil.rmtree(path, ignore_errors=True) + elif (name.endswith('.part') and age > 86400) or (name.endswith('.data') and age > 7 * 86400): + os.remove(path) + elif name.endswith('.apk'): + apks.append((st.st_mtime, st.st_size, path)) + except OSError: + pass + total = sum(size for _, size, _ in apks) + for mtime, size, path in sorted(apks): + if total <= APK_CAP: + break + if now - mtime < 3600: # may be about to be installed + continue + try: + os.remove(path) + total -= size + except OSError: + pass + + def checked_url(url, hosts): try: p = urllib.parse.urlsplit(url) @@ -153,6 +189,7 @@ def apk(url, hosts, digest=None, name=None): raise SourceError('Download is not an APK') path = os.path.join(cache(), actual + '.apk') os.replace(tmp, path) + prune() return {'apk': path, 'obb': [], 'sha256': actual, 'verified': bool(digest)} except urllib.error.HTTPError as e: if e.code in (403, 429): diff --git a/ui/apk_sources/fdroid.py b/ui/apk_sources/fdroid.py index a88d6e2..4ca4307 100644 --- a/ui/apk_sources/fdroid.py +++ b/ui/apk_sources/fdroid.py @@ -614,7 +614,9 @@ def download(source, entry_id, version_code=None): sha = version['sha256'] path = frame_host.cache_dir('apk-sources', sha + '.apk') try: - if not path.exists() or _sha256(path) != sha: + if path.exists() and _sha256(path) == sha: + os.utime(str(path)) # most recently used, for cache pruning + else: path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part') os.close(fd) @@ -626,6 +628,7 @@ def download(source, entry_id, version_code=None): finally: if os.path.exists(tmp): os.unlink(tmp) + _web.prune() return {'apk': str(path), 'obb': [], 'sha256': sha, 'verified': True} except SourceLimited as e: raise _limited(source, e) from e diff --git a/ui/apk_sources/search.py b/ui/apk_sources/search.py index 65f89ab..197596c 100644 --- a/ui/apk_sources/search.py +++ b/ui/apk_sources/search.py @@ -279,6 +279,8 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l def warm(): """Start every enabled source's index download in the background (server start, new repo).""" + from apk_sources import _web + _web.prune() items, _ = registry() for m, s in items: if s['enabled'] and not s.get('page_only'):