From 029c92bccb3cbdefc5dc531687b457a3bf86fa3f Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:11:04 +1000 Subject: [PATCH] F-Droid: serve an expired index as stale while refreshing in the background Searches no longer wait for (or fail on) a refresh of an expired index; the store notes which sources show saved listings. A failed refresh keeps the old index and is retried after 10 minutes. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/apk-repos.md | 4 ++- tests/test_apk_search.py | 12 ++++++-- tests/test_fdroid_sources.py | 41 ++++++++++++++++++++++++-- ui/apk_sources/fdroid.py | 56 ++++++++++++++++++++++++++++++------ ui/apk_sources/search.py | 3 +- ui/index.html | 2 ++ 6 files changed, 104 insertions(+), 14 deletions(-) diff --git a/docs/apk-repos.md b/docs/apk-repos.md index ae3b535..25c762a 100644 --- a/docs/apk-repos.md +++ b/docs/apk-repos.md @@ -66,7 +66,9 @@ separate work. Built-in sources can be disabled but cannot be removed. Settings and pins live in `frame_host.data_dir('apk-repos.json')` (`~/Library/Application Support/Frame Control/apk-repos.json` on macOS). Authenticated reduced indexes and APKs live under -`frame_host.cache_dir('apk-sources')`; indexes refresh after 24 hours. +`frame_host.cache_dir('apk-sources')`; indexes refresh after 24 hours. An +expired index is still served (marked stale in the store) while it refreshes in +the background; a failed refresh is retried after 10 minutes. The existing catalogue's unverified index cache is never treated as authenticated. Rollback protection: each repository's newest accepted signed index timestamp diff --git a/tests/test_apk_search.py b/tests/test_apk_search.py index 24ab71e..ec92e54 100644 --- a/tests/test_apk_search.py +++ b/tests/test_apk_search.py @@ -87,9 +87,10 @@ class SearchTests(SettingsTest): self.assertEqual(search.search('other', timeout=.03)['sources'][1]['status'], 'loading') search.search('newest', timeout=.03) self.assertEqual(calls, ['']) + queued = search._pending['slow'] release.set() - result = search.search('newest', timeout=2) - self.assertEqual(result['sources'][1]['status'], 'ok') + self.assertTrue(queued['event'].wait(2)) + self.assertEqual(queued['entries'], []) self.assertEqual(calls, ['', 'newest']) # 'other' was superseded, never run finally: release.set() @@ -106,6 +107,13 @@ class SearchTests(SettingsTest): search.set_enabled('one', False) self.assertEqual(free, [True]) + def test_stale_source_status(self): + mod = fake() + mod.stale = lambda source: True + with patch.object(search, 'modules', return_value=([mod], [])): + status = search.search(timeout=1)['sources'][0] + self.assertEqual((status['status'], status['stale']), ('ok', True)) + def test_limited_source_status(self): from apk_sources import SourceLimited mods = [fake('busy', Mock(side_effect=SourceLimited('busy is limiting requests', 60)))] diff --git a/tests/test_fdroid_sources.py b/tests/test_fdroid_sources.py index 7e550d0..a6b72e6 100644 --- a/tests/test_fdroid_sources.py +++ b/tests/test_fdroid_sources.py @@ -75,8 +75,9 @@ class Repositories(unittest.TestCase): self.v1 = False self.corrupt = None self.files = {} - _web._limited.clear() - self.addCleanup(_web._limited.clear) + for state in (_web._limited, fdroid._stale, fdroid._retry_at, fdroid._refreshing): + state.clear() + self.addCleanup(state.clear) def fetch(self, url, path, maximum): name = url.rsplit('/', 1)[-1] @@ -347,6 +348,42 @@ class Repositories(unittest.TestCase): self.assertEqual(opener.return_value.open.call_count, 1) self.fetch_mock = self.fetch_patch.start() + def expire(self, source): + cache = fdroid.frame_host.cache_dir('apk-sources', source['id'] + '.json') + old = cache.stat().st_mtime - fdroid.MAX_AGE - 1 + fdroid.os.utime(str(cache), (old, old)) + + def test_expired_index_served_stale_while_refreshing(self): + source = self.add() + self.expire(source) + before = self.fetch_mock.call_count + release = __import__('threading').Event() + def slow(url, path, maximum): + release.wait(5) + self.fetch(url, path, maximum) + self.fetch_mock.side_effect = slow + self.assertEqual(len(fdroid.search(source, 'example')), 1) # immediately, from the old index + self.assertTrue(fdroid.stale(source)) + refresh = fdroid._refreshing[source['id']] + fdroid.search(source, 'example') + self.assertIs(fdroid._refreshing.get(source['id']), refresh) # one refresh at a time + release.set() + refresh.join(5) + self.assertEqual(self.fetch_mock.call_count, before + 2) + self.assertFalse(fdroid.stale(source)) + + def test_failed_refresh_keeps_serving_stale_index(self): + source = self.add() + self.expire(source) + self.fetch_mock.side_effect = urllib.error.HTTPError(URL, 503, 'unavailable', None, None) + self.assertEqual(len(fdroid.search(source, 'example')), 1) + fdroid._refreshing[source['id']].join(5) + calls = self.fetch_mock.call_count + self.assertEqual(fdroid.details(source, 'org.example.app')['version_code'], 2) + self.assertTrue(fdroid.stale(source)) + self.assertNotIn(source['id'], fdroid._refreshing) # failed refresh waits before retrying + self.assertEqual(self.fetch_mock.call_count, calls) + def test_cached_index_does_not_cross_pins(self): source = self.add() source['fingerprint'] = '0' * 64 diff --git a/ui/apk_sources/fdroid.py b/ui/apk_sources/fdroid.py index ee23275..a88d6e2 100644 --- a/ui/apk_sources/fdroid.py +++ b/ui/apk_sources/fdroid.py @@ -27,6 +27,10 @@ KIND = 'fdroid' CACHE_VERSION = 2 _LOCK = threading.RLock() # settings only; never held while downloading _load_locks = {} +_refreshing = {} # source id -> background refresh thread +_retry_at = {} # source id -> time before which a failed refresh isn't retried +_stale = set() # source ids currently served from an expired index +MAX_AGE = 86400 # Published by the repository operators; a user repository without a pin uses TOFU. FDROID_PIN = '43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab' IZZY_PIN = '3bf0d6abfeae2f401707b6d966be743bf0eee49c2561b9ba39073711f628937a' @@ -438,6 +442,40 @@ def _v1(content, path): return (index.get('repo') or {}).get('timestamp') +def _cached(source, cache): + """(apps, pin, expired) from a cache matching this source, or None.""" + try: + saved = json.loads(cache.read_text()) + if (saved.get('version') == CACHE_VERSION and saved.get('fingerprint') == source.get('fingerprint') + and saved.get('url') == source['url']): + return saved['apps'], saved['fingerprint'], time.time() - cache.stat().st_mtime >= MAX_AGE + except (OSError, ValueError, KeyError, AttributeError): + pass + return None + + +def stale(source): + """True while results come from an expired index that is being (or failed to be) refreshed.""" + return source['id'] in _stale + + +def _refresh_later(source): + with _LOCK: + if source['id'] in _refreshing or time.time() < _retry_at.get(source['id'], 0): + return + def run(): + try: + _load(source, force=True) + except Exception: + with _LOCK: + _retry_at[source['id']] = time.time() + 600 + finally: + with _LOCK: + _refreshing.pop(source['id'], None) + _refreshing[source['id']] = thread = threading.Thread(target=run, daemon=True) + thread.start() + + def _source_lock(source_id): with _LOCK: return _load_locks.setdefault(source_id, threading.Lock()) @@ -448,15 +486,16 @@ def _load(source, force=False): raise SourceError('invalid source id') _url(source['url'], source.get('fingerprint')) cache = frame_host.cache_dir('apk-sources', source['id'] + '.json') + found = None if force else _cached(source, cache) + if found: + if found[2]: # expired: serve it now, marked stale, and refresh without blocking anyone + _stale.add(source['id']) + _refresh_later(source) + return found[:2] with _source_lock(source['id']): - if not force and cache.exists() and time.time() - cache.stat().st_mtime < 86400: - try: - saved = json.loads(cache.read_text()) - if (saved.get('version') == CACHE_VERSION and saved.get('fingerprint') == source.get('fingerprint') - and saved.get('url') == source['url']): - return saved['apps'], saved['fingerprint'] - except (OSError, ValueError, KeyError, AttributeError): - pass + found = None if force else _cached(source, cache) # another caller may have just loaded it + if found and not found[2]: + return found[:2] cache.parent.mkdir(parents=True, exist_ok=True) try: with tempfile.TemporaryDirectory(dir=str(cache.parent)) as tmp: @@ -486,6 +525,7 @@ def _load(source, force=False): apps = _reduce(raw, source) _write(cache, {'version': CACHE_VERSION, 'url': source['url'], 'fingerprint': pin, 'apps': apps}) _accept(source, timestamp, v2) + _stale.discard(source['id']) return apps, pin except SourceLimited as e: raise _limited(source, e) from e diff --git a/ui/apk_sources/search.py b/ui/apk_sources/search.py index ed9d622..65f89ab 100644 --- a/ui/apk_sources/search.py +++ b/ui/apk_sources/search.py @@ -237,6 +237,7 @@ def _start(key, task): try: task['entries'] = [dict(e, source=key, source_name=source['name'], trust=source.get('trust')) for e in module.search(source, query, limit=limit) if e.get('free') is True] + task['stale'] = bool(getattr(module, 'stale', lambda s: False)(source)) except Exception as e: task['error'] = str(e) task['limited'] = isinstance(e, SourceLimited) @@ -268,7 +269,7 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l elif 'error' in task: status.update(status='limited' if task.get('limited') else 'error', error=task['error']) else: - status.update(status='ok') + status.update(status='ok', stale=task['stale']) entries.extend(task['entries']) statuses.append(status) with _lock: diff --git a/ui/index.html b/ui/index.html index b44fc62..833500f 100644 --- a/ui/index.html +++ b/ui/index.html @@ -2010,6 +2010,8 @@ async function searchSources(quiet) { if(loading.length)notes.push(`Still fetching ${loading.join(" and ")}; more results will appear in a moment.`); if(unavailable.length)notes.push(`${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.`); result.sources.filter(s=>s.status==='limited').forEach(s=>notes.push(`${s.error}.`)); + const stale=result.sources.filter(s=>s.stale).map(s=>storeName(s.name)); + if(stale.length)notes.push(`Showing saved listings from ${stale.join(" and ")} while ${stale.length===1 ? "it refreshes" : "they refresh"}.`); $("sourceStatus").innerHTML=notes.map(esc).join(" ")+(result.elsewhere||[]).map(x=>` Browse ${esc(storeName(x.name))} ↗`).join(""); $("sourceStatus").hidden=!notes.length && !(result.elsewhere||[]).length; clearTimeout(sourceState.retry);