F-Droid: serve an expired index as stale while refreshing in the background

Searches no longer wait for (or fail on) a refresh of an expired index; the
store notes which sources show saved listings. A failed refresh keeps the old
index and is retried after 10 minutes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/apk-repos.md

@@ -66,7 +66,9 @@ separate work. Built-in sources can be disabled but cannot be removed.
 Settings and pins live in `frame_host.data_dir('apk-repos.json')`
 (`~/Library/Application Support/Frame Control/apk-repos.json` on macOS).
 Authenticated reduced indexes and APKs live under
-`frame_host.cache_dir('apk-sources')`; indexes refresh after 24 hours.
+`frame_host.cache_dir('apk-sources')`; indexes refresh after 24 hours. An
+expired index is still served (marked stale in the store) while it refreshes in
+the background; a failed refresh is retried after 10 minutes.
 The existing catalogue's unverified index cache is never treated as authenticated.
 
 Rollback protection: each repository's newest accepted signed index timestamp

tests/test_apk_search.py

@@ -87,9 +87,10 @@ class SearchTests(SettingsTest):
                 self.assertEqual(search.search('other', timeout=.03)['sources'][1]['status'], 'loading')
                 search.search('newest', timeout=.03)
                 self.assertEqual(calls, [''])
+                queued = search._pending['slow']
                 release.set()
-                result = search.search('newest', timeout=2)
-                self.assertEqual(result['sources'][1]['status'], 'ok')
+                self.assertTrue(queued['event'].wait(2))
+                self.assertEqual(queued['entries'], [])
                 self.assertEqual(calls, ['', 'newest'])  # 'other' was superseded, never run
         finally:
             release.set()
@@ -106,6 +107,13 @@ class SearchTests(SettingsTest):
             search.set_enabled('one', False)
         self.assertEqual(free, [True])
 
+    def test_stale_source_status(self):
+        mod = fake()
+        mod.stale = lambda source: True
+        with patch.object(search, 'modules', return_value=([mod], [])):
+            status = search.search(timeout=1)['sources'][0]
+        self.assertEqual((status['status'], status['stale']), ('ok', True))
+
     def test_limited_source_status(self):
         from apk_sources import SourceLimited
         mods = [fake('busy', Mock(side_effect=SourceLimited('busy is limiting requests', 60)))]

tests/test_fdroid_sources.py

@@ -75,8 +75,9 @@ class Repositories(unittest.TestCase):
         self.v1 = False
         self.corrupt = None
         self.files = {}
-        _web._limited.clear()
-        self.addCleanup(_web._limited.clear)
+        for state in (_web._limited, fdroid._stale, fdroid._retry_at, fdroid._refreshing):
+            state.clear()
+            self.addCleanup(state.clear)
 
     def fetch(self, url, path, maximum):
         name = url.rsplit('/', 1)[-1]
@@ -347,6 +348,42 @@ class Repositories(unittest.TestCase):
             self.assertEqual(opener.return_value.open.call_count, 1)
         self.fetch_mock = self.fetch_patch.start()
 
+    def expire(self, source):
+        cache = fdroid.frame_host.cache_dir('apk-sources', source['id'] + '.json')
+        old = cache.stat().st_mtime - fdroid.MAX_AGE - 1
+        fdroid.os.utime(str(cache), (old, old))
+
+    def test_expired_index_served_stale_while_refreshing(self):
+        source = self.add()
+        self.expire(source)
+        before = self.fetch_mock.call_count
+        release = __import__('threading').Event()
+        def slow(url, path, maximum):
+            release.wait(5)
+            self.fetch(url, path, maximum)
+        self.fetch_mock.side_effect = slow
+        self.assertEqual(len(fdroid.search(source, 'example')), 1)  # immediately, from the old index
+        self.assertTrue(fdroid.stale(source))
+        refresh = fdroid._refreshing[source['id']]
+        fdroid.search(source, 'example')
+        self.assertIs(fdroid._refreshing.get(source['id']), refresh)  # one refresh at a time
+        release.set()
+        refresh.join(5)
+        self.assertEqual(self.fetch_mock.call_count, before + 2)
+        self.assertFalse(fdroid.stale(source))
+
+    def test_failed_refresh_keeps_serving_stale_index(self):
+        source = self.add()
+        self.expire(source)
+        self.fetch_mock.side_effect = urllib.error.HTTPError(URL, 503, 'unavailable', None, None)
+        self.assertEqual(len(fdroid.search(source, 'example')), 1)
+        fdroid._refreshing[source['id']].join(5)
+        calls = self.fetch_mock.call_count
+        self.assertEqual(fdroid.details(source, 'org.example.app')['version_code'], 2)
+        self.assertTrue(fdroid.stale(source))
+        self.assertNotIn(source['id'], fdroid._refreshing)  # failed refresh waits before retrying
+        self.assertEqual(self.fetch_mock.call_count, calls)
+
     def test_cached_index_does_not_cross_pins(self):
         source = self.add()
         source['fingerprint'] = '0' * 64

ui/apk_sources/fdroid.py

@@ -27,6 +27,10 @@ KIND = 'fdroid'
 CACHE_VERSION = 2
 _LOCK = threading.RLock()  # settings only; never held while downloading
 _load_locks = {}
+_refreshing = {}  # source id -> background refresh thread
+_retry_at = {}  # source id -> time before which a failed refresh isn't retried
+_stale = set()  # source ids currently served from an expired index
+MAX_AGE = 86400
 # Published by the repository operators; a user repository without a pin uses TOFU.
 FDROID_PIN = '43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab'
 IZZY_PIN = '3bf0d6abfeae2f401707b6d966be743bf0eee49c2561b9ba39073711f628937a'
@@ -438,6 +442,40 @@ def _v1(content, path):
     return (index.get('repo') or {}).get('timestamp')
 
 
+def _cached(source, cache):
+    """(apps, pin, expired) from a cache matching this source, or None."""
+    try:
+        saved = json.loads(cache.read_text())
+        if (saved.get('version') == CACHE_VERSION and saved.get('fingerprint') == source.get('fingerprint')
+                and saved.get('url') == source['url']):
+            return saved['apps'], saved['fingerprint'], time.time() - cache.stat().st_mtime >= MAX_AGE
+    except (OSError, ValueError, KeyError, AttributeError):
+        pass
+    return None
+
+
+def stale(source):
+    """True while results come from an expired index that is being (or failed to be) refreshed."""
+    return source['id'] in _stale
+
+
+def _refresh_later(source):
+    with _LOCK:
+        if source['id'] in _refreshing or time.time() < _retry_at.get(source['id'], 0):
+            return
+        def run():
+            try:
+                _load(source, force=True)
+            except Exception:
+                with _LOCK:
+                    _retry_at[source['id']] = time.time() + 600
+            finally:
+                with _LOCK:
+                    _refreshing.pop(source['id'], None)
+        _refreshing[source['id']] = thread = threading.Thread(target=run, daemon=True)
+    thread.start()
+
+
 def _source_lock(source_id):
     with _LOCK:
         return _load_locks.setdefault(source_id, threading.Lock())
@@ -448,15 +486,16 @@ def _load(source, force=False):
         raise SourceError('invalid source id')
     _url(source['url'], source.get('fingerprint'))
     cache = frame_host.cache_dir('apk-sources', source['id'] + '.json')
+    found = None if force else _cached(source, cache)
+    if found:
+        if found[2]:  # expired: serve it now, marked stale, and refresh without blocking anyone
+            _stale.add(source['id'])
+            _refresh_later(source)
+        return found[:2]
     with _source_lock(source['id']):
-        if not force and cache.exists() and time.time() - cache.stat().st_mtime < 86400:
-            try:
-                saved = json.loads(cache.read_text())
-                if (saved.get('version') == CACHE_VERSION and saved.get('fingerprint') == source.get('fingerprint')
-                        and saved.get('url') == source['url']):
-                    return saved['apps'], saved['fingerprint']
-            except (OSError, ValueError, KeyError, AttributeError):
-                pass
+        found = None if force else _cached(source, cache)  # another caller may have just loaded it
+        if found and not found[2]:
+            return found[:2]
         cache.parent.mkdir(parents=True, exist_ok=True)
         try:
             with tempfile.TemporaryDirectory(dir=str(cache.parent)) as tmp:
@@ -486,6 +525,7 @@ def _load(source, force=False):
                 apps = _reduce(raw, source)
                 _write(cache, {'version': CACHE_VERSION, 'url': source['url'], 'fingerprint': pin, 'apps': apps})
                 _accept(source, timestamp, v2)
+                _stale.discard(source['id'])
                 return apps, pin
         except SourceLimited as e:
             raise _limited(source, e) from e

ui/apk_sources/search.py

@@ -237,6 +237,7 @@ def _start(key, task):
         try:
             task['entries'] = [dict(e, source=key, source_name=source['name'], trust=source.get('trust'))
                                for e in module.search(source, query, limit=limit) if e.get('free') is True]
+            task['stale'] = bool(getattr(module, 'stale', lambda s: False)(source))
         except Exception as e:
             task['error'] = str(e)
             task['limited'] = isinstance(e, SourceLimited)
@@ -268,7 +269,7 @@ def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, l
         elif 'error' in task:
             status.update(status='limited' if task.get('limited') else 'error', error=task['error'])
         else:
-            status.update(status='ok')
+            status.update(status='ok', stale=task['stale'])
             entries.extend(task['entries'])
         statuses.append(status)
         with _lock:

ui/index.html

@@ -2010,6 +2010,8 @@ async function searchSources(quiet) {
     if(loading.length)notes.push(`Still fetching ${loading.join(" and ")}; more results will appear in a moment.`);
     if(unavailable.length)notes.push(`${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.`);
     result.sources.filter(s=>s.status==='limited').forEach(s=>notes.push(`${s.error}.`));
+    const stale=result.sources.filter(s=>s.stale).map(s=>storeName(s.name));
+    if(stale.length)notes.push(`Showing saved listings from ${stale.join(" and ")} while ${stale.length===1 ? "it refreshes" : "they refresh"}.`);
     $("sourceStatus").innerHTML=notes.map(esc).join(" ")+(result.elsewhere||[]).map(x=>` <a href="${esc(x.url)}" target="_blank" rel="noopener">Browse ${esc(storeName(x.name))} ↗</a>`).join("");
     $("sourceStatus").hidden=!notes.length && !(result.elsewhere||[]).length;
     clearTimeout(sourceState.retry);