F-Droid: serve an expired index as stale while refreshing in the background

Searches no longer wait for (or fail on) a refresh of an expired index; the
store notes which sources show saved listings. A failed refresh keeps the old
index and is retried after 10 minutes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/test_apk_search.py

@@ -87,9 +87,10 @@ class SearchTests(SettingsTest):
                 self.assertEqual(search.search('other', timeout=.03)['sources'][1]['status'], 'loading')
                 search.search('newest', timeout=.03)
                 self.assertEqual(calls, [''])
+                queued = search._pending['slow']
                 release.set()
-                result = search.search('newest', timeout=2)
-                self.assertEqual(result['sources'][1]['status'], 'ok')
+                self.assertTrue(queued['event'].wait(2))
+                self.assertEqual(queued['entries'], [])
                 self.assertEqual(calls, ['', 'newest'])  # 'other' was superseded, never run
         finally:
             release.set()
@@ -106,6 +107,13 @@ class SearchTests(SettingsTest):
             search.set_enabled('one', False)
         self.assertEqual(free, [True])
 
+    def test_stale_source_status(self):
+        mod = fake()
+        mod.stale = lambda source: True
+        with patch.object(search, 'modules', return_value=([mod], [])):
+            status = search.search(timeout=1)['sources'][0]
+        self.assertEqual((status['status'], status['stale']), ('ok', True))
+
     def test_limited_source_status(self):
         from apk_sources import SourceLimited
         mods = [fake('busy', Mock(side_effect=SourceLimited('busy is limiting requests', 60)))]

tests/test_fdroid_sources.py

@@ -75,8 +75,9 @@ class Repositories(unittest.TestCase):
         self.v1 = False
         self.corrupt = None
         self.files = {}
-        _web._limited.clear()
-        self.addCleanup(_web._limited.clear)
+        for state in (_web._limited, fdroid._stale, fdroid._retry_at, fdroid._refreshing):
+            state.clear()
+            self.addCleanup(state.clear)
 
     def fetch(self, url, path, maximum):
         name = url.rsplit('/', 1)[-1]
@@ -347,6 +348,42 @@ class Repositories(unittest.TestCase):
             self.assertEqual(opener.return_value.open.call_count, 1)
         self.fetch_mock = self.fetch_patch.start()
 
+    def expire(self, source):
+        cache = fdroid.frame_host.cache_dir('apk-sources', source['id'] + '.json')
+        old = cache.stat().st_mtime - fdroid.MAX_AGE - 1
+        fdroid.os.utime(str(cache), (old, old))
+
+    def test_expired_index_served_stale_while_refreshing(self):
+        source = self.add()
+        self.expire(source)
+        before = self.fetch_mock.call_count
+        release = __import__('threading').Event()
+        def slow(url, path, maximum):
+            release.wait(5)
+            self.fetch(url, path, maximum)
+        self.fetch_mock.side_effect = slow
+        self.assertEqual(len(fdroid.search(source, 'example')), 1)  # immediately, from the old index
+        self.assertTrue(fdroid.stale(source))
+        refresh = fdroid._refreshing[source['id']]
+        fdroid.search(source, 'example')
+        self.assertIs(fdroid._refreshing.get(source['id']), refresh)  # one refresh at a time
+        release.set()
+        refresh.join(5)
+        self.assertEqual(self.fetch_mock.call_count, before + 2)
+        self.assertFalse(fdroid.stale(source))
+
+    def test_failed_refresh_keeps_serving_stale_index(self):
+        source = self.add()
+        self.expire(source)
+        self.fetch_mock.side_effect = urllib.error.HTTPError(URL, 503, 'unavailable', None, None)
+        self.assertEqual(len(fdroid.search(source, 'example')), 1)
+        fdroid._refreshing[source['id']].join(5)
+        calls = self.fetch_mock.call_count
+        self.assertEqual(fdroid.details(source, 'org.example.app')['version_code'], 2)
+        self.assertTrue(fdroid.stale(source))
+        self.assertNotIn(source['id'], fdroid._refreshing)  # failed refresh waits before retrying
+        self.assertEqual(self.fetch_mock.call_count, calls)
+
     def test_cached_index_does_not_cross_pins(self):
         source = self.add()
         source['fingerprint'] = '0' * 64