From 7c439fdf28cb1e50efaab006b3e5189c6ea933c6 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 22:09:55 +1000 Subject: [PATCH] Store: per-host backoff after 403/429 honouring Retry-After A host that answers 403/429 is left alone until its Retry-After (or GitHub's rate-limit reset; default 10 minutes). Meanwhile cached data is served, or the source reports 'limited' with its own name, e.g. 'GitHub is limiting requests; try again in 10 minutes'. Covers _web reads/downloads and F-Droid fetches. Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/test_apk_more_sources.py | 42 +++++++++++++++++- tests/test_fdroid_sources.py | 19 +++++++- ui/apk_sources/_web.py | 79 +++++++++++++++++++++++++++++----- ui/apk_sources/fdroid.py | 39 ++++++++++++----- ui/apk_sources/github.py | 5 ++- ui/apk_sources/itch.py | 2 +- ui/index.html | 1 + 7 files changed, 159 insertions(+), 28 deletions(-) diff --git a/tests/test_apk_more_sources.py b/tests/test_apk_more_sources.py index 4c788f1..ab3e540 100644 --- a/tests/test_apk_more_sources.py +++ b/tests/test_apk_more_sources.py @@ -18,6 +18,8 @@ class PublisherSources(unittest.TestCase): self.network = patch('urllib.request.OpenerDirector.open', side_effect=AssertionError('network in test')) self.network.start() self.addCleanup(self.network.stop) + _web._limited.clear() + self.addCleanup(_web._limited.clear) def test_curated_search_is_offline(self): self.assertEqual(github.search(github.sources()[0], 'hello')[0]['id'], 'KhronosGroup/OpenXR-SDK-Source') @@ -138,13 +140,49 @@ class PublisherSources(unittest.TestCase): self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') self.assertEqual(op.call_count, 1) error = urllib.error.HTTPError('https://api.github.com/x', 403, 'limited', {}, None) - with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'FRAME_GITHUB_TOKEN'): - _web.read('https://api.github.com/x', ('api.github.com',)) + with patch.object(_web, 'open_url', side_effect=error), patch.dict(os.environ, {'FRAME_GITHUB_TOKEN': ''}), \ + self.assertRaisesRegex(SourceError, 'FRAME_GITHUB_TOKEN'): + github._api('/x') with patch.object(_web, 'open_url', side_effect=error), patch.object(_web.time, 'time', return_value=1e12): self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') # throttled: stale copy with patch.object(_web, 'read', return_value=b''), self.assertRaises(SourceError): github._api('/x') + def test_backoff_honours_retry_after_per_host(self): + from apk_sources import SourceLimited + from email.utils import formatdate + now = [1e9] + clock = patch.object(_web.time, 'time', side_effect=lambda: now[0]) + clock.start() + self.addCleanup(clock.stop) + error = urllib.error.HTTPError('https://itch.io/a', 429, 'slow down', {'Retry-After': '120'}, None) + with patch.object(_web, 'open_url', side_effect=error) as op: + with self.assertRaisesRegex(SourceLimited, '^itch.io is limiting requests; try again in 2 minutes$'): + itch.search(itch.sources()[0], '') + with self.assertRaises(SourceLimited) as caught: # other URLs on the host wait too + _web.read('https://itch.io/b', ('itch.io',), name='itch.io') + self.assertEqual(op.call_count, 1) + self.assertAlmostEqual(caught.exception.retry_after, 120) + with self.assertRaises(SourceLimited): + _web.apk('https://itch.io/c.apk', ('itch.io',)) + self.assertEqual(op.call_count, 1) + with patch.object(_web, 'open_url', return_value=io.BytesIO(b'fresh')): + self.assertEqual(_web.read('https://api.github.com/x', ('api.github.com',)), b'fresh') # other hosts unaffected + now[0] += 121 + with patch.object(_web, 'open_url', return_value=io.BytesIO(b'feed')): + self.assertEqual(_web.read('https://itch.io/b', ('itch.io',)), b'feed') + cases = [({}, 600), ({'Retry-After': formatdate(now[0] + 300, usegmt=True)}, 300), + ({'X-RateLimit-Remaining': '0', 'X-RateLimit-Reset': str(int(now[0]) + 60)}, 60)] + for headers, expected in cases: + with self.subTest(headers=headers): + _web._limited.clear() + self.assertAlmostEqual(_web.throttle('https://h.test/x', headers), expected, delta=1) + self.assertAlmostEqual(_web.wait_time('https://h.test/y'), expected, delta=1) + error = urllib.error.HTTPError('https://api.github.com/x', 403, 'limited', {}, None) + with patch.object(_web, 'open_url', side_effect=error), patch.dict(os.environ, {'FRAME_GITHUB_TOKEN': ''}), \ + self.assertRaisesRegex(SourceLimited, '^GitHub is limiting requests; try again in 10 minutes .*TOKEN'): + github._api('/y') + if __name__ == '__main__': unittest.main() diff --git a/tests/test_fdroid_sources.py b/tests/test_fdroid_sources.py index afcd140..7e550d0 100644 --- a/tests/test_fdroid_sources.py +++ b/tests/test_fdroid_sources.py @@ -10,7 +10,7 @@ import urllib.error import zipfile sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) -from apk_sources import SourceError, fdroid +from apk_sources import SourceError, SourceLimited, _web, fdroid FIXTURES = Path(__file__).parent / 'fixtures' / 'fdroid' PIN = (FIXTURES / 'fingerprint.txt').read_text().strip() @@ -69,12 +69,14 @@ class Repositories(unittest.TestCase): net = patch.object(fdroid.urllib.request, 'build_opener', side_effect=AssertionError('network forbidden')) net.start() self.addCleanup(net.stop) - mock = patch.object(fdroid, '_fetch', side_effect=self.fetch) + mock = self.fetch_patch = patch.object(fdroid, '_fetch', side_effect=self.fetch) self.fetch_mock = mock.start() self.addCleanup(mock.stop) self.v1 = False self.corrupt = None self.files = {} + _web._limited.clear() + self.addCleanup(_web._limited.clear) def fetch(self, url, path, maximum): name = url.rsplit('/', 1)[-1] @@ -332,6 +334,19 @@ class Repositories(unittest.TestCase): stream = io.BytesIO(self.files['entry.jar']) self.assertIn(b'index', fdroid._jar(stream, 'entry.json', None)[0]) # index-v1.jar may still use SHA-1 + def test_rate_limited_host_backs_off(self): + source = dict(self.add(), name='My repo') + self.fetch_patch.stop() + error = urllib.error.HTTPError(URL, 429, 'slow down', {'Retry-After': '300'}, None) + with patch.object(fdroid.urllib.request, 'build_opener') as opener: + opener.return_value.open.side_effect = error + with self.assertRaisesRegex(SourceLimited, '^My repo is limiting requests; try again in 5 minutes$'): + fdroid._load(source, force=True) + with self.assertRaisesRegex(SourceLimited, 'My repo'): + fdroid.download(source, 'org.example.app', 1) + self.assertEqual(opener.return_value.open.call_count, 1) + self.fetch_mock = self.fetch_patch.start() + def test_cached_index_does_not_cross_pins(self): source = self.add() source['fingerprint'] = '0' * 64 diff --git a/ui/apk_sources/_web.py b/ui/apk_sources/_web.py index d86e843..1ed80f1 100644 --- a/ui/apk_sources/_web.py +++ b/ui/apk_sources/_web.py @@ -1,10 +1,50 @@ """Small HTTPS cache and APK downloader for public publisher sources.""" -import hashlib, os, tempfile, time, urllib.error, urllib.parse, urllib.request, zipfile +import hashlib, os, tempfile, threading, time, urllib.error, urllib.parse, urllib.request, zipfile +from email.utils import parsedate_to_datetime import frame_host -from . import SourceError +from . import SourceError, SourceLimited UA = 'FrameControl/0.1' +BACKOFF = 600 # seconds to leave a host alone after 403/429 without Retry-After +_limited = {} # host -> time.time() before which we don't contact it +_limited_lock = threading.Lock() + + +def _host(url): + return (urllib.parse.urlsplit(url).hostname or '').lower() + + +def throttle(url, headers=None): + """Remember that url's host asked us to back off; return the delay in seconds.""" + headers = headers or {} + now, delay = time.time(), None + value = (headers.get('Retry-After') or '').strip() + if value.isdigit(): + delay = int(value) + elif value: + try: + delay = parsedate_to_datetime(value).timestamp() - now + except (TypeError, ValueError, OverflowError): + pass + reset = headers.get('X-RateLimit-Reset') or '' + if delay is None and headers.get('X-RateLimit-Remaining') == '0' and reset.isdigit(): + delay = int(reset) - now # GitHub + delay = min(max(delay if delay is not None else BACKOFF, 1), 6 * 3600) + with _limited_lock: + _limited[_host(url)] = max(_limited.get(_host(url), 0), now + delay) + return delay + + +def wait_time(url): + with _limited_lock: + return max(0, _limited.get(_host(url), 0) - time.time()) + + +def limited_error(name, seconds, hint=''): + minutes = max(1, int(round(seconds / 60))) + return SourceLimited('%s is limiting requests; try again in %d minute%s%s' + % (name, minutes, '' if minutes == 1 else 's', hint), seconds) def cache(): @@ -42,12 +82,24 @@ def open_url(url, hosts, headers=None): urllib.request.Request(url, headers={'User-Agent': UA, **(headers or {})}), timeout=60) -def read(url, hosts, headers=None, ttl=3600): +def read(url, hosts, headers=None, ttl=3600, name=None, hint=''): path = os.path.join(cache(), hashlib.sha256(url.encode()).hexdigest() + '.data') + name = name or _host(url) or 'The source' + + def cached(): + if os.path.isfile(path): # throttled: an older copy beats no results + with open(path, 'rb') as f: + return f.read() try: if os.path.isfile(path) and time.time() - os.path.getmtime(path) < ttl: with open(path, 'rb') as f: return f.read() + wait = wait_time(url) + if wait: + data = cached() + if data is None: + raise limited_error(name, wait, hint) + return data with open_url(url, hosts, headers) as r: data = r.read(8 * 1024 * 1024 + 1) if len(data) > 8 * 1024 * 1024: @@ -63,19 +115,22 @@ def read(url, hosts, headers=None, ttl=3600): return data except urllib.error.HTTPError as e: if e.code in (403, 429): - if os.path.isfile(path): # throttled: an older copy beats no results - with open(path, 'rb') as f: - return f.read() - host = urllib.parse.urlsplit(url).hostname or 'The source' - hint = ' (set FRAME_GITHUB_TOKEN to raise the limit)' if host.endswith('github.com') else '' - raise SourceError(host + ' is limiting requests right now; try again later' + hint) from e + delay = throttle(url, e.headers) + data = cached() + if data is None: + raise limited_error(name, delay, hint) from e + return data raise SourceError('Source HTTP error: ' + str(e.code)) from e except (OSError, ValueError) as e: raise SourceError('Could not read source: ' + str(e)) from e -def apk(url, hosts, digest=None): +def apk(url, hosts, digest=None, name=None): tmp = None + name = name or _host(url) + wait = wait_time(url) + if wait: + raise limited_error(name, wait) try: fd, tmp = tempfile.mkstemp(dir=cache(), suffix='.part') h = hashlib.sha256() @@ -99,6 +154,10 @@ def apk(url, hosts, digest=None): path = os.path.join(cache(), actual + '.apk') os.replace(tmp, path) return {'apk': path, 'obb': [], 'sha256': actual, 'verified': bool(digest)} + except urllib.error.HTTPError as e: + if e.code in (403, 429): + raise limited_error(name, throttle(url, e.headers)) from e + raise SourceError('Could not download APK: HTTP error ' + str(e.code)) from e except (OSError, ValueError, zipfile.BadZipFile) as e: raise SourceError('Could not download APK: ' + str(e)) from e finally: diff --git a/ui/apk_sources/fdroid.py b/ui/apk_sources/fdroid.py index 9244a8a..ee23275 100644 --- a/ui/apk_sources/fdroid.py +++ b/ui/apk_sources/fdroid.py @@ -18,7 +18,7 @@ import zipfile if __package__ in (None, ''): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from apk_sources import SourceError +from apk_sources import SourceError, SourceLimited, _web import frame_host from frame_apk_sign import _der_parts, _cert_key, der from frame_catalog import _IndexReader, _reduce_index, _sha256 @@ -85,17 +85,30 @@ class _HTTPSRedirect(urllib.request.HTTPRedirectHandler): def _fetch(url, path, maximum): + host = urllib.parse.urlsplit(url).hostname + wait = _web.wait_time(url) + if wait: + raise _web.limited_error(host, wait) request = urllib.request.Request(url, headers={'User-Agent': 'FrameControl/1.0'}) - with urllib.request.build_opener(_HTTPSRedirect()).open(request, timeout=60) as r, open(path, 'wb') as f: - total = 0 - while True: - chunk = r.read(1 << 20) - if not chunk: - break - total += len(chunk) - if total > maximum: - raise SourceError('repository file exceeds size limit') - f.write(chunk) + try: + with urllib.request.build_opener(_HTTPSRedirect()).open(request, timeout=60) as r, open(path, 'wb') as f: + total = 0 + while True: + chunk = r.read(1 << 20) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise SourceError('repository file exceeds size limit') + f.write(chunk) + except urllib.error.HTTPError as e: + if e.code in (403, 429): + raise _web.limited_error(host, _web.throttle(url, e.headers)) from e + raise + + +def _limited(source, error): + return _web.limited_error(source['name'], error.retry_after or _web.BACKOFF) def _children(item): @@ -474,6 +487,8 @@ def _load(source, force=False): _write(cache, {'version': CACHE_VERSION, 'url': source['url'], 'fingerprint': pin, 'apps': apps}) _accept(source, timestamp, v2) return apps, pin + except SourceLimited as e: + raise _limited(source, e) from e except SourceError: raise except (OSError, ValueError, KeyError, TypeError, IndexError) as e: @@ -572,6 +587,8 @@ def download(source, entry_id, version_code=None): if os.path.exists(tmp): os.unlink(tmp) return {'apk': str(path), 'obb': [], 'sha256': sha, 'verified': True} + except SourceLimited as e: + raise _limited(source, e) from e except OSError as e: raise SourceError('cannot download APK: ' + str(e)) from e diff --git a/ui/apk_sources/github.py b/ui/apk_sources/github.py index 757bec2..08fcf57 100644 --- a/ui/apk_sources/github.py +++ b/ui/apk_sources/github.py @@ -25,7 +25,8 @@ def _api(path): if token: headers['Authorization'] = 'Bearer ' + token try: - return json.loads(_web.read(API + path, ('api.github.com',), headers)) + return json.loads(_web.read(API + path, ('api.github.com',), headers, name='GitHub', + hint='' if token else ' (set FRAME_GITHUB_TOKEN to raise the limit)')) except (ValueError, TypeError) as e: raise SourceError('Invalid GitHub response') from e @@ -112,4 +113,4 @@ def download(source, entry_id, version_code=None): expected = 'https://github.com/' + entry['id'] + '/releases/download/' if not v['url'].startswith(expected): raise SourceError('APK URL does not belong to the curated publisher') - return _web.apk(v['url'], HOSTS, v['sha256']) + return _web.apk(v['url'], HOSTS, v['sha256'], name='GitHub') diff --git a/ui/apk_sources/itch.py b/ui/apk_sources/itch.py index 3ef904c..fdb70c1 100644 --- a/ui/apk_sources/itch.py +++ b/ui/apk_sources/itch.py @@ -54,7 +54,7 @@ def search(source, query, limit=50): if tag not in FEEDS: raise SourceError('Unsupported itch.io feed') url = 'https://itch.io/games/free/platform-android/tag-' + tag + '.xml' - for entry in _parse(source, _web.read(url, ('itch.io',))): + for entry in _parse(source, _web.read(url, ('itch.io',), name='itch.io')): entries.setdefault(entry['id'], entry) words = query.lower().split() return [e for e in entries.values() if all(w in (e['name'] + ' ' + e['summary']).lower() diff --git a/ui/index.html b/ui/index.html index 1e6a933..b44fc62 100644 --- a/ui/index.html +++ b/ui/index.html @@ -2009,6 +2009,7 @@ async function searchSources(quiet) { const notes=[]; if(loading.length)notes.push(`Still fetching ${loading.join(" and ")}; more results will appear in a moment.`); if(unavailable.length)notes.push(`${unavailable.join(" and ")} ${unavailable.length===1 ? "isn't" : "aren't"} responding right now. You can still explore the other sources.`); + result.sources.filter(s=>s.status==='limited').forEach(s=>notes.push(`${s.error}.`)); $("sourceStatus").innerHTML=notes.map(esc).join(" ")+(result.elsewhere||[]).map(x=>` Browse ${esc(storeName(x.name))} ↗`).join(""); $("sourceStatus").hidden=!notes.length && !(result.elsewhere||[]).length; clearTimeout(sourceState.retry);