From 113216cc0e6569cfb099af5e3b5d3be447f6fd63 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:03:27 +1000 Subject: [PATCH 001/122] Add curated GitHub APK releases and itch.io VR feed listings Survey publisher consent and access limits; add cached sources, recorded fixtures and local APK proof. Co-Authored-By: GPT-6 Astra --- .claude/.gitignore | 6 + .claude/NOTES-more-sources.md | 44 ++ .claude/proof-more-sources.log | 36 ++ .claude/prove-more-sources.py | 24 + .claude/tests-more-sources.log | 5 + docs/apk-sources.md | 96 ++++ tests/fixtures/more_sources/README.md | 15 + .../fixtures/more_sources/brush-releases.json | 88 ++++ .../more_sources/itch-author-robots.txt | 12 + tests/fixtures/more_sources/itch-feed.txt | 11 + tests/fixtures/more_sources/itch-robots.txt | 11 + .../more_sources/khronos-releases.json | 410 ++++++++++++++++++ tests/fixtures/more_sources/topic.json | 19 + tests/fixtures/more_sources/tux-releases.json | 17 + tests/test_apk_more_sources.py | 108 +++++ ui/apk_sources/_web.py | 101 +++++ ui/apk_sources/github.py | 110 +++++ ui/apk_sources/github_curated.json | 29 ++ ui/apk_sources/itch.py | 72 +++ 19 files changed, 1214 insertions(+) create mode 100644 .claude/.gitignore create mode 100644 .claude/NOTES-more-sources.md create mode 100644 .claude/proof-more-sources.log create mode 100644 .claude/prove-more-sources.py create mode 100644 .claude/tests-more-sources.log create mode 100644 docs/apk-sources.md create mode 100644 tests/fixtures/more_sources/README.md create mode 100644 tests/fixtures/more_sources/brush-releases.json create mode 100644 tests/fixtures/more_sources/itch-author-robots.txt create mode 100644 tests/fixtures/more_sources/itch-feed.txt create mode 100644 tests/fixtures/more_sources/itch-robots.txt create mode 100644 tests/fixtures/more_sources/khronos-releases.json create mode 100644 tests/fixtures/more_sources/topic.json create mode 100644 tests/fixtures/more_sources/tux-releases.json create mode 100644 tests/test_apk_more_sources.py create mode 100644 ui/apk_sources/_web.py create mode 100644 ui/apk_sources/github.py create mode 100644 ui/apk_sources/github_curated.json create mode 100644 ui/apk_sources/itch.py diff --git a/.claude/.gitignore b/.claude/.gitignore new file mode 100644 index 0000000..4f4d046 --- /dev/null +++ b/.claude/.gitignore @@ -0,0 +1,6 @@ +/proof-cache/ +/amazon.html +/aptoide-terms.html +/itch-game.html +/meta.html +/uptodown-terms.html diff --git a/.claude/NOTES-more-sources.md b/.claude/NOTES-more-sources.md new file mode 100644 index 0000000..b699e4d --- /dev/null +++ b/.claude/NOTES-more-sources.md @@ -0,0 +1,44 @@ +# more-sources + +Scope: only this worktree/branch; no delegation, SSH, installation, push, PR or +issue writes. Parent performs integration and independent review per brief. + +Decisions: implement GitHub curated public APK releases and itch.io RSS page +links. itch robots exclude keyed download pages used by /tmp/vrapk/itchdl.py; +no bypass. Topic results are not automatically trusted. Curated Open Brush and +SuperTux prereleases must be explicitly allowed; discovered during fixture tests. +No shared files changed. Default GitHub search makes no network request. + +Evidence (2026-09-28): +- Read all six prerequisite files and the full brief. +- Fetched itch terms, root/author robots and OpenXR RSS with FrameControl UA. +- Anonymous GitHub returned 403 rate-limit; authenticated gh API reads succeeded. +- GitHub fixtures record upstream Khronos, Open Brush and SuperTux releases. +- `python3 .claude/prove-more-sources.py`: GitHub search/download succeeded, + published SHA-256 matched; `python3 ui/frame_android.py info` exited 0. + Overall script exit 1 because subsequent itch RSS request returned HTTP 429. + A second invocation had the same outcome; no further live itch retries. +- Download: `.claude/proof-cache/f24bbe8ba6f6339fca658628868ba8189cbc33390d6ac508f69d76fb67b5fa34.apk`. + Package org.khronos.openxr.hello_xr.vulkan; 1.1.63/1063; API 24; arm64; + OpenXR. No runtime compatibility claim. +- Initial whole suite: 174 tests, 1 failed (prerelease handling). After explicit + curated prerelease support: 174 tests passed, exit 0. Final rerun below. + +Unverified: live itch search completion (429), itch download/info (disallowed +flow), Open Brush/SuperTux downloads, topic live adapter search (API fixture +capture succeeded), headset runtime, UI integration, independent provider +review (explicit implementation brief prohibits delegation and assigns review +to parent). No claim that any reviewer/model participated. + +Research HTML and local downloaded APKs are local evidence only, not committed. +Recorded fixtures retain public source data, not credentials. The proof script +uses gh's token in memory and overrides cache into this worktree. + +Final verification: +- Python version: 3.9.6. +- `python3 -m unittest discover -s tests > .claude/tests-more-sources.log 2>&1`: + 174 tests passed in 5.569s, real exit 0. +- `git diff --check`: exit 0. +- Final artifact: docs/apk-sources.md plus two source modules, a private HTTPS + helper, curated JSON, recorded fixtures and tests. Evidence scripts/logs stay + in .claude; research HTML and APK cache are explicitly ignored there. diff --git a/.claude/proof-more-sources.log b/.claude/proof-more-sources.log new file mode 100644 index 0000000..83e67d0 --- /dev/null +++ b/.claude/proof-more-sources.log @@ -0,0 +1,36 @@ +org.khronos.openxr.hello_xr.vulkan · 1.1.63 (code 1063) +Minimum: Android 7.0 (API 24) +ABIs: arm64-v8a, armeabi-v7a, x86, x86_64 +Lepton can install this APK. Features may still need services Lepton lacks. +VR app +Uses OpenXR (good). +GitHub search: [{"source": "github", "id": "KhronosGroup/OpenXR-SDK-Source", "package": null, "name": "hello_xr", "summary": "Khronos OpenXR sample (Vulkan and OpenGL ES)", "icon": null, "images": {"icon": null, "banner": null, "screenshots": []}, "page": "https://github.com/KhronosGroup/OpenXR-SDK-Source", "version": null, "version_code": null, "min_sdk": null, "abis": null, "vr": true, "size": null, "free": true, "license": "Apache-2.0", "updated": null, "downloadable": true}] +GitHub download: {"apk": "/Users/saphid/projects/steam-frame-moresrc/.claude/proof-cache/f24bbe8ba6f6339fca658628868ba8189cbc33390d6ac508f69d76fb67b5fa34.apk", "obb": [], "sha256": "f24bbe8ba6f6339fca658628868ba8189cbc33390d6ac508f69d76fb67b5fa34", "verified": true} +info exit: 0 +Traceback (most recent call last): + File "/Users/saphid/projects/steam-frame-moresrc/ui/apk_sources/_web.py", line 47, in read + with open_url(url, hosts, headers) as r: + File "/Users/saphid/projects/steam-frame-moresrc/ui/apk_sources/_web.py", line 37, in open_url + return urllib.request.build_opener(Redirect(hosts)).open( + File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/urllib/request.py", line 523, in open + response = meth(req, response) + File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/urllib/request.py", line 632, in http_response + response = self.parent.error( + File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/urllib/request.py", line 561, in error + return self._call_chain(*args) + File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/urllib/request.py", line 494, in _call_chain + result = func(*args) + File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/urllib/request.py", line 641, in http_error_default + raise HTTPError(req.full_url, code, msg, hdrs, fp) +urllib.error.HTTPError: HTTP Error 429: Too Many Requests + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/Users/saphid/projects/steam-frame-moresrc/.claude/prove-more-sources.py", line 18, in + entries = itch.search(itch.sources()[0], 'off nominal') + File "/Users/saphid/projects/steam-frame-moresrc/ui/apk_sources/itch.py", line 58, in search + for entry in _parse(source, _web.read(url, ('itch.io',))): + File "/Users/saphid/projects/steam-frame-moresrc/ui/apk_sources/_web.py", line 62, in read + raise SourceError('Source refused access or reached its rate limit; try later (GitHub accepts FRAME_GITHUB_TOKEN)') from e +apk_sources.SourceError: Source refused access or reached its rate limit; try later (GitHub accepts FRAME_GITHUB_TOKEN) diff --git a/.claude/prove-more-sources.py b/.claude/prove-more-sources.py new file mode 100644 index 0000000..85d4e49 --- /dev/null +++ b/.claude/prove-more-sources.py @@ -0,0 +1,24 @@ +import json, os, subprocess, sys +from pathlib import Path +sys.path.insert(0, str(Path.cwd() / 'ui')) +from apk_sources import github, itch, _web +proof = Path.cwd() / '.claude' / 'proof-cache' +proof.mkdir(exist_ok=True) +_web.cache = lambda: str(proof) +token = subprocess.run(['gh', 'auth', 'token'], capture_output=True, text=True, check=True).stdout.strip() +os.environ['FRAME_GITHUB_TOKEN'] = token +s = github.sources()[0] +entries = github.search(s, 'hello') +print('GitHub search:', json.dumps(entries)) +r = github.download(s, entries[0]['id']) +print('GitHub download:', json.dumps(r)) +p = subprocess.run(['python3', 'ui/frame_android.py', 'info', r['apk']]) +print('info exit:', p.returncode) +assert p.returncode == 0 +entries = itch.search(itch.sources()[0], 'off nominal') +print('itch.io search:', json.dumps(entries)) +assert entries and entries[0]['downloadable'] is False +try: + itch.download(itch.sources()[0], entries[0]['id']) +except Exception as e: + print('itch.io download correctly refused:', e) diff --git a/.claude/tests-more-sources.log b/.claude/tests-more-sources.log new file mode 100644 index 0000000..8834d11 --- /dev/null +++ b/.claude/tests-more-sources.log @@ -0,0 +1,5 @@ +.............................................................................................................................................................................. +---------------------------------------------------------------------- +Ran 174 tests in 5.569s + +OK diff --git a/docs/apk-sources.md b/docs/apk-sources.md new file mode 100644 index 0000000..15aa0b5 --- /dev/null +++ b/docs/apk-sources.md @@ -0,0 +1,96 @@ +# Developer-consented APK sources + +Surveyed 2026-09-28. Free access is not proof of redistribution permission or +Frame compatibility. These adapters fetch only public publisher releases or +link to publisher pages. They do not acquire store entitlements, defeat access +checks, install anything, or rehost APKs. See [VR compatibility](vr-apks.md). + +| Source | Developer consent and automated-access position | API/feed; VR coverage | Decision | +|---|---|---|---| +| [itch.io](https://itch.io/docs/legal/terms) | Publishers warrant distribution rights (§4). Users may access content through the service; this is not blanket scraping permission. Main robots excludes `/game/download/`; author subdomains exclude `/*/download/`. No challenge bypass. | Public free Android RSS for `openxr` and `oculus-quest`; substantial indie VR. Server API is mostly authenticated publisher/account functionality, not a general anonymous store-download API. | Implement RSS search, artwork and page links; `downloadable: False`. The supplied free-download script follows keyed download pages excluded by robots, so it is not shipped. | +| [GitHub releases](https://docs.github.com/en/rest/releases/releases) | Maintainers publish assets; curated repositories below establish provenance. Public hosting or an open-source topic alone does not establish rights to every uploaded binary. Use supported REST API under [API terms](https://docs.github.com/en/site-policy/github-terms/github-terms-of-service#h-api-terms), not HTML crawling. | Releases API includes APK assets and sometimes SHA-256. Topic search finds OpenXR/Quest projects. 60 unauthenticated requests/hour; authenticated user limits are generally 5,000/hour, with separate search/secondary limits. | Implement curated downloads and explicit topic discovery. Unreviewed topic results are page-only. | +| [Uptodown](https://www.uptodown.com/aboutus) | Developer distribution program exists, but that does not prove publisher authorization for every catalog item. [Privacy policy](https://www.uptodown.com/aboutus/privacy) explicitly describes protection against automated access. General automation permission was not established. | Broad Android catalog, limited VR focus; no supported public consumer-download API established in this survey. | Page links only; no downloader. Do not infer consent from an unchanged APK signature. | +| [APKPure](https://apkpure.com/terms) | Third-party APK catalog; individual publisher consent and automation rights were not established. Terms request returned HTTP 403; no bypass attempted. | Broad Android coverage, incidental VR; internal endpoints are not permission to automate. | Exclude automatic indexing/downloading; user may open site. | +| [APKMirror](https://www.apkmirror.com/faq/) | Publisher-signed files and a free-app policy are not a blanket developer-consent or automation grant. FAQ request returned HTTP 403, so current terms could not be confirmed. | General Android/version archive; APK bundles often need another installer; little VR focus. No supported consumer-download API established. | Page links only, no scraping or bundle conversion. | +| [Aptoide](https://en.aptoide.com/company/legal) | Terms define an app supplier as developer, owner or authorized distributor; user stores still require per-item provenance. API availability alone does not settle third-party access rights. | API ecosystem and general Android catalog; weak VR focus. | Defer until a publisher-owned store and its API terms can be approved. No blanket community-store downloader. | +| [Amazon Appstore](https://developer.amazon.com/docs/app-submission/understanding-submission.html) | Official developer submissions; store account, device and license rules apply. Publisher submission APIs do not authorize public binary extraction. | Fire-device distribution; Android-device Appstore support ended in 2025; little Quest relevance. | Official product links only; no account or entitlement extraction. | +| [PICO / ByteDance store](https://developer.picoxr.com/document/distribute) | Official publisher channel with store/device entitlements. No public unauthenticated binary-download grant established; documentation request encountered a redirect error. | Strong standalone VR; PICO builds may depend on PICO services/extensions. | Store links only. A developer's independently published GitHub/itch build can qualify separately. | +| [Meta Horizon Store / former App Lab](https://www.meta.com/experiences/) | Official developer submissions. A free store entitlement is still an entitlement; no license bypass or authenticated store extraction. App Lab was folded into the main store in 2024. | Strongest Quest coverage; no supported anonymous APK-download API established. | Store links only; independently distributed free builds use their publisher source. | +| [Khronos samples](https://github.com/KhronosGroup/OpenXR-SDK-Source) | Official upstream, Apache-2.0 sample; developer-published release APKs. GitHub API terms apply. | `hello_xr` Vulkan/OpenGL ES APKs; excellent OpenXR diagnostics. | Included in GitHub curated list, Vulkan variant selected. | +| [Meta OpenXR samples](https://github.com/meta-quest/Meta-OpenXR-SDK) | Official upstream; check each sample's license. Source availability does not imply a published APK, and some samples require Meta extensions/services. | Source/build examples, inconsistent ready-made APK releases. | Link to upstream; add specific free APKs only after release/provenance review. | +| [Godot XR demos](https://github.com/GodotVR/godot-xr-tools) | Official project source and publisher demo pages; licenses and dependencies vary by demo. | OpenXR examples on GitHub/itch. Older Godot builds can fail on Lepton's missing clipboard service. | Covered by source discovery; no compatibility promise from an OpenXR tag. | + +The table distinguishes observed restrictions from unknown permission. An +unverified policy is a reason to defer automation, not a claim that a site is +unlawful. Only the two implemented source kinds are registered by their own +`sources()` functions; the other rows are recommendations, not new UI entries. + +## Adapters + +`ui/apk_sources/github.py` uses `github_curated.json`: Khronos `hello_xr`, +[Open Brush](https://github.com/icosa-foundation/open-brush), and +[SuperTux 3D](https://github.com/SgtBilko76/SuperTux-3D). These have official +OpenXR project/release evidence, not a blanket claim of headset compatibility. +Open Brush's compatibility evidence is recorded in [vr-apks.md](vr-apks.md). +Open Brush and SuperTux publish the selected builds as prereleases; curated +opt-ins preserve that label in version records. Exact APK filename patterns +avoid downloading desktop archives or alternate non-Quest builds. +[OpenSaberPlus](https://github.com/arpruss/OpenSaberPlus) was examined but not +curated: GitHub reports its license as `NOASSERTION`, and current OpenXR APK +provenance was not established in this pass. + +Default GitHub search is offline against this small list. Queries +`topic:openxr`, `topic:oculus-quest`, and `topic:quest` explicitly call repository +search. Results outside the curated list stay page-only, even if a repository +claims an open-source license. This prevents an arbitrary tagged mirror from +becoming a trusted downloader. Extend the curated JSON after provenance review. + +Set optional `FRAME_GITHUB_TOKEN` in the process environment for a higher API +quota. Tokens are sent only to `api.github.com`, never written to the cache, +never sent to asset hosts, and removed on redirects. The adapter does not +read `gh` credentials automatically. Metadata is cached for one hour under +`frame_host.cache_dir('apk-sources', 'publisher')`. A cold details request +fetches at most ten releases. Rate-limit errors are surfaced without retry +loops. Asset IDs and release tags are not Android version codes: metadata +leaves the latter unknown and rejects a requested `version_code` rather than +silently fetching a different build. + +`itch.py` exposes separate OpenXR and Quest feed sources, so one feed's failure +does not suppress the other at the aggregator level. Queries filter the current +feed window locally: this is not an exhaustive historical itch search. Only +explicit zero-price Android entries are returned. Covers are exposed in +`images`; absent screenshots, APK version, ABI and minimum SDK stay unknown. +GitHub's release metadata has no standard app artwork field, so artwork stays +empty rather than presenting a repository-owner avatar as an app icon. VR is +based on curated evidence or a VR-specific feed/topic, not a compatibility claim. + +Downloads stream to unique temporary files, require an APK manifest entry, +restrict HTTPS origins and redirects, and enforce a 2 GiB ceiling. `verified` +means the downloaded SHA-256 matches GitHub's published digest. Without such a +digest, the computed SHA-256 is returned with `verified: False`; neither value +claims publisher-signature validation. Installation must inspect the APK as +usual. OBBs, split APKs, paid assets and external release-body download links +are unsupported. + +## Evidence and limits + +On this Mac, Python 3.9 downloaded the real Khronos Vulkan 1.1.63 APK through +the GitHub adapter, matched its published SHA-256 +`f24bbe8ba6f6339fca658628868ba8189cbc33390d6ac508f69d76fb67b5fa34`, and +`python3 ui/frame_android.py info ` exited 0: package +`org.khronos.openxr.hello_xr.vulkan`, version code 1063, minimum API 24, +arm64-v8a present, OpenXR detected. No Frame connection or installation occurred. + +The itch OpenXR RSS was fetched successfully and recorded as a fixture. +Subsequent live adapter search encountered HTTP 429; it is not claimed as a +successful live end-to-end search. Fixture search finds Off Nominal and parses +nine Android entries from the ten-item feed (one has only an HTML platform). +A real itch download and APK inspection were deliberately not performed: +robots restrictions take precedence over that requested proof. No current +policy text is claimed verified where the table records failed access. + +Tests use recorded, reduced API/RSS fixtures with network access blocked in +the new test class. They cover selection, prereleases, unknown topic results, +paid/non-Android exclusion, URL restrictions, redirect credential removal, +caching, rate limits, checksum mismatch, non-APK rejection and partial-file +cleanup. See `.claude/NOTES-more-sources.md` for commands and local evidence. diff --git a/tests/fixtures/more_sources/README.md b/tests/fixtures/more_sources/README.md new file mode 100644 index 0000000..be6d3c8 --- /dev/null +++ b/tests/fixtures/more_sources/README.md @@ -0,0 +1,15 @@ +Recorded 2026-09-28 from public publisher endpoints using FrameControl/0.1 or +`gh api`. JSON fixtures are reduced to fields consumed by the adapters; API +values are unchanged. No token, cookies or signed download URL is included. + +- `*-releases.json`: `/repos/{repo}/releases?per_page=10`, first two releases, + for KhronosGroup/OpenXR-SDK-Source, icosa-foundation/open-brush and + SgtBilko76/SuperTux-3D (one release). +- `topic.json`: `/search/repositories?q=topic:openxr+archived:false&sort=stars&per_page=3`. +- `itch-feed.txt`: `https://itch.io/games/free/platform-android/tag-openxr.xml`. +- `itch-robots.txt`: `https://itch.io/robots.txt`. +- `itch-author-robots.txt`: `https://godotvr.itch.io/robots.txt`. + +The synthetic ZIP in tests is only a transport/integrity fixture, not an +installable APK. Actual APK parsing was verified separately on the downloaded +Khronos Vulkan sample; see docs/apk-sources.md. diff --git a/tests/fixtures/more_sources/brush-releases.json b/tests/fixtures/more_sources/brush-releases.json new file mode 100644 index 0000000..c17f800 --- /dev/null +++ b/tests/fixtures/more_sources/brush-releases.json @@ -0,0 +1,88 @@ +[ + { + "tag_name": "2.32.29", + "draft": false, + "prerelease": true, + "published_at": "2026-09-26T17:49:28Z", + "assets": [ + { + "id": 591143285, + "name": "OpenBrush_Android_2.32.29.apk", + "size": 314602794, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.29/OpenBrush_Android_2.32.29.apk", + "digest": "sha256:f20361b830803a2bf53e2af648bba59ee17bc4615bde534dbf6c4f0012bbd291" + }, + { + "id": 591143287, + "name": "OpenBrush_Desktop_2.32.29.zip", + "size": 380735034, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.29/OpenBrush_Desktop_2.32.29.zip", + "digest": "sha256:3b680b07ca0b8def579fe194916aa7db4d007c3094c4dea818124776098c9b9a" + }, + { + "id": 591143282, + "name": "OpenBrush_Linux_2.32.29.zip", + "size": 364906490, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.29/OpenBrush_Linux_2.32.29.zip", + "digest": "sha256:e380934f77d2b1441179424193a75f7b4b5793b34761c04cbf2d87bad9f8fc16" + }, + { + "id": 591143283, + "name": "OpenBrush_Mac_2.32.29.dmg", + "size": 373196471, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.29/OpenBrush_Mac_2.32.29.dmg", + "digest": "sha256:5d544d0a64bed1cae65173fcfa7ada069d4f81b42385e806e99cc4427ef3513c" + }, + { + "id": 591143286, + "name": "OpenBrush_Quest_2.32.29.apk", + "size": 314603546, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.29/OpenBrush_Quest_2.32.29.apk", + "digest": "sha256:57cd7b9067689060451494e55dc06276f934a992f5cbbd44d965069903937ba6" + } + ] + }, + { + "tag_name": "2.32.28", + "draft": false, + "prerelease": true, + "published_at": "2026-09-26T14:42:27Z", + "assets": [ + { + "id": 590837298, + "name": "OpenBrush_Android_2.32.28.apk", + "size": 314603450, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.28/OpenBrush_Android_2.32.28.apk", + "digest": "sha256:1a3af1a194c4348ef67c8ea61d9a8258e2490cdfe1f760cbc3c69f2978a6a082" + }, + { + "id": 590837301, + "name": "OpenBrush_Desktop_2.32.28.zip", + "size": 380738236, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.28/OpenBrush_Desktop_2.32.28.zip", + "digest": "sha256:c2de5424bc022951f0e0bdbd652ede549a1e60d9cfbf41c730ea43d16d97262f" + }, + { + "id": 590837297, + "name": "OpenBrush_Linux_2.32.28.zip", + "size": 364907046, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.28/OpenBrush_Linux_2.32.28.zip", + "digest": "sha256:29daf410347b3ca36e47808e31172270df8040ba7decc83be2bdcd51de895e06" + }, + { + "id": 590837296, + "name": "OpenBrush_Mac_2.32.28.dmg", + "size": 373195966, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.28/OpenBrush_Mac_2.32.28.dmg", + "digest": "sha256:2f352d766450c993fdcae8a8552878a6a976d32d675246b63c81bb1b326fd20d" + }, + { + "id": 590837295, + "name": "OpenBrush_Quest_2.32.28.apk", + "size": 314604234, + "browser_download_url": "https://github.com/icosa-foundation/open-brush/releases/download/2.32.28/OpenBrush_Quest_2.32.28.apk", + "digest": "sha256:9a3af6a6e838dd8bd201e549c5570f67db794df940e960390aeeae93235d702e" + } + ] + } +] diff --git a/tests/fixtures/more_sources/itch-author-robots.txt b/tests/fixtures/more_sources/itch-author-robots.txt new file mode 100644 index 0000000..5dacc48 --- /dev/null +++ b/tests/fixtures/more_sources/itch-author-robots.txt @@ -0,0 +1,12 @@ +User-agent: Mediapartners-Google +Disallow: + +User-agent: * +Disallow: /*/download/ +Disallow: /*/rh/ +Disallow: /*/rp/ +Disallow: /-/ + +Sitemap: https://itch.io/sitemap.xml + +# vim: set ft=robots: diff --git a/tests/fixtures/more_sources/itch-feed.txt b/tests/fixtures/more_sources/itch-feed.txt new file mode 100644 index 0000000..87a9e72 --- /dev/null +++ b/tests/fixtures/more_sources/itch-feed.txt @@ -0,0 +1,11 @@ +Top free games for Android tagged openxr - itch.iohttps://itch.io/games/free/platform-android/tag-openxrhttps://leandrodreamer.itch.io/open-saberOpen Saber [Free] [Rhythm] [Windows] [Linux] [Android]Open Saberhttps://img.itch.zone/aW1nLzEzMzgzMzgzLmdpZg==/original/P7L1sC.gif$0.00USDhttps://leandrodreamer.itch.io/open-saber]]>Thu, 07 Sep 2023 02:11:50 GMTThu, 07 Sep 2023 02:11:50 GMTWed, 08 Jan 2025 02:24:29 GMTyeshttps://absyo.itch.io/off-nominalOff Nominal [Free] [Puzzle] [Windows] [Linux] [Android]Off Nominalhttps://img.itch.zone/aW1nLzMwMjk0MDA1LnBuZw==/315x250%23c/QSbOIy.png$0.00USDhttps://absyo.itch.io/off-nominal]]>Fri, 25 Sep 2026 19:54:48 GMTFri, 25 Sep 2026 19:54:48 GMTSun, 27 Sep 2026 23:25:20 GMTyesyesyeshttps://somar-project.itch.io/somar-projectSomar-project [Free] [Educational] [Android]Somar-projecthttps://img.itch.zone/aW1nLzIwNDM2MzM1LnBuZw==/315x250%23c/Xswj5t.png$0.00USDhttps://somar-project.itch.io/somar-project]]>Wed, 26 Mar 2025 17:17:58 GMTWed, 26 Mar 2025 17:17:58 GMTFri, 28 Mar 2025 15:52:59 GMTyeshttps://5imon.itch.io/buggenesisBug Genesis VR [Free] [Interactive Fiction] [Windows] [Android]Bug Genesis VRhttps://img.itch.zone/aW1nLzIxMzYzODczLmpwZw==/315x250%23c/LVpznb.jpg$0.00USDhttps://5imon.itch.io/buggenesis]]>Sun, 25 May 2025 20:22:49 GMTSun, 25 May 2025 20:22:49 GMTSun, 25 May 2025 20:37:39 GMTyesyeshttps://benmclean.itch.io/wolfsharpWolfSharp [Free] [Shooter] [Windows] [Linux] [Android]WolfSharphttps://img.itch.zone/aW1nLzI4NzMyNjQyLnBuZw==/315x250%23c/heg%2BmL.png$0.00USDhttps://benmclean.itch.io/wolfsharp]]>Sat, 25 Jul 2026 12:16:01 GMTSat, 25 Jul 2026 12:16:01 GMTSat, 25 Jul 2026 13:45:38 GMTyesyesyeshttps://mimekunst.itch.io/winter-solitude-vrWinter Solitude VR [Free] [Simulation] [Windows] [macOS] [Linux] [Android]Winter Solitude VRhttps://img.itch.zone/aW1nLzE0NDA4NzQxLnBuZw==/315x250%23c/EOaygC.png$0.00USDhttps://mimekunst.itch.io/winter-solitude-vr]]>Tue, 19 Dec 2023 19:19:59 GMTTue, 19 Dec 2023 19:19:59 GMTWed, 20 Dec 2023 22:49:23 GMTyesyesyesyeshttps://salmondev.itch.io/evil-miner-vrEVIL MINER VR [Free] [Other] [Windows] [Android]EVIL MINER VRhttps://img.itch.zone/aW1nLzIxNjY2NDA0LnBuZw==/315x250%23c/n4bF8N.png$0.00USDhttps://salmondev.itch.io/evil-miner-vr]]>Fri, 13 Jun 2025 16:48:01 GMTFri, 13 Jun 2025 16:48:01 GMTSun, 15 Jun 2025 15:19:53 GMTyesyeshttps://robinhuud.itch.io/winter-challenge-north-pole-defenseNorth Pole Defense VR [Free] [Action] [Android]North Pole Defense VRhttps://img.itch.zone/aW1nLzc2NzU1ODMucG5n/315x250%23c/71b4aj.png$0.00USDhttps://robinhuud.itch.io/winter-challenge-north-pole-defense]]>Thu, 16 Dec 2021 01:33:33 GMTThu, 16 Dec 2021 01:33:33 GMTThu, 16 Dec 2021 01:51:29 GMTyeshttps://envemos.itch.io/ambly-native-xrAmbly Native XR [Free] [Linux] [Android]Ambly Native XRhttps://img.itch.zone/aW1nLzI4NzEwMTc0LmpwZw==/315x250%23c/4XHeya.jpg$0.00USDhttps://envemos.itch.io/ambly-native-xr]]>Wed, 22 Jul 2026 18:38:55 GMTWed, 22 Jul 2026 18:38:55 GMTFri, 07 Aug 2026 16:04:20 GMTyesyeshttps://andyman404.itch.io/glow-up-gardenGlow Up Garden (VR) [Free] [Action] [Windows] [Android]Glow Up Garden (VR)https://img.itch.zone/aW1nLzE2NDE3NjE3LmpwZw==/315x250%23c/nHkM4g.jpg$0.00USDhttps://andyman404.itch.io/glow-up-garden]]>Mon, 03 Jun 2024 20:36:53 GMTMon, 03 Jun 2024 20:36:53 GMTTue, 04 Jun 2024 04:20:37 GMTyesyes \ No newline at end of file diff --git a/tests/fixtures/more_sources/itch-robots.txt b/tests/fixtures/more_sources/itch-robots.txt new file mode 100644 index 0000000..3b8cbfc --- /dev/null +++ b/tests/fixtures/more_sources/itch-robots.txt @@ -0,0 +1,11 @@ +User-agent: * +Disallow: /embed/ +Disallow: /embed-upload/ +Disallow: /search +Disallow: /checkout/ +Disallow: /game/download/ +Disallow: /bundle/download/ +Disallow: /register-for-purchase/ +Disallow: /email-feedback/ + +Sitemap: https://itch.io/sitemap.xml diff --git a/tests/fixtures/more_sources/khronos-releases.json b/tests/fixtures/more_sources/khronos-releases.json new file mode 100644 index 0000000..5de075d --- /dev/null +++ b/tests/fixtures/more_sources/khronos-releases.json @@ -0,0 +1,410 @@ +[ + { + "tag_name": "release-1.1.63", + "draft": false, + "prerelease": false, + "published_at": "2026-09-02T21:22:32Z", + "assets": [ + { + "id": 541779948, + "name": "apilayer_api_dump-1.1.63.aar", + "size": 5120886, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_api_dump-1.1.63.aar", + "digest": "sha256:9cab975cc8df3a99f6530f47a1fbabfa0527109a138f5a3ef5150672a658c61c" + }, + { + "id": 541779969, + "name": "apilayer_api_dump-1.1.63.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_api_dump-1.1.63.aar.asc", + "digest": "sha256:419ec65d3f526c617176f272d8a481488181ade017cb05ef42205cb2c5a86f05" + }, + { + "id": 541779983, + "name": "apilayer_api_dump-1.1.63.pom", + "size": 1462, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_api_dump-1.1.63.pom", + "digest": "sha256:9a5b3e470830ae471fe317bc63f43b33bc063458239238fe27e234232c45ff28" + }, + { + "id": 541780002, + "name": "apilayer_api_dump-1.1.63.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_api_dump-1.1.63.pom.asc", + "digest": "sha256:7cbf9f1af504ca68fc36e0985dadb74d1fbf4d2bbdcde3e00bfe9ea6168fc4a8" + }, + { + "id": 541780126, + "name": "apilayer_best_practices_validation-1.1.63.aar", + "size": 484596, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_best_practices_validation-1.1.63.aar", + "digest": "sha256:0c56cb3dc0b093b28f4e5490820d3cb3f7b201b48444134f347455d4ee99abd2" + }, + { + "id": 541780150, + "name": "apilayer_best_practices_validation-1.1.63.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_best_practices_validation-1.1.63.aar.asc", + "digest": "sha256:7eb9ab3efe939c367e4661d5a75836c1d9e0799ab627e99211253546935defa7" + }, + { + "id": 541780162, + "name": "apilayer_best_practices_validation-1.1.63.pom", + "size": 1487, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_best_practices_validation-1.1.63.pom", + "digest": "sha256:97d410936f5f051ab2a73cdac196c744ac56b2dde6279a5e46e944ed61316147" + }, + { + "id": 541780192, + "name": "apilayer_best_practices_validation-1.1.63.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_best_practices_validation-1.1.63.pom.asc", + "digest": "sha256:7f9fa0cd33627c4ecc2a4410e53dedadb5731b3cddae59a3c0d4fcd467b3472d" + }, + { + "id": 541780025, + "name": "apilayer_core_validation-1.1.63.aar", + "size": 6386964, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_core_validation-1.1.63.aar", + "digest": "sha256:9663ce94a5076b6707502cf5503bac456987ccf1f39a3f7c8f350d4521db8647" + }, + { + "id": 541780057, + "name": "apilayer_core_validation-1.1.63.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_core_validation-1.1.63.aar.asc", + "digest": "sha256:c6e75df848ca6d436fe24f680bde73a9e69972b67eef46e49aba4b77dec366e9" + }, + { + "id": 541780090, + "name": "apilayer_core_validation-1.1.63.pom", + "size": 1455, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_core_validation-1.1.63.pom", + "digest": "sha256:6aa9337f5e645baf489c05e562cd074dc696bad87db70a0cdd661dffc63b2c89" + }, + { + "id": 541780108, + "name": "apilayer_core_validation-1.1.63.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/apilayer_core_validation-1.1.63.pom.asc", + "digest": "sha256:0fe8ded9fdf0660bf28a544ae405b9b58682a8d9648dacedf4e4ceef2f827869" + }, + { + "id": 541777706, + "name": "hello_xr-OpenGLES-release-1.1.63.apk", + "size": 9557049, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/hello_xr-OpenGLES-release-1.1.63.apk", + "digest": "sha256:7c96022ac002cb0c72e3cd14c11a817767b7b5dbdf5a1d1c85d0c083b5719056" + }, + { + "id": 541777527, + "name": "hello_xr-Vulkan-release-1.1.63.apk", + "size": 9557441, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/hello_xr-Vulkan-release-1.1.63.apk", + "digest": "sha256:f24bbe8ba6f6339fca658628868ba8189cbc33390d6ac508f69d76fb67b5fa34" + }, + { + "id": 541800362, + "name": "OpenXR-SDK-Source-release-1.1.63.tar.gz", + "size": 4857593, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/OpenXR-SDK-Source-release-1.1.63.tar.gz", + "digest": "sha256:a3b97a36f11abe256a7ea1668a0a468aac9b738e94bea6b468f0ae31ad537a46" + }, + { + "id": 541800378, + "name": "OpenXR-SDK-Source-release-1.1.63.tar.gz.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/OpenXR-SDK-Source-release-1.1.63.tar.gz.asc", + "digest": "sha256:4f97028306ae219f599e9960adbf9bb072e8da2bfbedffd1f0de312516a61f87" + }, + { + "id": 541821806, + "name": "OpenXR.Loader.1.1.63.nupkg", + "size": 1916162, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/OpenXR.Loader.1.1.63.nupkg", + "digest": "sha256:4e5a50a8807ef66f25180ff224e7d8150b594aa8ee4b07590f9ade55a8e98703" + }, + { + "id": 541821827, + "name": "OpenXR.Loader.1.1.63.nupkg.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/OpenXR.Loader.1.1.63.nupkg.asc", + "digest": "sha256:9d66a6e958f7c2d5c4a0a4aef8df90a7beecab13547440c9fa2069979eab7ac7" + }, + { + "id": 541779892, + "name": "openxr_loader_for_android-1.1.63-sources.jar", + "size": 1141287, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63-sources.jar", + "digest": "sha256:6f964ad09c4afa3f42f451cada86e61503392b018660b22dd34b61dfbf71a555" + }, + { + "id": 541779920, + "name": "openxr_loader_for_android-1.1.63-sources.jar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63-sources.jar.asc", + "digest": "sha256:b98cba20f5c3b202b887307cb19196f4459f895413e55b68823a606f50d045fa" + }, + { + "id": 541779720, + "name": "openxr_loader_for_android-1.1.63.aar", + "size": 4170279, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63.aar", + "digest": "sha256:622419d2f6741c3443a3beb4779af0764318edd01830de967f24c741ebcded73" + }, + { + "id": 541779762, + "name": "openxr_loader_for_android-1.1.63.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63.aar.asc", + "digest": "sha256:3bb68b26d7def68b4fe8506bf09f302116290c2de9cf91fdfdba753978bff5ed" + }, + { + "id": 541779849, + "name": "openxr_loader_for_android-1.1.63.pom", + "size": 1598, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63.pom", + "digest": "sha256:c98f38fa8acf4cf1bd8bcb40774f9815ae6ed9da94c8a7be2ed9db7815c85404" + }, + { + "id": 541779867, + "name": "openxr_loader_for_android-1.1.63.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_for_android-1.1.63.pom.asc", + "digest": "sha256:1c2625f5b889c7ed967c8a6010294ca94bbd96091d879b2fc989c6f40f9a6af1" + }, + { + "id": 541793000, + "name": "openxr_loader_macos-1.1.63.zip", + "size": 826298, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_macos-1.1.63.zip", + "digest": "sha256:b243eebcdfa8683d17ccc8cfbfb9036be94b3f5a22b3eb73c39da0877694a3ab" + }, + { + "id": 541793027, + "name": "openxr_loader_macos-1.1.63.zip.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_macos-1.1.63.zip.asc", + "digest": "sha256:3e95172aabc4bd2537a7125060abbb8efbdd0cee19bdf1bf30cbd9736b7dcbd2" + }, + { + "id": 541784717, + "name": "openxr_loader_windows-1.1.63.zip", + "size": 31961521, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_windows-1.1.63.zip", + "digest": "sha256:01c631aeabbfe0879540f77ef833416c532a20746285b494630160c23588b771" + }, + { + "id": 541784760, + "name": "openxr_loader_windows-1.1.63.zip.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.63/openxr_loader_windows-1.1.63.zip.asc", + "digest": "sha256:e9384e2a94d82c5059d1d83c57d0da585056d95e393255048b0955b0ce69b3fc" + } + ] + }, + { + "tag_name": "release-1.1.62", + "draft": false, + "prerelease": false, + "published_at": "2026-08-01T01:32:30Z", + "assets": [ + { + "id": 500510340, + "name": "apilayer_api_dump-1.1.62.aar", + "size": 4800443, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_api_dump-1.1.62.aar", + "digest": "sha256:2a7c2d1bb14d94dfed8c1aaf170a9dda649756477fbcba4a143c76d08a50bed8" + }, + { + "id": 500510366, + "name": "apilayer_api_dump-1.1.62.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_api_dump-1.1.62.aar.asc", + "digest": "sha256:7ab53e3c1fcaabf49561737fe8ed5df9ad9a5a761615f05e1d5eed2799e85c5f" + }, + { + "id": 500510378, + "name": "apilayer_api_dump-1.1.62.pom", + "size": 1462, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_api_dump-1.1.62.pom", + "digest": "sha256:fcd4358d7582ce0787b96aacb9840afc086a28499767a6aa7491dbb682bcc921" + }, + { + "id": 500510385, + "name": "apilayer_api_dump-1.1.62.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_api_dump-1.1.62.pom.asc", + "digest": "sha256:4832ce5c8835d1880ae5adbc535b774d50f8c86f9870650a50fbf3b9993ec5fa" + }, + { + "id": 500510464, + "name": "apilayer_best_practices_validation-1.1.62.aar", + "size": 482607, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_best_practices_validation-1.1.62.aar", + "digest": "sha256:6d1a369ba367049aff23ae554b284ccc17cb3be8832869de0c1d151228a26502" + }, + { + "id": 500510477, + "name": "apilayer_best_practices_validation-1.1.62.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_best_practices_validation-1.1.62.aar.asc", + "digest": "sha256:658ecbb7f871e97ed0d659ed55b27ca6ff6cc6e1853699498ee7b1d5583d7313" + }, + { + "id": 500510480, + "name": "apilayer_best_practices_validation-1.1.62.pom", + "size": 1487, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_best_practices_validation-1.1.62.pom", + "digest": "sha256:571d7c5587075e83ca0a4d2382d87515de614ab8c34416a82a1b9b1a7eb5f9c0" + }, + { + "id": 500510489, + "name": "apilayer_best_practices_validation-1.1.62.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_best_practices_validation-1.1.62.pom.asc", + "digest": "sha256:342d0ed7080e92399dbc8648df4fe9378086718f1abc73f79f3a52de211ed36e" + }, + { + "id": 500510395, + "name": "apilayer_core_validation-1.1.62.aar", + "size": 5910024, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_core_validation-1.1.62.aar", + "digest": "sha256:5692ccd5563a0963c4d842614af11d7c280960687a221643a46266ef968c4d70" + }, + { + "id": 500510422, + "name": "apilayer_core_validation-1.1.62.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_core_validation-1.1.62.aar.asc", + "digest": "sha256:1e39ff48d4cd87231d931515968317c717a6811da84585650f0623c49329ec41" + }, + { + "id": 500510432, + "name": "apilayer_core_validation-1.1.62.pom", + "size": 1455, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_core_validation-1.1.62.pom", + "digest": "sha256:1917e5cee9b979c9032c7819c386ea62e4498c30ffbbcf0c25578ebcd0c1e441" + }, + { + "id": 500510453, + "name": "apilayer_core_validation-1.1.62.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/apilayer_core_validation-1.1.62.pom.asc", + "digest": "sha256:8aed84009daa5e388b7d179ab90837e9537b4f762a1676aab922e03dc633ed18" + }, + { + "id": 497398718, + "name": "hello_xr-OpenGLES-release-1.1.62.apk", + "size": 9548745, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/hello_xr-OpenGLES-release-1.1.62.apk", + "digest": "sha256:da5e421795b801684cab50156c572422b73cd98fc8c75aeeb968962b43a2ab46" + }, + { + "id": 497398704, + "name": "hello_xr-Vulkan-release-1.1.62.apk", + "size": 9549137, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/hello_xr-Vulkan-release-1.1.62.apk", + "digest": "sha256:a154b2353983f8c0cb1827ddf51d7e80fc105085253fe524502f35c5ddac3284" + }, + { + "id": 500514717, + "name": "OpenXR-SDK-Source-release-1.1.62.tar.gz", + "size": 4834887, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/OpenXR-SDK-Source-release-1.1.62.tar.gz", + "digest": "sha256:977073d7f4c0d1af8ab975f57e4b6ffd1c4e9209be66075812b890576e0e1e5f" + }, + { + "id": 500514735, + "name": "OpenXR-SDK-Source-release-1.1.62.tar.gz.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/OpenXR-SDK-Source-release-1.1.62.tar.gz.asc", + "digest": "sha256:3ea4d6e47da6a8b3480931636af3e85eb2e0cddaf582153ee97973a8b05a5214" + }, + { + "id": 500514501, + "name": "OpenXR.Loader.1.1.62.nupkg", + "size": 1902954, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/OpenXR.Loader.1.1.62.nupkg", + "digest": "sha256:6bb16b4dbe3c11f29605def2def5b7d1177784c0403dc9a24bc2e13d7eb82604" + }, + { + "id": 500514512, + "name": "OpenXR.Loader.1.1.62.nupkg.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/OpenXR.Loader.1.1.62.nupkg.asc", + "digest": "sha256:386dfd33e9c2db9c9c37d881b77eec9b74d181fda394b47c473b2bce37fd7005" + }, + { + "id": 500510319, + "name": "openxr_loader_for_android-1.1.62-sources.jar", + "size": 1110593, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62-sources.jar", + "digest": "sha256:b83318394b30bb129b069dd854de2b1e21df4376dda1a7fa342aeaff17a71d3d" + }, + { + "id": 500510327, + "name": "openxr_loader_for_android-1.1.62-sources.jar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62-sources.jar.asc", + "digest": "sha256:838b5f5a23329eb7f0e13afde7a7d6ae73b439c7cbf6d3ec0af3e65efd7d5bd6" + }, + { + "id": 500510263, + "name": "openxr_loader_for_android-1.1.62.aar", + "size": 4158815, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62.aar", + "digest": "sha256:c03c689fed9a48f9394af953660982c998b00f6d2d2d8d150bc3f890c75a7465" + }, + { + "id": 500510280, + "name": "openxr_loader_for_android-1.1.62.aar.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62.aar.asc", + "digest": "sha256:883ccab775776c65ee35bf3120553f6a3f0f47de2dd661e074469e98d3caea46" + }, + { + "id": 500510292, + "name": "openxr_loader_for_android-1.1.62.pom", + "size": 1598, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62.pom", + "digest": "sha256:9b1047158a416984fd6d60c472da09bb324aec74709f83a1516435917fa05064" + }, + { + "id": 500510305, + "name": "openxr_loader_for_android-1.1.62.pom.asc", + "size": 215, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_for_android-1.1.62.pom.asc", + "digest": "sha256:9dd7d3f79ad76a48f709f55d8c205892ae30f70b10a44c4afbd51f50603c4478" + }, + { + "id": 500514048, + "name": "openxr_loader_macos-1.1.62.zip", + "size": 817438, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_macos-1.1.62.zip", + "digest": "sha256:400bf9ab932d04cf8a315fe8e63d5cd9824015b64ad2e5d72b2ffc086c54313c" + }, + { + "id": 500514061, + "name": "openxr_loader_macos-1.1.62.zip.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_macos-1.1.62.zip.asc", + "digest": "sha256:034a3575bbee545926dc59558aa5d62ea9fc2de9e23c426ac640cbb68bd8db88" + }, + { + "id": 500513308, + "name": "openxr_loader_windows-1.1.62.zip", + "size": 30975472, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_windows-1.1.62.zip", + "digest": "sha256:800ec772e2f9448a26ab9f579f4914d984346dd9d0d7c007841abe21d2c8ff2f" + }, + { + "id": 500513351, + "name": "openxr_loader_windows-1.1.62.zip.asc", + "size": 870, + "browser_download_url": "https://github.com/KhronosGroup/OpenXR-SDK-Source/releases/download/release-1.1.62/openxr_loader_windows-1.1.62.zip.asc", + "digest": "sha256:8117563bfc5092895e17112366cb954ca78f84964e5c09526dfd69656c1713fd" + } + ] + } +] diff --git a/tests/fixtures/more_sources/topic.json b/tests/fixtures/more_sources/topic.json new file mode 100644 index 0000000..129a59a --- /dev/null +++ b/tests/fixtures/more_sources/topic.json @@ -0,0 +1,19 @@ +{ + "items": [ + { + "full_name": "LWJGL/lwjgl3", + "name": "lwjgl3", + "description": "LWJGL is a Java library that enables cross-platform access to popular native APIs useful in the development of graphics (OpenGL, Vulkan, bgfx), audio (OpenAL, Opus), parallel computing (OpenCL, CUDA) and XR (OpenVR, LibOVR, OpenXR) applications." + }, + { + "full_name": "sahibzada-allahyar/YC-Killer", + "name": "YC-Killer", + "description": "A library of enterprise-grade AI agents designed to democratize artificial intelligence and provide free, open-source alternatives to overvalued Y Combinator startups." + }, + { + "full_name": "bjornbytes/lovr", + "name": "lovr", + "description": "Lua Virtual Reality Framework" + } + ] +} diff --git a/tests/fixtures/more_sources/tux-releases.json b/tests/fixtures/more_sources/tux-releases.json new file mode 100644 index 0000000..2d482fa --- /dev/null +++ b/tests/fixtures/more_sources/tux-releases.json @@ -0,0 +1,17 @@ +[ + { + "tag_name": "Beta0.2", + "draft": false, + "prerelease": true, + "published_at": "2026-09-23T14:51:47Z", + "assets": [ + { + "id": 583977968, + "name": "SuperTux-Beta0.2-quest-pico-arm64-v8a.apk", + "size": 294152462, + "browser_download_url": "https://github.com/SgtBilko76/SuperTux-3D/releases/download/Beta0.2/SuperTux-Beta0.2-quest-pico-arm64-v8a.apk", + "digest": "sha256:63287e5d6f1866193e0d4730bf4a2ba87fd2fbdbe6317bd6bd24b96a8ddc9963" + } + ] + } +] diff --git a/tests/test_apk_more_sources.py b/tests/test_apk_more_sources.py new file mode 100644 index 0000000..a21dd90 --- /dev/null +++ b/tests/test_apk_more_sources.py @@ -0,0 +1,108 @@ +import hashlib, io, json, os, sys, tempfile, unittest, urllib.error, urllib.request, zipfile +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) +from apk_sources import SourceError, github, itch, _web + +FIX = Path(__file__).parent / 'fixtures' / 'more_sources' + + +class PublisherSources(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.cache = patch.object(_web, 'cache', return_value=self.tmp.name) + self.cache.start() + self.addCleanup(self.cache.stop) + self.network = patch('urllib.request.OpenerDirector.open', side_effect=AssertionError('network in test')) + self.network.start() + self.addCleanup(self.network.stop) + + def test_curated_search_is_offline(self): + self.assertEqual(github.search(github.sources()[0], 'hello')[0]['id'], 'KhronosGroup/OpenXR-SDK-Source') + self.assertEqual(github.search(github.sources()[0], '', 0), []) + + def test_real_releases(self): + for key, repo in [('khronos', 'KhronosGroup/OpenXR-SDK-Source'), + ('brush', 'icosa-foundation/open-brush'), ('tux', 'SgtBilko76/SuperTux-3D')]: + with patch.object(github, '_api', return_value=json.loads((FIX / (key + '-releases.json')).read_text())): + e = github.details(github.sources()[0], repo) + self.assertTrue(e['downloadable']) + self.assertTrue(e['versions'][0]['name'].endswith('.apk')) + self.assertIsNone(e['version_code']) + with self.assertRaises(SourceError): + github.download(github.sources()[0], repo, 123) + + def test_topic_results_need_approval(self): + data = json.loads((FIX / 'topic.json').read_text()) + with patch.object(github, '_api', return_value=data): + entries = github.search(github.sources()[0], 'topic:openxr') + self.assertTrue(entries) + self.assertTrue(any(not e['downloadable'] for e in entries)) + self.assertFalse(github.details(github.sources()[0], 'unknown/project')['downloadable']) + with self.assertRaises(SourceError): + github.search(github.sources()[0], 'topic:piracy') + + def test_feed_free_android_only_and_deduplicated(self): + with patch.object(_web, 'read', return_value=(FIX / 'itch-feed.txt').read_bytes()): + entries = itch.search(itch.sources()[0], '') + self.assertEqual(len(entries), 9) + e = itch.details(itch.sources()[0], entries[0]['id']) + self.assertTrue(e['vr']) + self.assertTrue(e['images']['banner']) + self.assertFalse(e['downloadable']) + self.assertEqual(itch.search(itch.sources()[0], 'off nominal')[0]['name'], 'Off Nominal') + with self.assertRaises(SourceError): + itch.download(itch.sources()[0], entries[0]['id']) + with self.assertRaises(SourceError): + itch._parse(itch.sources()[0], b'not xml') + + def test_paid_and_unsafe_feed(self): + raw = (FIX / 'itch-feed.txt').read_bytes().replace(b'$0.00', b'$1.00') + self.assertEqual(itch._parse(itch.sources()[0], raw), []) + raw = (FIX / 'itch-feed.txt').read_bytes().replace(b'https://absyo.itch.io', b'http://localhost') + self.assertFalse(any(e['name'] == 'Off Nominal' for e in itch._parse(itch.sources()[0], raw))) + + def test_download_hash_and_cleanup(self): + b = io.BytesIO() + with zipfile.ZipFile(b, 'w') as z: + z.writestr('AndroidManifest.xml', b'fixture') + raw = b.getvalue() + digest = hashlib.sha256(raw).hexdigest() + with patch.object(_web, 'open_url', return_value=io.BytesIO(raw)): + result = _web.apk('https://github.com/owner/repo/file.apk', github.HOSTS, digest) + self.assertTrue(result['verified']) + self.assertEqual(Path(result['apk']).read_bytes(), raw) + with patch.object(_web, 'open_url', return_value=io.BytesIO(raw)): + self.assertFalse(_web.apk('https://github.com/file.apk', github.HOSTS)['verified']) + for content, expected in [(raw, '0' * 64), (b'html challenge', None)]: + with patch.object(_web, 'open_url', return_value=io.BytesIO(content)), self.assertRaises(SourceError): + _web.apk('https://github.com/file.apk', github.HOSTS, expected) + self.assertFalse(list(Path(self.tmp.name).glob('*.part'))) + + def test_origin_and_redirect(self): + for url in ['http://github.com/x', 'https://evil.test/x', 'https://user@github.com/x']: + with self.assertRaises(SourceError): + _web.checked_url(url, github.HOSTS) + req = urllib.request.Request('https://api.github.com/x', headers={'Authorization': 'Bearer secret'}) + handler = _web.Redirect(('api.github.com', 'github.com')) + redirected = handler.redirect_request(req, None, 302, '', {}, 'https://github.com/x') + self.assertFalse(redirected.has_header('Authorization')) + with self.assertRaises(SourceError): + handler.redirect_request(req, None, 302, '', {}, 'https://evil.test/x') + + def test_cache_rate_limit_and_invalid_json(self): + with patch.object(_web, 'open_url', return_value=io.BytesIO(b'index')) as op: + self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') + self.assertEqual(_web.read('https://itch.io/test', ('itch.io',)), b'index') + self.assertEqual(op.call_count, 1) + error = urllib.error.HTTPError('https://api.github.com/x', 403, 'limited', {}, None) + with patch.object(_web, 'open_url', side_effect=error), self.assertRaisesRegex(SourceError, 'rate limit'): + _web.read('https://api.github.com/x', ('api.github.com',)) + with patch.object(_web, 'read', return_value=b''), self.assertRaises(SourceError): + github._api('/x') + + +if __name__ == '__main__': + unittest.main() diff --git a/ui/apk_sources/_web.py b/ui/apk_sources/_web.py new file mode 100644 index 0000000..1ed8d4a --- /dev/null +++ b/ui/apk_sources/_web.py @@ -0,0 +1,101 @@ +"""Small HTTPS cache and APK downloader for public publisher sources.""" +import hashlib, os, tempfile, time, urllib.error, urllib.parse, urllib.request, zipfile + +import frame_host +from . import SourceError + +UA = 'FrameControl/0.1' + + +def cache(): + path = frame_host.cache_dir('apk-sources', 'publisher') + os.makedirs(path, exist_ok=True) + return str(path) + + +def checked_url(url, hosts): + try: + p = urllib.parse.urlsplit(url) + port = p.port + except (ValueError, TypeError) as e: + raise SourceError('Source returned an invalid URL') from e + if p.scheme != 'https' or p.username or p.password or port not in (None, 443) or p.hostname not in hosts: + raise SourceError('Source returned an unexpected download URL') + return url + + +class Redirect(urllib.request.HTTPRedirectHandler): + def __init__(self, hosts): + self.hosts = hosts + + def redirect_request(self, req, fp, code, msg, headers, newurl): + checked_url(newurl, self.hosts) + result = super().redirect_request(req, fp, code, msg, headers, newurl) + if result: + result.remove_header('Authorization') + return result + + +def open_url(url, hosts, headers=None): + checked_url(url, hosts) + return urllib.request.build_opener(Redirect(hosts)).open( + urllib.request.Request(url, headers={'User-Agent': UA, **(headers or {})}), timeout=60) + + +def read(url, hosts, headers=None, ttl=3600): + path = os.path.join(cache(), hashlib.sha256(url.encode()).hexdigest() + '.data') + try: + if os.path.isfile(path) and time.time() - os.path.getmtime(path) < ttl: + with open(path, 'rb') as f: + return f.read() + with open_url(url, hosts, headers) as r: + data = r.read(8 * 1024 * 1024 + 1) + if len(data) > 8 * 1024 * 1024: + raise SourceError('Source index is too large') + fd, tmp = tempfile.mkstemp(dir=cache(), suffix='.part') + try: + with os.fdopen(fd, 'wb') as f: + f.write(data) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.remove(tmp) + return data + except urllib.error.HTTPError as e: + if e.code in (403, 429): + raise SourceError('Source refused access or reached its rate limit; try later (GitHub accepts FRAME_GITHUB_TOKEN)') from e + raise SourceError('Source HTTP error: ' + str(e.code)) from e + except (OSError, ValueError) as e: + raise SourceError('Could not read source: ' + str(e)) from e + + +def apk(url, hosts, digest=None): + tmp = None + try: + fd, tmp = tempfile.mkstemp(dir=cache(), suffix='.part') + h = hashlib.sha256() + with os.fdopen(fd, 'wb') as f, open_url(url, hosts) as r: + size = 0 + while True: + chunk = r.read(1 << 20) + if not chunk: + break + size += len(chunk) + if size > 2 * 1024 ** 3: + raise SourceError('APK exceeds the 2 GiB download limit') + h.update(chunk) + f.write(chunk) + actual = h.hexdigest() + if digest and actual != digest: + raise SourceError('SHA-256 mismatch; download discarded') + with zipfile.ZipFile(tmp) as z: + if 'AndroidManifest.xml' not in z.namelist(): + raise SourceError('Download is not an APK') + path = os.path.join(cache(), actual + '.apk') + os.replace(tmp, path) + return {'apk': path, 'obb': [], 'sha256': actual, 'verified': bool(digest)} + except (OSError, ValueError, zipfile.BadZipFile) as e: + raise SourceError('Could not download APK: ' + str(e)) from e + finally: + if tmp and os.path.exists(tmp): + os.remove(tmp) diff --git a/ui/apk_sources/github.py b/ui/apk_sources/github.py new file mode 100644 index 0000000..6c7516a --- /dev/null +++ b/ui/apk_sources/github.py @@ -0,0 +1,110 @@ +"""Curated publisher releases; optional topic discovery is page-only.""" +import fnmatch, json, os, re, urllib.parse + +from . import SourceError, _web + +KIND = 'github' +API = 'https://api.github.com' +TOPICS = ('oculus-quest', 'openxr', 'quest') +HOSTS = ('github.com', 'release-assets.githubusercontent.com', 'objects.githubusercontent.com') + + +def sources(): + return [{'id': KIND, 'kind': KIND, 'name': 'GitHub releases', 'url': 'https://github.com', + 'builtin': True, 'enabled': True, 'trust': 'community'}] + + +def _curated(): + with open(os.path.join(os.path.dirname(__file__), 'github_curated.json')) as f: + return json.load(f) + + +def _api(path): + headers = {'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'} + token = os.environ.get('FRAME_GITHUB_TOKEN') + if token: + headers['Authorization'] = 'Bearer ' + token + try: + return json.loads(_web.read(API + path, ('api.github.com',), headers)) + except (ValueError, TypeError) as e: + raise SourceError('Invalid GitHub response') from e + + +def _entry(source, c, approved=True): + return {'source': source['id'], 'id': c['repo'], 'package': None, + 'name': c['name'], 'summary': c.get('summary') or '', 'icon': None, + 'images': {'icon': None, 'banner': None, 'screenshots': []}, + 'page': 'https://github.com/' + c['repo'], 'version': None, 'version_code': None, + 'min_sdk': None, 'abis': None, 'vr': c.get('vr'), 'size': None, + 'free': True if approved else None, 'license': c.get('license'), 'updated': None, + 'downloadable': approved} + + +def search(source, query, limit=50): + limit = max(0, min(int(limit), 100)) + if not limit: + return [] + if query.startswith('topic:'): + topic = query[6:].strip() + if topic not in TOPICS: + raise SourceError('Choose topic:oculus-quest, topic:openxr or topic:quest') + data = _api('/search/repositories?' + urllib.parse.urlencode( + {'q': 'topic:' + topic + ' archived:false', 'sort': 'stars', 'per_page': limit})) + curated = {c['repo'].lower(): c for c in _curated()} + out = [] + for repo in data.get('items', []): + c = curated.get(repo['full_name'].lower()) + entry = _entry(source, c or {'repo': repo['full_name'], 'name': repo['name'], + 'summary': repo.get('description'), 'vr': True}, bool(c)) + # Repository owners' avatars are not app artwork. + out.append(entry) + return out + words = query.lower().split() + return [_entry(source, c) for c in _curated() + if all(w in (c['name'] + ' ' + c['repo'] + ' ' + c['summary']).lower() + for w in words)][:limit] + + +def details(source, entry_id): + c = next((c for c in _curated() if c['repo'].lower() == entry_id.lower()), None) + if not c: + if not re.fullmatch(r'[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+', entry_id): + raise SourceError('Invalid GitHub repository') + entry = _entry(source, {'repo': entry_id, 'name': entry_id}, False) + return {**entry, 'versions': []} + entry = _entry(source, c) + releases = _api('/repos/' + c['repo'] + '/releases?per_page=10') + versions = [] + for release in releases: + if release.get('draft') or (release.get('prerelease') and not c.get('allow_prerelease')): + continue + assets = [a for a in release.get('assets', []) if + fnmatch.fnmatch(a['name'].lower(), c['asset_pattern'].lower())] + for asset in assets: + digest = asset.get('digest') or '' + versions.append({'version': release['tag_name'], 'version_code': None, + 'asset_id': asset['id'], 'name': asset['name'], 'min_sdk': None, + 'prerelease': bool(release.get('prerelease')), + 'size': asset.get('size'), 'updated': release.get('published_at'), + 'url': asset['browser_download_url'], + 'sha256': digest[7:] if re.fullmatch(r'sha256:[0-9a-f]{64}', digest) else None}) + entry['versions'] = versions + entry['downloadable'] = bool(versions) + if versions: + entry.update({k: versions[0][k] for k in ('version', 'size', 'updated')}) + return entry + + +def download(source, entry_id, version_code=None): + entry = details(source, entry_id) + versions = entry['versions'] + if not versions: + raise SourceError('No approved APK release; open the publisher page') + # GitHub asset IDs and tags are not Android version codes. + if version_code is not None: + raise SourceError('GitHub does not publish Android version codes; download the latest release') + v = versions[0] + expected = 'https://github.com/' + entry['id'] + '/releases/download/' + if not v['url'].startswith(expected): + raise SourceError('APK URL does not belong to the curated publisher') + return _web.apk(v['url'], HOSTS, v['sha256']) diff --git a/ui/apk_sources/github_curated.json b/ui/apk_sources/github_curated.json new file mode 100644 index 0000000..9f7fc14 --- /dev/null +++ b/ui/apk_sources/github_curated.json @@ -0,0 +1,29 @@ +[ + { + "repo": "KhronosGroup/OpenXR-SDK-Source", + "name": "hello_xr", + "summary": "Khronos OpenXR sample (Vulkan and OpenGL ES)", + "license": "Apache-2.0", + "vr": true, + "asset_pattern": "hello_xr-Vulkan-*.apk", + "allow_prerelease": false + }, + { + "repo": "icosa-foundation/open-brush", + "name": "Open Brush", + "summary": "Open source spatial painting", + "license": "Apache-2.0", + "vr": true, + "asset_pattern": "*Quest*.apk", + "allow_prerelease": true + }, + { + "repo": "SgtBilko76/SuperTux-3D", + "name": "SuperTux 3D", + "summary": "OpenXR stereoscopic platform game for Quest and PICO", + "license": "GPL-3.0", + "vr": true, + "asset_pattern": "*Quest*.apk", + "allow_prerelease": true + } +] diff --git a/ui/apk_sources/itch.py b/ui/apk_sources/itch.py new file mode 100644 index 0000000..3ef904c --- /dev/null +++ b/ui/apk_sources/itch.py @@ -0,0 +1,72 @@ +"""Free Android VR RSS listings. Download pages stay in the publisher's UI.""" +import html, re, urllib.parse, xml.etree.ElementTree as ET +from email.utils import parsedate_to_datetime + +from . import SourceError, _web + +KIND = 'itch' +FEEDS = ('openxr', 'oculus-quest') + + +def sources(): + return [{'id': KIND if tag == 'openxr' else 'itch-quest', 'kind': KIND, + 'name': 'itch.io (' + tag + ')', 'url': 'https://itch.io', 'tag': tag, + 'builtin': True, 'enabled': True, 'trust': 'community'} for tag in FEEDS] + + +def _parse(source, data): + try: + root = ET.fromstring(data) + except ET.ParseError as e: + raise SourceError('Invalid itch.io RSS feed') from e + entries = [] + for item in root.findall('./channel/item'): + page = item.findtext('link') or '' + p = urllib.parse.urlsplit(page) + if p.scheme != 'https' or not (p.hostname or '').endswith('.itch.io') or p.username or ':' in p.netloc: + continue + if item.findtext('price') != '$0.00' or item.findtext('platforms/android') != 'yes': + continue + cover = item.findtext('imageurl') or None + if cover and not cover.startswith('https://img.itch.zone/'): + cover = None + updated = None + try: + updated = parsedate_to_datetime(item.findtext('updateDate')).date().isoformat() + except (ValueError, TypeError, AttributeError): + pass + entries.append({'source': source['id'], 'id': page, 'package': None, + 'name': item.findtext('plainTitle') or item.findtext('title') or page, + 'summary': html.unescape(re.sub('<[^>]*>', '', item.findtext('description') or '')).strip(), + 'icon': cover, 'images': {'icon': cover, 'banner': cover, 'screenshots': []}, + 'page': page, 'version': None, 'version_code': None, 'min_sdk': None, + 'abis': None, 'vr': True, 'size': None, 'free': True, 'license': None, + 'updated': updated, 'downloadable': False}) + return entries + + +def search(source, query, limit=50): + limit = max(0, min(int(limit), 100)) + if not limit: + return [] + entries = {} + tag = source.get('tag', 'openxr') + if tag not in FEEDS: + raise SourceError('Unsupported itch.io feed') + url = 'https://itch.io/games/free/platform-android/tag-' + tag + '.xml' + for entry in _parse(source, _web.read(url, ('itch.io',))): + entries.setdefault(entry['id'], entry) + words = query.lower().split() + return [e for e in entries.values() if all(w in (e['name'] + ' ' + e['summary']).lower() + for w in words)][:limit] + + +def details(source, entry_id): + entry = next((e for e in search(source, '', 100) if e['id'] == entry_id), None) + if not entry: + raise SourceError('Game is not in the current free Android VR feed; open its publisher page') + return {**entry, 'versions': []} + + +def download(source, entry_id, version_code=None): + raise SourceError('Open the itch.io publisher page to download; automated download pages are disallowed by robots rules') From f1b12a6eb624d9bb2c058e2130b05e374ae32846 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:04:48 +1000 Subject: [PATCH 002/122] Add unified APK source search and source management Co-Authored-By: GPT-6 Astra --- .claude/NOTES-apk-search.md | 32 ++++ tests/fixtures/apk-search/entries.json | 7 + tests/search_preview.py | 27 +++ tests/test_apk_search.py | 195 +++++++++++++++++++++ ui/apk_sources/_demo.py | 34 ++++ ui/apk_sources/search.py | 226 +++++++++++++++++++++++++ ui/index.html | 96 ++++++++++- ui/server.py | 66 +++++++- 8 files changed, 676 insertions(+), 7 deletions(-) create mode 100644 .claude/NOTES-apk-search.md create mode 100644 tests/fixtures/apk-search/entries.json create mode 100644 tests/search_preview.py create mode 100644 tests/test_apk_search.py create mode 100644 ui/apk_sources/_demo.py create mode 100644 ui/apk_sources/search.py diff --git a/.claude/NOTES-apk-search.md b/.claude/NOTES-apk-search.md new file mode 100644 index 0000000..ec4b0dd --- /dev/null +++ b/.claude/NOTES-apk-search.md @@ -0,0 +1,32 @@ +# APK search + +- Scope: dynamic source aggregator, additive server APIs, Android search view; no device calls. +- Source metadata stays unknown when missing. Unknown-package names only group with other unknown packages (avoids ambiguous package attribution). +- Queries use daemon workers with at most one in-flight request per source; deadlines don't wait for stuck workers. +- Independent review delegated to parent per task brief; no workers launched. + +## Implementation + +- `ui/apk_sources/search.py`: discovers KIND modules, parallel daemon queries (12 s deadline), bounded per-source workers, grouped offers, fit/rank/filter logic, persisted source enablement, repo adapter and install adapter. +- `ui/server.py`: GET `/api/sources`, GET `/api/search?q=&vr=true|false&installable=true|false&source=`, POST `/api/sources/install` and POST `/api/sources` with actions `add`, `remove`, `enable`. +- `ui/index.html`: unified Find apps in Android tab, source/VR/Flat/Installable chips, offer picker, metadata, jobs, Sources panel. Old catalogue DOM remains hidden for existing report/icon code; only the unified search is visible. +- `_demo.py` is discovered only with `FRAME_APK_SEARCH_DEMO=1`; it cannot download. `tests/search_preview.py` serves the real UI/search endpoints and rejects all device endpoints and writes. + +## Evidence + +- `python3 -m unittest discover -s tests`: 179 tests passed on Python 3.9.6, exit 0. Log `/tmp/apk-search-suite.log`. +- `node --check /tmp/apk-search-evidence/ui.js`: exit 0; script extracted from index.html. +- `git diff --check`: exit 0. +- Python 3.9 grammar parse: search.py, _demo.py, server.py, test_apk_search.py and search_preview.py passed; actual suite interpreter also Python 3.9.6. +- Started `FRAME_APK_SEARCH_DEMO=1 python3 tests/search_preview.py` locally; used T3 preview at http://127.0.0.1:8795/#android (1280x800). +- Browser assertions: two grouped apps, source picker updates install choice, VR excludes flat, Flat excludes VR, source filter keeps one offer, search query narrows results, pending install disables its button after re-render. Passed. +- Inspected screenshots: `/tmp/apk-search-evidence/search.png`, `/tmp/apk-search-evidence/sources.png`. Other panels show intentional device-access errors in the preview. +- Tests cover dynamic module discovery, grouping (including ambiguous names), ranking, fit, timeout/failure isolation, persistent enablement, endpoint validation, repo callbacks, background install with mocked frame_android.install, conditional artwork, and OBB support/refusal. + +## Parent integration / unverified + +- No source modules from sibling branches are present yet. Real network listings/downloads, signatures, merged repo persistence and physical installation were not exercised. No SSH/device installation performed. +- Assumes future `frame_android.install_obb(package, paths)`; reconcile with vr-library worker's actual signature. Artwork is passed only when install explicitly declares that parameter. OBB-required apps fail before APK installation when helper is absent. +- `app/package.json` currently copies ui with only `*.py` / `*.html`; parent must include `apk_sources/**/*.py` in packaged resources when integrating source workers. Kept package config outside this worker's scope. +- No cross-provider reviewer launched: task explicitly forbids delegation and assigns integration/review to the parent. +- Source metadata unknowns do not imply compatibility or verification. Unknown-package entries only group with other unknown-package entries with the same normalized name. diff --git a/tests/fixtures/apk-search/entries.json b/tests/fixtures/apk-search/entries.json new file mode 100644 index 0000000..1b1f236 --- /dev/null +++ b/tests/fixtures/apk-search/entries.json @@ -0,0 +1,7 @@ +[ + {"source":"one","id":"brush","package":"org.brush","name":"Open Brush","free":true,"downloadable":true,"verified":true,"version":"1","version_code":1,"min_sdk":29,"abis":["arm64-v8a"],"vr":true,"updated":"2025-01-01"}, + {"source":"two","id":"brush2","package":"org.brush","name":"Open Brush","free":true,"downloadable":true,"verified":false,"version":"2","version_code":2,"min_sdk":29,"abis":["arm64-v8a"],"vr":true,"updated":"2026-01-01"}, + {"source":"one","id":"other","package":"org.other","name":"Open Brush","free":true,"downloadable":true,"min_sdk":31,"abis":["arm64-v8a"],"vr":true}, + {"source":"one","id":"unknown","package":null,"name":"Pocket Radio!","free":true,"downloadable":false,"vr":false}, + {"source":"two","id":"unknown2","package":null,"name":"pocket radio","free":true,"downloadable":false,"vr":false} +] diff --git a/tests/search_preview.py b/tests/search_preview.py new file mode 100644 index 0000000..3e0bf5a --- /dev/null +++ b/tests/search_preview.py @@ -0,0 +1,27 @@ +"""Local demo server for UI checks; rejects every device endpoint. + +FRAME_APK_SEARCH_DEMO=1 python3 tests/search_preview.py +""" +import os +from pathlib import Path +import sys +from urllib.parse import urlparse +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) +import server + + +class Preview(server.Handler): + def do_GET(self): + if urlparse(self.path).path not in ('/', '/index.html', '/api/search', '/api/sources', '/api/job', '/api/host'): + self.send_json({'error': 'Device access disabled in search preview'}, 503) + return + super().do_GET() + + def do_POST(self): + self.send_json({'error': 'Writes disabled in search preview'}, 403) + + +if __name__ == '__main__': + if os.environ.get('FRAME_APK_SEARCH_DEMO') != '1': + sys.exit('Set FRAME_APK_SEARCH_DEMO=1') + server.ThreadingHTTPServer(('127.0.0.1', 8795), Preview).serve_forever() diff --git a/tests/test_apk_search.py b/tests/test_apk_search.py new file mode 100644 index 0000000..9775e07 --- /dev/null +++ b/tests/test_apk_search.py @@ -0,0 +1,195 @@ +import http.client +import json +from pathlib import Path +import sys +import tempfile +import threading +import time +import types +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) +from apk_sources import search, SourceError +import server + +ENTRIES = json.loads((Path(__file__).parent / 'fixtures/apk-search/entries.json').read_text()) + + +def fake(source_id='one', fn=None): + return types.SimpleNamespace(KIND=source_id, sources=lambda: [dict(id=source_id, name=source_id, + enabled=True, trust='official', builtin=True)], + search=fn or (lambda s, q, limit=50: [e for e in ENTRIES if e['source'] == source_id]), + details=lambda s, i: ENTRIES[0], + download=Mock(return_value={'apk': '/fake.apk', 'obb': []})) + + +class SettingsTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + p = patch.object(search, 'settings_path', return_value=Path(self.tmp.name) / 'enabled.json') + p.start() + self.addCleanup(p.stop) + search._running.clear() + search._status.clear() + + +class SearchTests(SettingsTest): + def test_group_does_not_merge_distinct_or_unknown_packages(self): + result = search.group(ENTRIES) + self.assertEqual(len(result), 3) + self.assertEqual(sorted(len(a['offers']) for a in result), [1, 2, 2]) + same_name = dict(ENTRIES[0], package=None) + self.assertEqual(len(search.group(ENTRIES[:1] + [same_name])), 2) + + def test_rank_exact_and_installable_and_verified(self): + result = search.group(ENTRIES, 'Open Brush') + self.assertEqual(result[0]['package'], 'org.brush') + self.assertEqual(result[0]['offers'][0]['source'], 'one') + self.assertEqual(result[1]['package'], 'org.other') + self.assertEqual(len(search.group(ENTRIES, vr=False)), 1) + self.assertEqual(len(search.group(ENTRIES, installable=True)), 1) + + def test_fit_unknown_and_native_free_and_vr_hints(self): + self.assertIsNone(search.fit({})['installable']) + self.assertTrue(search.fit({'min_sdk': 23, 'abis': []})['installable']) + self.assertFalse(search.fit({'min_sdk': 23, 'abis': ['x86']})['installable']) + self.assertIn('Legacy VrApi', search.fit({'engine': 'VrApi'})['reasons'][0]) + + def test_timeout_and_failure_leave_other_results(self): + release = threading.Event() + calls = [] + def slow(s, q, limit=50): + calls.append(q) + release.wait(2) + return [] + mods = [fake(), fake('slow', slow), fake('broken', Mock(side_effect=SourceError('offline')))] + try: + with patch.object(search, 'modules', return_value=(mods, [])): + started = time.monotonic() + result = search.search(timeout=.03) + self.assertLess(time.monotonic() - started, .3) + self.assertTrue(result['apps']) + self.assertEqual([s['status'] for s in result['sources']], ['ok', 'timed out', 'error']) + search.search('other', timeout=.03) + self.assertEqual(calls, ['']) + finally: + release.set() + + def test_disable_persists_and_prevents_queries_and_installs(self): + mod = fake() + with patch.object(search, 'modules', return_value=([mod], [])): + search.set_enabled('one', False) + self.assertFalse(search.sources()[0]['enabled']) + self.assertEqual(search.search()['apps'], []) + with self.assertRaisesRegex(SourceError, 'disabled'): + search.install('one', 'brush') + + def test_install_passes_metadata_artwork_and_obb(self): + mod = fake() + mod.download.return_value.update(obb=['main.obb'], artwork={'hero': '/hero.png'}, icon_png=b'png') + def install(apk, name=None, icon_png=None, source=None, artwork=None): + self.assertEqual((apk, name, icon_png, source, artwork), + ('/fake.apk', 'Open Brush', b'png', 'one', {'hero': '/hero.png'})) + return {'package': 'org.brush'} + with patch.object(search, 'modules', return_value=([mod], [])), \ + patch.object(server.frame_android, 'install_obb', create=True) as obb: + # An actual function exposes the future signature for inspection. + with patch.object(server.frame_android, 'install', install): + search.install('one', 'brush', 1) + obb.assert_called_once_with('org.brush', ['main.obb']) + mod.download.assert_called_once_with(mod.sources()[0] | {'status': 'not searched'}, 'brush', version_code=1) + + def test_discovery_and_demo_are_opt_in(self): + module = fake() + with patch.object(search.pkgutil, 'iter_modules', return_value=[types.SimpleNamespace(name='example')]), \ + patch.object(search.importlib, 'import_module', return_value=module), \ + patch.dict(search.os.environ, {'FRAME_APK_SEARCH_DEMO': '0'}): + self.assertEqual(search.modules(), ([module], [])) + with patch.object(search.pkgutil, 'iter_modules', return_value=[]), \ + patch.dict(search.os.environ, {'FRAME_APK_SEARCH_DEMO': '0'}): + self.assertEqual(search.modules(), ([], [])) + + def test_newest_compatible_then_official_offer(self): + first = dict(ENTRIES[0], verified=False, trust='community') + newer = dict(first, source='new', version_code=2, updated='2026-01-01') + official = dict(newer, source='official', trust='official') + incompatible = dict(newer, source='blocked', verified=True, min_sdk=40) + offers = search.group([first, incompatible, newer, official])[0]['offers'] + self.assertEqual([e['source'] for e in offers], ['official', 'new', 'one', 'blocked']) + + def test_missing_obb_support_stops_before_install(self): + mod = fake() + mod.download.return_value['obb'] = ['main.obb'] + with patch.object(search, 'modules', return_value=([mod], [])), \ + patch.object(server.frame_android, 'install') as install, \ + patch.dict(server.frame_android.__dict__): + server.frame_android.__dict__.pop('install_obb', None) + with self.assertRaisesRegex(SourceError, 'OBB'): + search.install('one', 'brush') + install.assert_not_called() + + def test_listing_cannot_download(self): + mod = fake() + mod.details = lambda s, i: dict(ENTRIES[0], downloadable=False) + with patch.object(search, 'modules', return_value=([mod], [])): + with self.assertRaisesRegex(SourceError, 'developer page'): + search.install('one', 'brush') + mod.download.assert_not_called() + + +class EndpointTests(SettingsTest): + def setUp(self): + super().setUp() + self.mod = fake() + p = patch.object(search, 'modules', return_value=([self.mod], [])) + p.start() + self.addCleanup(p.stop) + self.httpd = server.ThreadingHTTPServer(('127.0.0.1', 0), server.Handler) + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + self.addCleanup(self.httpd.server_close) + self.addCleanup(self.httpd.shutdown) + + def request(self, method, path, body=None): + c = http.client.HTTPConnection('127.0.0.1', self.httpd.server_port) + c.request(method, path, json.dumps(body) if body is not None else None, + {'X-Frame-UI': '1', 'Content-Type': 'application/json'}) + r = c.getresponse() + result = r.status, json.loads(r.read()) + c.close() + return result + + def test_http_search_and_validation(self): + self.assertEqual(self.request('GET', '/api/sources')[1]['sources'][0]['id'], 'one') + self.assertTrue(self.request('GET', '/api/search?q=Brush&vr=true')[1]['apps']) + self.assertEqual(self.request('GET', '/api/search?vr=invalid')[0], 400) + self.assertEqual(self.request('GET', '/api/search?source=missing')[0], 400) + for body in ({'source': 'one'}, {'source': 'one', 'id': 'brush', 'version_code': True}): + self.assertEqual(self.request('POST', '/api/sources/install', body)[0], 400) + + def test_http_install_background_job(self): + with patch.object(server.frame_android, 'install', return_value={'package': 'org.brush'}) as install: + status, reply = self.request('POST', '/api/sources/install', {'source': 'one', 'id': 'brush'}) + self.assertEqual(status, 200) + for _ in range(100): + job = self.request('GET', '/api/job?id=' + reply['job'])[1] + if job['done']: + break + time.sleep(.01) + self.assertTrue(job['done']) + self.assertIsNone(job['error']) + install.assert_called_once_with('/fake.apk', name='Open Brush', icon_png=None, source='one') + + def test_http_repository_management(self): + self.assertEqual(self.request('POST', '/api/sources', {'action': 'enable', 'source': 'one', 'enabled': False})[0], 200) + code, reply = self.request('POST', '/api/sources', {'action': 'add', 'url': 'https://repo.example/repo'}) + self.assertEqual(code, 400) + self.assertIn('not available', reply['error']) + mod = fake('fdroid') + mod.add_repo, mod.remove_repo, mod.set_enabled = Mock(), Mock(), Mock() + with patch.object(search, 'modules', return_value=([mod], [])): + self.assertEqual(self.request('POST', '/api/sources', {'action': 'add', 'url': 'https://repo.example/repo'})[0], 200) + mod.add_repo.assert_called_once_with(url='https://repo.example/repo', fingerprint=None, name=None) + self.assertEqual(self.request('POST', '/api/sources', {'action': 'remove', 'source': 'fdroid'})[0], 200) + mod.remove_repo.assert_called_once_with(source_id='fdroid') diff --git a/ui/apk_sources/_demo.py b/ui/apk_sources/_demo.py new file mode 100644 index 0000000..1080368 --- /dev/null +++ b/ui/apk_sources/_demo.py @@ -0,0 +1,34 @@ +"""Opt-in local UI fixtures: FRAME_APK_SEARCH_DEMO=1. Never downloads.""" +from apk_sources import SourceError + +KIND = 'demo' + + +def sources(): + return [{'id': 'demo-' + key, 'kind': KIND, 'name': name + ' (demo)', 'enabled': True, + 'builtin': True, 'trust': trust, 'url': 'https://example.invalid'} + for key, name, trust in [('fdroid', 'F-Droid', 'community'), + ('sidequest', 'SideQuest', 'official'), ('itch', 'itch.io', 'community')]] + + +def search(source, query, limit=50): + if source['id'] == 'demo-itch': + raise SourceError('Demo: source temporarily unavailable') + entries = [{'id': 'brush', 'package': 'org.demo.brush', 'name': 'Open Brush', + 'summary': 'Paint in VR — demonstration listing only', 'version': '2.32', + 'version_code': 232, 'min_sdk': 29, 'abis': ['arm64-v8a'], 'vr': True, + 'engine': 'Unity OpenXR', 'size': 125000000, 'updated': '2026-09-25', + 'verified': source['id'] == 'demo-fdroid', 'free': True, 'downloadable': True}, + {'id': 'radio', 'package': 'org.demo.radio', 'name': 'Pocket Radio', + 'summary': 'Listen in a flat Android panel', 'version': '1.0', 'version_code': 1, + 'min_sdk': 24, 'abis': [], 'vr': False, 'size': 3000000, 'free': True, + 'downloadable': False, 'page': 'https://example.invalid/radio'}] + return [e for e in entries if query.lower() in e['name'].lower()][:limit] + + +def details(source, entry_id): + return next(e for e in search(source, '') if e['id'] == entry_id) + + +def download(source, entry_id, version_code=None): + raise SourceError('Demo sources cannot download or install apps') diff --git a/ui/apk_sources/search.py b/ui/apk_sources/search.py new file mode 100644 index 0000000..c4aeb93 --- /dev/null +++ b/ui/apk_sources/search.py @@ -0,0 +1,226 @@ +"""Parallel APK search and source selection. No device access during searches.""" +import importlib +import inspect +import json +import os +import pkgutil +import re +import threading +import time +import unicodedata + +import frame_host +from apk_sources import SourceError + +_lock = threading.RLock() +_running = {} +_status = {} +TIMEOUT = 12 + + +def modules(): + import apk_sources + found, errors = [], [] + for item in pkgutil.iter_modules(apk_sources.__path__): + if item.name == 'search' or item.name.startswith('_'): + continue + try: + module = importlib.import_module('apk_sources.' + item.name) + if getattr(module, 'KIND', None): + found.append(module) + except Exception as e: + errors.append({'id': item.name, 'name': item.name, 'enabled': False, + 'trust': 'unknown', 'status': 'error', 'error': str(e)}) + if os.environ.get('FRAME_APK_SEARCH_DEMO') == '1': + found.append(importlib.import_module('apk_sources._demo')) + return found, errors + + +def settings_path(): + return frame_host.data_dir('apk-sources', 'enabled.json') + + +def overrides(): + try: + return json.loads(settings_path().read_text()) + except FileNotFoundError: + return {} + + +def registry(): + result = [] + mods, errors = modules() + with _lock: + enabled = overrides() + for module in mods: + try: + for source in module.sources(): + source = dict(source) + source['enabled'] = enabled.get(source['id'], source.get('enabled', True)) + source.update(_status.get(source['id'], {'status': 'not searched'})) + result.append((module, source)) + except Exception as e: + errors.append({'id': module.KIND, 'name': module.KIND, 'enabled': False, + 'trust': 'unknown', 'status': 'error', 'error': str(e)}) + return result, errors + + +def sources(): + items, errors = registry() + return [s for _, s in items] + errors + + +def resolve(source_id): + for module, source in registry()[0]: + if source['id'] == source_id: + return module, source + raise SourceError('Unknown source') + + +def set_enabled(source_id, enabled): + module, source = resolve(source_id) + with _lock: + if hasattr(module, 'set_enabled'): + module.set_enabled(source_id, enabled) + values = overrides() + values[source_id] = enabled + path = settings_path() + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix('.tmp') + tmp.write_text(json.dumps(values)) + tmp.replace(path) + return {'message': source['name'] + (' enabled' if enabled else ' disabled')} + + +def manage_repo(action, **kwargs): + mods, _ = modules() + module = next((m for m in mods if m.KIND == 'fdroid'), None) + if module is None or not hasattr(module, action): + raise SourceError('User repositories are not available in this build') + return getattr(module, action)(**kwargs) + + +def normalise(name): + return ' '.join(re.findall(r'\w+', unicodedata.normalize('NFKC', name or '').casefold())) + + +def fit(entry): + sdk, abis = entry.get('min_sdk'), entry.get('abis') + reasons = [] + blocked = (sdk is not None and sdk > 30) or (abis is not None and bool(abis) and 'arm64-v8a' not in abis) + if sdk is not None and sdk > 30: + reasons.append('Needs Android API %s; Lepton supports 30' % sdk) + if abis and 'arm64-v8a' not in abis: + reasons.append('No arm64-v8a build') + known = sdk is not None and abis is not None + hints = ' '.join(str(entry.get(k) or '') for k in ('engine', 'vr_engine', 'vr_hints', 'vr_issues')).lower() + if 'vrapi' in hints: + reasons.append('Legacy VrApi requires a translator') + if 'godot' in hints: + reasons.append('Older Godot builds can crash on the missing clipboard service') + if 'openxr' in hints: + reasons.append('OpenXR candidate; required extensions still need checking') + if entry.get('vr'): + reasons.append('VR runtime compatibility is not guaranteed') + return {'installable': False if blocked else True if known else None, + 'verdict': "Won't install" if blocked else 'Installable' if known else 'Compatibility unknown', + 'reasons': reasons} + + +def offer_rank(entry): + compatible = entry['fit']['installable'] is True + return (entry.get('downloadable') is True and entry['fit']['installable'] is not False, + entry.get('verified') is True, compatible, + str(entry.get('updated') or '') if compatible else '', + (entry.get('version_code') or 0) if compatible else 0, + entry.get('trust') == 'official') + + +def group(entries, query='', vr=None, installable=False): + groups = {} + for entry in entries: + entry = dict(entry, fit=fit(entry)) + if vr is not None and entry.get('vr') is not vr: + continue + if installable and entry['fit']['installable'] is not True: + continue + key = ('package', entry['package']) if entry.get('package') else ('name', normalise(entry.get('name'))) + if not key[1]: + key = ('id', entry['source'], entry['id']) + groups.setdefault(key, []).append(entry) + result = [] + for offers in groups.values(): + offers.sort(key=offer_rank, reverse=True) + best = offers[0] + result.append({'name': best.get('name'), 'package': best.get('package'), + 'summary': best.get('summary'), 'offers': offers}) + q = normalise(query) + result.sort(key=lambda a: (not any(normalise(o.get('name')) == q for o in a['offers']), + not any(o['fit']['installable'] is True for o in a['offers']), + not any(normalise(o.get('name')).startswith(q) for o in a['offers']), + normalise(a['name']))) + return result + + +def _launch(module, source, query, limit): + key = source['id'] + with _lock: + old = _running.get(key) + if old and not old['event'].is_set(): + return old if old['query'] == query else None + task = {'event': threading.Event(), 'query': query, 'started': time.monotonic()} + _running[key] = task + def run(): + try: + task['entries'] = [dict(e, source=key, source_name=source['name'], trust=source.get('trust')) + for e in module.search(source, query, limit=limit) if e.get('free') is True] + except Exception as e: + task['error'] = str(e) + finally: + task['event'].set() + threading.Thread(target=run, daemon=True).start() + return task + + +def search(query='', vr=None, source=None, installable=False, timeout=TIMEOUT, limit=50): + items, errors = registry() + if source and source not in [s['id'] for _, s in items]: + raise SourceError('Unknown source') + tasks = [(s, _launch(m, s, query, limit)) for m, s in items + if s['enabled'] and (not source or s['id'] == source)] + entries, statuses = [], list(errors) + for s, task in tasks: + status = {'id': s['id'], 'name': s['name']} + if task is None or not task['event'].wait(max(0, task['started'] + timeout - time.monotonic())): + status.update(status='timed out') + elif 'error' in task: + status.update(status='error', error=task['error']) + else: + status.update(status='ok') + entries.extend(task['entries']) + statuses.append(status) + with _lock: + _status[s['id']] = {k: v for k, v in status.items() if k not in ('id', 'name')} + return {'apps': group(entries, query, vr, installable), 'sources': statuses} + + +def install(source_id, entry_id, version_code=None): + import frame_android + module, source = resolve(source_id) + if not source['enabled']: + raise SourceError('This source is disabled') + entry = module.details(source, entry_id) + if entry.get('free') is not True or entry.get('downloadable') is not True: + raise SourceError('This app must be obtained from its developer page') + downloaded = module.download(source, entry_id, version_code=version_code) + obb = downloaded.get('obb') or entry.get('obb') or [] + if obb and not hasattr(frame_android, 'install_obb'): + raise SourceError('This app needs OBB data; this build cannot install it yet') + kwargs = {'name': entry.get('name'), 'icon_png': downloaded.get('icon_png') or entry.get('icon_png'), + 'source': source['name']} + if 'artwork' in inspect.signature(frame_android.install).parameters: + kwargs['artwork'] = downloaded.get('artwork') or entry.get('artwork') + result = frame_android.install(downloaded['apk'], **kwargs) + if obb: + frame_android.install_obb(result['package'], obb) + return result diff --git a/ui/index.html b/ui/index.html index 073e2b7..97b68e5 100644 --- a/ui/index.html +++ b/ui/index.html @@ -70,6 +70,15 @@ body.mobile .desk-only { display: none; } + .source-repo-form { display:grid; gap:10px; margin-top:14px; } + .source-repo-form input { display:block; width:100%; margin-top:5px; } + .source-offer { border-top:1px solid var(--line); padding-top:10px; margin-top:10px; } + #searchGrid .source-offer select { display:block; max-width:100%; width:100%; margin-top:5px; } + #searchFilters [aria-pressed=true], #sourceFilters [aria-pressed=true] { box-shadow:inset 0 0 0 1px var(--link); color:var(--link); } + #searchGrid h3, #searchGrid p { margin:4px 0; } + #searchGrid .row { margin-top:0; } + #searchGrid .source-offer { margin-top:auto; } + #sourceStatus { margin-bottom:10px; } /* ---- pages: one per tab; the header nav switches between them ---- */ .page { display: flex; flex-direction: column; gap: 26px; } .page:not(.on) { display: none; } @@ -121,7 +130,7 @@ .seg { display: inline-flex; background: rgba(0,0,0,.3); border-radius: 3px; padding: 2px; } .seg button { background: transparent; height: 28px; font-size: 12.5px; letter-spacing: .6px; text-transform: uppercase; } .seg button.on { background: var(--btn-hi); color: var(--bright); } - input[type=text], input[type=password], textarea { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; + input[type=text], input[type=search], input[type=url], input[type=password], textarea { width: 100%; background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; padding: 9px 11px; font: inherit; } textarea { resize: vertical; min-height: 76px; } input:focus, textarea:focus { outline: none; border-color: var(--blue); background: rgba(0,0,0,.4); } @@ -282,8 +291,8 @@ .and-btns { flex-basis: 100%; padding-left: 46px; } .and-icon { width: 36px; height: 36px; border-radius: 8px; flex: none; background: rgba(0,0,0,.25); object-fit: cover; } .cat-tools { display: flex; gap: 8px; flex-wrap: wrap; align-items: center; } - .cat-tools input[type=text] { flex: 1 1 240px; width: auto; } - .cat-tools select { background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; + .cat-tools input[type=text], .cat-tools input[type=search] { flex: 1 1 240px; width: auto; } + .cat-tools select, .source-offer select { background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; padding: 9px 10px; font: inherit; font-size: 13px; max-width: 220px; } .vchips { display: flex; gap: 6px; flex-wrap: wrap; margin: 10px 0 14px; } .vchip { height: 26px; font-size: 12px; padding: 0 10px; opacity: .55; } @@ -357,7 +366,7 @@ button { height: 38px; } button.small { height: 32px; } .actions button { height: 46px; } - .cat-tools select { max-width: none; flex: 1 1 100%; } + .cat-tools select, .source-offer select { max-width: none; flex: 1 1 100%; } /* The on-screen keyboard covers the bottom of the screen; the tab bar would ride on top of it. */ body.typing nav { display: none; } } @@ -575,7 +584,22 @@
-

Find apps

+

Find apps

+
+
+
+
+
+
Sources +
+
+ + + +
+
+
Installable means Android 11 and arm64 requirements match. It does not guarantee the app runs. Unknown facts stay unknown.
+
@@ -1780,6 +1804,66 @@ document.body.addEventListener("click", async e => { }); loadOwned(); +// ---- All APK sources ---- +const sourceState = { vr: null, installable: false, source: "", apps: [], request: 0 }; +function sourcePage(url) { try { const u = new URL(url); return ["https:", "http:"].includes(u.protocol) ? u.href : ""; } catch { return ""; } } +async function loadSources() { + try { + const { sources } = await api("/api/sources"); + $("sourceFilters").innerHTML = `` + sources.filter(s => s.enabled).map(s => ``).join(""); + $("sourcesList").innerHTML = sources.map(s => `
${s.builtin === false ? `` : ""}
`).join("") || `
No source modules are available in this build.
`; + } catch (e) { $("sourceError").textContent = e.message; } +} +async function searchSources() { + const request = ++sourceState.request; + $("sourceStatus").textContent = "Searching enabled sources…"; + const params = new URLSearchParams({ q: $("sourceQ").value, source: sourceState.source, installable: String(sourceState.installable) }); + if (sourceState.vr !== null) params.set("vr", String(sourceState.vr)); + try { + const result = await api("/api/search?" + params); + if (request !== sourceState.request) return; + sourceState.apps = result.apps; + $("searchCount").textContent = `${result.apps.length} apps`; + $("sourceStatus").textContent = result.sources.map(s => `${s.name}: ${s.status === "error" ? s.error : s.status}`).join(" · ") || "No enabled sources"; + $("searchGrid").innerHTML = result.apps.map((a, i) => `

${esc(a.name || "Unnamed app")}

${esc(a.package || "Package unknown")}

${esc(a.summary || "")}

${[...new Set(a.offers.map(o => o.source_name))].map(n => `${esc(n)}`).join(" ")}
`).join("") || `
No matching apps. Try another search or enable more sources.
`; + result.apps.forEach((_, i) => renderOffer(i, 0)); + await loadSources(); + } catch (e) { if (request === sourceState.request) $("sourceStatus").textContent = e.message; } +} +function renderOffer(i, j) { + const o = sourceState.apps[i].offers[j], page = sourcePage(o.page), busy = installing.has(`${o.source}:${o.id}`); + $("offerInfo" + i).innerHTML = `

${esc(o.fit.verdict)} · API ${esc(o.min_sdk ?? "unknown")} · ${esc(o.abis == null ? "ABI unknown" : o.abis.join(", ") || "No native code")}
Version code ${esc(o.version_code ?? "unknown")} · ${o.size == null ? "Size unknown" : (o.size / 1048576).toFixed(1) + " MB"} · Updated ${esc(o.updated || "unknown")}

${o.fit.reasons.map(esc).join(" · ")}

${o.downloadable && o.fit.installable !== false ? `` : ""}${page ? `Developer page` : ""}
`; +} +$("sourceSearch").onsubmit = e => { e.preventDefault(); searchSources(); }; +$("searchFilters").onclick = e => { + const b = e.target.closest("[data-filter]"); if (!b) return; + if (b.dataset.filter === "installable") sourceState.installable = !sourceState.installable; + else { const value = b.dataset.filter === "vr"; sourceState.vr = sourceState.vr === value ? null : value; } + $("searchFilters").querySelectorAll("button").forEach(b => b.setAttribute("aria-pressed", b.dataset.filter === "installable" ? sourceState.installable : sourceState.vr === (b.dataset.filter === "vr"))); + searchSources(); +}; +$("sourceFilters").onclick = e => { const b = e.target.closest("[data-source]"); if (b) { sourceState.source = b.dataset.source; searchSources(); } }; +$("searchGrid").onchange = e => { if (e.target.matches("[data-offer]")) renderOffer(+e.target.dataset.offer, +e.target.value); }; +$("searchGrid").onclick = async e => { + const b = e.target.closest("[data-source-install]"); if (!b) return; + const o = sourceState.apps[+b.dataset.sourceInstall].offers[+b.dataset.choice]; + if (installing.has(`${o.source}:${o.id}`)) return; + b.disabled = true; b.textContent = "Downloading and installing…"; + const result = await runJob(`Install ${o.name}`, `${o.source}:${o.id}`, () => api("/api/sources/install", {source: o.source, id: o.id, version_code: o.version_code ?? null})); + b.disabled = false; b.textContent = result ? "Installed" : "Retry install"; + if (result) loadAndroid(); + if (!b.isConnected) searchSources(); +}; +async function changeSource(body) { + $("sourceError").textContent = "Saving…"; + try { const result = await api("/api/sources", body); $("sourceError").textContent = result.message; sourceState.source = ""; await loadSources(); await searchSources(); } + catch (e) { $("sourceError").textContent = e.message; await loadSources(); } +} +$("sourcesList").onchange = e => { if (e.target.matches("[data-enable]")) changeSource({action: "enable", source: e.target.dataset.enable, enabled: e.target.checked}); }; +$("sourcesList").onclick = e => { const b = e.target.closest("[data-remove]"); if (b) changeSource({action: "remove", source: b.dataset.remove}); }; +$("addSource").onsubmit = e => { e.preventDefault(); const data = new FormData(e.target); changeSource({action: "add", url: data.get("url"), fingerprint: data.get("fingerprint")}); }; +loadSources().then(searchSources); + // ---- Android apps (own Lepton instance each) + catalogue ---- const VLABEL = { works: "Works on Frame", likely: "Should work", maybe: "Might work", unlikely: "Probably crashes", no: "Won't work" }; const cat = { apps: null, q: "", cat: "", v: new Set(["works", "likely"]), shown: 0, results: [] }; diff --git a/ui/server.py b/ui/server.py index 8000468..cb85c85 100755 --- a/ui/server.py +++ b/ui/server.py @@ -37,6 +37,7 @@ from urllib.parse import parse_qs, unquote, urlparse sys.path.insert(0, str(Path(__file__).resolve().parent)) import frame_android # noqa: E402 +from apk_sources import search as apk_search, SourceError # noqa: E402 import frame_apk_versions # noqa: E402 import frame_catalog # noqa: E402 import frame_host # noqa: E402 @@ -1227,7 +1228,66 @@ def _sweep_one(prefix, d): pass -POST = {"/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard, +def source_text(body, key, optional=False): + value = body.get(key) + if optional and value in (None, ''): + return None + if not isinstance(value, str) or not value.strip() or len(value) > 2000: + raise Failure('Provide a valid ' + key, 400) + return value.strip() + + +def source_search(query): + args = parse_qs(query) + q = args.get('q', [''])[0] + vr = args.get('vr', [''])[0] + installable = args.get('installable', [''])[0] + if len(q) > 500 or vr not in ('', 'true', 'false', '1', '0') or installable not in ('', 'true', 'false', '1', '0'): + raise Failure('Invalid search filters', 400) + try: + return apk_search.search(q, vr=None if not vr else vr in ('true', '1'), + source=args.get('source', [None])[0], installable=installable in ('true', '1')) + except SourceError as e: + raise Failure(str(e), 400) + + +def source_install(body): + source, entry = source_text(body, 'source'), source_text(body, 'id') + version = body.get('version_code') + if version is not None and (type(version) is not int or version < 0): + raise Failure('version_code must be a non-negative integer', 400) + try: + _, selected = apk_search.resolve(source) + if not selected['enabled']: + raise SourceError('This source is disabled') + except SourceError as e: + raise Failure(str(e), 400) + return start_job('Install ' + entry, lambda: apk_search.install(source, entry, version)) + + +def source_manage(body): + action = body.get('action') + try: + if action == 'enable': + if type(body.get('enabled')) is not bool: + raise Failure('enabled must be true or false', 400) + return apk_search.set_enabled(source_text(body, 'source'), body['enabled']) + if action == 'add': + url = source_text(body, 'url') + if urlparse(url).scheme != 'https' or not urlparse(url).hostname or urlparse(url).username: + raise Failure('Use an HTTPS repository URL without credentials', 400) + apk_search.manage_repo('add_repo', url=url, fingerprint=source_text(body, 'fingerprint', True), + name=source_text(body, 'name', True)) + return {'message': 'Repository added'} + if action == 'remove': + apk_search.manage_repo('remove_repo', source_id=source_text(body, 'source')) + return {'message': 'Repository removed'} + raise Failure('Unknown source action', 400) + except SourceError as e: + raise Failure(str(e), 400) + + +POST = {"/api/sources": source_manage, "/api/sources/install": source_install, "/api/android/display": android_display, "/api/android": android, "/api/titles": titles, "/api/launch": launch, "/api/steam": steam, "/api/volume": set_volume, "/api/clipboard": clipboard, "/api/flatpak": flatpak, "/api/open": open_thing, "/api/shots/save": save_shots, "/api/webinstall/check": webinstall_check, "/api/webinstall/start": webinstall_start, "/api/webinstall/cancel": webinstall_cancel} @@ -1335,6 +1395,10 @@ class Handler(BaseHTTPRequestHandler): self.send_json({"os": "SteamOS", "fileManager": None, "computer": DEVICE, "mobile": True} if LOCAL else {"os": frame_host.NAME, "fileManager": frame_host.FILE_MANAGER, "computer": "Mac" if frame_host.MAC else "PC"}) + elif path == "/api/sources": + self.send_json({"sources": apk_search.sources()}) + elif path == "/api/search": + self.send_json(source_search(url.query)) elif path == "/api/apk-versions": self.send_json(apk_versions(url.query)) elif path == "/api/android": From f4dbb1180c8e1934cd7c114dd917c9410194b5d0 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:05:59 +1000 Subject: [PATCH 003/122] Add OBB transfers, private app-data backups and SideQuest source policy Borrow expansion-file and save-management features with offline verification. Keep SideQuest page-only under its current access terms; document research, integration limits and device acceptance gaps. Co-Authored-By: GPT-6 Astra --- .claude/NOTES-sidequest.md | 55 +++++++ docs/apks.md | 8 + docs/sidequest.md | 142 ++++++++++++++++++ frame/android/app-data.py | 139 +++++++++++++++++ tests/fixtures/sidequest-policy.json | 18 +++ tests/test_frame_android_data.py | 213 +++++++++++++++++++++++++++ tests/test_sidequest.py | 36 +++++ ui/apk_sources/sidequest.py | 41 ++++++ ui/frame_android.py | 24 +++ ui/frame_android_data.py | 133 +++++++++++++++++ 10 files changed, 809 insertions(+) create mode 100644 .claude/NOTES-sidequest.md create mode 100644 docs/sidequest.md create mode 100644 frame/android/app-data.py create mode 100644 tests/fixtures/sidequest-policy.json create mode 100644 tests/test_frame_android_data.py create mode 100644 tests/test_sidequest.py create mode 100644 ui/apk_sources/sidequest.py create mode 100644 ui/frame_android_data.py diff --git a/.claude/NOTES-sidequest.md b/.claude/NOTES-sidequest.md new file mode 100644 index 0000000..8e085e0 --- /dev/null +++ b/.claude/NOTES-sidequest.md @@ -0,0 +1,55 @@ +# SideQuest implementation notes + +2026-09-28. Worktree steam-frame-sidequest, branch sidequest. No delegation, +Frame mutations, installs, launches, pushes or issue edits. + +- Read shared source interface, APK/VR docs, catalogue README and Python backend. +- SideQuest robots: crawl delay 3; disallow /search/, /user/*, /sideload/*. +- Current /terms Angular text (main-4MMXZRXL.js): Prohibited Activities (i) + prohibits scraping; (xi) limits access to provided/authorised technologies; + (xii) forbids bypass. Public API address is not permission for a third-party + integration. Page-only source; no automated store downloads or metadata crawl. +- api.sidequestvr.com/robots.txt returned HTTP 403. First shared JS chunk also + returned 403. No attempt to bypass either response. +- Public SideQuest desktop source cloned inside .claude/research for inspection. + /install-from-key takes a website-issued token and returns apps[].urls[] with + provider APK/OBB/Github Release/Mod and link_url. Do not reproduce token flow. +- Two SSH read-only attempts to frame timed out (exit 255). Exact host-side + /sdcard mapping cannot be claimed. internal/ is private app data, + not evidence of an OBB mapping. Use the running container's /sdcard path for + OBB writes, without launching it; backup only documented internal/. +- Scope: OBB helper/CLI, private-data backup/restore helper/CLI, compliant + SideQuest page-only source. No search UI, artwork installer or install edits. +- Cross-provider review not launched: task explicitly forbids delegation; + parent brief reserves integration and review for the parent. + +## Implementation and verification + +- Added ui/frame_android_data.py and frame/android/app-data.py; additive wrappers + and CLI branches only in frame_android.py (install/_install unchanged). +- OBB transfers to an already-running named container, SHA-256 check before + per-file rename. No claimed host sdcard mapping or device persistence. +- Backup/restore covers private internal/ only, requires a stopped + instance, uses podman unshare, validates archive paths/types/package/instance, + preserves numeric owners/modes, retains the prior data directory on restore. +- SideQuest adapter intentionally raises a page-only SourceError on search and + download; details gives a numeric listing page with unknown facts, images + schema and downloadable=False. Needs aggregate search to surface the error. +- docs/sidequest.md contains source links, feature comparison, command examples, + terms/robots findings and outstanding device/API questions. +- Fixture tests/fixtures/sidequest-policy.json records observed policy excerpts; + no API response is fabricated. +- `python3 -m unittest discover -s tests`: final run 181 tests, OK (exit 0). + Includes real local shell execution of the OBB checksum/publish sequence, + rejecting a changed input without replacing the previous file; archive + round-trip/retained previous save, 0600 backup, malformed archive rejection. + No test contacts the network or Frame. +- `ast.parse(..., feature_version=(3,9))`: four implementation files passed. + Runtime python3 is Xcode Python 3.9. +- `python3 ui/frame_android.py install-obb` and `... backup-data`: both exit 1 + with the intended required-arguments error, without contacting Frame. +- `git diff --check`: passed before commit. +- No SideQuest game downloaded and no `info ` run: terms blocked that + requested E2E. No Frame app was installed or launched; no live OBB, namespace + ownership or restore acceptance test. Independent review reserved for parent, + per this task's explicit no-delegation instruction. diff --git a/docs/apks.md b/docs/apks.md index 8bca60a..54d2fdf 100644 --- a/docs/apks.md +++ b/docs/apks.md @@ -325,3 +325,11 @@ because gamescope scales Lepton's surface to fit the same panel. Also unverified whether the settings survive the app or its Lepton instance relaunching. Lepton Development rebuilds its Android data on exit, so there they probably don't. + +## Expansion files and save backups + +SideQuest-inspired CLI helpers install local OBB files into an already-running +app instance and back up/restore a stopped instance's private app data. See +[SideQuest features and limits](sidequest.md) for commands, archive scope and +verification status. These paths have offline coverage; real Frame storage and +permissions remain unverified. They do not change APK install or launch behavior. diff --git a/docs/sidequest.md b/docs/sidequest.md new file mode 100644 index 0000000..eba0f02 --- /dev/null +++ b/docs/sidequest.md @@ -0,0 +1,142 @@ +# SideQuest and Frame Control + +Researched 2026-09-28. SideQuest is both a Quest discovery website and a desktop +sideloading/device-management app. Its Quest labels are **not** evidence that a +game works on Lepton: inspect the APK for arm64/OpenXR, Android API requirements, +VrApi and Meta services (see [VR APKs](vr-apks.md)). + +## Features worth borrowing + +Desktop evidence is the public [SideQuest source at af2ac70](https://github.com/SideQuestVR/SideQuest/tree/af2ac7043db122bca3c8db18f2b58f1660e9befb), +especially [ADB operations](https://github.com/SideQuestVR/SideQuest/blob/af2ac7043db122bca3c8db18f2b58f1660e9befb/desktop-app/src/app/adb-client.service.ts), +[drag and drop](https://github.com/SideQuestVR/SideQuest/blob/af2ac7043db122bca3c8db18f2b58f1660e9befb/desktop-app/src/app/drag-and-drop.service.ts), +and the [legacy repository index](https://github.com/SideQuestVR/SideQuest/blob/af2ac7043db122bca3c8db18f2b58f1660e9befb/desktop-app/src/app/packages/package.service.ts). +Website evidence: [SideQuest](https://sidequestvr.com/) and its public Angular +bundle `main-4MMXZRXL.js`, inspected locally without browser automation. +No SideQuest implementation code was copied. + +| SideQuest feature | Frame Control before this change | Borrow? / effort | +|---|---|---| +| Store descriptions, screenshots, banners, trailers, ratings | F-Droid names, icons, compatibility verdicts and reports; no equivalent rich VR store | Yes, from authorised sources; medium. Search and library workers own presentation/artwork. | +| OBB expansion-file install | APK-only install | **Implemented helper and CLI**, medium. Essential for games whose assets are separate from the APK. | +| App-data backup/restore | Persistent instances and optional keep-data uninstall, no portable save archive | **Implemented private-data helper and CLI**, medium. Back up before updates or experiments. | +| File manager (list, upload, download, remove) | General Send to Frame, no Android file browser | Useful later, medium; requires clear instance selection and scoped paths. | +| Installed-app management (launch, uninstall, backup) | List, launch, stop, remove, probe | Already mostly covered. Backup added here. | +| Update notices / account library | Compatible-version lookup; no source-aware installed update notices | Useful later, medium; needs original version code and source identity recorded on install. | +| Custom repositories | Built-in F-Droid catalogue and compatible-version indexes | Separate user-repos worker. Legacy SideQuest source has a fixed SideQuestRepos index; arbitrary current custom-repo support was not verified. | +| Drag-and-drop APK/OBB install | APK drag-and-drop already works | OBB backend added here; future UI can call it. UI drop wiring is not included. | +| Tags, price, headset filters, reviews | Text search and Lepton verdicts, not Quest headset metadata | Useful, medium; search worker owns filters. Keep source headset claims distinct from tested Frame compatibility. | +| Screenshot/video capture and streaming | Frame screenshots/VR capture already present | Reuse existing tools; do not port Quest capture commands. | +| Device settings and ADB utilities | Frame/Android display settings, SSH and own-instance tools | Borrow selectively; Quest CPU/GPU presets and wireless-ADB setup do not map directly to Lepton. | + +Priority: expansion files, then save backup/restore. Rich discovery and update +notices follow once a permitted metadata source and source/version persistence +are available. This patch deliberately exposes CLI/backend operations, leaving +shared UI, install(), Steam artwork and launch behavior to sibling work. + +## SideQuest as a source: page-only + +[Terms](https://sidequestvr.com/terms), “Prohibited Activities”, (i) prohibits +copying/distributing/disclosing the Service including automated or non-automated +“scraping”; (xi) prohibits content access through means other than those provided +or authorised by the Service; (xii) prohibits bypassing access restrictions. +The terms describe downloading developer-posted games through the Service, but +do not establish permission for this third-party API integration. + +[robots.txt](https://sidequestvr.com/robots.txt) requests a three-second crawl +delay and disallows `/search/`, `/user/*` and `/sideload/*`. Robots permission +would not override the terms. The API host's robots request returned HTTP 403; +a request for the first shared website JS chunk also returned 403. No bypass, +account token, cookies, browser session or private endpoint was used. + +The homepage publishes `https://api.sidequestvr.com` and +`https://cdn.sidequestvr.com`. The website bundle calls `searchApps(...)` and +`getApp(id, null)`; their actual HTTP search/detail routes could not be established +from the retrieved bundle. Do not invent endpoints. The open-source desktop +[install flow](https://github.com/SideQuestVR/SideQuest/blob/af2ac7043db122bca3c8db18f2b58f1660e9befb/electron/app.ts) +POSTs `{token: ...}` to `/install-from-key`. It consumes +`data.apps[].urls[]`, with `provider` values including `APK`, `OBB`, +`Github Release` and `Mod`, and `link_url`. This is a website-issued install-key +flow, not evidence of an anonymous download API. It is not implemented here. + +`ui/apk_sources/sidequest.py` implements the shared interface conservatively: + +- `sources()` marks SideQuest `page_only` and explains why. +- `search()` raises a user-readable `SourceError` with the browse URL (zero + limit returns no rows). It does not invent app results or report a false + “no matching games”. The aggregate search UI should surface this source error. +- `details()` accepts a numeric listing id and returns its canonical page link, + `downloadable: False`, empty versions/tags/headsets and the `images` shape + `{icon: None, banner: None, screenshots: []}`. Name is explicitly a listing id; + unknown facts, including free/VR status, stay `None`. +- `download()` refuses with that page link. Paid/external listings cannot be + downloaded by this adapter either. No downloads means no verification claim. + +The JSON fixture records policy evidence, **not a purported live app response**. +No listing metadata, artwork URLs, or OBB download URLs were scraped. +The requested real SideQuest → OpenXR APK → `frame_android.py info` test is +**blocked by the terms**, and was not performed. No alternate source is silently +substituted. A future integration needs SideQuest's permission or an expressly +supported third-party API, plus recorded search/detail/download fixtures, +free/direct-download classification, and size/hash verification. A calculated +local SHA-256 alone must not be called publisher verification. + +## OBB files + +```sh +python3 ui/frame_android.py install-obb org.example.game main.42.org.example.game.obb +python3 ui/frame_android.py install-obb org.example.game main.42.org.example.game.obb patch.42.org.example.game.obb +``` + +Install the APK first. The named instance must already be running; the helper +never launches an app or uses Lepton Development. It requires standard +`main|patch...obb` filenames and nonempty files, validates +the entire batch before transfer, streams each file through SSH into that +instance, checks its SHA-256 **inside Android**, then renames it into +`/sdcard/Android/obb//`. `verified: True` here means transfer integrity +against the local input, not publisher authentication. Publication is atomic per +file, not for the whole batch; retry after a partial batch failure. Existing OBBs +with different version codes remain. The filename version must match the game; +the current install metadata does not expose its version code for comparison. +Restart the game yourself after the transfer if it cached missing expansion data. + +Both read-only SSH attempts to the Frame timed out. Therefore the exact +host-side `/sdcard` mapping and persistence of expansion data were **not verified**. +`compatdata//internal/` is documented as `/data/data/`; +it must not be mistaken for `/sdcard`. Using Android's path avoids guessing a +host layout, but device verification across restart/update is still required. +No OBB file was installed on the Frame during this work. + +## Private app-data backups + +```sh +python3 ui/frame_android.py stop org.example.game +python3 ui/frame_android.py backup-data org.example.game ./game-save.tar.gz +python3 ui/frame_android.py restore-data org.example.game ./game-save.tar.gz +``` + +Keep the instance stopped throughout either operation; do not launch it from +Steam concurrently. The remote guard fails if Podman cannot enumerate containers +or reports that instance running. The helpers use `podman unshare` to read/write +Android's mapped ownership without changing the live data's permissions. + +The archive covers **only** `compatdata//internal/`, not the +APK, external `/sdcard/Android/data`, OBBs, keystore, or the full Android snapshot. +It contains a package/instance manifest and regular files/directories. Backups +are private (0600), validated before publication, and never overwrite an existing +backup. Keep them safe: app data can contain credentials and is not encrypted. + +Restore checks the package and instance, rejects absolute/traversing/duplicate +paths, links and devices, caps files at 100,000 and content at 20 GiB, and validates +again on the Frame. It extracts into a separate directory, preserves numeric +ownership, ordinary modes and timestamps, then swaps the private-data directory. +Setuid/setgid bits are not restored. The previous directory remains beside it as +`..before-restore-`; the returned `previous` path identifies +it. This is an additional recovery copy, not an automatic deletion policy. + +Locally verified: archive round trip including recovery copy, malformed archive +rejection, transfer command construction and failure handling. Not verified: +real Frame UID mappings/permissions, Android app-level recovery, live FUSE OBB +writes or persistence. Backups reject symlinks/special files; an app requiring +those needs a separately designed backup format. These CLI features still need +a real-device acceptance pass before being exposed as a polished UI workflow. diff --git a/frame/android/app-data.py b/frame/android/app-data.py new file mode 100644 index 0000000..1ecc141 --- /dev/null +++ b/frame/android/app-data.py @@ -0,0 +1,139 @@ +"""Private-data archives, run under podman unshare on the Frame. Stdlib only.""" +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import sys +import tarfile +import tempfile +import time + +MAX_BYTES = 20 * 1024 ** 3 +MAX_FILES = 100000 + + +def inspect_archive(path, package, instance): + names, total, manifest = set(), 0, None + with tarfile.open(path, 'r:gz') as archive: + for member in archive: + name = member.name + parts = PurePosixPath(name).parts + if (not parts or name.startswith('/') or '..' in parts or + name != '/'.join(parts) or name in names or '\\' in name): + raise ValueError('unsafe or duplicate archive path') + if name == 'data' and not member.isdir(): + raise ValueError('data root must be a directory') + names.add(name) + if len(names) > MAX_FILES or not (member.isdir() or member.isfile()): + raise ValueError('archive has too many files, links or special files') + if member.uid < 0 or member.gid < 0 or member.uid > 65535 or member.gid > 65535: + raise ValueError('archive owner outside Android user namespace') + total += member.size + if total > MAX_BYTES: + raise ValueError('archive exceeds 20 GiB') + if name == 'manifest.json' and member.isfile() and member.size <= 4096: + manifest = json.load(archive.extractfile(member)) + elif parts[0] != 'data': + raise ValueError('unexpected archive member') + if (not isinstance(manifest, dict) or manifest.get('format') != 1 or + manifest.get('package') != package or manifest.get('instance') != instance or + 'data' not in names): + raise ValueError('backup does not match this package and instance') + return {'files': len(names) - 1, 'bytes': total, 'package': package, 'instance': instance} + + +def backup(root, package, instance, output): + import io + source = root / package + if source.is_symlink() or not source.is_dir(): + raise ValueError('private app data does not exist or is a symlink') + count, total = 0, 0 + + def checked(member): + nonlocal count, total + count += 1 + total += member.size + if not (member.isdir() or member.isfile()) or count > MAX_FILES or total > MAX_BYTES: + raise ValueError('private data contains links/special files or exceeds backup limits') + return member + + manifest = json.dumps({'format': 1, 'package': package, 'instance': instance}).encode() + with tarfile.open(fileobj=output, mode='w|gz', dereference=False) as archive: + member = tarfile.TarInfo('manifest.json') + member.size, member.mode = len(manifest), 0o600 + archive.addfile(member, io.BytesIO(manifest)) + archive.add(str(source), arcname='data', filter=checked) + + +def restore(root, package, instance, input_stream): + source = root / package + if source.is_symlink() or not source.is_dir(): + raise ValueError('private app data does not exist or is a symlink') + with tempfile.TemporaryDirectory(prefix='.frame-restore-', dir=str(root)) as work: + work = Path(work) + archive_path = work / 'backup.tar.gz' + with archive_path.open('wb') as output: + size = 0 + while True: + chunk = input_stream.read(1024 * 1024) + if not chunk: + break + size += len(chunk) + if size > MAX_BYTES: + raise ValueError('compressed backup exceeds 20 GiB') + output.write(chunk) + result = inspect_archive(archive_path, package, instance) + stage = work / 'stage' + stage.mkdir(mode=0o700) + with tarfile.open(archive_path, 'r:gz') as archive: + directories = [] + for member in archive: + if member.name == 'manifest.json': + continue + target = stage / member.name + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + directories.append((target, member)) + else: + target.parent.mkdir(parents=True, exist_ok=True) + with archive.extractfile(member) as src, target.open('xb') as dst: + shutil.copyfileobj(src, dst, 1024 * 1024) + apply_metadata(target, member) + for target, member in reversed(directories): + apply_metadata(target, member) + previous = root / ('.' + package + '.before-restore-' + str(time.time_ns())) + source.rename(previous) + try: + (stage / 'data').rename(source) + except BaseException: + previous.rename(source) + raise + result['previous'] = str(previous) + return result + + +def apply_metadata(path, member): + os.chown(str(path), member.uid, member.gid) + os.chmod(str(path), member.mode & 0o777) + os.utime(str(path), (member.mtime, member.mtime)) + + +def main(): + action, package, instance = sys.argv[1:] + instance = int(instance) + root = Path.home() / '.local/share/Steam/steamapps/compatdata' / str(instance) / 'internal' + if root.is_symlink() or root.resolve() != root.absolute(): + raise ValueError('private-data directory traverses a symlink') + if action == 'backup': + backup(root, package, instance, sys.stdout.buffer) + elif action == 'restore': + print(json.dumps(restore(root, package, instance, sys.stdin.buffer))) + else: + raise ValueError('unknown app-data action') + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, tarfile.TarError) as error: + sys.exit(str(error)) diff --git a/tests/fixtures/sidequest-policy.json b/tests/fixtures/sidequest-policy.json new file mode 100644 index 0000000..dd08588 --- /dev/null +++ b/tests/fixtures/sidequest-policy.json @@ -0,0 +1,18 @@ +{ + "recorded": "2026-09-28", + "robots": { + "url": "https://sidequestvr.com/robots.txt", + "user_agent": "*", + "crawl_delay": 3, + "disallow": ["/search/", "/user/*", "/sideload/*"], + "sitemap": "https://sidequestvr.com/sitemap_index.xml" + }, + "api_robots": {"url": "https://api.sidequestvr.com/robots.txt", "status": 403}, + "terms": { + "url": "https://sidequestvr.com/terms", + "bundle": "https://sidequestvr.com/main-4MMXZRXL.js", + "prohibited_activities_i": "copy, distribute, or disclose any part of the Service in any medium, including without limitation by any automated or non-automated scraping", + "prohibited_activities_xi": "access any content on the Service through any technology or means other than those provided or authorized by the Service" + }, + "note": "Policy evidence, not a fabricated API response. No app metadata or download fixture was collected after discovering the restriction." +} diff --git a/tests/test_frame_android_data.py b/tests/test_frame_android_data.py new file mode 100644 index 0000000..3f4c3fe --- /dev/null +++ b/tests/test_frame_android_data.py @@ -0,0 +1,213 @@ +import io +import json +import os +from pathlib import Path +import runpy +import shlex +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'ui')) +import frame_android as android +import frame_android_data as data + +REMOTE = runpy.run_path(str(data.REMOTE)) +PKG = 'org.example.game' +META = {'package': PKG, 'instance': 2800000001} + + +class ObbTests(unittest.TestCase): + def test_invalid_files_never_contact_frame(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(android, 'ssh') as ssh: + for name in ('game.obb', 'main.1.org.other.game.obb', 'main.x.' + PKG + '.obb'): + path = Path(tmp) / name + path.write_bytes(b'content') + with self.assertRaises(android.FrameError): + data.install_obb(PKG, [path]) + with self.assertRaises(android.FrameError): + data.install_obb('../game', []) + with self.assertRaises(android.FrameError): + data.install_obb(PKG, []) + ssh.assert_not_called() + + def test_streams_to_correct_instance_and_checks_hash_before_rename(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / ('main.7.' + PKG + '.obb') + path.write_bytes(b'expansion payload') + calls = [] + def stream(command, src=None, dst=None): + calls.append(command) + self.assertEqual(src.read(), b'expansion payload') + with patch.object(android, '_meta_or_fail', return_value=META), \ + patch.object(android, 'ssh', return_value='lepton-steamlaunch-2800000001\n'), \ + patch.object(data, '_stream', side_effect=stream): + result = data.install_obb(PKG, [path]) + self.assertTrue(result['verified']) + self.assertIn('podman exec -i lepton-steamlaunch-2800000001', calls[0]) + self.assertIn('/sdcard/Android/obb/' + PKG, calls[0]) + self.assertLess(calls[0].index('sha256sum'), calls[0].index('; mv')) + self.assertIn(result['obb'][0]['sha256'], calls[0]) + + def test_stopped_instance_and_failed_transfer(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / ('patch.7.' + PKG + '.obb') + path.write_bytes(b'patch') + with patch.object(android, '_meta_or_fail', return_value=META), \ + patch.object(android, 'ssh', return_value=''), patch.object(data, '_stream') as stream: + with self.assertRaisesRegex(android.FrameError, 'start this app'): + data.install_obb(PKG, [path]) + stream.assert_not_called() + with patch.object(subprocess, 'run', return_value=subprocess.CompletedProcess([], 1, b'', b'bad hash')): + with self.assertRaisesRegex(android.FrameError, 'bad hash'): + data._stream('command') + + + @unittest.skipUnless(shutil.which("sh") and shutil.which("shasum"), "shell checksum tools unavailable") + def test_android_shell_publish_and_hash_failure(self): + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / ('main.7.' + PKG + '.obb') + source.write_bytes(b'good expansion') + output = Path(tmp) / 'sdcard/Android/obb' / PKG / source.name + tools_dir = Path(tmp) / 'bin' + tools_dir.mkdir() + checksum = tools_dir / 'sha256sum' + checksum.write_text('#!/bin/sh\nexec shasum -a 256 "$@"\n') + checksum.chmod(0o700) + corrupt = False + def stream(command, src=None, dst=None): + script = shlex.split(command)[-1].replace('/sdcard/', tmp + '/sdcard/') + if corrupt: + source.write_bytes(b'corrupt expansion') + result = subprocess.run(['sh', '-c', script], stdin=src, capture_output=True, + env=dict(os.environ, PATH=str(tools_dir) + ':' + os.environ['PATH'])) + if result.returncode: + raise android.FrameError('checksum failed') + with patch.object(android, '_meta_or_fail', return_value=META), \ + patch.object(android, 'ssh', return_value='lepton-steamlaunch-2800000001'), \ + patch.object(data, '_stream', side_effect=stream): + data.install_obb(PKG, [source]) + self.assertEqual(output.read_bytes(), b'good expansion') + corrupt = True + with self.assertRaises(android.FrameError): + data.install_obb(PKG, [source]) + self.assertEqual(output.read_bytes(), b'good expansion') + self.assertEqual(list(output.parent.glob('*.part')), []) + + +class BackupTests(unittest.TestCase): + def test_roundtrip_and_retains_previous_data(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / PKG + (source / 'files').mkdir(parents=True) + (source / 'files/save').write_bytes(b'original save') + archive = io.BytesIO() + REMOTE['backup'](root, PKG, META['instance'], archive) + (source / 'files/save').write_bytes(b'new save') + archive.seek(0) + # Current user's uid/gid in this local test; no elevated execution. + result = REMOTE['restore'](root, PKG, META['instance'], archive) + self.assertEqual((source / 'files/save').read_bytes(), b'original save') + self.assertEqual((Path(result['previous']) / 'files/save').read_bytes(), b'new save') + + def make_archive(self, path, members, package=PKG): + with tarfile.open(path, 'w:gz') as archive: + payload = json.dumps({'format': 1, 'package': package, 'instance': META['instance']}).encode() + member = tarfile.TarInfo('manifest.json') + member.size = len(payload) + archive.addfile(member, io.BytesIO(payload)) + root = tarfile.TarInfo('data') + root.type = tarfile.DIRTYPE + archive.addfile(root) + for name, kind in members: + member = tarfile.TarInfo(name) + member.type = kind + member.linkname = '/tmp/escape' + archive.addfile(member) + + def test_rejects_wrong_package_traversal_links_devices_duplicates(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'bad.tar.gz' + cases = [('../escape', tarfile.REGTYPE), ('/absolute', tarfile.REGTYPE), + ('data/link', tarfile.SYMTYPE), ('data/link', tarfile.LNKTYPE), + ('data/device', tarfile.CHRTYPE), ('data', tarfile.DIRTYPE), + ('other/file', tarfile.REGTYPE), ('data/../escape', tarfile.REGTYPE)] + for member in cases: + self.make_archive(path, [member]) + with self.assertRaises(ValueError, msg=str(member)): + REMOTE['inspect_archive'](path, PKG, META['instance']) + self.make_archive(path, [], package='org.other.game') + with self.assertRaisesRegex(ValueError, 'does not match'): + REMOTE['inspect_archive'](path, PKG, META['instance']) + + def test_failed_backup_leaves_no_archive_and_existing_is_preserved(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'backup.tar.gz' + with patch.object(android, '_meta_or_fail', return_value=META), \ + patch.object(data, '_stream', side_effect=android.FrameError('offline')): + with self.assertRaises(android.FrameError): + data.backup_data(PKG, path) + self.assertEqual(list(Path(tmp).iterdir()), []) + path.write_bytes(b'keep') + with self.assertRaisesRegex(android.FrameError, 'already exists'): + data.backup_data(PKG, path) + self.assertEqual(path.read_bytes(), b'keep') + + def test_guard_does_not_hide_podman_failure(self): + command = data._data_command('backup', META) + self.assertIn('|| exit 1', command) + self.assertIn('stop the app', command) + self.assertIn('podman unshare python3', command) + self.assertNotIn('|| true', command) + + def test_bad_restore_is_rejected_before_transfer(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / 'bad.tar.gz' + self.make_archive(path, [('../escape', tarfile.REGTYPE)]) + with patch.object(android, '_meta_or_fail', return_value=META), patch.object(data, '_stream') as stream: + with self.assertRaises(android.FrameError): + data.restore_data(PKG, path) + stream.assert_not_called() + + + def test_successful_backup_is_private_and_inspectable(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / PKG).mkdir() + (root / PKG / 'save').write_bytes(b'checkpoint') + destination = root / 'backup.tar.gz' + def stream(command, src=None, dst=None): + REMOTE['backup'](root, PKG, META['instance'], dst) + with patch.object(android, '_meta_or_fail', return_value=META), \ + patch.object(data, '_stream', side_effect=stream): + result = data.backup_data(PKG, destination) + self.assertEqual(destination.stat().st_mode & 0o777, 0o600) + self.assertEqual(result['sha256'], data._sha256(destination)) + self.assertEqual(result['files'], 2) + + def test_rejected_restore_keeps_existing_data(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / PKG).mkdir() + (root / PKG / 'save').write_bytes(b'keep') + archive = root / 'bad.tar.gz' + self.make_archive(archive, [('data/link', tarfile.SYMTYPE)]) + with archive.open('rb') as source, self.assertRaises(ValueError): + REMOTE['restore'](root, PKG, META['instance'], source) + self.assertEqual((root / PKG / 'save').read_bytes(), b'keep') + self.assertFalse(list(root.glob('.frame-restore-*'))) + self.assertFalse(list(root.glob('.*.before-restore-*'))) + + def test_archive_root_must_be_a_directory(self): + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / 'bad.tar.gz' + with tarfile.open(archive, 'w:gz') as target: + target.addfile(tarfile.TarInfo('data')) + with self.assertRaisesRegex(ValueError, 'directory'): + REMOTE['inspect_archive'](archive, PKG, META['instance']) diff --git a/tests/test_sidequest.py b/tests/test_sidequest.py new file mode 100644 index 0000000..495d685 --- /dev/null +++ b/tests/test_sidequest.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'ui')) +from apk_sources import SourceError, sidequest + + +class SideQuestTests(unittest.TestCase): + def test_policy_is_recorded_and_source_is_page_only(self): + fixture = json.loads((Path(__file__).parent / 'fixtures/sidequest-policy.json').read_text()) + self.assertIn('/search/', fixture['robots']['disallow']) + self.assertIn('scraping', fixture['terms']['prohibited_activities_i']) + self.assertTrue(sidequest.sources()[0]['page_only']) + + @patch('urllib.request.urlopen', side_effect=AssertionError('network forbidden')) + def test_unknown_listing_never_claims_free_or_downloadable(self, _urlopen): + source = sidequest.sources()[0] + entry = sidequest.details(source, '123') + self.assertEqual(entry['page'], 'https://sidequestvr.com/app/123') + self.assertFalse(entry['downloadable']) + self.assertIsNone(entry['free']) + self.assertIsNone(entry['vr']) + self.assertEqual(entry['images']['screenshots'], []) + with self.assertRaisesRegex(SourceError, 'page-only'): + sidequest.download(source, '123') + with self.assertRaisesRegex(SourceError, 'page-only'): + sidequest.search(source, 'open saber') + self.assertEqual(sidequest.search(source, 'open saber', 0), []) + + def test_invalid_ids(self): + for value in ('../123', '1?paid=false', '１', '', '1/2', '1' * 13): + with self.assertRaises(SourceError): + sidequest.details(sidequest.sources()[0], value) diff --git a/ui/apk_sources/sidequest.py b/ui/apk_sources/sidequest.py new file mode 100644 index 0000000..dfed2ee --- /dev/null +++ b/ui/apk_sources/sidequest.py @@ -0,0 +1,41 @@ +"""SideQuest page links only: its terms do not authorise third-party scraping. + +No API calls, cached listings or automated downloads. See docs/sidequest.md. +""" +from . import SourceError + +KIND = 'sidequest' +URL = 'https://sidequestvr.com' +REASON = ('SideQuest is page-only: its terms restrict scraping and unauthorised ' + 'access. Browse and download with SideQuest, then import a developer-provided APK.') + + +def sources(): + return [{'id': KIND, 'kind': KIND, 'name': 'SideQuest', 'url': URL, + 'builtin': True, 'enabled': True, 'trust': 'community', + 'page_only': True, 'reason': REASON}] + + +def search(source, query, limit=50): + # Do not invent catalogue results or interpret a query as a verified free app. + if limit <= 0: + return [] + raise SourceError(REASON + ' ' + URL + '/apps') + + +def details(source, entry_id): + entry_id = str(entry_id) + if not entry_id.isascii() or not entry_id.isdecimal() or len(entry_id) > 12: + raise SourceError('SideQuest listing ids must be numeric') + return {'source': source['id'], 'id': entry_id, 'package': None, + 'name': 'SideQuest listing ' + entry_id, 'summary': REASON, + 'icon': None, 'page': URL + '/app/' + entry_id, 'version': None, + 'version_code': None, 'min_sdk': None, 'abis': None, 'vr': None, + 'size': None, 'free': None, 'license': None, 'updated': None, + 'downloadable': False, 'versions': [], 'tags': [], 'headsets': [], + 'images': {'icon': None, 'banner': None, 'screenshots': []}} + + +def download(source, entry_id, version_code=None): + entry = details(source, entry_id) + raise SourceError(REASON + ' ' + entry['page']) diff --git a/ui/frame_android.py b/ui/frame_android.py index 499e99e..80c39a0 100644 --- a/ui/frame_android.py +++ b/ui/frame_android.py @@ -8,6 +8,7 @@ Lepton Development, which wipes its apps on exit. See docs/apks.md. Python stdlib only. CLI: python3 ui/frame_android.py install APK [--vr|--flat] [--no-xr-compat] | info APK | versions APK-or-PKG + install-obb PKG OBB [OBB ...] | backup-data PKG ARCHIVE | restore-data PKG ARCHIVE patch SRC DST [--add NAME=PATH ...] | list | launch PKG | stop PKG | remove PKG | probe PKG """ import json, os, re, shlex, shutil, struct, subprocess, sys, threading, time, zlib @@ -333,6 +334,21 @@ def patch(src, dst, add=None): raise FrameError(str(e)) from e +def install_obb(pkg, paths): + import frame_android_data + return frame_android_data.install_obb(pkg, paths) + + +def backup_data(pkg, destination): + import frame_android_data + return frame_android_data.backup_data(pkg, destination) + + +def restore_data(pkg, archive): + import frame_android_data + return frame_android_data.restore_data(pkg, archive) + + def main(): cmd, *args = sys.argv[1:] or ['help'] try: @@ -368,6 +384,14 @@ def main(): with open(path, 'rb') as f: additions[entry] = f.read() r = patch(opts.src, opts.dst, additions) + elif cmd == 'install-obb': + if len(args) < 2: + raise FrameError('install-obb requires PACKAGE OBB [OBB ...]') + r = install_obb(args[0], args[1:]) + elif cmd in ('backup-data', 'restore-data'): + if len(args) != 2: + raise FrameError(cmd + ' requires PACKAGE ARCHIVE.tar.gz') + r = (backup_data if cmd == 'backup-data' else restore_data)(*args) elif cmd == 'list': r = list_apps() elif cmd in ('launch', 'stop', 'probe'): diff --git a/ui/frame_android_data.py b/ui/frame_android_data.py new file mode 100644 index 0000000..5a2e31a --- /dev/null +++ b/ui/frame_android_data.py @@ -0,0 +1,133 @@ +"""Expansion files and stopped-instance private-data backups. Python stdlib only.""" +import hashlib +import json +import os +from pathlib import Path +import re +import runpy +import shlex +import subprocess +import tempfile +import uuid + +import frame_android as android + +REMOTE = Path(android.ROOT) / 'frame/android/app-data.py' + + +def _stream(command, src=None, dst=None): + try: + result = subprocess.run(['ssh', *android.SSH_OPTS, android.FRAME, command], + stdin=src if src else subprocess.DEVNULL, + stdout=dst if dst else subprocess.PIPE, + stderr=subprocess.PIPE, timeout=1800) + except subprocess.TimeoutExpired: + raise android.FrameError('app-data transfer timed out') + except OSError as error: + raise android.FrameError('app-data transfer failed: ' + str(error)) + if result.returncode: + raise android.FrameError(result.stderr.decode(errors='replace').strip()[-600:] or + 'app-data transfer failed') + return result.stdout + + +def _sha256(path): + digest = hashlib.sha256() + with open(path, 'rb') as src: + for chunk in iter(lambda: src.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def _meta(package): + if not android.PKG_RE.fullmatch(package or ''): + raise android.FrameError('invalid package name') + meta = android._meta_or_fail(package) + if meta['package'] != package: + raise android.FrameError('installed app metadata has a different package') + return meta + + +def install_obb(package, paths): + if not android.PKG_RE.fullmatch(package or ''): + raise android.FrameError('invalid package name') + paths = [Path(p).resolve() for p in paths] + if not paths: + raise android.FrameError('select at least one OBB file') + names = set() + for path in paths: + if (not re.fullmatch(r'(main|patch)\.[0-9]+\.' + re.escape(package) + r'\.obb', path.name) + or not path.is_file() or path.stat().st_size == 0 or path.name in names): + raise android.FrameError('OBB must be a nonempty main/patch..' + package + '.obb file') + names.add(path.name) + with android._install_lock: + meta = _meta(package) + container = 'lepton-steamlaunch-' + str(int(meta['instance'])) + # No implicit launch. Android resolves /sdcard, never an assumed host path. + running = android.ssh('podman ps --format "{{.Names}}"').splitlines() + if container not in running: + raise android.FrameError('start this app instance before installing OBB data') + dest = '/sdcard/Android/obb/' + package + results = [] + for path in paths: + digest = _sha256(path) + part = dest + '/.frame-' + uuid.uuid4().hex + '.part' + target = dest + '/' + path.name + script = (f'set -eu; mkdir -p {dest}; umask 002; ' + f'trap "rm -f {part}" EXIT; cat > {part}; ' + f'test "$(sha256sum {part} | cut -d " " -f 1)" = {digest}; ' + f'chmod 664 {part}; mv {part} {target}') + command = shlex.join(['podman', 'exec', '-i', container, '/system/bin/sh', '-c', script]) + with path.open('rb') as src: + _stream(command, src=src) + results.append({'name': path.name, 'path': target, 'sha256': digest}) + return {'package': package, 'instance': meta['instance'], 'obb': results, + 'verified': True} + + +def _data_command(action, meta): + container = 'lepton-steamlaunch-' + str(int(meta['instance'])) + # Fail closed if podman cannot enumerate containers; don't mistake errors for stopped. + guard = ('running=$(podman ps --format "{{.Names}}") || exit 1; ' + f'if printf "%s\\n" "$running" | grep -Fxq {shlex.quote(container)}; then ' + 'echo "stop the app before backup or restore" >&2; exit 1; fi; ') + return guard + shlex.join(['podman', 'unshare', 'python3', '-c', REMOTE.read_text(), + action, meta['package'], str(int(meta['instance']))]) + + +def backup_data(package, destination): + destination = Path(destination).expanduser().absolute() + if destination.exists(): + raise android.FrameError('backup destination already exists') + with android._install_lock: + meta = _meta(package) + fd, temporary = tempfile.mkstemp(prefix='.frame-backup-', dir=str(destination.parent)) + try: + with os.fdopen(fd, 'wb') as dst: + _stream(_data_command('backup', meta), dst=dst) + result = _inspect(temporary, meta) + # Exclusive publication: a concurrently created backup is never overwritten. + os.link(temporary, str(destination)) + return dict(result, path=str(destination), sha256=_sha256(destination)) + finally: + os.unlink(temporary) + + +def _inspect(path, meta): + import tarfile + try: + return runpy.run_path(str(REMOTE))['inspect_archive'](path, meta['package'], int(meta['instance'])) + except (OSError, EOFError, ValueError, tarfile.TarError) as error: + raise android.FrameError('invalid app-data backup: ' + str(error)) + + +def restore_data(package, archive): + with android._install_lock: + meta = _meta(package) + _inspect(archive, meta) + with open(archive, 'rb') as src: + result = _stream(_data_command('restore', meta), src=src) + try: + return json.loads(result) + except (ValueError, TypeError): + raise android.FrameError('could not read restore result; inspect app data before retrying') From 784a48f218abdfecbd3c98cdf230eb9d7f26c4fe Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:06:06 +1000 Subject: [PATCH 004/122] Add authenticated F-Droid user repositories and management CLI Verify pinned JAR/CMS signatures, v2 index hashes and APK downloads; support signed v1 fallback and persist TOFU identities. Reuse the catalogue reducer and document repository publishing with offline and live verification evidence. Co-Authored-By: GPT-6 Astra --- .claude/NOTES-user-repos.md | 55 +++ .claude/user-repos-proof.json | 50 +++ docs/apk-repos.md | 131 +++++++ tests/fixtures/fdroid/README.md | 14 + tests/fixtures/fdroid/entry.jar | Bin 0 -> 1834 bytes tests/fixtures/fdroid/example.apk | 1 + tests/fixtures/fdroid/fingerprint.txt | 1 + tests/fixtures/fdroid/index-v1.jar | Bin 0 -> 1917 bytes tests/fixtures/fdroid/index-v2.json | 1 + tests/fixtures/fdroid/izzy-entry.jar | Bin 0 -> 3300 bytes tests/test_fdroid_sources.py | 176 ++++++++++ ui/apk_sources/fdroid.py | 474 ++++++++++++++++++++++++++ 12 files changed, 903 insertions(+) create mode 100644 .claude/NOTES-user-repos.md create mode 100644 .claude/user-repos-proof.json create mode 100644 docs/apk-repos.md create mode 100644 tests/fixtures/fdroid/README.md create mode 100644 tests/fixtures/fdroid/entry.jar create mode 100644 tests/fixtures/fdroid/example.apk create mode 100644 tests/fixtures/fdroid/fingerprint.txt create mode 100644 tests/fixtures/fdroid/index-v1.jar create mode 100644 tests/fixtures/fdroid/index-v2.json create mode 100644 tests/fixtures/fdroid/izzy-entry.jar create mode 100644 tests/test_fdroid_sources.py create mode 100644 ui/apk_sources/fdroid.py diff --git a/.claude/NOTES-user-repos.md b/.claude/NOTES-user-repos.md new file mode 100644 index 0000000..6833762 --- /dev/null +++ b/.claude/NOTES-user-repos.md @@ -0,0 +1,55 @@ +# User repositories + +Scope: ui/apk_sources/fdroid.py, fixture tests and docs/apk-repos.md. No headset access. +No delegation or independent reviewer launched: task explicitly forbids delegation; +parent integrates and reviews. Existing catalogue API stays unchanged. + +Decisions: +- Support F-Droid signed v2 and v1, HTTPS only, RSA PKCS#1 CMS/JAR verification. +- Pin operator certificate fingerprints. Without a supplied fingerprint, verify + the complete signature chain of hashes on first use, then persist that signer. +- Reuse frame_catalog._IndexReader and _reduce_index; keep authenticated source + cache separate from legacy unauthenticated catalogue cache to avoid laundering trust. +- Sources list compatible builds (Android <=30, arm64 or no native code), as + existing catalogue reducer does. This is compatibility filtering, not a runtime guarantee. +- No Obtainium export import or new unsigned JSON format in this source. + +Research: downloaded F-Droid API/setup docs, Obtainium and SideQuest READMEs, +Izzy repo page and entry.jar via HTTPS. Izzy's published fingerprint matched +pure-Python CMS validation: 3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A. + +## Final verification + +All commands below ran in /Users/saphid/projects/steam-frame-userrepo on user-repos. + +- `python3 --version`: Python 3.9.6. +- `python3 -m unittest discover -s tests -p test_fdroid_sources.py`: initially + 13 tests, OK, exit 0. Added a cache-corruption test afterward. +- Final `python3 -m unittest discover -s tests`: 180 tests in 6.866s, OK, + exit 0 (includes 14 source tests and existing catalogue/version tests). +- `python3 ui/apk_sources/fdroid.py add 'https://apt.izzysoft.de/fdroid/repo?fingerprint=3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A' --name 'IzzyOnDroid verification'`: exit 0; + saved fdroid-user-57e98c13877f14fdea65 with the published pin. +- `python3 ui/apk_sources/fdroid.py search fdroid-user-57e98c13877f14fdea65 'tinymusicplayer'`: + exit 0; com.martinmimigames.tinymusicplayer, version 1.3 / code 4, + GPL-3.0-only, 16,520 bytes. +- `python3 ui/apk_sources/fdroid.py download fdroid-user-57e98c13877f14fdea65 com.martinmimigames.tinymusicplayer`: + exit 0, verified true. Independent hashlib readback matched + d7bcb24d101b04beb3394b695b24be4e2c3d6ed702f1d0e06bc4dd707f64d86a. +- Direct `_fetch` + `_jar` calls on F-Droid main/archive entry.jar: exit 0; + both matched the published 43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab pin. +- `.claude/user-repos-proof.json` retains live CLI results and APK readback. + Live APK remains in the per-user apk-sources cache; the added source remains + in the per-user apk-repos.json, as requested for the real add/search/download run. + +Not verified: headset installation/runtime, native UI/server integration (sibling +worker), real v1-only server (offline signed fixture covers fallback), executing +the fdroidserver publishing instructions, full F-Droid main/archive index/APK +downloads (their live signed entry jars were checked). No independent reviewer +was run because this task forbids delegation and assigns review to the parent. + +Known limits / follow-up questions: only one RSA-2048–8192 JAR signer supported; +no ECDSA/DSA/PSS or section-only SF signatures; no index timestamp rollback or +expiry policy, automated key rotation or cross-process settings-write locking. +The settings API serializes threads and publishes atomically. Should a later +change add explicit rollback policy and broader JAR algorithms? Parent may +choose UI wording for TOFU; this source already returns trust_on_first_use. diff --git a/.claude/user-repos-proof.json b/.claude/user-repos-proof.json new file mode 100644 index 0000000..71d737d --- /dev/null +++ b/.claude/user-repos-proof.json @@ -0,0 +1,50 @@ +{ + "add": { + "id": "fdroid-user-57e98c13877f14fdea65", + "kind": "fdroid", + "name": "IzzyOnDroid verification", + "url": "https://apt.izzysoft.de/fdroid/repo/", + "builtin": false, + "enabled": true, + "trust": "user", + "fingerprint": "3bf0d6abfeae2f401707b6d966be743bf0eee49c2561b9ba39073711f628937a", + "trust_on_first_use": false + }, + "search": [ + { + "source": "fdroid-user-57e98c13877f14fdea65", + "id": "com.martinmimigames.tinymusicplayer", + "package": "com.martinmimigames.tinymusicplayer", + "name": "Tiny Music Player", + "summary": "Android 1.0+ minimal (", + "icon": "https://apt.izzysoft.de/fdroid/repo/com.martinmimigames.tinymusicplayer/en-US/icon.png", + "page": "https://martinmimigames.github.io/projects/tiny-music-player/index.html", + "vr": null, + "free": true, + "license": "GPL-3.0-only", + "downloadable": true, + "version": "1.3", + "version_code": 4, + "min_sdk": 1, + "abis": [], + "size": 16520, + "updated": "2023-02-04" + } + ], + "download": { + "apk": "/Users/saphid/Library/Caches/Frame Control/apk-sources/d7bcb24d101b04beb3394b695b24be4e2c3d6ed702f1d0e06bc4dd707f64d86a.apk", + "obb": [], + "sha256": "d7bcb24d101b04beb3394b695b24be4e2c3d6ed702f1d0e06bc4dd707f64d86a", + "verified": true + }, + "independent_readback": { + "size": 16520, + "sha256": "d7bcb24d101b04beb3394b695b24be4e2c3d6ed702f1d0e06bc4dd707f64d86a" + }, + "python": "3.9.6", + "suite": "python3 -m unittest discover -s tests: 180 tests, 6.866s, OK, exit 0", + "builtins_entry_signatures": { + "fdroid": "43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab", + "fdroid-archive": "43238d512c1e5eb2d6569f4a3afbf5523418b82e0a3ed1552770abb9a9c9ccab" + } +} diff --git a/docs/apk-repos.md b/docs/apk-repos.md new file mode 100644 index 0000000..6e87879 --- /dev/null +++ b/docs/apk-repos.md @@ -0,0 +1,131 @@ +# APK repositories + +Frame Control supports **F-Droid-format repositories**, including F-Droid, +F-Droid archive, IzzyOnDroid and user-provided HTTPS repositories. Repository +indexes are authenticated before their apps appear. Search lists builds with +Android API ≤30 and arm64-v8a or no native libraries, using the same streaming +reducer as the existing catalogue. This does not guarantee an app works in Lepton. + +## Formats considered + +| Format | Users and purpose | Support in this source | +|---|---|---| +| F-Droid v2 | F-Droid, IzzyOnDroid, self-hosted fdroidserver repositories; consumed by F-Droid clients including Droid-ify and Neo Store | Preferred: signed `entry.jar` authenticates `entry.json`; its SHA-256 authenticates `index-v2.json`, which supplies APK SHA-256 hashes | +| F-Droid v1 | Older F-Droid servers and clients | Fallback: verify `index-v1.jar`, then read its signed `index-v1.json` | +| Obtainium configurations / exports | Obtainium users share app URLs plus source-specific filters and update settings; exports can contain a list of app configuration objects | Not imported here: configurations describe how to find releases, not one signed repository index | +| SideQuest listings / custom feeds | SideQuest's own app discovery and installation service | No interoperable signed custom-repository specification was established from the public project documentation examined; SideQuest needs its own adapter | +| GitHub release lists | Developers publish APK assets on release pages; community lists link to projects | Not a repository standard: asset naming, build selection and publisher verification vary; handled separately from this F-Droid source | +| Minimal JSON list | A private list could contain package, title, APK URL and SHA-256 | Deliberately not introduced: unsigned hashes downloaded alongside files do not authenticate their publisher; another bespoke signing/update protocol would duplicate F-Droid | + +Research references (checked 2026-09-28): + +- [F-Droid APIs](https://f-droid.org/docs/All_our_APIs/) and + [repository setup](https://f-droid.org/docs/Setup_an_F-Droid_App_Repo/). +- [F-Droid signing keys](https://f-droid.org/docs/Release_Channels_and_Signing_Keys/) + and [IzzyOnDroid's repository page and fingerprint](https://apt.izzysoft.de/fdroid/). +- [Droid-ify](https://github.com/Droid-ify/client) and + [Neo Store](https://github.com/NeoApplications/Neo-Store). +- [Obtainium](https://github.com/ImranR98/Obtainium), its + [configuration/deep-link format](https://wiki.obtainium.imranr.dev/deep_links/), + and [community app configurations](https://apps.obtainium.imranr.dev/). +- [SideQuest's public client](https://github.com/SideQuestVR/SideQuest). + The absence of a specification in these materials is not proof that no + historical or private custom-feed format exists. + +## Add a repository in Frame Control + +From the Frame Control checkout, use its source-management CLI: + +```sh +python3 ui/apk_sources/fdroid.py add 'https://example.org/fdroid/repo?fingerprint=YOUR_64_HEX_CERTIFICATE_FINGERPRINT' --name 'My apps' +python3 ui/apk_sources/fdroid.py list +python3 ui/apk_sources/fdroid.py search SOURCE_ID 'music' +python3 ui/apk_sources/fdroid.py download SOURCE_ID org.example.app +python3 ui/apk_sources/fdroid.py remove SOURCE_ID +``` + +Replace `SOURCE_ID` with the `id` printed by `add` or `list`. `--fingerprint` +can also supply the pin. `fdroidrepos://example.org/fdroid/repo?fingerprint=…` +links are accepted and converted to HTTPS. Conflicting fingerprints are refused. +A URL must identify the repository directory, not its website or an index file. + +Adding fetches and validates the complete index **before saving** the source. +Without a fingerprint, Frame Control verifies the JAR signature and remembers +its signer: trust on first use (TOFU). This establishes continuity with the +first server response, not independent publisher identity. Obtain the published +fingerprint through a trusted channel when possible. Re-adding an existing URL +preserves its pin; changing it requires deliberately removing and re-adding it. + +The API for the search/server integration is in `ui/apk_sources/fdroid.py`: +`add_repo(url, fingerprint=None, name=None)`, `remove_repo(source_id)`, +`set_enabled(source_id, enabled)`, and `user_repos()`. The module also exposes +`sources`, `search`, `details`, and `download` from the shared source contract. +This change supplies the CLI and API; the integrated source-management UI is +separate work. Built-in sources can be disabled but cannot be removed. + +Settings and pins live in `frame_host.data_dir('apk-repos.json')` +(`~/Library/Application Support/Frame Control/apk-repos.json` on macOS). +Authenticated reduced indexes and APKs live under +`frame_host.cache_dir('apk-sources')`; indexes refresh after 24 hours. +The existing catalogue's unverified index cache is never treated as authenticated. + +## Publish your own repository + +Only publish free APKs you own or have the developer's permission to distribute. +Do not publish paid app mirrors or bypass store licences. Check distribution +terms before adding someone else's repository; this module does not infer legal +permission from a signature or automatically audit a repository's terms. + +Install a current [fdroidserver](https://f-droid.org/docs/Installing_the_Server_and_Repo_Tools/) +and its documented Android/Java dependencies on the publishing machine, then: + +```sh +mkdir my-fdroid +cd my-fdroid +fdroid init +# Set repo_url in config.yml to https://example.org/fdroid/repo +# Also set repo_name and repo_description; keep the generated signing key safe. +cp /path/to/your-free-app.apk repo/ +fdroid update --create-metadata +# Review the generated metadata (name, summary, licence, source and website). +fdroid update +``` + +Serve the generated **repo directory** at that HTTPS URL, including APKs, +icons, `entry.jar`, `index-v2.json` and `index-v1.jar`. Do not publish the +private signing keystore or configuration passwords. Configure fdroidserver's +`serverwebroot` and run `fdroid deploy` for managed publication, or copy the +public directory with your existing deployment tool. Publish the SHA-256 +repository certificate fingerprint displayed by fdroidserver in a link such as +`https://example.org/fdroid/repo?fingerprint=…`. + +Keep the repository signing key backed up: changing it breaks existing pins. +For updates, add the new APK, edit metadata as needed, run `fdroid update` and +publish again. Test the published URL with Frame Control's `add`, `search` and +`download` commands. The above publisher setup is documented from fdroidserver; +it was not executed as part of this implementation. + +## Verification and limits + +The stdlib verifier supports one RSA PKCS#1 v1.5 JAR/CMS signer with a key of +2048–8192 bits; SHA-256/384/512 and legacy SHA-1 digest encodings are +recognized. It checks the signer certificate pin, the signature over `.SF`, +the whole-manifest digest, and the manifest's digest of the JSON member. +ECDSA, DSA, RSA-PSS, multiple signers and section-only `.SF` manifests are +rejected. Certificates are pinned identities, not validated as Web PKI chains. +HTTPS certificates are separately checked by Python's normal TLS validation. + +v1 fallback occurs only when `entry.jar` returns HTTP 404 or 410. Signature, +fingerprint, index hash, TLS and server errors never trigger an unsigned +fallback. APKs are cached by SHA-256 and checked again before reuse. Here, +`verified: true` means the bytes match the signed repository's APK hash; it +is not an independent APK publisher-signature or runtime compatibility verdict. +There is no repository timestamp rollback/expiry policy or automated signing-key +rotation yet. An old correctly signed index can still validate. + +Offline fixtures exercise v2, v1, TOFU, pin changes, disabled sources, cache +reuse, URL rejection and corruption of every signature/hash layer. On the Mac, +the real IzzyOnDroid repository was added with its published pin, searched for +Tiny Music Player, and its 16,520-byte APK downloaded with SHA-256 +`d7bcb24d101b04beb3394b695b24be4e2c3d6ed702f1d0e06bc4dd707f64d86a`. +No headset connection or installation was performed. diff --git a/tests/fixtures/fdroid/README.md b/tests/fixtures/fdroid/README.md new file mode 100644 index 0000000..01b7eb6 --- /dev/null +++ b/tests/fixtures/fdroid/README.md @@ -0,0 +1,14 @@ +# F-Droid verification fixtures + +`entry.jar` and `index-v1.jar` are synthetic RSA-2048/SHA-256 signed JARs, +including CMS signed attributes. `fingerprint.txt` identifies their throwaway +certificate. Their JSON describes org.example.app; `example.apk` is deliberately +plain test data, not an installable app. The v2 index includes incompatible +Android-31 and x86-only versions to exercise the shared reducer. + +`izzy-entry.jar` was recorded from +https://apt.izzysoft.de/fdroid/repo/entry.jar on 2026-09-28. Its certificate +fingerprint matches the operator's published fingerprint: +3BF0D6ABFEAE2F401707B6D966BE743BF0EEE49C2561B9BA39073711F628937A. +It exercises an independent production JAR/CMS encoder without network access. +The index it references is not needed by this signature-only fixture test. diff --git a/tests/fixtures/fdroid/entry.jar b/tests/fixtures/fdroid/entry.jar new file mode 100644 index 0000000000000000000000000000000000000000..b73864ad0eddc09ab576a3dad6528fb1f46b7512 GIT binary patch literal 1834 zcmZ{ldpHwn9LHCq3rRwv&Phj1?2=`PIXToOo1vkda+!NBg{UNJF3E8$bu!m%R!v7T zluKEda!RsW8tLY85<7$~+SbL|na+7m<(%`r&-Zzs-}^q__x(J-KfbRQ7N}tW005Q% zfR`NuO!S74(dsHjt+`zs%EuYv=7EOcoITvoC~qGq4n2e8aow>HXtxd)VkO`S>+zd< zHV=)w7@5ZMaEw&S89Tw<9WboyUnk7pBUSF|I3VCtCYNk1@_>#?Mj>{XZ zEpQiN!5bl%-mk&FfbdjP%jWQbaF#(sKaaG`x=ZOI;#=ov<9QGnFk>xuntR+^D*>NK zsdF*&ubV ziLRBQgUuPiOv>~m^|VAB=NMT_Hiz!!$wj$}F_7_*rcER#J*MCeuQTmiXWsd=*B|1E zRq(0Qy-K~QJ1zlFfyDJB>LWxES2z1 zKv8Y(f;h~B>bJ;5EJvEN#-`01tg&x1pH$2VIT9H7y1SRlQiZWkqG2B!EF4|zZUsM@ zR8quEnuHW51+nplTvP)db6S4o=hP`9QIUJi9=oR&|UnM|J2cV8o^S5?pKvx1K+$Fv)d zR+zW=lVXIfoNy9sYuvVCP-WRiZfo7q$%1&qjjG$mEb+uKBTH@t0j}cM_9o>iQz^#w z_nKf8+fqq~pYkKqWmdm(>W5cdaz(M?-_~>=kq(ZVAqk5~@H)EkZO78ljuU&XCA)}w zCK8rm_+*c3hPp$(a!GbV!XHYxU!_^f!%VCn#^6BALnC`cczL4lx#O)-4t^1^r2S9j z<}`TGsnAhayz1`tt@pN@9VSb%`YH^x$z>yLZ#a>OGh|iHx^~sPG#NDIQ%UL+p{u2( zR(R~8gk)%RVq7dLm#_7k>55FE`2^dkCW0{eY;`|g`)S7d{H&G~9F#7S@;BAD(A3us^7@OPxochAq<&u(iq!9VYRa;p<~zI}d^KiiWvzihb|R&&wZnTC2Q K)ay5&sy_i1JRfWT literal 0 HcmV?d00001 diff --git a/tests/fixtures/fdroid/example.apk b/tests/fixtures/fdroid/example.apk new file mode 100644 index 0000000..e53f5b7 --- /dev/null +++ b/tests/fixtures/fdroid/example.apk @@ -0,0 +1 @@ +Fixture APK payload, deliberately not installable. diff --git a/tests/fixtures/fdroid/fingerprint.txt b/tests/fixtures/fdroid/fingerprint.txt new file mode 100644 index 0000000..494e9b0 --- /dev/null +++ b/tests/fixtures/fdroid/fingerprint.txt @@ -0,0 +1 @@ +0e87b227cd414d7093fb150fda81f1754900f3abc810e6785d8e0767c6eb798a diff --git a/tests/fixtures/fdroid/index-v1.jar b/tests/fixtures/fdroid/index-v1.jar new file mode 100644 index 0000000000000000000000000000000000000000..6ea58d2a306f7904f24be201d4d86587c52724cc GIT binary patch literal 1917 zcmZ{l4>TLr9>+siTNb5st+IcmHIY9J!X`RTw`fU%CMr{e_%l=+|62bdSZSvXb;#Dx zgs4J z>X`umfXx8CYp%gtkk+CTnv9}}ja?kt&mHRN?E%NRdwY7I34Sn~#{y0&!D67W9uNi! zmPzFe_>k#=1MNa!KYlk!tggeTmqN#EQ6;QJ>$C@F1#tijp{ zQP@hFDxS3eT};ADPV~G3Q3@e15pxHdPRA-jAYi7#XuGkBO9@ zb5yenjKpCdfyzE0Rc#}j?Z^Golu_*Mz5erp;|im> zqA0YU`@9dX8Zx~yxjmtk8sdmfcsb0ebVjjPfK&_MTi#Il6o1Du9|^B->6;{~?w!V! zqZGyEFXI>sfv>u@g^#qHCUfcYO2&oRsW{i@y39Q=f2q2+U}@UMJh^#ahMNgj)-Dxi zf8s8!$bS7cKBUHeChO>`$xMI|>N~p{Y#Rym##R=k<{PJ1Pm&Hh10vAtXTbq8z$2O4`DbNu<|3eG=Y!gy4(^ zeC^g=*8|RX$&Zw)^r>cjBGqjvq_IieTML&fuyh)wDkcbE}g9EhAnxCKv= zBPl6eF|yd28$9&&v7(|gHnIja%%yQ&J#V#3nQw}>=;xBo<9Y1EA1<2ML(Y0Z z3`#8u&GK4qbvycWyTqyrqo(->f85!;JGUDNBcf+WeXCob<_(Sr`&8E#ZkOJ)N5e76 z8Pgf2Ie?ZkA^MeLrv3fR$E7FF8}kew9#v z4FVb13E|zKo=aE+tB_PV=?N(*3(ov5kbgG^O&>aMJux+Z-MhSPWq>D1j1>{az^7TM zl9^hA?@7YfYi&U{2P>;y5khF`4{km^$8cU4QFNyxuU1a4#i9C{-YK((od(udY73utz>X!wQ6gpgx1Qc-oc&A~=3?;;rnaIcA^stHJk0Wf z+E_#kH|1_yYuTUMsS-%2{e~xH^zN|%!ym7TP#@pC45*|I5*I}seYXrSVFF{NS$T+ON`L^xw9lsf>ORQ}{ zw)k?MNz$I8dc_NCU6#4n?2vBtk>Uv_XXDsB$boPDOTR>Rrc=sBD6Qj$eiZ?m;#=bU0B3qH{p{&U^n5>12EmSke z&e*0gStk2Bmi+4b&$slS|9#JW?>X3U z1ppjQ002x!w+MG%Z^7Ly(BQu$N|`2A+Gox`rY3p6>y>fW;64M zMJ$C%AfC?(nBY@yiU`?%eC5C&o$vQ;2bnXc%Nu^Rp;>2G@J8=-8w`tr=U7Qs*bR12 zi~U;p&O&n|jd1@r0wL>%a&VzML!zaYb(89Y$JC(x%6PGdjaV~ZHN?58E<=mzubZZ) zmo1E<)SLR3VV$8XY7%Zr3huWE201F1i9o(edmvp6Jq}fag%d zE)M~*A6tM8^b`#wSv;VMG%^;Q(zjhZ6rh<2!igBTG39TGD%Z;}v_smi6IOu9r{<3j z%(upj%(2Z{VB3_d>hw^wuYxw)rjJV^yn?->T_B++6%?c2sK|oHggc|d_-n}-Nw`J9 zLKMfnX4KHK=UuK8!or9TWV;|puKEJz!#JmLvU?Ho64NxXlwE>kZ{{NqOKHt#B)AdJ zcE4b-jCqi~o;Ot*#BI$kjPB8DyjDiftCVSWR}dz`VX^sacy z)1R7Ml|LrjeYg=65_qF4mGxWsxydB?T4fU2*yTy4uhV*a&yzYcY-+lF5T&ikq~Y85 z!MXitQ=W^#zw+x_i{#93Gfj$v6H0piAj{!1_%didboueJL004`e7dddj8OqET$t)! z+$Tym{3?U_cE;^?PW8nCt0cN4dN!Wn$WJ5IV0@c&C)~Dl>$@5x2W@udAfsyh6+w9+ zO<8EQ`{eEsr`vn=C4Rb)LO0792&+U)xs+tq=j<29G8%{<)ui`O z+HCzOy@}jWpVh+7&cX!j3(W?z(!-1+HQYiGn1MU~i^xyTTIpIYK6BrdOVjSZvpLKP z?ed^>@9*yJ%EA6@e}+HXpN*+`zWS)`yN~SDkr`;gp)$IL+H!_k8b-3F+EzFya~Swc zD6wJs!N?n+{u4`aU+}fWKA3(9q08v@d}QY9O$u}Tg!N=w5+*w!MfLMMx18r$fFChUk++cE)Dh10m+>U zLLLdd(p&qViaxG>X@R~ptAC9YENj+o=A~w-CQlz$$E``mo`q^IMwMCMCU(u>pPvY1 zcCqTynk)E^{YVUaS8#ll`QHqik>?7ejvhDa$k_gep@}IpF4I(U=AZ znO_Udb22tD(3g2*YUf;9P*LELZTIYz%a{q$xF$9dguG^{PtOrHR%hAIQWiYxBwq7W z*|9qvHD$EEsy?gU^PWFz{aVsH4dS?@*YH>WQvhzIY#cd4)CUzx0}u|?Zhq&{VRcMZ z5C#Su%S9bp)2)5%RaOLDjt~u3Sr@bgutbhwXNPwC*M@=3L3RmkFErNM1c5N?JF@mQ zQqb-9%eJS)pkhp#XBL*{z4&EhBAe%OdhYaQIK@6q6@{|o`T>Kj*Y|Bm*cgFu@*!$x z#+(5p4Y>RT)$9NUNCuecA5B3=TLxB*=ugsn|AN-=nTxM^UCh%1GSLQg#f8s z5sbc#IWSxa!(IUD9jiU?D)wB;UobqM@)d6vca~YQ0L1DjCztAf&StbqCqb-3tnTXQ zDr#Fa2*ffx3b|Tslh`*MzI}?t2eX&F7*nL{^>)p7@I_%O3T#@`2$+UCphJi4(srKd zP;;2@`7>#cCew>f>|sMI(dbNb#-LiG^3%$ZlYTSQ^d_5&dV222j;3l-BMZ|-{_J)L z_0vJ$aD~%_!-Yac!cgOU_PS|$NW}bx_cCNl#x#FkhyO#cYnc`)AxSlS{ya{M;}y5D zn4034zZsLyRcYa2f8#J(gxjDRX`|McNlwv%Vi?>#mjBg*zSoA)cj&r1ZU4=ecTkg& zHB}0fI8l}k$9uq%=e2B#Av4aBPxO|iif4AL;=~HhDF;6Zex9M4k$YRaLB1vzjj?+S z->+=CQdA5pgux#Oz&;4smR46~gbRG>2G!qe2uVfYTbijDyFCe#5&3qJ`vPLmL-TNQ z<>A00m4OvD+i%P~c|f*rc|N-B$ZsI`w$#JNxxU5!z3YwxR6V5mYxQj>5_z9(qHCH9 zOb4K$_jUqR59qqR+R4=^iXIutHAYAbl|1sbFk%Qg6!m@`zw=-*Vt@3dqr3COP2Hq- z1Z(^9tq=T=B>Hx)W<;@T2egbQhp=% zt%#0Y=Qc(vN_C^K{KX5RD?6NL7&=sOYlAjcYbihcxR(kk?lPTdbq#ALIqucHxkE(l ztwMP*VJ}?I>!`Gb(r0GL6#oSoDt+?H0(MWh`AqQy;JwUqv)ZgAUVKsPnI`)RBFGNZ zv73d-!zQ+r-&eWINBNY~+k`b22niZxAx3z7Zf`inAurhNm$W?`zk0UhEQOms zK88)?4pq@vC?x3vMdO0)W`Yul!#L5S^2WpbWkcw)-QpN>X9VXV6#U26y%RA;C7$CK z9+vuLP6`{!yt>Du(u<>WwZ|qpvn*6*cV9N=^NM-iiUsbyr^f;eB)!<+JVr#W3d4Bg zlU{qVw+=!T2;TSX&A^VB9S70<>KBhs@&?Vz2M<;xDah!W=ro|hfvEDu*1#X+21MWs zx|<*bLRb|otswt#ayqr?VC3aP-4$-0dQ95V%GDFwk`e9Sb*o61Zv|k@YgLF3>NRhs zPrO#G7VL-{kB5Ioyy8|$Ib?_&!f^|gX7kp3I}^9mu8-$!%3mGbQ6iY^ND=z>B=Vfx z)LYwWvu32#sj@i^g76&PspAcvbN88^Ruxsy1;SrHrIUBS95CvjjiRH?esF_#Gs6!x zXAk=r^^S+{7yH{|ThyjARmv5Z3OE0dV2rYYB!vVlW@Au=D_u?;$;Ne*LZZ^hWcE03eb`@+I4vMF0Vm=X6JXl z8B&yL;QP8W!-2GRAux~iB{r^epooVPdeZ2NWH{-f=oz*Nt)gL)ixEc{m74lc>R8^m{)km&D=RFrzgp&eRF=)9p|lQLgR^puIu3FK5%IFR z0N9~063;$Si+1LZFTd*#li&fCeLN2AtzsZ0<7-A;xtG$y(sg4FqVRc?J!q<&>|!dI zC%+|HGl_oCsx3=J(y>~^z0s@psgS7mSk;t-&2CKyZ%Xc@O!kym%?!J%N@FG8X}-0K z#TBGL#;pEUH337Z2VSu3&<2Fl^Vw?+&%f=AC3wKZTfz~GtA=HRdlt7%?$;oD`2Rgw zLzx=uT#hCx^2mM)>c6yu0n|`eTgwzKYoNVufctzyUqD{omH=` zfNn{k;tivX_^uV^kN)dmS^TChYl&VReL4dGrcXo5dF=lS`=bPZt?GwA-QP0*7w%{N z|3?x1iTZgS9(@!3;`m>E^gH@@wedHac 1: + raise SourceError('only one fingerprint query parameter is supported') + pin = _fingerprint(fingerprint) if fingerprint else None + if pins: + linked = _fingerprint(pins[0]) + if pin and pin != linked: + raise SourceError('conflicting fingerprints') + pin = linked + return urllib.parse.urlunsplit(('https', p.netloc.lower(), p.path.rstrip('/') + '/', '', '')), pin + + +def _child(base, name): + name = str(name).lstrip('/') + decoded = urllib.parse.unquote(name) + if not name or '\\' in decoded or any(x in ('.', '..') for x in decoded.split('/')): + raise SourceError('unsafe repository file name') + url = urllib.parse.urljoin(base, name) + if not url.startswith(base) or urllib.parse.urlsplit(url).query or urllib.parse.urlsplit(url).fragment: + raise SourceError('repository file is outside its repository') + return url + + +class _HTTPSRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + if urllib.parse.urlsplit(newurl).scheme != 'https': + raise SourceError('refusing non-HTTPS redirect') + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _fetch(url, path, maximum): + request = urllib.request.Request(url, headers={'User-Agent': 'FrameControl/1.0'}) + with urllib.request.build_opener(_HTTPSRedirect()).open(request, timeout=60) as r, open(path, 'wb') as f: + total = 0 + while True: + chunk = r.read(1 << 20) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise SourceError('repository file exceeds size limit') + f.write(chunk) + + +def _children(item): + return _der_parts(item[1]) + + +def _cms(data, content): + outer = _der_parts(data) + if len(outer) != 1: + raise ValueError('invalid CMS wrapper') + wrapper = _children(outer[0]) + if wrapper[0][1].hex() != '2a864886f70d010702': + raise ValueError('not CMS SignedData') + fields = _children(_children(wrapper[1])[0]) + certs = next(_children(f) for f in fields[3:] if f[0] == 0xa0) + signers = _children(fields[-1]) + if len(signers) != 1: + raise ValueError('exactly one repository signer required') + signer = _children(signers[0]) + sid = _children(signer[1]) + matching = [] + for cert in certs: + tbs = _children(_children(cert)[0]) + offset = 1 if tbs[0][0] == 0xa0 else 0 + if tbs[offset][1] == sid[1][1] and tbs[offset + 2][2] == sid[0][2]: + matching.append(cert[2]) + if len(matching) != 1: + raise ValueError('missing or ambiguous signer certificate') + cert = matching[0] + digest, prefix = _DIGESTS[_children(signer[2])[0][1].hex()] + at, signed = 3, content + if signer[at][0] == 0xa0: + attrs = {} + for attr in _children(signer[at]): + pair = _children(attr) + oid = pair[0][1].hex() + if oid in attrs: + raise ValueError('duplicate CMS attribute') + attrs[oid] = _children(pair[1]) + if attrs['2a864886f70d010904'][0][1] != hashlib.new(digest, content).digest(): + raise ValueError('CMS content digest mismatch') + if attrs['2a864886f70d010903'][0][1].hex() != '2a864886f70d010701': + raise ValueError('unexpected CMS content type') + signed = der(0x31, signer[at][1]) + at += 1 + algorithm = _children(signer[at])[0][1].hex() + allowed = {'sha1': '2a864886f70d010105', 'sha256': '2a864886f70d01010b', + 'sha384': '2a864886f70d01010c', 'sha512': '2a864886f70d01010d'} + if algorithm not in ('2a864886f70d010101', allowed[digest]): + raise ValueError('unsupported repository signature algorithm (RSA PKCS#1 required)') + n, e, _ = _cert_key(cert) + sig = signer[at + 1][1] + size = (n.bit_length() + 7) // 8 + if not 256 <= size <= 1024 or n % 2 != 1 or not 3 <= e <= 0xffffffff or e % 2 != 1 or len(sig) != size or int.from_bytes(sig, 'big') >= n: + raise ValueError('invalid RSA signature/key size') + value = bytes.fromhex(prefix) + hashlib.new(digest, signed).digest() + expected = b'\0\1' + b'\xff' * (size - len(value) - 3) + b'\0' + value + if pow(int.from_bytes(sig, 'big'), e, n).to_bytes(size, 'big') != expected: + raise ValueError('repository RSA signature mismatch') + return hashlib.sha256(cert).hexdigest() + + +def _sections(data): + sections = [] + for block in re.split(b'\r?\n\r?\n', data): + if not block: + continue + attrs = {} + for line in re.sub(b'\r?\n ', b'', block).splitlines(): + key, value = line.decode('utf-8').split(': ', 1) + key = key.lower() + if key in attrs: + raise ValueError('duplicate manifest attribute') + attrs[key] = value + sections.append(attrs) + return sections + + +def _digest_check(attrs, suffix, content): + for label, digest in (('sha-512', 'sha512'), ('sha-384', 'sha384'), ('sha-256', 'sha256'), ('sha1', 'sha1'), ('sha-1', 'sha1')): + if label + suffix in attrs: + if base64.b64decode(attrs[label + suffix], validate=True) != hashlib.new(digest, content).digest(): + raise ValueError('JAR digest mismatch') + return + raise ValueError('missing supported JAR digest') + + +def _jar(path, member, pin): + try: + with zipfile.ZipFile(path) as z: + names = z.namelist() + if len(names) > 64 or len(names) != len(set(names)) or any( + i.file_size > (1024 * 1024 if i.filename.upper().startswith('META-INF/') else 256 * 1024 * 1024) + for i in z.infolist()): + raise ValueError('duplicate or oversized JAR member') + blocks = [n for n in names if n.upper().startswith('META-INF/') and n.upper().endswith('.RSA')] + if len(blocks) != 1: + raise ValueError('exactly one RSA JAR signer required') + sf = z.read(blocks[0][:-4] + '.SF') + fingerprint = _cms(z.read(blocks[0]), sf) + if pin and fingerprint != pin: + raise ValueError('repository fingerprint mismatch') + manifest = z.read('META-INF/MANIFEST.MF') + _digest_check(_sections(sf)[0], '-digest-manifest', manifest) + entries = [s for s in _sections(manifest)[1:] if s.get('name') == member] + if len(entries) != 1: + raise ValueError('index is not uniquely signed') + content = z.read(member) + _digest_check(entries[0], '-digest', content) + return content, fingerprint + except (ValueError, KeyError, IndexError, StopIteration, RuntimeError, NotImplementedError, zipfile.BadZipFile) as e: + raise SourceError('invalid signed repository: ' + str(e)) from e + + +def _storage(): + return frame_host.data_dir('apk-repos.json') + + +def _read(): + try: + settings = json.loads(_storage().read_text()) + if not isinstance(settings, dict) or not isinstance(settings.get('repos'), list) or not isinstance(settings.get('enabled'), dict): + raise ValueError('invalid settings structure') + return settings + except FileNotFoundError: + return {'repos': [], 'enabled': {}} + except (OSError, ValueError) as e: + raise SourceError('cannot read repository settings: ' + str(e)) from e + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part') + try: + with os.fdopen(fd, 'w') as f: + json.dump(value, f, separators=(',', ':')) + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + + +def user_repos(): + with _LOCK: + return _read()['repos'] + + +def sources(): + builtins = [('fdroid', 'F-Droid', 'https://f-droid.org/repo/', FDROID_PIN), + ('fdroid-archive', 'F-Droid archive', 'https://f-droid.org/archive/', FDROID_PIN), + ('izzyondroid', 'IzzyOnDroid', 'https://apt.izzysoft.de/fdroid/repo/', IZZY_PIN)] + settings = _read() + return [dict(id=i, kind=KIND, name=n, url=u, fingerprint=p, builtin=True, + enabled=settings['enabled'].get(i, True), trust='community') + for i, n, u, p in builtins] + settings['repos'] + + +def _text(value): + if isinstance(value, dict): + return value.get('en-US') or value.get('en') or next(iter(value.values()), '') + return value or '' + + +def _reduce(path, source): + compatible = _reduce_index(path) + result = {} + with open(path, encoding='utf-8') as f: + reader = _IndexReader(f) + for key in reader.members(): + if key != 'packages': + reader.value() + continue + for pkg in reader.members(): + item = reader.value() + if pkg not in compatible: + continue + meta = item.get('metadata', {}) + files = {v['file'].get('name'): v for v in item.get('versions', {}).values() + if isinstance(v, dict) and isinstance(v.get('file'), dict)} + versions = [] + for v in compatible[pkg]: + original = files[v['name']] + versions.append(dict(v, size=original['file'].get('size'), updated=_date(original.get('added')))) + versions.sort(key=lambda v: (v['version_code'], v['abis'] == ['arm64-v8a']), reverse=True) + latest = versions[0] + icon = _text(meta.get('icon')) + result[pkg] = dict(source=source['id'], id=pkg, package=pkg, + name=_text(meta.get('name')) or pkg, summary=_text(meta.get('summary')), + icon=_child(source['url'], icon['name']) if isinstance(icon, dict) and icon.get('name') else None, + page=meta.get('webSite') or source['url'], vr=None, free=True, + license=meta.get('license'), downloadable=bool(latest.get('sha256')), + versions=versions, **{k: latest[k] for k in ('version', 'version_code', 'min_sdk', 'abis', 'size', 'updated')}) + return result + + +def _date(value): + return time.strftime('%Y-%m-%d', time.gmtime(value / 1000)) if isinstance(value, (int, float)) else None + + +def _v1(content, path): + index = json.loads(content) + apps = {a['packageName']: a for a in index['apps']} + packages = {} + for pkg, builds in index['packages'].items(): + app = apps.get(pkg, {}) + localized = app.get('localized', {}) + en = localized.get('en-US') or next(iter(localized.values()), {}) + meta = {k: en.get(k) or app.get(k) for k in ('name', 'summary', 'license', 'webSite')} + versions = {} + for i, v in enumerate(builds): + versions[str(i)] = {'manifest': {'versionName': v.get('versionName'), 'versionCode': v['versionCode'], + 'usesSdk': {'minSdkVersion': v.get('minSdkVersion', 1)}, 'nativecode': v.get('nativecode', [])}, + 'file': {'name': v['apkName'], 'sha256': v.get('hash') if v.get('hashType') == 'sha256' else None, + 'size': v.get('size')}, 'added': v.get('added')} + packages[pkg] = {'metadata': meta, 'versions': versions} + path.write_text(json.dumps({'packages': packages})) + + +def _load(source, force=False): + if not re.fullmatch(r'[a-z0-9-]+', source['id']): + raise SourceError('invalid source id') + _url(source['url'], source.get('fingerprint')) + cache = frame_host.cache_dir('apk-sources', source['id'] + '.json') + with _LOCK: + if not force and cache.exists() and time.time() - cache.stat().st_mtime < 86400: + try: + saved = json.loads(cache.read_text()) + if saved.get('fingerprint') == source.get('fingerprint') and saved.get('url') == source['url']: + return saved['apps'], saved['fingerprint'] + except (OSError, ValueError, KeyError, AttributeError): + pass + cache.parent.mkdir(parents=True, exist_ok=True) + try: + with tempfile.TemporaryDirectory(dir=str(cache.parent)) as tmp: + jar, raw = Path(tmp) / 'index.jar', Path(tmp) / 'index.json' + try: + _fetch(source['url'] + 'entry.jar', jar, 8 * 1024 * 1024) + except urllib.error.HTTPError as e: + if e.code not in (404, 410): + raise + _fetch(source['url'] + 'index-v1.jar', jar, 256 * 1024 * 1024) + content, pin = _jar(jar, 'index-v1.json', source.get('fingerprint')) + _v1(content, raw) + else: + content, pin = _jar(jar, 'entry.json', source.get('fingerprint')) + entry = json.loads(content)['index'] + _fetch(_child(source['url'], entry['name']), raw, 512 * 1024 * 1024) + if _sha256(raw) != entry['sha256'] or (entry.get('size') is not None and raw.stat().st_size != entry['size']): + raise SourceError('index SHA-256 or size mismatch') + apps = _reduce(raw, source) + _write(cache, {'url': source['url'], 'fingerprint': pin, 'apps': apps}) + return apps, pin + except SourceError: + raise + except (OSError, ValueError, KeyError, TypeError, IndexError) as e: + raise SourceError('cannot load repository: ' + str(e)) from e + + +def add_repo(url, fingerprint=None, name=None): + url, pin = _url(url, fingerprint) + with _LOCK: + settings = _read() + existing = next((s for s in settings['repos'] if s['url'] == url), None) + if existing: + if pin and pin != existing['fingerprint']: + raise SourceError('repository already has a different pinned fingerprint; remove it first') + pin = existing['fingerprint'] + source = dict(id='fdroid-user-' + hashlib.sha256(url.encode()).hexdigest()[:20], kind=KIND, + name=name or (existing or {}).get('name') or urllib.parse.urlsplit(url).hostname, + url=url, builtin=False, enabled=True, trust='user', fingerprint=pin) + _, source['fingerprint'] = _load(source, force=True) + source['trust_on_first_use'] = existing.get('trust_on_first_use', False) if existing else pin is None + settings['repos'] = [s for s in settings['repos'] if s['id'] != source['id']] + [source] + _write(_storage(), settings) + return source + + +def remove_repo(source_id): + with _LOCK: + settings = _read() + if not any(s['id'] == source_id for s in settings['repos']): + raise SourceError('unknown user repository') + settings['repos'] = [s for s in settings['repos'] if s['id'] != source_id] + _write(_storage(), settings) + + +def set_enabled(source_id, enabled): + if not isinstance(enabled, bool): + raise SourceError('enabled must be a boolean') + with _LOCK: + settings = _read() + source = next((s for s in sources() if s['id'] == source_id), None) + if not source: + raise SourceError('unknown repository') + if source['builtin']: + settings['enabled'][source_id] = enabled + else: + for s in settings['repos']: + if s['id'] == source_id: + s['enabled'] = enabled + _write(_storage(), settings) + + +def search(source, query, limit=50): + if not source.get('enabled', True): + return [] + apps, _ = _load(source) + words = query.casefold().split() + found = [a for a in apps.values() if all(w in (a['id'] + ' ' + a['name'] + ' ' + a['summary']).casefold() for w in words)] + found.sort(key=lambda a: (a['id'].casefold() != query.casefold(), a['name'].casefold())) + return [{k: v for k, v in a.items() if k != 'versions'} for a in found[:max(0, limit)]] + + +def details(source, entry_id): + if not source.get('enabled', True): + raise SourceError('repository is disabled') + apps, _ = _load(source) + if entry_id not in apps: + raise SourceError('app has no Lepton-compatible version in this repository') + return apps[entry_id] + + +def download(source, entry_id, version_code=None): + entry = details(source, entry_id) + version = next((v for v in entry['versions'] if version_code is None or str(v['version_code']) == str(version_code)), None) + if not version or not re.fullmatch('[0-9a-f]{64}', version.get('sha256') or ''): + raise SourceError('version is missing or has no SHA-256 digest') + sha = version['sha256'] + path = frame_host.cache_dir('apk-sources', sha + '.apk') + try: + if not path.exists() or _sha256(path) != sha: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), suffix='.part') + os.close(fd) + try: + _fetch(_child(source['url'], version['name']), tmp, 4 * 1024 ** 3) + if _sha256(tmp) != sha: + raise SourceError('APK SHA-256 mismatch; download discarded') + os.replace(tmp, path) + finally: + if os.path.exists(tmp): + os.unlink(tmp) + return {'apk': str(path), 'obb': [], 'sha256': sha, 'verified': True} + except OSError as e: + raise SourceError('cannot download APK: ' + str(e)) from e + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + add = sub.add_parser('add') + add.add_argument('url') + add.add_argument('--fingerprint') + add.add_argument('--name') + sub.add_parser('list') + remove = sub.add_parser('remove') + remove.add_argument('source') + for command in ('search', 'download'): + p = sub.add_parser(command) + p.add_argument('source') + p.add_argument('query' if command == 'search' else 'package') + args = parser.parse_args() + try: + if args.command == 'add': + result = add_repo(args.url, args.fingerprint, args.name) + elif args.command == 'list': + result = sources() + elif args.command == 'remove': + result = remove_repo(args.source) + else: + source = next((s for s in sources() if s['id'] == args.source), None) + if not source: + raise SourceError('unknown repository id; use list') + result = search(source, args.query) if args.command == 'search' else download(source, args.package) + print(json.dumps(result, indent=2)) + except SourceError as e: + parser.exit(1, 'error: ' + str(e) + '\n') + + +if __name__ == '__main__': + main() From a954fc83c9b9f41e2269200901e5d07c0f570cf0 Mon Sep 17 00:00:00 2001 From: saphid <4596216+saphid@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:13:58 +1000 Subject: [PATCH 005/122] Devices: several headsets, several addresses each, and live connection status Frame Control can now manage more than one Steam Frame, and reach each at any of several addresses (LAN IPs per network, its .local name, Tailscale). A connector in the server tries them all at once, picks the best one that answers, follows ssh -v through each stage (network, finding, SSH, identity, login) and streams that to the page. The header shows it live; a new Devices tab (key 5) manages headsets, addresses and network names. - ui/frame_devices.py: registry in devices.json, imported from the managed ~/.ssh/config blocks; per-headset host key pinning; config block updates. - ui/frame_network.py: gateway IP+MAC fingerprint, Wi-Fi name, Tailscale. - ui/frame_link.py: the connector, Test now, Tailscale/mDNS discovery, API. - server.py: ensure_master delegates to the connector; /api/connection, /api/connection/events (SSE), /api/devices. - Electron: headset switcher and Devices item in the Frame menu. - frame_connect.py --alias; FRAME_CONTROL_DATA_DIR / FRAME_CONTROL_SSH_DIR keep tests off real data. Co-Authored-By: Claude Opus 5.5 (1M context) --- app/main.js | 39 +- app/preload.js | 9 +- docs/devices.md | 167 ++++++ docs/frame-control.md | 24 +- tests/fakessh/ssh | 76 +++ tests/sandbox.py | 16 + tests/test_compat_db.py | 1 + tests/test_connect.py | 1 + tests/test_devices.py | 273 ++++++++++ tests/test_frame_apk.py | 1 + tests/test_frame_apk_versions.py | 1 + tests/test_frame_titles.py | 1 + tests/test_link.py | 337 ++++++++++++ tests/test_network.py | 147 +++++ tests/test_server.py | 5 +- tests/test_steam.py | 1 + tests/test_webinstall.py | 1 + ui/frame_connect.py | 18 +- ui/frame_devices.py | 620 +++++++++++++++++++++ ui/frame_host.py | 7 +- ui/frame_link.py | 905 +++++++++++++++++++++++++++++++ ui/frame_network.py | 310 +++++++++++ ui/index.html | 548 ++++++++++++++++++- ui/server.py | 150 +++-- 24 files changed, 3597 insertions(+), 61 deletions(-) create mode 100644 docs/devices.md create mode 100755 tests/fakessh/ssh create mode 100644 tests/sandbox.py create mode 100644 tests/test_devices.py create mode 100644 tests/test_link.py create mode 100644 tests/test_network.py create mode 100644 ui/frame_devices.py create mode 100644 ui/frame_link.py create mode 100644 ui/frame_network.py diff --git a/app/main.js b/app/main.js index bf6ff66..aac8d35 100644 --- a/app/main.js +++ b/app/main.js @@ -245,6 +245,23 @@ function fromUi(e) { ipcMain.handle("clipboard:read", (e) => fromUi(e) ? clipboard.readText() : ""); ipcMain.handle("connection:setup", (e) => { if (fromUi(e)) setUpConnection(); }); +// The page reports the headsets it knows (the server's Devices tab), so the Frame +// menu can switch between them. Only plain names and ids go into the menu. +const ALIAS_RE = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; +let devices = []; +ipcMain.on("devices:changed", (e, list) => { + if (!fromUi(e) || !Array.isArray(list)) return; + const next = list.slice(0, 20).filter(d => d && typeof d.id === "string" && ALIAS_RE.test(d.alias || "")) + .map(d => ({ id: d.id.slice(0, 80), name: String(d.name || d.alias).slice(0, 60), alias: d.alias, active: !!d.active })); + if (JSON.stringify(next) === JSON.stringify(devices)) return; + devices = next; + buildMenu(); +}); +const activeAlias = () => (devices.find(d => d.active) || {}).alias || FRAME; +function showDevices() { + if (win && url) win.webContents.executeJavaScript('location.hash = "devices"').catch(() => {}); +} + // frame-control://install links from websites (docs/web-install.md). They can // arrive before the window or server exists (macOS open-url on a cold launch), // so they wait here until the page asks for them. The page checks the link with @@ -326,13 +343,14 @@ async function runInTerminal(argv) { } } -async function setUpConnection() { - const alias = `FRAME_ALIAS=${FRAME}`; +// Set Up Connection for the headset in use (or another alias, from the Devices tab). +async function setUpConnection(name = activeAlias()) { + if (!ALIAS_RE.test(name)) return; + const alias = `FRAME_ALIAS=${name}`; if (IS_MAC) return runInTerminal(["env", alias, "zsh", path.join(SCRIPTS, "connect.sh")]); const py = python || await findPython({ ...process.env, PATH: await loginPath() }); - const setup = [py || "python3", ...PY_FLAGS, path.join(ROOT, "ui", "frame_connect.py")]; - // A new console inherits our environment on Windows; Linux terminals may not. - runInTerminal(IS_WIN ? setup : ["env", alias, ...setup]); + // --alias, since a new console on Windows (and some Linux terminals) doesn't get our environment. + runInTerminal([py || "python3", ...PY_FLAGS, path.join(ROOT, "ui", "frame_connect.py"), "--alias", name]); } function buildMenu() { @@ -343,8 +361,15 @@ function buildMenu() { { label: "Frame", submenu: [ - { label: "Set Up Connection…", click: setUpConnection }, - { label: IS_MAC ? "Open SSH in Terminal" : "Open SSH in a Terminal", click: () => runInTerminal(["ssh", FRAME]) }, + { label: "Set Up Connection…", click: () => setUpConnection() }, + { label: IS_MAC ? "Open SSH in Terminal" : "Open SSH in a Terminal", click: () => runInTerminal(["ssh", activeAlias()]) }, + { type: "separator" }, + ...(devices.length > 1 ? [{ + label: "Headset", + submenu: devices.map(d => ({ label: d.name, type: "radio", checked: d.active, + click: () => { if (win) win.webContents.send("use-device", d.id); } })), + }] : []), + { label: "Devices…", accelerator: "CmdOrCtrl+5", click: showDevices }, { type: "separator" }, { label: "Open in Browser", click: () => url && shell.openExternal(url) }, { label: "Restart Server", click: () => win ? restartServer() : createWindow() }, diff --git a/app/preload.js b/app/preload.js index 43bc34d..9ff2567 100644 --- a/app/preload.js +++ b/app/preload.js @@ -2,7 +2,8 @@ // to the Frame needs no pbpaste, PowerShell, xclip or wl-clipboard. Also tells // the page where a dropped file or folder lives, so a folder can be sideloaded // as a title without zipping it (the local server reads it from there). -// It can open Set Up Connection when the headset can't be reached. +// It can open Set Up Connection when the headset can't be reached, and keeps the +// Frame menu's list of headsets up to date. // It also receives frame-control://install links (docs/web-install.md): only // what the link asked for, never an install; the page asks the user first. const { contextBridge, ipcRenderer, webUtils } = require("electron"); @@ -10,6 +11,12 @@ const { contextBridge, ipcRenderer, webUtils } = require("electron"); contextBridge.exposeInMainWorld("frameApp", { readClipboard: () => ipcRenderer.invoke("clipboard:read"), setUpConnection: () => ipcRenderer.invoke("connection:setup"), + // The Frame menu's headset switcher: the page tells it the headsets, and hears picks. + devicesChanged: (list) => ipcRenderer.send("devices:changed", list), + onUseDevice: (cb) => { + ipcRenderer.removeAllListeners("use-device"); + ipcRenderer.on("use-device", (_e, id) => cb(String(id))); + }, pathForFile: (file) => { try { return webUtils.getPathForFile(file) || ""; } catch { return ""; } }, onInstallLink: (cb) => { ipcRenderer.removeAllListeners("install-link"); diff --git a/docs/devices.md b/docs/devices.md new file mode 100644 index 0000000..4f01b69 --- /dev/null +++ b/docs/devices.md @@ -0,0 +1,167 @@ +# Headsets, addresses and the connection + +Frame Control can manage more than one Steam Frame, and each headset can be +reached at more than one address: a LAN IP at home, another at the office, its +mDNS name (`frame.local`), its Tailscale IP or MagicDNS name. The **Devices** +tab (key 5) lists them, and the connection pill in the header shows what the +app is doing to reach the one in use, step by step, as it happens. + +The code is in three modules, all stdlib-only Python on your computer: + +| Module | What it does | +|---|---| +| `ui/frame_devices.py` | The registry: headsets, their addresses, networks; importing and updating `~/.ssh/config`; pinned host keys | +| `ui/frame_network.py` | Which network this computer is on, and Tailscale's state | +| `ui/frame_link.py` | The connector: finds the headset, keeps the SSH connection, publishes each stage; the Devices API | + +## Headsets + +Each headset keeps its own SSH alias, as Set Up Connection has always written +it: the first is `frame`, the next `frame-2`, and so on. Terminal's +`ssh frame-2` and the helper scripts (`FRAME_ALIAS=frame-2 scripts/push.sh …`) +work for each one. + +- **Nothing to migrate by hand.** On first start, the app imports every + `# >>> steam-frame (ALIAS) >>>` block in `~/.ssh/config` as a headset, with + the block's HostName as its first address. It also copies the host key your + `known_hosts` already trusts for that address into the app's own + `~/.ssh/frame-control_known_hosts`, so nobody is asked to trust it again. +- **Add a headset** runs Set Up Connection (`scripts/connect.sh` on macOS, + `ui/frame_connect.py --alias NAME` elsewhere) in a terminal with a new alias. + When it writes its block, the app picks the headset up by itself. If Set Up + Connection runs again and finds a headset somewhere new, that address is added + at the top of its list. +- **Use this headset** (or the switcher in the header, or the app's + **Frame → Headset** menu) moves the whole app to another headset; every panel + reloads from it. +- **Remove** forgets a headset. Its `~/.ssh/config` block stays unless you tick + the box; either way it isn't imported again unless Set Up Connection changes it. +- A plain `FRAME_ALIAS` that Set Up Connection never configured still works: the + app shows it as not set up and lets ssh's own config decide where it goes. + +## Addresses + +Each address has a kind (LAN, mDNS, Tailscale or Other, guessed from the address +and changeable), an optional label, the networks it has worked on, and when it +last worked with its round-trip time. + +When connecting, the app **tries all addresses at once** (TCP to the SSH port) +and ranks them: + +1. addresses that worked on the network this computer is on now; +2. mDNS names; +3. Tailscale addresses, if Tailscale is running here; +4. addresses not tried on this network yet; +5. addresses that only ever worked on other networks; +6. Tailscale addresses while Tailscale is off. + +Your order on the Devices tab breaks ties. The best-ranked address that answers +wins; one that answers first waits up to 0.35 s for a better-ranked one that is +still trying. If SSH to the winner fails in a way another address could fix +(a different device answered there, or the link dropped), the next one that +answered is tried. Every success records the network on that address, so next +time on that network it's tried first. + +**Test now** probes every address and tries SSH on each one that answers, without +disturbing the connection in use: "SSH works", "answered as a different +headset", "refused this computer's key", or why it didn't answer. **Find on +Tailscale** lists your tailnet's devices (likely headsets first, from `tailscale +status --json`, including the Mac app's own CLI) with buttons to add their +MagicDNS name or IP. **Find on this network** asks mDNS for SteamOS devkit +services and checks `ALIAS.local` and `frame.local`. + +## Networks + +A network is told apart by its default gateway: the router's IP address plus its +hardware (MAC) address, read with `route`/`arp` (macOS), `ip route`/`ip neigh` +(Linux) or `route print`/`arp -a` (Windows). That works on wired networks, and +on macOS 14 and later, which hides the Wi-Fi name from apps without Location +permission. Where the system does share the Wi-Fi name, it's shown, and you can +name any network yourself ("Home Wi-Fi") on the Devices tab. + +The app rereads the gateway every 5 seconds and Tailscale's state every +30 seconds. Changing networks reconnects. + +## The connection, stage by stage + +The connector runs in the server (`frame_link.Link`) and moves through: + +1. **Checking this computer's network**: gateway, Wi-Fi, this computer's IP, Tailscale. +2. **Finding the headset**: each address resolving, trying, answered in N ms, + no answer, refused, or can't be found. +3. **Opening SSH** to the address that answered. +4. **Checking the headset's identity**: the host key must match the one pinned + for this headset. +5. **Logging in** as the headset's user. +6. **Connected** via network N, address A, round trip T; or **failed** at a stage + with the reason in plain words and a countdown to the next try (5, 10, 20, + then every 30 seconds). Retry now skips the wait. + +Stages 3 to 5 come from following `ssh -v` as it runs. On macOS and Linux the +connection is an SSH ControlMaster that every command shares; when it dies (the +headset slept or left the network) the connector notices and starts again. On +Windows, where OpenSSH can't share a connection, the same handshake runs once +and each command then connects on its own; a command that can't reach the +headset makes the connector start again. + +Once connected, every `ssh`, `scp` and `rsync` the app runs gets +`-o HostName=
-o HostKeyAlias=frame-control- +-o UserKnownHostsFile=~/.ssh/frame-control_known_hosts -o User=… -o Port=…`. The +alias's block in `~/.ssh/config` is also updated to the last address that +worked (and to the user and port you set), so Terminal's `ssh frame` and the +scripts follow. + +**Host keys are pinned per headset, not per address.** Your own `known_hosts` +is keyed by address, so a different device answering at a remembered IP (a DHCP +lease that moved) would look like a new host there. The app keys its own +known_hosts by headset instead: a different device answering at one of its +addresses is refused, and the pill says so. A headset's first connection trusts +the key it shows, as Set Up Connection does. After reinstalling SteamOS the +headset has a new key; **Forget identity** on the Devices tab lets the next +connection save the new one. + +## API + +All under the usual `/api/` guards (loopback `Host`, `X-Frame-UI` header). + +| Request | Returns | +|---|---| +| `GET /api/connection` | The connection state: `phase` (connecting, connected, failed), `device`, `network`, `stages`, `probes`, `via`, `error`, `retry_at`, `tests`, `version` | +| `GET /api/connection/events` | The same as server-sent events, one each time it changes (the page reads it with `fetch`, since `EventSource` can't send the header) | +| `GET /api/devices` | Headsets, the current network, known networks, the next free alias | +| `GET /api/devices/tailscale?id=` | Tailscale peers, likely headsets first | +| `GET /api/devices/mdns?id=` | Headsets found on this network | +| `POST /api/devices` | `{"action": ...}`: `use`, `update` (name, user, port), `remove`, `address-add`, `address-update`, `address-remove`, `address-move`, `test`, `forget-identity`, `name-network`, `setup` (alias, optional host), `retry` | + +Every host, alias and user is checked against strict patterns before it's +stored, because they end up in ssh arguments and `~/.ssh/config`; nothing goes +through a shell. + +## The registry file + +`devices.json` in the app's data folder (`~/Library/Application Support/Frame +Control` on macOS, `%APPDATA%\Frame Control` on Windows, +`~/.local/share/frame-control` on Linux). It's plain JSON so the iPhone app can +share the format later (it still connects to one host; see +[iphone.md](iphone.md)): + +```json +{"version": 1, "active": "f67f8b7e", + "devices": [{"id": "f67f8b7e", "name": "Steam Frame", "alias": "frame", "user": "steamos", "port": 22, + "identity_files": ["~/.ssh/id_ed25519_frame"], + "addresses": [{"host": "frame.local", "kind": "mdns", "label": "", + "networks": ["n-e0998baa61"], "last_ok": 1790593550.4, "last_rtt_ms": 0.9}]}], + "networks": {"n-e0998baa61": {"name": "Home Wi-Fi", "ssid": null, "gateway": "192.168.1.1", + "gateway_mac": "b4:fb:e4:b5:67:55", "wifi": true, "last_seen": 1790593550.0}}} +``` + +A network id is `n-` and the first 10 hex digits of SHA-1 of `gateway|mac`. + +## Tests + +`tests/test_devices.py`, `tests/test_network.py` and `tests/test_link.py` run +with the other unit tests. They use a stand-in `ssh` (`tests/fakessh/ssh`) that +prints what `ssh -v` prints and plays a ControlMaster, real sockets on this +computer for the addresses, and temporary folders for `~/.ssh` +(`FRAME_CONTROL_SSH_DIR`) and the app data (`FRAME_CONTROL_DATA_DIR`), so they +never touch yours. diff --git a/docs/frame-control.md b/docs/frame-control.md index 299c57c..f73c484 100644 --- a/docs/frame-control.md +++ b/docs/frame-control.md @@ -17,13 +17,16 @@ python3 ui/server.py # anywhere: then open http://127.0.0.1:47810 ## Features -The window has four tabs: **Home** (headset view, status, screenshots), +The window has five tabs: **Home** (headset view, status, screenshots), **Games** (installed games, sideloaded titles, getting games), **Android** (apps, -the catalogue, display settings, reports) and **Tools** (sending files and text, -Flatpaks, remote and power). Keys 1–4 switch between them. Files can be dropped -anywhere in the window. When the Frame can't be reached, one banner says why in -plain words and the app retries every few seconds, filling everything in once it -answers. Flatpak and Android installs run in the background; the bottom bar +the catalogue, display settings, reports), **Tools** (sending files and text, +Flatpaks, remote and power) and **Devices** (your headsets and their addresses). +Keys 1–5 switch between them. Files can be dropped anywhere in the window. A +connection pill in the header always shows which headset, which network this +computer is on, the address in use or being tried, and each step of connecting +as it happens; click it for the whole timeline. When the Frame can't be +reached, a banner says why in plain words, what was tried, and counts down to +the next try, filling everything in once it answers. Flatpak and Android installs run in the background; the bottom bar counts them while they run. - **Headset view**: what the lenses show, as SteamVR composites it (the room, @@ -72,6 +75,11 @@ counts them while they run. Frame clipboard. - **Flatpaks**: install and remove them (quick picks: Moonlight, Firefox, VLC, Remmina). +- **Devices**: several headsets, each with several addresses (LAN IPs per + network, its `.local` mDNS name, its Tailscale IP or MagicDNS name). The app + tries them all at once and learns which worked on which network. Add, edit, + reorder and test addresses, find a headset on Tailscale or on this network, + name your networks, and switch headsets. See [devices.md](devices.md). - **One-click tools**: SSH or SFTP in a terminal window, Steam Link, and remote desktop (Windows App on macOS, Remote Desktop on Windows, Remmina or FreeRDP on Linux). Sleep, restart and shut down open a terminal window because SteamOS @@ -92,7 +100,9 @@ trusts root certificates already in the Windows store. The server is Python stdlib only and listens on 127.0.0.1. It rejects requests with a non-local `Host` header, and any `/api/` request without a custom header, so other websites can't drive it or read captures. Everything reaches -the headset through the `frame` SSH alias. On macOS and Linux it keeps one +the headset through its SSH alias (`frame` for the first one), pointed at the +address that answered with `-o HostName=` (`ui/frame_link.py`, described in +[devices.md](devices.md)). On macOS and Linux it keeps one multiplexed SSH connection open, so status and each capture take about 0.3 s. Windows' OpenSSH can't share a connection, so there each request connects on its own and the app is a little slower. What differs between the three diff --git a/tests/fakessh/ssh b/tests/fakessh/ssh new file mode 100755 index 0000000..af0ebb4 --- /dev/null +++ b/tests/fakessh/ssh @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""A stand-in for OpenSSH's ssh, for tests/test_link.py: prints what `ssh -v` prints at +each step of a connection, and plays a ControlMaster. What each HostName does comes +from $FAKESSH_HOSTS (JSON: host -> "ok", "wrong" (a different host key) or "denied"); +every call is appended to $FAKESSH_LOG as a JSON line. POSIX only.""" +import json +import os +import signal +import sys +import time + +args = sys.argv[1:] +with open(os.environ["FAKESSH_LOG"], "a") as f: + f.write(json.dumps(args) + "\n") +hosts = json.loads(os.environ.get("FAKESSH_HOSTS", "{}")) +opts = {} +i = 0 +while i < len(args) and args[i].startswith("-"): + if args[i] in ("-o", "-O", "-p", "-l"): + key = args[i] + val = args[i + 1] + if key == "-o": + k, _, v = val.partition("=") + opts[k.lower()] = v + else: + opts[key] = val + i += 2 + else: + opts[args[i]] = True + i += 1 +alias = args[i] if i < len(args) else "" +host = opts.get("hostname", alias).replace("%%", "%") +marker = os.path.join(os.environ["FAKESSH_DIR"], "master-" + host.replace("/", "_")) +say = lambda s: (sys.stderr.write(s + "\n"), sys.stderr.flush()) + +if "-G" in opts: + print(f"hostname {alias}\nport 22\nuser tester") + sys.exit(0) +if opts.get("-O") == "check": + sys.exit(0 if os.path.exists(marker) else 255) +if opts.get("-O") == "exit": + if os.path.exists(marker): + os.unlink(marker) + sys.exit(0) + +what = hosts.get(host) +if what is None: + say(f"ssh: Could not resolve hostname {host}: nodename nor servname provided, or not known") + sys.exit(255) +say(f"debug1: Connecting to {host} [127.0.0.1] port {opts.get('port', 22)}.") +say("debug1: Connection established.") +say(f"debug1: Authenticating to {host}:22 as '{opts.get('user', 'tester')}'") +say("debug1: Server host key: ssh-ed25519 SHA256:fakefakefakefakefakefakefakefakefakefakefak") +if what == "wrong": + say("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@") + say("@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @") + say("Host key verification failed.") + sys.exit(255) +say(f"debug1: Host '{opts.get('hostkeyalias', host)}' is known and matches the ED25519 host key.") +say("debug1: Next authentication method: publickey") +if what == "denied": + say(f"tester@{host}: Permission denied (publickey).") + sys.exit(255) +say(f'Authenticated to {host} ([127.0.0.1]:22) using "publickey".') +if opts.get("controlmaster") == "yes": + open(marker, "w").close() + def bye(*_): + if os.path.exists(marker): + os.unlink(marker) + sys.exit(0) + signal.signal(signal.SIGTERM, bye) + while True: + time.sleep(0.2) + if not os.path.exists(marker): + sys.exit(0) +sys.exit(0) diff --git a/tests/sandbox.py b/tests/sandbox.py new file mode 100644 index 0000000..5d2a3f0 --- /dev/null +++ b/tests/sandbox.py @@ -0,0 +1,16 @@ +"""Imported first by every test module: nothing a test does reaches this person's +app data, their telemetry, or the shared compatibility database. + +Must run before any ui module is imported, since those read these at import time. +""" +import atexit +import os +import shutil +import tempfile + +_dir = tempfile.mkdtemp(prefix="frame-control-tests-") +atexit.register(shutil.rmtree, _dir, ignore_errors=True) +os.environ["FRAME_CONTROL_DATA_DIR"] = _dir +os.environ["FRAME_CONTROL_TELEMETRY"] = "0" +# A maintainer's machine holds the database key; send anything that slips through nowhere. +os.environ["FRAME_COMPAT_DB_URL"] = "http://127.0.0.1:9" diff --git a/tests/test_compat_db.py b/tests/test_compat_db.py index 7081d79..37208d5 100644 --- a/tests/test_compat_db.py +++ b/tests/test_compat_db.py @@ -2,6 +2,7 @@ Run: python3 -m unittest discover -s tests """ +import sandbox # noqa: F401 (first: keeps tests off real data and services) import os import sys import tempfile diff --git a/tests/test_connect.py b/tests/test_connect.py index b3366bd..0e8972a 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -3,6 +3,7 @@ steamos-devkit-service, the ~/.ssh/config block, and the mDNS output parsers. Run: python3 -m unittest discover -s tests """ +import sandbox # noqa: F401 (first: keeps tests off real data and services) import json import socket import sys diff --git a/tests/test_devices.py b/tests/test_devices.py new file mode 100644 index 0000000..83c187c --- /dev/null +++ b/tests/test_devices.py @@ -0,0 +1,273 @@ +"""frame_devices: the headset registry, importing ~/.ssh/config, address order, pinned +host keys and input checks. Everything works in temporary folders. + +Run: python3 -m unittest discover -s tests +""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "ui")) + +import frame_devices as fd # noqa: E402 + +CONFIG = """Host lxso1 + HostName 192.168.1.109 + +# >>> steam-frame (frame) >>> +Host frame + HostName frame.tail1234.ts.net + User steamos + IdentityFile ~/.ssh/id_ed25519_frame + IdentityFile ~/.ssh/id_rsa_frame_devkit + IdentitiesOnly yes + ServerAliveInterval 30 +Host * +# <<< steam-frame (frame) <<< +# >>> steam-frame (frame-2) >>> +Host frame-2 + HostName 192.168.1.60 + Port 2222 + User deck + IdentityFile ~/.ssh/id_ed25519_frame +Host * +# <<< steam-frame (frame-2) <<< +Host * + ServerAliveInterval 60 +""" +KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIID6kdLfZZmdTqS1snKfTESTKEYTESTKEYTESTKEYTESTKE" + + +class Base(unittest.TestCase): + def setUp(self): + self.dir = Path(tempfile.mkdtemp(prefix="frame-devices-")) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + self.ssh = self.dir / "ssh" + self.ssh.mkdir() + (self.ssh / "config").write_text(CONFIG) + old = os.environ.get("FRAME_CONTROL_SSH_DIR") + os.environ["FRAME_CONTROL_SSH_DIR"] = str(self.ssh) + self.addCleanup(lambda: os.environ.__setitem__("FRAME_CONTROL_SSH_DIR", old) if old + else os.environ.pop("FRAME_CONTROL_SSH_DIR", None)) + self.reg = fd.Registry(self.dir / "devices.json") + + +class Validation(unittest.TestCase): + def test_hosts(self): + for good in ("frame.local", "192.168.1.40", "fd7a:115c:a1e0::5928:ae55", "fe80::1%en0", "frame-2.tail1234.ts.net"): + self.assertEqual(fd.check_host(good), good) + for bad in ("", " ", "-oProxyCommand=sh", "a b", "frame;id", "frame\nHost *", "frame..local", "$(id)", + "frame%en0", "x" * 300, None, 5, "frame/../x"): + with self.assertRaises(fd.DeviceError, msg=repr(bad)): + fd.check_host(bad) + + def test_names(self): + self.assertEqual(fd.check_alias("frame-2"), "frame-2") + for bad in ("", "-F", "frame 2", "frame\n", "a" * 65, None): + with self.assertRaises(fd.DeviceError): + fd.check_alias(bad) + with self.assertRaises(fd.DeviceError): + fd.check_user(bad) + for bad in ("0", "65536", "x", None, "22; id"): + with self.assertRaises(fd.DeviceError): + fd.check_port(bad) + self.assertEqual(fd.check_port("2222"), 2222) + with self.assertRaises(fd.DeviceError): + fd.check_text("line\nbreak", "label") + with self.assertRaises(fd.DeviceError): + fd.check_kind("wifi") + + def test_ipv6_zone_is_escaped_for_ssh(self): + self.assertEqual(fd.ssh_host("fe80::1%en0"), "fe80::1%%en0") + + +class Migration(Base): + def test_blocks_are_parsed(self): + blocks = fd.parse_blocks(CONFIG) + self.assertEqual([b["alias"] for b in blocks], ["frame", "frame-2"]) + self.assertEqual(blocks[0]["hostname"], "frame.tail1234.ts.net") + self.assertEqual(blocks[0]["identity_files"], ["~/.ssh/id_ed25519_frame", "~/.ssh/id_rsa_frame_devkit"]) + self.assertEqual((blocks[1]["port"], blocks[1]["user"]), (2222, "deck")) + + def test_existing_headsets_are_imported_once(self): + self.assertTrue(self.reg.sync_from_config(seed=False)) + devices = self.reg.devices() + self.assertEqual([d["alias"] for d in devices], ["frame", "frame-2"]) + frame, second = devices + self.assertEqual(frame["name"], "Steam Frame") + self.assertEqual(frame["addresses"][0]["host"], "frame.tail1234.ts.net") + self.assertEqual(frame["addresses"][0]["kind"], "tailscale") + self.assertEqual((second["user"], second["port"]), ("deck", 2222)) + self.assertEqual(self.reg.active(), frame["id"]) + self.assertFalse(self.reg.sync_from_config(seed=False)) # nothing new + # It's all on disk, in the documented shape. + data = json.loads((self.dir / "devices.json").read_text()) + self.assertEqual(data["version"], 1) + self.assertEqual(len(data["devices"]), 2) + self.assertEqual(fd.Registry(self.dir / "devices.json").devices(), self.reg.devices()) + + def test_setup_finding_a_new_address_adds_it(self): + self.reg.sync_from_config(seed=False) + (self.ssh / "config").write_text(CONFIG.replace("HostName frame.tail1234.ts.net", "HostName 192.168.1.237")) + self.assertTrue(self.reg.sync_from_config(seed=False)) + hosts = [a["host"] for a in self.reg.by_alias("frame")["addresses"]] + self.assertEqual(hosts, ["192.168.1.237", "frame.tail1234.ts.net"]) # the new one first + + def test_removed_headset_stays_removed_until_setup_changes_it(self): + self.reg.sync_from_config(seed=False) + second = self.reg.by_alias("frame-2") + self.reg.remove_device(second["id"]) + self.reg.sync_from_config(seed=False) + self.assertIsNone(self.reg.by_alias("frame-2")) + self.reg.undismiss("frame-2") # Set Up Connection run for it from the Devices tab + self.reg.sync_from_config(seed=False) + self.assertIsNotNone(self.reg.by_alias("frame-2")) + + def test_corrupt_registry_is_ignored(self): + (self.dir / "bad.json").write_text("{not json") + self.assertEqual(fd.Registry(self.dir / "bad.json").devices(), []) + (self.dir / "evil.json").write_text(json.dumps({"devices": [ + {"id": "x1", "alias": "-oProxyCommand=id", "addresses": []}, + {"id": "x2", "alias": "ok", "addresses": [{"host": "a b", "kind": "lan"}, {"host": "frame.local", "kind": "mdns"}]}]})) + devices = fd.Registry(self.dir / "evil.json").devices() + self.assertEqual([d["alias"] for d in devices], ["ok"]) + self.assertEqual([a["host"] for a in devices[0]["addresses"]], ["frame.local"]) + + +class ConfigRewrite(Base): + def test_hostname_user_and_port_change_only_inside_the_block(self): + cfg = self.ssh / "config" + self.assertTrue(fd.rewrite_block("frame", hostname="192.168.1.237")) + text = cfg.read_text() + self.assertIn(" HostName 192.168.1.237\n", text) + self.assertEqual(text.replace("192.168.1.237", "frame.tail1234.ts.net"), CONFIG) # nothing else moved + self.assertFalse(fd.rewrite_block("frame", hostname="192.168.1.237")) # no change, no write + self.assertTrue(fd.rewrite_block("frame", port=2200, user="deck")) + block = fd.parse_blocks(cfg.read_text())[0] + self.assertEqual((block["port"], block["user"], block["hostname"]), (2200, "deck", "192.168.1.237")) + self.assertTrue(fd.rewrite_block("frame-2", port=22)) # back to the default: the line goes + self.assertEqual(fd.parse_blocks(cfg.read_text())[1]["port"], 22) + self.assertNotIn("Port 22\n", cfg.read_text()) + self.assertIn("HostName 192.168.1.109", cfg.read_text()) # other hosts untouched + if os.name != "nt": + self.assertEqual(cfg.stat().st_mode & 0o777, 0o600) + + def test_zone_is_escaped_and_read_back(self): + fd.rewrite_block("frame", hostname="fe80::1%en0") + self.assertIn("HostName fe80::1%%en0", (self.ssh / "config").read_text()) + self.assertEqual(fd.parse_blocks((self.ssh / "config").read_text())[0]["hostname"], "fe80::1%en0") + + def test_missing_block_is_left_alone(self): + self.assertFalse(fd.rewrite_block("frame-9", hostname="10.0.0.1")) + self.assertFalse(fd.remove_block("frame-9")) + self.assertTrue(fd.remove_block("frame-2")) + self.assertEqual([b["alias"] for b in fd.parse_blocks((self.ssh / "config").read_text())], ["frame"]) + + +@unittest.skipUnless(shutil.which("ssh-keygen"), "needs ssh-keygen") +class Pins(Base): + def test_seed_copies_the_trusted_key_under_the_device_alias(self): + (self.ssh / "known_hosts").write_text(f"frame.tail1234.ts.net {KEY}\nother.example {KEY}X\n") + self.assertFalse(fd.pinned("d1")) + self.assertTrue(fd.seed_pin("d1", ["frame.tail1234.ts.net"])) + self.assertTrue(fd.pinned("d1")) + self.assertEqual(fd.known_hosts().read_text(), f"frame-control-d1 {KEY}\n") + self.assertTrue(fd.seed_pin("d1", ["frame.tail1234.ts.net"])) # idempotent + self.assertEqual(fd.known_hosts().read_text().count("\n"), 1) + self.assertFalse(fd.seed_pin("d2", ["never-seen.example"])) + self.assertTrue(fd.forget_pin("d1")) + self.assertFalse(fd.pinned("d1")) + + def test_hashed_and_non_default_port_entries(self): + kh = self.ssh / "known_hosts" + kh.write_text(f"[frame.local]:2222 {KEY}\n") + subprocess.run(["ssh-keygen", "-H", "-f", str(kh)], capture_output=True, check=True) + self.assertFalse(fd.seed_pin("d3", ["frame.local"])) # port 22: not that entry + self.assertTrue(fd.seed_pin("d3", ["frame.local"], port=2222)) + self.assertIn(f"frame-control-d3 {KEY}", fd.known_hosts().read_text()) + + def test_known_hosts_option_uses_the_override(self): + self.assertEqual(fd.known_hosts_opt(), str(self.ssh / "frame-control_known_hosts")) + + +class Registry(Base): + def test_address_editing(self): + d = self.reg.add_device("frame-3", hosts=["192.168.1.40"]) + a = self.reg.add_address(d["id"], "frame-3.local", label="mDNS") + self.assertEqual(a["kind"], "mdns") + self.reg.add_address(d["id"], "100.100.1.1", kind="tailscale", label="Tailscale") + with self.assertRaises(fd.DeviceError): + self.reg.add_address(d["id"], "frame-3.local") # already there + with self.assertRaises(fd.DeviceError): + self.reg.add_address(d["id"], "frame-3.local; id") + self.reg.move_address(d["id"], "100.100.1.1", -1) + self.reg.move_address(d["id"], "100.100.1.1", -1) + self.reg.move_address(d["id"], "100.100.1.1", -1) # already first: stays + hosts = lambda: [x["host"] for x in self.reg.get(d["id"])["addresses"]] + self.assertEqual(hosts(), ["100.100.1.1", "192.168.1.40", "frame-3.local"]) + self.reg.record_success(d["id"], "192.168.1.40", "n-home", 3.2) + self.reg.update_address(d["id"], "192.168.1.40", label="Home") + self.assertEqual(self.reg.get(d["id"])["addresses"][1]["networks"], ["n-home"]) # a label keeps what it learned + self.reg.update_address(d["id"], "192.168.1.40", new_host="192.168.1.41") + moved = self.reg.get(d["id"])["addresses"][1] + self.assertEqual((moved["host"], moved["networks"], moved["last_ok"]), ("192.168.1.41", [], None)) + self.reg.remove_address(d["id"], "192.168.1.41") + self.assertEqual(hosts(), ["100.100.1.1", "frame-3.local"]) + with self.assertRaises(fd.DeviceError): + self.reg.remove_address(d["id"], "nope") + + def test_devices(self): + a = self.reg.add_device("frame") + b = self.reg.add_device("frame-2", name="Office") + self.assertEqual(self.reg.active(), a["id"]) + with self.assertRaises(fd.DeviceError): + self.reg.add_device("frame") + self.reg.set_active(b["id"]) + self.assertEqual(self.reg.update_device(b["id"], name="Desk", user="deck", port="2222")["port"], 2222) + with self.assertRaises(fd.DeviceError): + self.reg.update_device(b["id"], user="bad user") + self.reg.remove_device(b["id"]) + self.assertEqual(self.reg.active(), a["id"]) + with self.assertRaises(fd.DeviceError): + self.reg.get(b["id"]) + + def test_networks_get_names(self): + net = {"id": "n-1", "gateway": "192.168.1.1", "gateway_mac": "aa:bb:cc:dd:ee:ff", "ssid": None, "wifi": True} + self.assertEqual(self.reg.network_name(net), "Wi-Fi via 192.168.1.1") + self.reg.record_network(net) + self.reg.name_network("n-1", "Home Wi-Fi") + self.assertEqual(self.reg.network_name(net), "Home Wi-Fi") + self.assertEqual(self.reg.network_name(dict(net, id="n-2", ssid="Cafe")), "Cafe") + self.assertEqual(self.reg.network_name(None), "No network") + with self.assertRaises(fd.DeviceError): + self.reg.name_network("n-unknown", "x") + + +class Order(unittest.TestCase): + def addr(self, host, kind, networks=()): + return {"host": host, "kind": kind, "networks": list(networks)} + + def test_known_here_then_mdns_then_tailscale_then_the_rest(self): + addrs = [self.addr("10.1.1.5", "lan", ["n-office"]), self.addr("192.168.1.40", "lan"), + self.addr("100.64.1.2", "tailscale"), self.addr("frame.local", "mdns"), + self.addr("192.168.1.237", "lan", ["n-home"])] + order = [a["host"] for a, _ in fd.order_addresses(addrs, "n-home", True)] + self.assertEqual(order, ["192.168.1.237", "frame.local", "100.64.1.2", "192.168.1.40", "10.1.1.5"]) + # Tailscale off: its addresses go last. + order = [a["host"] for a, _ in fd.order_addresses(addrs, "n-home", False)] + self.assertEqual(order[-1], "100.64.1.2") + # On an unknown network nothing has worked yet; the user's order breaks ties. + ranked = fd.order_addresses(addrs, None, True) + self.assertEqual([a["host"] for a, _ in ranked], ["frame.local", "100.64.1.2", "192.168.1.40", "10.1.1.5", "192.168.1.237"]) + self.assertEqual(ranked[0][1], "mDNS name") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_frame_apk.py b/tests/test_frame_apk.py index 1af14f9..ebe7b0b 100644 --- a/tests/test_frame_apk.py +++ b/tests/test_frame_apk.py @@ -1,4 +1,5 @@ """frame_apk against a small APK built here: binary manifest plus resource table.""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) import io import os import struct diff --git a/tests/test_frame_apk_versions.py b/tests/test_frame_apk_versions.py index 8cf7701..ae19175 100644 --- a/tests/test_frame_apk_versions.py +++ b/tests/test_frame_apk_versions.py @@ -1,4 +1,5 @@ """Offline version lookup with small index-v2 fixtures.""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) import io import json import os diff --git a/tests/test_frame_titles.py b/tests/test_frame_titles.py index 55d080d..acb9454 100644 --- a/tests/test_frame_titles.py +++ b/tests/test_frame_titles.py @@ -1,4 +1,5 @@ """frame_titles without a headset: executable headers, launch targets, zips, runtimes.""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) import json import os import shutil diff --git a/tests/test_link.py b/tests/test_link.py new file mode 100644 index 0000000..a2575b4 --- /dev/null +++ b/tests/test_link.py @@ -0,0 +1,337 @@ +"""frame_link: finding a headset among its addresses and following each stage of +connecting, with a stand-in ssh (tests/fakessh/ssh) and real sockets on this computer. +Also the server's /api/connection, its event stream, and /api/devices. + +Run: python3 -m unittest discover -s tests +""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) +import http.client +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "ui")) + +import frame_devices as fd # noqa: E402 +import frame_link as fl # noqa: E402 +import frame_network as fn # noqa: E402 + +FAKESSH = ROOT / "tests" / "fakessh" +NET = {"id": "n-test", "gateway": "192.168.1.1", "gateway_mac": "aa:bb:cc:dd:ee:ff", "interface": "en0", + "ssid": None, "wifi": True, "local_ip": "192.168.1.9", "tailscale": {"up": False, "installed": False}} + + +def explain(msg): + """A cut-down server.unreachable, so this needs no server import.""" + if "Could not resolve" in msg: + return "Can't find the Frame on the network." + if "refused" in msg: + return "The Frame refused the connection." + if "timed out" in msg.lower(): + return "The Frame isn't answering." + if "Permission denied" in msg: + return "The Frame didn't accept this computer's SSH key." + return None + + +class Probe(unittest.TestCase): + def test_answers_refusals_and_unknown_names(self): + with socket.socket() as srv: + srv.bind(("127.0.0.1", 0)) + srv.listen(4) + port = srv.getsockname()[1] + seen = [] + res = fl.probe("127.0.0.1", port, update=lambda **f: seen.append(f["state"])) + self.assertEqual(res["state"], "answered") + self.assertEqual(res["ip"], "127.0.0.1") + self.assertIsInstance(res["rtt_ms"], float) + self.assertEqual(seen, ["resolving", "trying"]) + self.assertEqual(fl.probe("127.0.0.1", port, timeout=2)["state"], "refused") # closed now + self.assertEqual(fl.probe("frame-control-test.invalid", 22, timeout=2)["state"], "unresolved") + + def test_failed_probes_read_like_ssh(self): + # So the server's UNREACHABLE table words them like any other ssh failure. + self.assertIn("Could not resolve hostname x", fl.probe_raw("x", 22, {"state": "unresolved"})) + self.assertIn("port 22: Connection refused", fl.probe_raw("x", 22, {"state": "refused"})) + self.assertIn("Operation timed out", fl.probe_raw("x", 22, {"state": "timeout"})) + + +class Pick(unittest.TestCase): + def pick(self, results, tried=()): + return fl.Link.pick(results, set(tried), threading.Condition(), time.monotonic() + 5) + + def test_best_ranked_answer_wins(self): + now = time.monotonic() + ok = lambda t=now: {"state": "answered", "t": t} + no = {"state": "timeout", "t": now} + self.assertEqual(self.pick([no, ok(), ok()]), 1) + self.assertEqual(self.pick([no, ok(), ok()], tried=[1]), 2) + self.assertIsNone(self.pick([no, no])) + # A worse-ranked answer waits PREFER for a better one still trying, then goes. + t0 = time.monotonic() + self.assertEqual(self.pick([None, ok(time.monotonic())]), 1) + self.assertGreaterEqual(time.monotonic() - t0, fl.PREFER - 0.05) + + def test_gives_up_on_slow_lookups_at_the_deadline(self): + t0 = time.monotonic() + results = [None] + self.assertIsNone(fl.Link.pick(results, set(), threading.Condition(), time.monotonic() + 0.3)) + self.assertLess(time.monotonic() - t0, 2) + self.assertEqual(results[0]["state"], "timeout") + + +@unittest.skipIf(os.name == "nt", "the stand-in ssh is a POSIX script") +class Connecting(unittest.TestCase): + def setUp(self): + self.dir = Path(tempfile.mkdtemp(prefix="frame-link-")) + self.addCleanup(shutil.rmtree, self.dir, ignore_errors=True) + (self.dir / "ssh").mkdir() + self.log = self.dir / "calls.jsonl" + env = {"FRAME_CONTROL_SSH_DIR": str(self.dir / "ssh"), "FAKESSH_LOG": str(self.log), + "FAKESSH_DIR": str(self.dir), "PATH": f"{FAKESSH}{os.pathsep}{os.environ['PATH']}"} + patcher = mock.patch.dict(os.environ, env) + patcher.start() + self.addCleanup(patcher.stop) + for name, value in (("current_network", lambda *a, **k: dict(NET)), ("fingerprint", lambda: ("192.168.1.1", "en0", "aa:bb:cc:dd:ee:ff"))): + p = mock.patch.object(fn, name, value) + p.start() + self.addCleanup(p.stop) + self.srv = socket.socket() + self.srv.bind(("127.0.0.1", 0)) + self.srv.listen(16) + self.addCleanup(self.srv.close) + self.port = self.srv.getsockname()[1] + self.reg = fd.Registry(self.dir / "devices.json") + self.routes = [] + self.link = fl.Link(self.reg, env_alias=None, mux_base=["ssh", "-o", "BatchMode=yes", "-o", "ControlPath=x"], + control="x", apply=lambda alias, opts: self.routes.append((alias, list(opts))), + explain=explain) + self.addCleanup(self.link.stop) + + def hosts(self, mapping): + os.environ["FAKESSH_HOSTS"] = json.dumps(mapping) + + def calls(self): + return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] + + def device(self, *hosts): + d = self.reg.add_device("frame-t", port=self.port, hosts=[]) + for h in hosts: + self.reg.add_address(d["id"], h, kind="lan") + return d + + def test_falls_through_to_the_address_that_is_really_the_headset(self): + # Tried in this order: a name that doesn't resolve, a different device, the headset. + d = self.device("nothing.invalid", "127.0.0.1", "localhost") + self.hosts({"127.0.0.1": "wrong", "localhost": "ok"}) + self.link.connect(["start"]) + s = self.link.snapshot() + self.assertEqual(s["phase"], "connected", s["error"]) + self.assertEqual(s["via"]["host"], "localhost") + self.assertEqual([st["state"] for st in s["stages"]], ["done"] * 5) + rows = {p["host"]: p for p in s["probes"]} + self.assertEqual(rows["nothing.invalid"]["state"], "unresolved") + self.assertEqual(rows["127.0.0.1"]["state"], "sshfailed") + self.assertIn("different headset", rows["127.0.0.1"]["detail"]) + # Every ssh command was pointed at the winner, with the host key pinned per device. + alias, opts = self.routes[-1] + self.assertEqual(alias, "frame-t") + self.assertIn(f"HostName=localhost", opts) + self.assertIn(f"HostKeyAlias=frame-control-{d['id']}", opts) + self.assertIn(f"Port={self.port}", opts) + master = [c for c in self.calls() if "ControlMaster=yes" in c][-1] + self.assertIn("StrictHostKeyChecking=accept-new", master) # first connection: nothing pinned yet + # It learned: localhost works on this network. + learned = {a["host"]: a for a in self.reg.get(d["id"])["addresses"]} + self.assertEqual(learned["localhost"]["networks"], ["n-test"]) + self.assertEqual(learned["127.0.0.1"]["networks"], []) + self.assertTrue(self.link.alive()) + self.link.close_master() + self.assertFalse(any(p.name.startswith("master-") for p in self.dir.iterdir())) + + def test_stages_are_published_as_they_happen(self): + self.device("localhost") + self.hosts({"localhost": "ok"}) + steps, versions = [], [] + real = self.link.stage + + def stage(sid, state, detail=None): + real(sid, state, detail) + steps.append((sid, state)) + versions.append(self.link.snapshot()["version"]) + self.link.stage = stage + before = self.link.snapshot()["version"] + self.assertIsNone(self.link.wait(before, 0.05)) # nothing new yet + self.link.connect(["start"]) + started = [sid for sid, state in steps if state == "active"] + self.assertEqual(list(dict.fromkeys(started)), ["network", "find", "ssh", "identity", "login"]) + self.assertEqual([sid for sid, state in steps if state == "done"][-3:], ["ssh", "identity", "login"]) + self.assertEqual(versions, sorted(versions)) # every step is a new version for the page + self.assertEqual(self.link.wait(before, 1)["phase"], "connected") + + def test_nothing_answers(self): + self.device("nothing.invalid", "also-nothing.invalid") + self.link.connect(["start"]) + s = self.link.snapshot() + self.assertEqual(s["phase"], "failed") + self.assertEqual(s["error"]["stage"], "find") + self.assertEqual(s["error"]["message"], "Can't find the Frame on the network.") + self.assertGreater(s["retry_at"], time.time()) + self.assertEqual([st["state"] for st in s["stages"]][:2], ["done", "failed"]) + + def test_refused_key_stops_at_login(self): + self.device("localhost") + self.hosts({"localhost": "denied"}) + self.link.connect(["start"]) + s = self.link.snapshot() + self.assertEqual((s["phase"], s["error"]["stage"]), ("failed", "login")) + self.assertIn("SSH key", s["error"]["message"]) + + def test_pinned_identity_is_checked_strictly(self): + d = self.device("localhost") + (self.dir / "ssh" / "frame-control_known_hosts").write_text(f"frame-control-{d['id']} ssh-ed25519 AAAA\n") + self.hosts({"localhost": "ok"}) + self.link.connect(["start"]) + master = [c for c in self.calls() if "ControlMaster=yes" in c][-1] + self.assertIn("StrictHostKeyChecking=yes", master) + + def test_a_bare_alias_lets_ssh_config_decide(self): + self.link.override = "frame-bare" + self.hosts({"frame-bare": "ok"}) # the stand-in ssh has no config: the alias is the host + with mock.patch.object(fl, "ssh_g", return_value=("localhost", self.port, "tester")): + self.link.connect(["start"]) + s = self.link.snapshot() + self.assertEqual(s["phase"], "connected", s["error"]) + self.assertTrue(s["device"]["transient"]) + self.assertEqual(self.routes[-1], ("frame-bare", [])) # no HostName override, ssh's own known_hosts + + def test_test_now_checks_every_address_without_touching_the_connection(self): + d = self.device("127.0.0.1", "localhost", "nothing.invalid") + (self.dir / "ssh" / "frame-control_known_hosts").write_text(f"frame-control-{d['id']} ssh-ed25519 AAAA\n") + self.hosts({"127.0.0.1": "wrong", "localhost": "ok"}) + self.link.test(d["id"]) + rows = {r["host"]: r for r in self.link.snapshot()["tests"][d["id"]]["rows"]} + self.assertEqual(rows["localhost"]["ssh"], "ok") + self.assertEqual(rows["127.0.0.1"]["ssh"], "wrong") + self.assertEqual(rows["nothing.invalid"]["state"], "unresolved") + self.assertEqual(self.routes, []) + self.assertTrue(all("ControlPath=none" in c for c in self.calls())) + + def test_devices_api_checks_everything(self): + d = self.device("localhost") + bad = [{"action": "address-add", "id": d["id"], "host": "-oProxyCommand=touch /tmp/x"}, + {"action": "address-add", "id": d["id"], "host": "a\nHost *"}, + {"action": "address-add", "id": d["id"], "host": "frame.local", "kind": "wifi"}, + {"action": "update", "id": d["id"], "user": "root; id"}, + {"action": "update", "id": d["id"], "port": 0}, + {"action": "address-move", "id": d["id"], "host": "localhost", "delta": 5}, + {"action": "setup", "alias": "-F/etc/passwd"}, + {"action": "setup", "alias": "frame-9", "host": "$(id)"}, + {"action": "use", "id": "nope"}, + {"action": "explode"}] + for body in bad: + with self.assertRaises(fd.DeviceError, msg=body): + fl.devices_action(self.link, body, open_setup=lambda *a: self.fail("setup ran")) + opened = [] + out = fl.devices_action(self.link, {"action": "setup", "alias": "frame-9", "host": "192.168.1.50"}, + open_setup=lambda alias, host: opened.append((alias, host)) or "a terminal") + self.assertEqual(opened, [("frame-9", "192.168.1.50")]) + self.assertIn("frame-9", out["message"]) + self.assertEqual(out["active"], d["id"]) + self.assertEqual(fl.next_alias(self.link), "frame") + + +@unittest.skipIf(os.name == "nt", "the stand-in ssh is a POSIX script") +class ServerConnection(unittest.TestCase): + """The real server, a Set Up Connection block in a stand-in ~/.ssh, and the stand-in ssh.""" + + @classmethod + def setUpClass(cls): + cls.dir = Path(tempfile.mkdtemp(prefix="frame-link-server-")) + ssh_dir = cls.dir / "ssh" + ssh_dir.mkdir() + cls.srv = socket.socket() # the "headset's" port 22 + cls.srv.bind(("127.0.0.1", 0)) + cls.srv.listen(16) + (ssh_dir / "config").write_text("# >>> steam-frame (frame) >>>\nHost frame\n HostName localhost\n" + f" Port {cls.srv.getsockname()[1]}\n" + " User steamos\nHost *\n# <<< steam-frame (frame) <<<\n") + env = {**os.environ, "PYTHONDONTWRITEBYTECODE": "1", "FRAME_CONTROL_SSH_DIR": str(ssh_dir), + "FRAME_CONTROL_DATA_DIR": str(cls.dir / "data"), "FAKESSH_LOG": str(cls.dir / "calls.jsonl"), + "FAKESSH_DIR": str(cls.dir), "FAKESSH_HOSTS": json.dumps({"localhost": "ok"}), + "PATH": f"{FAKESSH}{os.pathsep}{os.environ['PATH']}"} + env.pop("FRAME_ALIAS", None) + cls.log = tempfile.TemporaryFile() + cls.proc = subprocess.Popen([sys.executable, str(ROOT / "ui" / "server.py"), "--port", "0"], env=env, + stdout=subprocess.PIPE, stderr=cls.log, text=True) + cls.port = int(cls.proc.stdout.readline().split("127.0.0.1:")[1].split()[0]) + + @classmethod + def tearDownClass(cls): + cls.proc.terminate() + cls.proc.wait(timeout=15) + cls.proc.stdout.close() + cls.log.close() + cls.srv.close() + shutil.rmtree(cls.dir, ignore_errors=True) + + def request(self, method, path, body=None, key="1"): + conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=20) + conn.request(method, path, body=json.dumps(body).encode() if body is not None else None, + headers={"X-Frame-UI": key, "Content-Type": "application/json"}) + r = conn.getresponse() + data = json.loads(r.read() or b"{}") + conn.close() + return r.status, data + + def wait_connected(self): + for _ in range(100): + status, s = self.request("GET", "/api/connection") + if s.get("phase") in ("connected", "failed"): + return s + time.sleep(0.1) + self.fail(f"never connected: {s}") + + def test_imports_the_headset_and_connects_through_its_port(self): + s = self.wait_connected() + self.assertEqual(s["phase"], "connected", s["error"]) + self.assertEqual(s["via"]["host"], "localhost") + self.assertEqual(s["device"]["alias"], "frame") + self.assertEqual(s["device"]["name"], "Steam Frame") + self.assertEqual(s["probes"][0]["host"], "localhost") + status, devices = self.request("GET", "/api/devices") + self.assertEqual(status, 200) + self.assertEqual([d["alias"] for d in devices["devices"]], ["frame"]) + self.assertEqual(devices["nextAlias"], "frame-2") + + def test_events_stream_the_state(self): + conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=20) + conn.request("GET", "/api/connection/events", headers={"X-Frame-UI": "1"}) + r = conn.getresponse() + self.assertEqual(r.status, 200) + self.assertEqual(r.getheader("Content-Type"), "text/event-stream") + line = r.fp.readline() + self.assertTrue(line.startswith(b"data: "), line) + self.assertIn("stages", json.loads(line[6:])) + conn.close() + + def test_guards_and_validation(self): + self.assertEqual(self.request("GET", "/api/connection", key="")[0], 403) + self.assertEqual(self.request("GET", "/api/devices", key="nope")[0], 403) + self.assertEqual(self.request("POST", "/api/devices", {"action": "address-add", "id": "x", "host": "a;b"})[0], 400) + self.assertEqual(self.request("POST", "/api/devices", {"action": "setup", "alias": "-oProxyCommand=x"})[0], 400) + self.assertEqual(self.request("POST", "/api/devices", {"action": "nope"})[0], 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_network.py b/tests/test_network.py new file mode 100644 index 0000000..7a14419 --- /dev/null +++ b/tests/test_network.py @@ -0,0 +1,147 @@ +"""frame_network's parsers, with what macOS, Linux and Windows print. + +Run: python3 -m unittest discover -s tests +""" +import sandbox # noqa: F401 (first: keeps tests off real data and services) +import json +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "ui")) + +import frame_network as fn # noqa: E402 + +MAC_ROUTE = """ route to: default +destination: default + mask: default + gateway: 192.168.1.1 + interface: en0 + flags: +""" +MAC_ARP = "? (192.168.1.1) at b4:fb:e4:1:87:3f on en0 ifscope [ethernet]\n" +MAC_ARP_INCOMPLETE = "? (192.168.1.1) at (incomplete) on en0 ifscope [ethernet]\n" +MAC_SUMMARY = """ { + BSSID : + ConnectionID : 1 + InterfaceType : WiFi + LinkStatusActive : TRUE + NetworkID : + SSID : + Security : WPA2_PSK +}""" +MAC_SUMMARY_NAMED = MAC_SUMMARY.replace("SSID : \n Security", "SSID : Home Net\n Security") +MAC_SUMMARY_WIRED = " {\n InterfaceType : Ethernet\n LinkStatusActive : TRUE\n}" + +LINUX_ROUTE = """default via 10.0.0.1 dev wlp2s0 proto dhcp src 10.0.0.23 metric 600 +default via 192.168.50.1 dev enp3s0 proto dhcp src 192.168.50.9 metric 100 +""" +LINUX_NEIGH = "192.168.50.1 dev enp3s0 lladdr 00:11:22:aa:bb:cc REACHABLE\n" +NMCLI = "no:Neighbour\nyes:Cafe\\: upstairs\nno:\n" + +WIN_ROUTE = """=========================================================================== +Interface List + 12...00 15 5d 01 02 03 ......Intel(R) Wi-Fi 6 AX201 160MHz +=========================================================================== + +IPv4 Route Table +=========================================================================== +Active Routes: +Network Destination Netmask Gateway Interface Metric + 0.0.0.0 0.0.0.0 192.168.0.254 192.168.0.40 50 + 0.0.0.0 0.0.0.0 192.168.1.1 192.168.1.50 35 +=========================================================================== +Persistent Routes: + None +""" +WIN_ARP = """ +Interface: 192.168.1.50 --- 0xc + Internet Address Physical Address Type + 192.168.1.1 b4-fb-e4-b5-67-55 dynamic +""" +NETSH = """ +There is 1 interface on the system: + + Name : Wi-Fi + Description : Intel(R) Wi-Fi 6 AX201 160MHz + State : connected + SSID : Office 5G + BSSID : 12:34:56:78:9a:bc + Network type : Infrastructure +""" +NETSH_OFF = NETSH.replace("State : connected", "State : disconnected") + +TAILSCALE = json.dumps({ + "BackendState": "Running", + "CurrentTailnet": {"Name": "example.github"}, + "Self": {"HostName": "laptop", "DNSName": "laptop.tail1234.ts.net.", "TailscaleIPs": ["fd7a:115c:a1e0::1", "100.101.102.103"]}, + "Peer": {"nodekey:1": {"HostName": "frame", "DNSName": "frame.tail1234.ts.net.", "OS": "linux", "Online": True, + "TailscaleIPs": ["100.113.174.84", "fd7a:115c:a1e0::5928:ae55"]}, + "nodekey:2": {"HostName": "phone", "DNSName": "phone.tail1234.ts.net.", "OS": "iOS", "Online": False, + "TailscaleIPs": ["100.77.1.2"]}}, +}) + + +class Parsers(unittest.TestCase): + def test_macos(self): + self.assertEqual(fn.parse_route_macos(MAC_ROUTE), ("192.168.1.1", "en0")) + self.assertEqual(fn.parse_route_macos("route: writing to routing socket: not in table\n"), (None, None)) + self.assertEqual(fn.parse_arp_macos(MAC_ARP, "192.168.1.1"), "b4:fb:e4:01:87:3f") # padded + self.assertIsNone(fn.parse_arp_macos(MAC_ARP_INCOMPLETE, "192.168.1.1")) + self.assertIsNone(fn.parse_arp_macos(MAC_ARP, "192.168.1.10")) + self.assertEqual(fn.parse_summary_macos(MAC_SUMMARY), (None, True)) # no Location permission + self.assertEqual(fn.parse_summary_macos(MAC_SUMMARY_NAMED), ("Home Net", True)) + self.assertEqual(fn.parse_summary_macos(MAC_SUMMARY_WIRED), (None, False)) + + def test_linux(self): + self.assertEqual(fn.parse_route_linux(LINUX_ROUTE), ("192.168.50.1", "enp3s0")) # lowest metric + self.assertEqual(fn.parse_route_linux(""), (None, None)) + self.assertEqual(fn.parse_neigh_linux(LINUX_NEIGH, "192.168.50.1"), "00:11:22:aa:bb:cc") + self.assertIsNone(fn.parse_neigh_linux("192.168.50.1 dev enp3s0 FAILED\n", "192.168.50.1")) + self.assertEqual(fn.parse_nmcli(NMCLI), "Cafe: upstairs") + self.assertIsNone(fn.parse_nmcli("no:Neighbour\n")) + + def test_windows(self): + self.assertEqual(fn.parse_route_windows(WIN_ROUTE), ("192.168.1.1", "192.168.1.50")) + self.assertEqual(fn.parse_arp_windows(WIN_ARP, "192.168.1.1"), "b4:fb:e4:b5:67:55") + self.assertEqual(fn.parse_netsh(NETSH), "Office 5G") # not the BSSID + self.assertIsNone(fn.parse_netsh(NETSH_OFF)) + + def test_mac_addresses(self): + self.assertEqual(fn.norm_mac("B4-FB-E4-B5-67-55"), "b4:fb:e4:b5:67:55") + for bad in ("", "(incomplete)", "ff:ff:ff:ff:ff:ff", "00:00:00:00:00:00", "b4:fb:e4:b5:67", "zz:fb:e4:b5:67:55"): + self.assertIsNone(fn.norm_mac(bad), bad) + + def test_network_id_is_stable_and_needs_both_parts(self): + a = fn.network_id("192.168.1.1", "b4:fb:e4:b5:67:55") + self.assertEqual(a, fn.network_id("192.168.1.1", "b4:fb:e4:b5:67:55")) + self.assertTrue(a.startswith("n-")) + self.assertNotEqual(a, fn.network_id("192.168.1.1", "b4:fb:e4:b5:67:56")) # same IP, another router + self.assertIsNone(fn.network_id("192.168.1.1", None)) + self.assertIsNone(fn.network_id(None, "b4:fb:e4:b5:67:55")) + + def test_tailscale(self): + ts = fn.parse_tailscale(TAILSCALE) + self.assertTrue(ts["up"]) + self.assertEqual(ts["ip"], "100.101.102.103") + self.assertEqual(ts["name"], "laptop.tail1234.ts.net") + self.assertEqual(ts["tailnet"], "example.github") + frame = ts["peers"][0] + self.assertEqual((frame["name"], frame["dns"], frame["os"], frame["online"]), + ("frame", "frame.tail1234.ts.net", "linux", True)) + self.assertFalse(fn.parse_tailscale(json.dumps({"BackendState": "Stopped", "Self": {}}))["up"]) + self.assertEqual(fn.parse_tailscale("not json"), {"up": False, "peers": []}) + self.assertEqual(fn.parse_tailscale("[]"), {"up": False, "peers": []}) + + def test_address_kinds(self): + cases = {"frame.local": "mdns", "frame.local.": "mdns", "frame.tail1234.ts.net": "tailscale", + "100.113.174.84": "tailscale", "fd7a:115c:a1e0::5928:ae55": "tailscale", + "192.168.1.40": "lan", "10.0.0.5": "lan", "fe80::1%en0": "lan", + "frame.example.com": "manual", "8.8.8.8": "manual"} + for host, kind in cases.items(): + self.assertEqual(fn.guess_kind(host), kind, host) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_server.py b/tests/test_server.py index ec389bd..31167d3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -5,6 +5,7 @@ request guards and input validation, which all run before any SSH call. Run: python3 -m unittest discover -s tests """ +import sandbox # noqa: F401 (first: keeps tests off real data and services) import http.client import io import json @@ -33,7 +34,9 @@ class ServerGuards(unittest.TestCase): @classmethod def setUpClass(cls): cls.port = free_port() - env = {**os.environ, "FRAME_ALIAS": "frame-control-test.invalid", "PYTHONDONTWRITEBYTECODE": "1"} + cls.ssh_dir = tempfile.mkdtemp(prefix="frame-control-ssh-") # an empty ~/.ssh: no headsets set up + env = {**os.environ, "FRAME_ALIAS": "frame-control-test.invalid", "PYTHONDONTWRITEBYTECODE": "1", + "FRAME_CONTROL_SSH_DIR": cls.ssh_dir} cls.log = tempfile.TemporaryFile() cls.proc = subprocess.Popen([sys.executable, str(ROOT / "ui" / "server.py"), "--port", str(cls.port)], env=env, stdout=cls.log, stderr=subprocess.STDOUT) diff --git a/tests/test_steam.py b/tests/test_steam.py index d29d975..615bf1e 100644 --- a/tests/test_steam.py +++ b/tests/test_steam.py @@ -2,6 +2,7 @@ Run: python3 -m unittest discover -s tests """ +import sandbox # noqa: F401 (first: keeps tests off real data and services) import json import subprocess import sys diff --git a/tests/test_webinstall.py b/tests/test_webinstall.py index 1858297..e197b49 100644 --- a/tests/test_webinstall.py +++ b/tests/test_webinstall.py @@ -4,6 +4,7 @@ the localhost-testing rule allows. Run: python3 -m unittest discover -s tests """ +import sandbox # noqa: F401 (first: keeps tests off real data and services) import hashlib import json import os diff --git a/ui/frame_connect.py b/ui/frame_connect.py index 9b75ded..6aefc4b 100644 --- a/ui/frame_connect.py +++ b/ui/frame_connect.py @@ -6,8 +6,9 @@ asking for the Developer Mode password once. The Linux and Windows twin of scripts/connect.sh (which the Mac app uses); same config block, so either can re-run over the other. Idempotent. -Usage: python3 ui/frame_connect.py [HOST_OR_IP[:PORT]] -Env: FRAME_USER (default steamos), FRAME_ALIAS (default frame) +Usage: python3 ui/frame_connect.py [--alias NAME] [HOST_OR_IP[:PORT]] +Env: FRAME_USER (default steamos), FRAME_ALIAS (default frame; --alias wins, for + terminals that don't pass the environment on, like Windows' `start`) """ import base64 import json @@ -367,9 +368,22 @@ def pair_with_devkit(host, port, user): return chosen[0], "paired, but key login still fails" +def use_alias(alias): + """--alias: set up another headset under its own ~/.ssh/config alias (Devices tab).""" + global FRAME_ALIAS, BEGIN, END + if not NAME_RE.fullmatch(alias): + sys.exit(f"--alias must be a plain name, not {alias!r}") + FRAME_ALIAS = alias + BEGIN = f"# >>> steam-frame ({FRAME_ALIAS}) >>>" + END = f"# <<< steam-frame ({FRAME_ALIAS}) <<<" + + def main(argv): if argv and argv[0] in ("-h", "--help"): sys.exit(__doc__) + if len(argv) >= 2 and argv[0] == "--alias": + use_alias(argv[1]) + argv = argv[2:] say("==> Looking for the Steam Frame") found = pick_host(argv[0] if argv else None) while not found: diff --git a/ui/frame_devices.py b/ui/frame_devices.py new file mode 100644 index 0000000..3b5a517 --- /dev/null +++ b/ui/frame_devices.py @@ -0,0 +1,620 @@ +"""The headsets Frame Control knows, and the addresses each can be reached at. + +One headset can answer at several addresses: a LAN IP at home, another in the +office, its mDNS name (frame.local), its Tailscale IP or MagicDNS name. The +registry keeps them all, learns which worked on which network, and hands the +connector (frame_link.py) an order to try them in. + +Stored as JSON in frame_host.data_dir("devices.json"). The format is plain so the +iPhone app can share it later; docs/devices.md describes it: + + {"version": 1, "active": "", + "devices": [{"id", "name", "alias", "user", "port", "identity_files", + "addresses": [{"host", "kind": lan|mdns|tailscale|manual, "label", + "networks": [network ids it worked on], "last_ok", "last_rtt_ms"}]}], + "networks": {"": {"name", "ssid", "gateway", "gateway_mac", "last_seen"}}} + +Headsets set up before this existed live only in ~/.ssh/config, in the managed +`# >>> steam-frame (ALIAS) >>>` blocks that scripts/connect.sh and +ui/frame_connect.py write; they're imported from there, so nobody has to add +them again. Each device keeps its alias: Terminal's `ssh frame` and the helper +scripts go on working, and the connector rewrites the block's HostName to the +last address that worked, so they follow it. + +Host keys are pinned per headset, not per address: ssh gets +`-o HostKeyAlias=frame-control-` and a known_hosts file of our own, so a +different device answering at a remembered IP is caught. + +Python stdlib only. +""" +import copy +import json +import os +import re +import secrets +import subprocess +import threading +import time +from pathlib import Path + +import frame_host +import frame_network + +VERSION = 1 +# Everything here can end up in ssh arguments or ~/.ssh/config, so nothing that +# could start an option, add a line, or carry a directive. +NAME_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,63}") +HOST_RE = re.compile(r"[A-Za-z0-9:][A-Za-z0-9.:-]{0,252}(%[A-Za-z0-9._-]{1,32})?") +TEXT_MAX = 60 +KINDS = ("lan", "mdns", "tailscale", "manual") +KIND_LABEL = {"lan": "Local network", "mdns": "mDNS (.local)", "tailscale": "Tailscale", "manual": "Other"} +DEFAULT_USER = "steamos" + + +class DeviceError(ValueError): + """Bad input from the page; the server answers 400 with the message.""" + + +def ssh_dir(): + """~/.ssh, or $FRAME_CONTROL_SSH_DIR in tests so they never touch the real one.""" + return Path(os.environ.get("FRAME_CONTROL_SSH_DIR") or Path.home() / ".ssh") + + +def ssh_config(): + return ssh_dir() / "config" + + +def known_hosts(): + return ssh_dir() / "frame-control_known_hosts" + + +def known_hosts_opt(): + """How ssh is told about our known_hosts file. `~` rather than the full path when it's + the usual place, so a home folder with a space in its name can't split the option.""" + return "~/.ssh/frame-control_known_hosts" if not os.environ.get("FRAME_CONTROL_SSH_DIR") else str(known_hosts()) + + +def host_key_alias(device_id): + return f"frame-control-{device_id}" + + +# ---- validation ---------------------------------------------------------------- + +def check_alias(alias): + if not isinstance(alias, str) or not NAME_RE.fullmatch(alias): + raise DeviceError("The SSH alias must be a plain name: letters, digits, dot, dash or underscore") + return alias + + +def check_user(user): + if not isinstance(user, str) or not NAME_RE.fullmatch(user): + raise DeviceError("The user name must be letters, digits, dot, dash or underscore") + return user + + +def check_host(host): + host = host.strip() if isinstance(host, str) else host + if (not isinstance(host, str) or not HOST_RE.fullmatch(host) or ".." in host + or ("%" in host and ":" not in host.split("%")[0])): # a zone only follows an IPv6 address + raise DeviceError(f"{host!r} isn't a host name or IP address") + return host + + +def check_port(port): + try: + port = int(port) + except (TypeError, ValueError): + raise DeviceError("The port must be a number") from None + if not 1 <= port <= 65535: + raise DeviceError("The port must be between 1 and 65535") + return port + + +def check_text(text, what): + text = (text or "").strip() if isinstance(text, (str, type(None))) else None + if text is None or len(text) > TEXT_MAX or re.search(r"[\x00-\x1f\x7f]", text): + raise DeviceError(f"The {what} must be plain text of at most {TEXT_MAX} characters") + return text + + +def check_kind(kind): + if kind not in KINDS: + raise DeviceError(f"The kind must be one of {', '.join(KINDS)}") + return kind + + +def ssh_host(host): + """A host for ssh's HostName, which expands %-tokens: an IPv6 zone's % is doubled.""" + return host.replace("%", "%%") + + +# ---- ~/.ssh/config's managed blocks --------------------------------------------- + +BLOCK_RE = re.compile(r"# >>> steam-frame \((" + NAME_RE.pattern + r")\) >>>") + + +def begin_mark(alias): + return f"# >>> steam-frame ({alias}) >>>" + + +def end_mark(alias): + return f"# <<< steam-frame ({alias}) <<<" + + +def parse_blocks(text): + """The managed blocks: [{"alias", "hostname", "user", "port", "identity_files"}].""" + blocks, cur = [], None + for line in text.splitlines(): + m = BLOCK_RE.fullmatch(line.strip()) + if m: + cur = {"alias": m.group(1), "hostname": None, "user": None, "port": 22, "identity_files": []} + continue + if cur is None: + continue + if line.strip() == end_mark(cur["alias"]): + blocks.append(cur) + cur = None + continue + f = line.split(None, 1) + if len(f) != 2: + continue + key, value = f[0].lower(), f[1].strip() + if key == "hostname" and cur["hostname"] is None: + cur["hostname"] = value.replace("%%", "%") + elif key == "user" and cur["user"] is None: + cur["user"] = value + elif key == "port" and value.isdigit(): + cur["port"] = int(value) + elif key == "identityfile": + cur["identity_files"].append(value) + return blocks + + +def read_config(path=None): + path = Path(path or ssh_config()) + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError): + return "" + + +def _write_config(path, text): + """Swap the file in whole (same as frame_connect.write_config), keeping it private.""" + tmp = path.with_name("config.frame-control.tmp") + tmp.write_text(text, encoding="utf-8") + if not frame_host.WINDOWS: + tmp.chmod(0o600) + for attempt in range(20): # Windows: a running ssh.exe can hold the file for a moment + try: + os.replace(tmp, path) + return + except PermissionError: + time.sleep(0.25) + tmp.unlink() + raise OSError(f"{path} stayed locked by another program") + + +def rewrite_block(alias, path=None, hostname=None, user=None, port=None): + """Change HostName, User or Port inside ALIAS's managed block, leaving the rest of the + file alone. -> True if the file changed. Does nothing if there's no such block.""" + path = Path(path or ssh_config()) + text = read_config(path) + lines = text.splitlines() + begin, end = begin_mark(alias), end_mark(alias) + if begin not in lines or end not in lines: + return False + i, j = lines.index(begin), lines.index(end) + if j < i: + return False + block = lines[i:j] + want = {"hostname": ssh_host(hostname) if hostname else None, "user": user, + "port": str(port) if port else None} + out, seen = [], set() + for line in block: + f = line.split(None, 1) + key = f[0].lower() if f else "" + if key in want and want[key] is not None and key not in seen: + seen.add(key) + if key == "port" and want[key] == "22": + continue # the default; connect.sh leaves it out + out.append(f" {f[0]} {want[key]}") + else: + out.append(line) + if want["port"] and want["port"] != "22" and "port" not in seen: + at = next((n + 1 for n, line in enumerate(out) if line.split(None, 1)[:1] == ["HostName"]), 2) + out.insert(at, f" Port {want['port']}") + new = lines[:i] + out + lines[j:] + if new == lines: + return False + _write_config(path, "\n".join(new) + "\n") + return True + + +def remove_block(alias, path=None): + path = Path(path or ssh_config()) + lines = read_config(path).splitlines() + begin, end = begin_mark(alias), end_mark(alias) + if begin not in lines or end not in lines: + return False + i, j = lines.index(begin), lines.index(end) + if j < i: + return False + _write_config(path, "\n".join(lines[:i] + lines[j + 1:]) + "\n") + return True + + +# ---- pinned host keys ------------------------------------------------------------- + +def _pin_lines(path=None): + try: + return Path(path or known_hosts()).read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError): + return [] + + +def pinned(device_id, path=None): + name = host_key_alias(device_id) + return any(line.split(None, 1)[0].split(",").count(name) for line in _pin_lines(path) + if line.strip() and not line.startswith("#")) + + +def seed_pin(device_id, hosts, port=22, sources=None, path=None): + """Copy the host keys ssh already trusts for one of `hosts` into our file under the + device's alias, so moving to per-device pinning asks nobody to trust anything again. + -> True if a key was pinned.""" + if pinned(device_id, path): + return True + sources = sources or [ssh_dir() / "known_hosts", ssh_dir() / "known_hosts2"] + name = host_key_alias(device_id) + for host in hosts: + wanted = host if port == 22 else f"[{host}]:{port}" + keys = [] + for src in sources: + if not Path(src).is_file(): + continue + try: + out = subprocess.run(["ssh-keygen", "-F", wanted, "-f", str(src)], capture_output=True, + stdin=subprocess.DEVNULL, text=True, timeout=10).stdout + except (OSError, subprocess.TimeoutExpired): + continue + for line in out.splitlines(): + f = line.split() + if len(f) >= 3 and not line.startswith("#") and not f[0].startswith("@"): + keys.append(f"{name} {f[1]} {f[2]}") + if keys: + target = Path(path or known_hosts()) + target.parent.mkdir(parents=True, exist_ok=True) + with open(target, "a", encoding="utf-8") as fh: + fh.write("\n".join(dict.fromkeys(keys)) + "\n") + if not frame_host.WINDOWS: + target.chmod(0o600) + return True + return False + + +def forget_pin(device_id, path=None): + """Drop a device's pinned keys, e.g. after SteamOS was reinstalled. The next connection + trusts whatever key the headset shows, as a first connection does.""" + target = Path(path or known_hosts()) + name = host_key_alias(device_id) + lines = _pin_lines(target) + kept = [line for line in lines if not (line.strip() and name in line.split(None, 1)[0].split(","))] + if kept != lines: + target.write_text("".join(line + "\n" for line in kept), encoding="utf-8") + return True + return False + + +# ---- address order -------------------------------------------------------------------- + +def order_addresses(addresses, network_id, tailscale_up): + """The order to try a device's addresses in, each with why it's there: + known to work on this network, then mDNS, then Tailscale if it's up, then the rest + (addresses that only ever worked elsewhere last). The user's order breaks ties.""" + def group(a): + nets = a.get("networks") or [] + if network_id and network_id in nets: + return 0, "worked on this network before" + if a["kind"] == "mdns": + return 1, "mDNS name" + if a["kind"] == "tailscale": + return (2, "Tailscale") if tailscale_up else (5, "Tailscale isn't running") + if nets: + return 4, "worked on another network" + return 3, "not tried on this network yet" + ranked = sorted(enumerate(addresses), key=lambda p: (group(p[1])[0], p[0])) + return [(a, group(a)[1]) for _, a in ranked] + + +# ---- the registry ------------------------------------------------------------------------ + +def new_address(host, kind=None, label=""): + host = check_host(host) + return {"host": host, "kind": check_kind(kind) if kind else frame_network.guess_kind(host), + "label": check_text(label, "label"), "networks": [], "last_ok": None, "last_rtt_ms": None} + + +class Registry: + """devices.json, loaded once and saved on every change. Thread-safe.""" + + def __init__(self, path=None, config=None): + self.path = Path(path or frame_host.data_dir("devices.json")) + self.config = Path(config) if config else None # None: ssh_config() at call time + self.lock = threading.RLock() + self.data = {"version": VERSION, "active": None, "devices": [], "networks": {}} + self.load() + + # -- storage -- + def load(self): + with self.lock: + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return + if isinstance(data, dict) and isinstance(data.get("devices"), list): + data.setdefault("networks", {}) + data.setdefault("active", None) + data["devices"] = [d for d in data["devices"] if self._sane(d)] + self.data = data + + @staticmethod + def _sane(d): + try: + check_alias(d["alias"]) + d["addresses"] = [a for a in d.get("addresses") or [] if isinstance(a, dict) and HOST_RE.fullmatch(a.get("host", "")) + and a.get("kind") in KINDS] + for a in d["addresses"]: + a.setdefault("networks", []) + a.setdefault("label", "") + return NAME_RE.fullmatch(d.get("id", "")) is not None + except (KeyError, TypeError, DeviceError): + return False + + def save(self): + with self.lock: + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_name(self.path.name + ".tmp") + tmp.write_text(json.dumps(self.data, indent=1), encoding="utf-8") + os.replace(tmp, self.path) + + def snapshot(self): + with self.lock: + return copy.deepcopy(self.data) + + # -- lookups -- + def devices(self): + with self.lock: + return copy.deepcopy(self.data["devices"]) + + def _find(self, device_id): + for d in self.data["devices"]: + if d["id"] == device_id: + return d + raise DeviceError("No such headset (it may have been removed)") + + def get(self, device_id): + with self.lock: + return copy.deepcopy(self._find(device_id)) + + def by_alias(self, alias): + with self.lock: + return next((copy.deepcopy(d) for d in self.data["devices"] if d["alias"] == alias), None) + + def active(self): + with self.lock: + return self.data.get("active") + + def set_active(self, device_id): + with self.lock: + self._find(device_id) + self.data["active"] = device_id + self.save() + + # -- devices -- + def add_device(self, alias, name=None, user=DEFAULT_USER, port=22, hosts=(), identity_files=()): + with self.lock: + check_alias(alias) + if any(d["alias"] == alias for d in self.data["devices"]): + raise DeviceError(f"There's already a headset with the alias {alias}") + ids = {d["id"] for d in self.data["devices"]} + device_id = secrets.token_hex(4) + while device_id in ids: + device_id = secrets.token_hex(4) + d = {"id": device_id, "name": check_text(name or ("Steam Frame" if alias == "frame" else alias), "name"), + "alias": alias, "user": check_user(user or DEFAULT_USER), "port": check_port(port), + "identity_files": [str(f) for f in identity_files][:8], "addresses": [], "config_host": None, + "added": time.time()} + for host in hosts: + if host and not any(a["host"] == host for a in d["addresses"]): + d["addresses"].append(new_address(host)) + self.data["devices"].append(d) + if not self.data.get("active"): + self.data["active"] = device_id + self.save() + return copy.deepcopy(d) + + def update_device(self, device_id, name=None, user=None, port=None): + """-> the device after the change. The caller mirrors user and port into ~/.ssh/config.""" + with self.lock: + d = self._find(device_id) + if name is not None: + d["name"] = check_text(name, "name") or d["alias"] + if user is not None: + d["user"] = check_user(user) + if port is not None: + d["port"] = check_port(port) + self.save() + return copy.deepcopy(d) + + def remove_device(self, device_id): + """Forget a headset. Its ~/.ssh/config block (if kept) isn't imported again + unless Set Up Connection changes it.""" + with self.lock: + d = self._find(device_id) + self.data["devices"].remove(d) + self.data.setdefault("dismissed", {})[d["alias"]] = d.get("config_host") or "" + if self.data.get("active") == device_id: + self.data["active"] = self.data["devices"][0]["id"] if self.data["devices"] else None + self.save() + return d + + # -- addresses -- + def _addr(self, d, host): + for a in d["addresses"]: + if a["host"] == host: + return a + raise DeviceError(f"{host} isn't one of this headset's addresses") + + def add_address(self, device_id, host, kind=None, label=""): + with self.lock: + d = self._find(device_id) + a = new_address(host, kind, label) + if any(x["host"] == a["host"] for x in d["addresses"]): + raise DeviceError(f"{a['host']} is already on the list") + if len(d["addresses"]) >= 32: + raise DeviceError("That's enough addresses for one headset") + d["addresses"].append(a) + self.save() + return copy.deepcopy(a) + + def update_address(self, device_id, host, new_host=None, kind=None, label=None): + with self.lock: + d = self._find(device_id) + a = self._addr(d, host) + if new_host is not None and new_host != host: + new_host = check_host(new_host) + if any(x["host"] == new_host for x in d["addresses"]): + raise DeviceError(f"{new_host} is already on the list") + a.update(host=new_host, networks=[], last_ok=None, last_rtt_ms=None) # a new place: learn again + if kind is not None: + a["kind"] = check_kind(kind) + if label is not None: + a["label"] = check_text(label, "label") + self.save() + return copy.deepcopy(a) + + def remove_address(self, device_id, host): + with self.lock: + d = self._find(device_id) + d["addresses"].remove(self._addr(d, host)) + self.save() + + def move_address(self, device_id, host, delta): + with self.lock: + d = self._find(device_id) + a = self._addr(d, host) + i = d["addresses"].index(a) + j = max(0, min(len(d["addresses"]) - 1, i + int(delta))) + d["addresses"].insert(j, d["addresses"].pop(i)) + self.save() + + def record_success(self, device_id, host, network_id, rtt_ms): + """Learn: this address worked on this network.""" + with self.lock: + try: + a = self._addr(self._find(device_id), host) + except DeviceError: + return + if network_id and network_id not in a["networks"]: + a["networks"] = (a["networks"] + [network_id])[-16:] + a["last_ok"] = time.time() + a["last_rtt_ms"] = rtt_ms + self.save() + + def undismiss(self, alias): + """Set Up Connection is about to run for this alias: import its block again.""" + with self.lock: + if self.data.get("dismissed", {}).pop(alias, None) is not None: + self.save() + + def set_config_host(self, device_id, host): + with self.lock: + try: + self._find(device_id)["config_host"] = host + except DeviceError: + return + self.save() + + # -- networks -- + def record_network(self, net): + """Remember a network we've seen (for naming it), keeping its user-given name.""" + if not net or not net.get("id"): + return + with self.lock: + known = self.data["networks"].get(net["id"]) or {"name": ""} + changed = (known.get("ssid") != (net.get("ssid") or known.get("ssid")) or + time.time() - (known.get("last_seen") or 0) > 3600 or "gateway" not in known) + known.update(ssid=net.get("ssid") or known.get("ssid"), gateway=net.get("gateway"), wifi=net.get("wifi"), + gateway_mac=net.get("gateway_mac"), last_seen=time.time()) + self.data["networks"][net["id"]] = known + if changed: + self.save() + + def name_network(self, network_id, name): + with self.lock: + if network_id not in self.data["networks"]: + raise DeviceError("That network hasn't been seen") + self.data["networks"][network_id]["name"] = check_text(name, "network name") + self.save() + + def network_name(self, net): + """What to call a network: the name given to it, its Wi-Fi name, or its router.""" + if not net: + return "No network" + with self.lock: + known = self.data["networks"].get(net.get("id") or "") or {} + if known.get("name"): + return known["name"] + ssid = net.get("ssid") or known.get("ssid") + if ssid: + return ssid + if net.get("gateway"): + return f"{'Wi-Fi' if net.get('wifi') else 'Network'} via {net['gateway']}" + return "No network" + + # -- ~/.ssh/config -- + def sync_from_config(self, seed=True): + """Import managed blocks we don't know yet, and pick up a HostName that Set Up + Connection changed since we last looked. -> True if anything changed.""" + blocks = parse_blocks(read_config(self.config)) + changed = False + with self.lock: + for b in blocks: + host = b["hostname"] if b["hostname"] and HOST_RE.fullmatch(b["hostname"]) else None + user = b["user"] if b["user"] and NAME_RE.fullmatch(b["user"]) else DEFAULT_USER + d = next((x for x in self.data["devices"] if x["alias"] == b["alias"]), None) + dismissed = self.data.get("dismissed", {}) + if d is None and b["alias"] in dismissed: + if dismissed[b["alias"]] == (host or ""): + continue # removed on the Devices tab; unchanged since + del dismissed[b["alias"]] + if d is None: + try: + d = self._find(self.add_device(b["alias"], user=user, port=b["port"], + identity_files=b["identity_files"])["id"]) + except DeviceError: + continue + if host: + d["addresses"].append(dict(new_address(host), label="From Set Up Connection")) + d["config_host"] = host + changed = True + if seed and host: + seed_pin(d["id"], [host], b["port"]) + elif host and host != d.get("config_host"): + # Set Up Connection ran again and found the headset somewhere new. + d["config_host"] = host + if not any(a["host"] == host for a in d["addresses"]): + d["addresses"].insert(0, dict(new_address(host), label="From Set Up Connection")) + if seed: + seed_pin(d["id"], [host], b["port"]) + changed = True + if d["identity_files"] != b["identity_files"] and b["identity_files"]: + d["identity_files"] = b["identity_files"][:8] + changed = True + d["managed"] = True + aliases = {b["alias"] for b in blocks} + for d in self.data["devices"]: + d["managed"] = d["alias"] in aliases + if changed: + self.save() + return changed diff --git a/ui/frame_host.py b/ui/frame_host.py index a60b82c..a3842f3 100644 --- a/ui/frame_host.py +++ b/ui/frame_host.py @@ -33,8 +33,11 @@ class HostError(RuntimeError): def data_dir(*parts): - """Per-user app data: ~/Library/Application Support, %APPDATA% or $XDG_DATA_HOME.""" - if MAC: + """Per-user app data: ~/Library/Application Support, %APPDATA% or $XDG_DATA_HOME + (or $FRAME_CONTROL_DATA_DIR, which the tests point at a throwaway directory).""" + if os.environ.get("FRAME_CONTROL_DATA_DIR"): + base = Path(os.environ["FRAME_CONTROL_DATA_DIR"]) + elif MAC: base = Path.home() / "Library" / "Application Support" / "Frame Control" elif WINDOWS: base = Path(os.environ.get("APPDATA") or Path.home() / "AppData" / "Roaming") / "Frame Control" diff --git a/ui/frame_link.py b/ui/frame_link.py new file mode 100644 index 0000000..79bc722 --- /dev/null +++ b/ui/frame_link.py @@ -0,0 +1,905 @@ +"""The connection to the active headset: which address to use, and every step of getting there. + +A background thread (Link) keeps one SSH connection to the active headset open +and publishes what it's doing, stage by stage, for the page's connection pill: + + 1. network checking this computer's network (gateway, Wi-Fi, Tailscale) + 2. find finding the headset: every address probed on port 22 at once + 3. ssh opening SSH to the address that answered + 4. identity checking the headset's identity (its pinned host key) + 5. login logging in as the device's user + then connected (network, address, round trip), or failed at a stage with a + plain reason and a countdown to the next try. + +Addresses go in the order frame_devices.order_addresses gives. All are probed +at once; the best-ranked one that answers wins, waiting a moment (PREFER) for a +better-ranked address that's still trying, happy-eyeballs style. If SSH to the +winner fails in a way another address could fix (a different device answered, +the link dropped), the next one that answered is tried. + +The server hands in `apply(alias, host_opts)`, which points every ssh, scp and +rsync it runs at the alias with `-o HostName=
` and friends, so they all +follow. Where ssh can share one connection (not Windows), the master connection +lives here; it reconnects when it dies, when this computer changes networks, and +when the page asks. + +Python stdlib only. Runs on this computer, never on the Frame. +""" +import copy +import queue +import re +import socket +import subprocess +import threading +import time + +import frame_devices +import frame_host +import frame_network + +PROBE_TIMEOUT = 4 # seconds for a TCP answer on port 22 +PREFER = 0.35 # how long an answer waits for a better-ranked address still trying +HANDSHAKE_TIMEOUT = 25 +TICK = 2 # the loop's heartbeat +NETWORK_EVERY = 5 # how often the network fingerprint is read +TAILSCALE_EVERY = 30 +RETRY = (5, 10, 20, 30) # seconds before automatic retries after a failure +REQUEST_GAP = 5 # a request may start a new attempt this long after the last one + +STAGES = [("network", "Checking this computer's network"), ("find", "Finding the headset"), + ("ssh", "Opening SSH"), ("identity", "Checking the headset's identity"), + ("login", "Logging in")] + +# What ssh -v prints at each step (OpenSSH on macOS, Linux and Windows). +CONNECTING = re.compile(r"Connecting to (\S+) \[([^\]]+)\] port (\d+)") +ESTABLISHED = re.compile(r"Connection established") +HOSTKEY = re.compile(r"Server host key: (\S+) (\S+)") +KNOWN = re.compile(r"is known and matches") +ADDED = re.compile(r"Permanently added") +CHANGED = re.compile(r"REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed") +UNKNOWN = re.compile(r"No \S+ host key is known for") +AUTH_START = re.compile(r"Authentications that can continue|Next authentication method") +AUTHED = re.compile(r"Authenticated to |Authentication succeeded") +DENIED = re.compile(r"Permission denied") + + +def now(): + return time.time() + + +def ssh_g(alias): + """(hostname, port, user) from `ssh -G ALIAS`, for a headset that's only an ssh alias.""" + try: + out = subprocess.run(["ssh", "-G", alias], capture_output=True, stdin=subprocess.DEVNULL, text=True, + timeout=10).stdout + except (OSError, subprocess.TimeoutExpired): + out = "" + got = {} + for line in out.splitlines(): + k, _, v = line.partition(" ") + if k in ("hostname", "port", "user") and k not in got: + got[k] = v.strip() + port = int(got["port"]) if got.get("port", "").isdigit() else 22 + return got.get("hostname") or alias, port, got.get("user") + + +def probe(host, port, timeout=PROBE_TIMEOUT, update=None): + """Try a TCP connection to host:port. -> {"state", "detail", "ip", "rtt_ms"}. + + state: answered, unresolved, timeout, refused, unreachable or error. update(fields) + reports progress (resolving, trying) as it happens.""" + update = update or (lambda **_: None) + update(state="resolving", detail="Looking up the name") + deadline = now() + timeout + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except (socket.gaierror, UnicodeError, OSError) as e: + return {"state": "unresolved", "detail": "Can't find this name on the network", "error": str(e)} + last = None + for family, kind, proto, _, addr in infos[:4]: + ip = addr[0] + left = deadline - now() + if left <= 0: + break + update(state="trying", detail=f"Trying {ip}", ip=ip) + s = socket.socket(family, kind, proto) + s.settimeout(left) + t0 = time.monotonic() + try: + s.connect(addr) + rtt = round((time.monotonic() - t0) * 1000, 1) + return {"state": "answered", "detail": f"Answered in {rtt:g} ms", "ip": ip, "rtt_ms": rtt} + except socket.timeout: + last = {"state": "timeout", "detail": "No answer", "ip": ip} + except ConnectionRefusedError: + last = {"state": "refused", "detail": "Refused: SSH isn't on at this address", "ip": ip} + except OSError as e: + last = {"state": "unreachable", "detail": f"Can't get there ({e.strerror or e})", "ip": ip} + finally: + s.close() + return last or {"state": "timeout", "detail": "No answer"} + + +def probe_raw(host, port, result): + """ssh's own wording for a failed probe, so the server's UNREACHABLE table explains it.""" + return {"unresolved": f"ssh: Could not resolve hostname {host}: not found", + "refused": f"ssh: connect to host {host} port {port}: Connection refused", + "unreachable": f"ssh: connect to host {host} port {port}: No route to host", + }.get(result["state"], f"ssh: connect to host {host} port {port}: Operation timed out") + + +class Link: + def __init__(self, registry, *, env_alias, mux_base, control, apply, explain): + self.reg = registry + self.override = env_alias # FRAME_ALIAS, if set: the headset this server starts on + self.mux_base = list(mux_base) # ["ssh", "-o", "BatchMode=yes", ControlPath...] + self.control = control # ControlPath, or None where ssh can't share connections + self.apply = apply # apply(alias, host_opts): point every ssh command at the headset + self.explain = explain # ssh error text -> plain reason, or None + self.cond = threading.Condition() + self.version = 0 + self.stopped = False + self.kicks = [] # reasons someone asked for a (re)connect + self.busy = False # the loop is handling kicks + self.state = {"phase": "idle", "reason": None, "device": None, "network": None, "stages": [], + "probes": [], "via": None, "error": None, "retry_at": None, "attempt": 0, + "started": None, "finished": None, "tests": {}, "devices_rev": 0} + self.master = None # the ssh ControlMaster process, if we started it + self.opts = [] # host options of the current connection + self.alias = None + self.fails = 0 + self.last_fp = None + self.last_attempt = 0 + self.config_mtime = None + self.thread = None + + # ---- publishing ---- + def publish(self, **fields): + with self.cond: + self.state.update(fields) + self.version += 1 + self.cond.notify_all() + + def snapshot(self): + with self.cond: + snap = copy.deepcopy(self.state) + snap["version"] = self.version + snap["now"] = now() + return snap + + def wait(self, version, timeout): + """The state once its version passes `version`, or None after `timeout` seconds.""" + with self.cond: + if not self.cond.wait_for(lambda: self.version > version or self.stopped, timeout): + return None + return self.snapshot() + + def stage(self, sid, state, detail=None): + """Move one stage along (pending -> active -> done or failed) and publish.""" + with self.cond: + for s in self.state["stages"]: + if s["id"] == sid: + if state == "active" and s["state"] != "active": + s["started"] = now() + if state in ("done", "failed", "skipped"): + s["ended"] = now() + s["started"] = s["started"] or s["ended"] + s["state"] = state + if detail is not None: + s["detail"] = detail + self.version += 1 + self.cond.notify_all() + + def probe_update(self, index, **fields): + with self.cond: + if index < len(self.state["probes"]): + self.state["probes"][index].update(fields) + self.version += 1 + self.cond.notify_all() + + def devices_changed(self): + with self.cond: + self.state["devices_rev"] += 1 + self.version += 1 + self.cond.notify_all() + + # ---- control from the server ---- + def start(self): + self.thread = threading.Thread(target=self.run, name="frame-link", daemon=True) + self.thread.start() + + def kick(self, reason): + with self.cond: + self.kicks.append(reason) + self.cond.notify_all() + + def stop(self): + with self.cond: + self.stopped = True + self.cond.notify_all() + self.close_master() + + def alive(self): + if self.state["phase"] != "connected": + return False + if not self.control: + return True + return self.master is None or self.master.poll() is None + + def ensure(self, wait=20): + """Called before a command: make sure a connection is up, or being tried. + + Waits (up to `wait` s) for an attempt already running, or starts one if the + last ended a while ago. Never raises: if the headset can't be reached, the + command runs anyway and fails with ssh's own error, as it always has.""" + with self.cond: + if self.stopped or self.alive(): + return + if self.state["phase"] != "connecting" and not self.kicks and now() - self.last_attempt > REQUEST_GAP: + self.kicks.append("request") + self.cond.wait_for(lambda: self.stopped or (not self.kicks and not self.busy and + self.state["phase"] != "connecting"), wait) + + def use(self, device_id): + """Switch to another headset.""" + self.reg.set_active(device_id) + self.override = None + self.devices_changed() + self.kick("switch") + + def lost(self, message): + """A command couldn't reach the headset (Windows has no master to watch).""" + if self.state["phase"] == "connected": + self.kick(f"lost: {message}") + + # ---- the device this server talks to ---- + def active_device(self): + """The active headset from the registry, or a stand-in for a bare ssh alias.""" + if self.override: + return self.reg.by_alias(self.override) or self.bare(self.override) + want = self.reg.active() + if want: + try: + return self.reg.get(want) + except frame_devices.DeviceError: + pass + devices = self.reg.devices() + return devices[0] if devices else self.bare("frame") + + @staticmethod + def bare(alias): + """A headset that's only an ssh alias (no Set Up Connection block): ssh's config decides.""" + return {"id": f"alias-{alias}", "name": alias, "alias": alias, "user": None, "port": None, + "addresses": [], "transient": True, "identity_files": []} + + def host_opts(self, device, host): + """What every ssh command adds to reach DEVICE at HOST.""" + if device.get("transient") or not host: + return [] + return ["-o", f"HostName={frame_devices.ssh_host(host)}", + "-o", f"HostKeyAlias={frame_devices.host_key_alias(device['id'])}", + "-o", f"UserKnownHostsFile={frame_devices.known_hosts_opt()}", + "-o", f"User={device['user']}", "-o", f"Port={device['port']}"] + + def public_device(self, d): + return {k: d.get(k) for k in ("id", "name", "alias", "user", "port", "transient")} + + # ---- the loop ---- + def run(self): + self.kick("start") + last_net = last_ts = 0 + while True: + with self.cond: + self.cond.wait_for(lambda: self.stopped or self.kicks, TICK) + if self.stopped: + return + reasons, self.kicks = self.kicks, [] + self.busy = bool(reasons) + try: + t = now() + if t - last_net >= NETWORK_EVERY: + last_net = t + fp = frame_network.fingerprint() + if self.last_fp is not None and fp[::2] != self.last_fp[::2]: + reasons.append("network") + self.last_fp = fp + self.watch_config() + if self.state["phase"] == "connected" and self.control and self.master is None \ + and not self.check(self.opts): + reasons.append("dropped") # a master we found open, not one we started + if t - last_ts >= TAILSCALE_EVERY and self.state["network"] and not reasons: + last_ts = t + self.refresh_network() + phase = self.state["phase"] + if phase == "connected" and not self.alive(): + reasons.append("dropped") + if phase == "failed" and self.state["retry_at"] and now() >= self.state["retry_at"]: + reasons.append("retry") + if reasons: + self.connect(reasons) + except Exception as e: # keep the loop alive whatever happens; say what went wrong + self.publish(phase="failed", error={"stage": "network", "message": f"{type(e).__name__}: {e}", + "raw": str(e)}, retry_at=now() + RETRY[-1]) + finally: + with self.cond: + self.busy = False + self.cond.notify_all() + + def watch_config(self): + """Set Up Connection may have added a headset or found a new address: pick it up.""" + try: + mtime = frame_devices.ssh_config().stat().st_mtime + except OSError: + mtime = None + if mtime != self.config_mtime: + self.config_mtime = mtime + if self.reg.sync_from_config(): + self.devices_changed() + + def refresh_network(self): + net = frame_network.current_network(self.last_fp) + self.reg.record_network(net) + net["name"] = self.reg.network_name(net) + self.publish(network=net) + + # ---- one attempt ---- + def connect(self, reasons): + why = self.describe(reasons) + self.last_attempt = now() + self.close_master() + device = self.active_device() + with self.cond: + self.state.update(phase="connecting", reason=why, device=self.public_device(device), via=None, + error=None, retry_at=None, attempt=self.state["attempt"] + 1, started=now(), + finished=None, probes=[], + stages=[{"id": i, "label": label, "state": "pending", "detail": "", + "started": None, "ended": None} for i, label in STAGES]) + self.version += 1 + self.cond.notify_all() + ok = False + try: + ok = self.attempt(device) + finally: + with self.cond: + self.state["finished"] = now() + if ok: + self.fails = 0 + self.state.update(phase="connected", retry_at=None, error=None) + else: + self.fails += 1 + self.state.update(phase="failed", + retry_at=now() + RETRY[min(self.fails, len(RETRY)) - 1]) + if not self.state["error"]: + self.state["error"] = {"stage": "find", "message": "Couldn't connect", "raw": ""} + self.version += 1 + self.cond.notify_all() + + @staticmethod + def describe(reasons): + for r in reasons: + if r == "network": + return "This computer changed networks" + if r == "dropped" or r.startswith("lost"): + return "The connection dropped" + if r == "switch": + return "Switched headset" + if "retry" in reasons: + return "Trying again" + if "start" in reasons: + return "Starting up" + return "Connecting" + + def fail(self, sid, message, raw=""): + self.stage(sid, "failed", message) + with self.cond: + self.state["error"] = {"stage": sid, "message": message, "raw": raw} + + def attempt(self, device): + # 1. this computer's network + self.stage("network", "active") + net = frame_network.current_network(self.last_fp) + self.reg.record_network(net) + net["name"] = self.reg.network_name(net) + ts = net.get("tailscale") or {} + self.publish(network=net) + bits = [net["name"]] + if net.get("local_ip"): + bits.append(f"this computer is {net['local_ip']}") + bits.append("Tailscale on" if ts.get("up") else "Tailscale off" if ts.get("installed") else "no Tailscale") + self.stage("network", "done" if net.get("gateway") or ts.get("up") else "failed", " · ".join(bits)) + if not net.get("gateway") and not ts.get("up"): + with self.cond: + self.state["error"] = {"stage": "network", "raw": "", + "message": "This computer isn't connected to a network."} + # Keep going anyway: a headset on a direct link or loopback could still answer. + + # 2. find the headset + self.stage("find", "active") + port = device.get("port") or 22 + if device.get("transient") or not device["addresses"]: + host, port, user = ssh_g(device["alias"]) + if user and not device.get("user"): + device["user"] = user + ranked = [({"host": host, "kind": frame_network.guess_kind(host), "label": "from ~/.ssh/config"}, + "from ~/.ssh/config")] + else: + ranked = frame_devices.order_addresses(device["addresses"], net.get("id"), bool(ts.get("up"))) + with self.cond: + self.state["probes"] = [{"host": a["host"], "kind": a["kind"], "label": a.get("label") or "", + "why": why, "state": "waiting", "detail": "Waiting", "ip": None, + "rtt_ms": None} for a, why in ranked] + self.stage("find", "active", f"Trying {len(ranked)} address{'es' * (len(ranked) != 1)} at once") + results = [None] * len(ranked) + done = threading.Condition() + + def run_probe(i, host): + res = probe(host, port, update=lambda **f: self.probe_update(i, **f)) + res.setdefault("ip", None) + res.setdefault("rtt_ms", None) + self.probe_update(i, **{k: res[k] for k in ("state", "detail", "ip", "rtt_ms")}) + with done: + if results[i] is None: # not already given up on + results[i] = dict(res, t=time.monotonic()) + done.notify_all() + + for i, (a, _) in enumerate(ranked): + threading.Thread(target=run_probe, args=(i, a["host"]), daemon=True).start() + + tried = set() + user = device.get("user") or "the headset's user" + deadline = time.monotonic() + PROBE_TIMEOUT + 1 + while True: + pick = self.pick(results, tried, done, deadline) + if pick is None: + break + if tried: # another address may do better (a different device answered, or it dropped) + for sid in ("ssh", "identity", "login"): + self.stage(sid, "pending", "") + tried.add(pick) + a = ranked[pick][0] + self.stage("find", "done", f"{a['host']} answered in {results[pick]['rtt_ms']:g} ms") + outcome = self.handshake(device, a, results[pick], user) + if outcome == "ok": + via = {"host": a["host"], "kind": a["kind"], "ip": results[pick]["ip"], + "rtt_ms": results[pick]["rtt_ms"], "why": ranked[pick][1], "network": net.get("id"), + "network_name": net["name"]} + self.publish(via=via) + self.learn(device, a["host"], net, results[pick]["rtt_ms"]) + return True + with self.cond: + why_not = (self.state["error"] or {}).get("message") or "SSH failed" + self.probe_update(pick, state="sshfailed", detail=why_not) + if outcome != "next": + return False + if tried: + return False # the last handshake already said why + # Nothing answered: explain with the most useful failure. + with self.cond: + for row in self.state["probes"]: + if row["state"] in ("waiting", "resolving", "trying"): + row.update(state="timeout", detail="No answer in time") + states = [r["state"] for r in results if r] + worst = next((s for s in ("refused", "timeout", "unreachable", "unresolved") if s in states), "timeout") + i = states.index(worst) if worst in states else 0 + raw = probe_raw(ranked[i][0]["host"], port, results[i] or {"state": worst}) + message = self.explain(raw) or "The Frame isn't answering." + self.fail("find", message, raw) + return False + + @staticmethod + def pick(results, tried, done, deadline=None): + """The next address to use: the best-ranked answer once every better-ranked + address has failed, or once it has waited PREFER seconds for them. None when + nothing (else) answered. Probes still going at `deadline` (a name lookup can + take longer than the connect timeout) count as no answer.""" + deadline = deadline or time.monotonic() + PROBE_TIMEOUT + 1 + with done: + while True: + if time.monotonic() >= deadline: + for i, r in enumerate(results): + if r is None: + results[i] = {"state": "timeout", "detail": "No answer", "ip": None, "rtt_ms": None, + "t": time.monotonic()} + answered = [i for i, r in enumerate(results) if r and r["state"] == "answered" and i not in tried] + pending = [i for i, r in enumerate(results) if r is None] + if answered: + best = answered[0] + better = [i for i in pending if i < best] + waited = time.monotonic() - results[best]["t"] + if not better or waited >= PREFER: + return best + done.wait(PREFER - waited) + elif not pending: + return None + else: + done.wait(min(0.5, max(0.01, deadline - time.monotonic()))) + + def learn(self, device, host, net, rtt): + if device.get("transient"): + return + self.reg.record_success(device["id"], host, net.get("id"), rtt) + # Terminal's `ssh ALIAS` and the helper scripts use ~/.ssh/config: point it here too. + try: + if frame_devices.rewrite_block(device["alias"], hostname=host, user=device["user"], + port=device["port"]): + self.config_mtime = frame_devices.ssh_config().stat().st_mtime + self.reg.set_config_host(device["id"], host) + except OSError: + pass # not fatal: the app itself doesn't need the file + self.devices_changed() + + # ---- SSH ---- + def check(self, opts, alias=None): + """Is a master connection up for these options? (`ssh -O check`)""" + if not self.control: + return False + try: + return subprocess.run([*self.mux_base, *opts, "-O", "check", alias or self.alias], capture_output=True, + stdin=subprocess.DEVNULL, timeout=5).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + def close_master(self): + proc, self.master = self.master, None + if self.control and self.alias: + try: + subprocess.run([*self.mux_base, *self.opts, "-O", "exit", self.alias], capture_output=True, + stdin=subprocess.DEVNULL, timeout=5) + except (OSError, subprocess.TimeoutExpired): + pass + if proc and proc.poll() is None: + proc.terminate() + try: + proc.wait(5) + except subprocess.TimeoutExpired: + proc.kill() + + def handshake(self, device, a, found, user): + """SSH to one address, following ssh -v through stages 3-5. + -> "ok", "next" (try another address) or "stop".""" + opts = self.host_opts(device, a["host"]) + alias = device["alias"] + self.alias, self.opts = alias, opts + self.apply(alias, opts) + target = f"{a['host']}" + (f" ({found['ip']})" if found.get("ip") and found["ip"] != a["host"] else "") + self.stage("ssh", "active", f"Opening SSH to {target}") + if self.control and self.check(opts, alias): + for sid in ("ssh", "identity", "login"): + self.stage(sid, "done", "Reusing the SSH connection that's already open") + return "ok" + extra = [] + if not device.get("transient"): + if frame_devices.pinned(device["id"]): + extra = ["-o", "StrictHostKeyChecking=yes"] + else: + # First connection since this headset was added: trust what it shows + # (as Set Up Connection does), and pin it from now on. + extra = ["-o", "StrictHostKeyChecking=accept-new"] + if self.control: + # No ConnectTimeout: with it, OpenSSH's master takes ~5s to open its socket. + argv = [*self.mux_base, *opts, *extra, "-v", "-o", "ControlMaster=yes", "-o", "ServerAliveInterval=5", + "-o", "ServerAliveCountMax=2", "-N", alias] + else: + argv = [*self.mux_base, *opts, *extra, "-v", "-o", "ConnectTimeout=10", alias, "true"] + try: + proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, **frame_host.DETACHED) + except OSError as e: + self.fail("ssh", f"Couldn't run ssh: {e}", str(e)) + return "stop" + lines = queue.Queue() + collecting = [True] + + def read(): + for raw in iter(proc.stderr.readline, b""): + if collecting[0]: + lines.put(raw.decode("utf-8", "replace").rstrip()) + lines.put(None) + proc.stderr.close() + threading.Thread(target=read, daemon=True).start() + + step, said, authed = "ssh", [], False + deadline = time.monotonic() + HANDSHAKE_TIMEOUT + mismatch = False + while True: + left = deadline - time.monotonic() + if left <= 0: + proc.kill() + self.fail(step, self.explain(f"Timed out talking to {alias}") or "The headset took too long to answer.", + f"Timed out talking to {alias}") + return "next" if step in ("ssh", "identity") else "stop" + try: + line = lines.get(timeout=min(left, 0.25)) + except queue.Empty: + line = "" + if authed and self.control and self.check(opts, alias): + break + if proc.poll() is not None and lines.empty(): + line = None + else: + continue + if line is None: # ssh exited + proc.wait() + if not self.control and proc.returncode == 0: + break + if authed and self.control and self.check(opts, alias): + break + return self.failed(step, said, mismatch, alias) + if not line.startswith("debug"): + said.append(line) + m = CONNECTING.search(line) + if m: + self.stage("ssh", "active", f"Opening SSH to {a['host']}" + + (f" ({m.group(2)})" if m.group(2) != a["host"] else "") + f", port {m.group(3)}") + elif ESTABLISHED.search(line): + self.stage("ssh", "done", f"Connected to {target}") + step = "identity" + self.stage("identity", "active", "Waiting for the headset's host key") + elif HOSTKEY.search(line): + m = HOSTKEY.search(line) + self.stage("identity", "active", f"It shows {m.group(1)} key {m.group(2)[:20]}…") + elif KNOWN.search(line): + self.stage("identity", "done", "Matches the identity saved for this headset") + step = "login" + self.stage("login", "active", f"Logging in as {user}") + elif ADDED.search(line): + self.stage("identity", "done", "First connection: saved this headset's identity") + step = "login" + self.stage("login", "active", f"Logging in as {user}") + elif (CHANGED.search(line) or UNKNOWN.search(line)) and not device.get("transient"): + mismatch = True # a bare alias keeps ssh's per-address check, and its wording + elif AUTH_START.search(line) and step != "login": + self.stage("identity", "done") + step = "login" + self.stage("login", "active", f"Logging in as {user}") + elif AUTHED.search(line): + authed = True + if step != "login": + self.stage("identity", "done") + self.stage("login", "done", f"Logged in as {user}") + step = "connected" + collecting[0] = False # the master keeps printing mux debug lines: drop them + for sid in ("ssh", "identity", "login"): + with self.cond: + pending = any(s["id"] == sid and s["state"] != "done" for s in self.state["stages"]) + if pending: + self.stage(sid, "done") + if self.control: + self.master = proc + return "ok" + + def failed(self, step, said, mismatch, alias): + text = "\n".join(said).strip() + if mismatch: + self.fail("identity", "This address answered as a different headset (its SSH identity doesn't match). " + "If SteamOS was reinstalled, use Forget Identity on the Devices tab.", text) + return "next" + if step == "login" or re.search(r"Permission denied", text): + self.fail("login", self.explain(text) or "The headset didn't accept this computer's key.", text) + return "stop" + if step == "connected": + self.fail("login", "Logged in, but the shared SSH connection didn't start.", text) + return "stop" + self.fail(step, self.explain(text) or (text.splitlines()[-1] if text else "ssh stopped"), text) + return "next" + + # ---- Test now ---- + def test(self, device_id): + """Probe every address of a headset and check SSH on the ones that answer, + without touching the live connection. Results stream into state["tests"].""" + device = self.reg.get(device_id) + started = now() + rows = [{"host": a["host"], "kind": a["kind"], "state": "waiting", "detail": "Waiting", "ip": None, + "rtt_ms": None, "ssh": None} for a in device["addresses"]] + + def put(**fields): + with self.cond: + self.state["tests"][device_id] = dict({"started": started, "done": False, "rows": rows}, **fields) + self.version += 1 + self.cond.notify_all() + + put() + net = self.state["network"] or {} + + def one(i, a): + res = probe(a["host"], device["port"], update=lambda **f: (rows[i].update(f), put())) + rows[i].update({k: res.get(k) for k in ("state", "detail", "ip", "rtt_ms")}) + put() + if res["state"] != "answered": + return + rows[i]["ssh"] = "checking" + put() + argv = [*self.mux_base[:3], "-o", "ControlPath=none", "-o", "ConnectTimeout=8", + *self.host_opts(device, a["host"]), "-o", "StrictHostKeyChecking=yes", device["alias"], "true"] + try: + r = subprocess.run(argv, capture_output=True, stdin=subprocess.DEVNULL, text=True, + errors="replace", timeout=20) + err = r.stderr.strip() + if r.returncode == 0: + rows[i].update(ssh="ok", detail=f"Answered in {res['rtt_ms']:g} ms · SSH works") + self.reg.record_success(device_id, a["host"], net.get("id"), res["rtt_ms"]) + elif UNKNOWN.search(err): + rows[i].update(ssh="unpinned", detail=f"Answered in {res['rtt_ms']:g} ms · identity not saved yet") + elif CHANGED.search(err): + rows[i].update(ssh="wrong", detail="Answered as a different headset") + elif DENIED.search(err): + rows[i].update(ssh="denied", detail="Answered, but refused this computer's key") + else: + rows[i].update(ssh="failed", detail=self.explain(err) or (err.splitlines() or ["SSH failed"])[-1]) + except (OSError, subprocess.TimeoutExpired): + rows[i].update(ssh="failed", detail="SSH took too long") + put() + + threads = [threading.Thread(target=one, args=(i, a), daemon=True) for i, a in enumerate(device["addresses"])] + for t in threads: + t.start() + for t in threads: + t.join(40) + put(done=True, finished=now()) + self.devices_changed() + + +# ---- the page's API: /api/devices ------------------------------------------------- + +def devices_view(link): + """Every headset with its addresses, the networks they worked on, and the current network.""" + snap = link.reg.snapshot() + active = link.active_device() + names = {nid: link.reg.network_name(dict(n, id=nid)) for nid, n in snap["networks"].items()} + devices = [] + if active.get("transient"): + devices.append(dict(link.public_device(active), active=True, addresses=[], managed=False, pinned=False)) + for d in snap["devices"]: + view = {k: v for k, v in d.items() if k not in ("config_host", "addresses")} + view["active"] = d["id"] == active["id"] + view["pinned"] = frame_devices.pinned(d["id"]) + view["addresses"] = [dict(a, network_names=[names.get(n, "an unnamed network") for n in a["networks"]]) + for a in d["addresses"]] + devices.append(view) + return {"devices": devices, "active": active["id"], "network": link.state["network"], + "networks": [dict(n, id=nid, display=names[nid]) for nid, n in snap["networks"].items()], + "kinds": frame_devices.KIND_LABEL} + + +def devices_action(link, body, open_setup): + """POST /api/devices {"action": ..., "id": device id, ...}. -> {"message", ...devices_view}.""" + reg = link.reg + action = body.get("action") + did = body.get("id") + active = link.active_device() + is_active = did == active["id"] + if action == "use": + d = reg.get(did) + link.use(did) + msg = f"Switched to {d['name']}" + elif action == "update": + d = reg.update_device(did, name=body.get("name"), user=body.get("user"), port=body.get("port")) + try: + frame_devices.rewrite_block(d["alias"], user=d["user"], port=d["port"]) + except OSError as e: + raise frame_devices.DeviceError(f"Saved, but couldn't update ~/.ssh/config: {e}") + if is_active: + link.kick("switch") + msg = f"Saved {d['name']}" + elif action == "remove": + d = reg.remove_device(did) + frame_devices.forget_pin(did) + removed = False + if body.get("config"): + try: + removed = frame_devices.remove_block(d["alias"]) + except OSError as e: + raise frame_devices.DeviceError(f"Removed, but couldn't edit ~/.ssh/config: {e}") + if is_active: + link.override = None + link.kick("switch") + msg = f"Removed {d['name']}" + (f" and its '{d['alias']}' entry in ~/.ssh/config" if removed else "") + elif action == "address-add": + a = reg.add_address(did, body.get("host"), body.get("kind") or None, body.get("label") or "") + if is_active and link.state["phase"] == "failed": + link.kick("retry") + msg = f"Added {a['host']}" + elif action == "address-update": + a = reg.update_address(did, body.get("host"), new_host=body.get("newHost"), kind=body.get("kind"), + label=body.get("label")) + msg = f"Saved {a['host']}" + elif action == "address-remove": + reg.remove_address(did, body.get("host")) + msg = f"Removed {body.get('host')}" + elif action == "address-move": + delta = body.get("delta") + if delta not in (-1, 1): + raise frame_devices.DeviceError("delta must be -1 or 1") + reg.move_address(did, body.get("host"), delta) + msg = "Moved" + elif action == "test": + d = reg.get(did) + if not d["addresses"]: + raise frame_devices.DeviceError("This headset has no addresses to test yet") + threading.Thread(target=link.test, args=(did,), daemon=True).start() + msg = f"Testing {len(d['addresses'])} address{'es' * (len(d['addresses']) != 1)}" + elif action == "forget-identity": + d = reg.get(did) + frame_devices.forget_pin(did) + if is_active: + link.kick("switch") + msg = f"Forgot {d['name']}'s SSH identity; the next connection saves the one it shows" + elif action == "name-network": + reg.name_network(body.get("network"), body.get("name")) + if link.state["network"]: + link.refresh_network() + msg = "Saved the network's name" + elif action == "setup": + alias = frame_devices.check_alias(body.get("alias")) + host = frame_devices.check_host(body["host"]) if body.get("host") else None + link.reg.undismiss(alias) + where = open_setup(alias, host) + msg = f"Opened Set Up Connection for '{alias}' in {where}" + elif action == "retry": + link.kick("retry") + msg = "Connecting…" + else: + raise frame_devices.DeviceError("unknown action") + link.devices_changed() + return dict(devices_view(link), message=msg) + + +def next_alias(link): + taken = {d["alias"] for d in link.reg.devices()} | {b["alias"] for b in frame_devices.parse_blocks( + frame_devices.read_config())} + if "frame" not in taken: + return "frame" + n = 2 + while f"frame-{n}" in taken: + n += 1 + return f"frame-{n}" + + +LIKELY = re.compile(r"frame|steam", re.I) + + +def tailscale_find(link, device_id=None): + """Tailscale peers that could be a headset, likely ones first, for "Find on Tailscale".""" + ts = frame_network.tailscale_status() + device = link.reg.get(device_id) if device_id else link.active_device() + known = {a["host"].rstrip(".").lower() for a in device.get("addresses") or []} + if not ts.get("installed"): + return {"up": False, "peers": [], "message": "Tailscale isn't installed on this computer."} + if not ts.get("up"): + return {"up": False, "peers": [], "message": "Tailscale isn't running on this computer. Start it, then look again."} + peers = [] + for p in ts["peers"]: + ip = next((i for i in p["ips"] if "." in i), p["ips"][0] if p["ips"] else None) + likely = p["os"] == "linux" and (LIKELY.search(p["name"]) or p["name"].lower() in ( + device["alias"].lower(), (device.get("name") or "").lower())) + peers.append({"name": p["name"], "dns": p["dns"], "ip": ip, "os": p["os"], "online": p["online"], + "likely": bool(likely), "added": bool({p["dns"].lower(), (ip or "").lower()} & known)}) + peers.sort(key=lambda p: (not p["likely"], p["os"] != "linux", not p["online"], p["name"].lower())) + return {"up": True, "peers": peers, "tailnet": ts.get("tailnet"), "message": None} + + +def mdns_find(link, device_id=None): + """Headsets on this network: SteamOS devkit services (mDNS) and .local.""" + device = link.reg.get(device_id) if device_id else link.active_device() + known = {a["host"].rstrip(".").lower() for a in device.get("addresses") or []} + try: + import frame_connect + found = frame_connect.discover_devkit() + except (ImportError, SystemExit, OSError): + found = [] + names = list(dict.fromkeys([h.rstrip(".") for h in found] + [f"{device['alias']}.local", "frame.local"])) + rows = [None] * len(names) + + def check(i, host): + res = probe(host, 22, timeout=3) + rows[i] = {"host": host, "state": res["state"], "ip": res.get("ip"), "rtt_ms": res.get("rtt_ms"), + "detail": res["detail"], "advertised": host in [h.rstrip(".") for h in found], + "added": host.lower() in known or (res.get("ip") or "").lower() in known} + threads = [threading.Thread(target=check, args=(i, h), daemon=True) for i, h in enumerate(names) + if frame_devices.HOST_RE.fullmatch(h)] + for t in threads: + t.start() + for t in threads: + t.join(8) + hosts = [r for r in rows if r and (r["advertised"] or r["state"] in ("answered", "refused"))] + return {"hosts": hosts, "tool": bool(frame_host.which("dns-sd") or frame_host.which("avahi-browse"))} diff --git a/ui/frame_network.py b/ui/frame_network.py new file mode 100644 index 0000000..252be4f --- /dev/null +++ b/ui/frame_network.py @@ -0,0 +1,310 @@ +"""Which network this computer is on, and whether Tailscale is up. + +Frame Control remembers which of a headset's addresses worked on which network, +so it needs a stable name for "this network". The Wi-Fi name (SSID) is the +friendly one, but macOS 14+ hides it from apps without Location permission, and +wired networks have none. So every network is identified by a fingerprint of +its default gateway: the router's IP and MAC address, which stay the same for a +given home or office network. The user can give a fingerprint a name. + +Runs on this computer (macOS, Linux, Windows). Python stdlib only; every probe +is a short command with a timeout, and each parser has fixtures in +tests/test_network.py. +""" +import hashlib +import ipaddress +import json +import os +import re +import socket +import subprocess +import time + +import frame_host + +TIMEOUT = 3 + + +def run(argv, timeout=TIMEOUT): + """A command's stdout, or "" if it's missing, fails or takes too long.""" + try: + r = subprocess.run(argv, capture_output=True, stdin=subprocess.DEVNULL, timeout=timeout, + **({"creationflags": subprocess.CREATE_NO_WINDOW} if frame_host.WINDOWS else {})) + except (OSError, subprocess.TimeoutExpired): + return "" + return r.stdout.decode("utf-8", "replace") if r.returncode == 0 else "" + + +def valid_ip(text): + try: + ipaddress.ip_address(text) + return True + except ValueError: + return False + + +def norm_mac(text): + """"b4:fb:e4:1:87:3f" or "B4-FB-E4-01-87-3F" -> "b4:fb:e4:01:87:3f"; None if it isn't a MAC.""" + parts = re.split(r"[:-]", (text or "").strip()) + if len(parts) != 6 or not all(re.fullmatch(r"[0-9A-Fa-f]{1,2}", p) for p in parts): + return None + mac = ":".join(p.lower().zfill(2) for p in parts) + return None if mac in ("00:00:00:00:00:00", "ff:ff:ff:ff:ff:ff") else mac + + +# ---- default gateway ------------------------------------------------------- + +def parse_route_macos(text): + """`route -n get default` -> (gateway, interface).""" + gw = re.search(r"^\s*gateway:\s*(\S+)", text, re.M) + iface = re.search(r"^\s*interface:\s*(\S+)", text, re.M) + gateway = gw.group(1) if gw and valid_ip(gw.group(1)) else None + return gateway, iface.group(1) if iface else None + + +def parse_route_linux(text): + """`ip -4 route show default` -> (gateway, interface) of the lowest-metric route.""" + best = None + for line in text.splitlines(): + m = re.search(r"^default via (\S+) dev (\S+)", line.strip()) + if not m or not valid_ip(m.group(1)): + continue + metric = re.search(r"\bmetric (\d+)", line) + key = int(metric.group(1)) if metric else 0 + if best is None or key < best[0]: + best = (key, m.group(1), m.group(2)) + return (best[1], best[2]) if best else (None, None) + + +def parse_route_windows(text): + """`route print -4 0.0.0.0` -> (gateway, local IP of the interface), lowest metric wins.""" + best = None + for line in text.splitlines(): + f = line.split() + if len(f) == 5 and f[0] == "0.0.0.0" and f[1] == "0.0.0.0" and valid_ip(f[2]) and f[4].isdigit(): + if best is None or int(f[4]) < best[0]: + best = (int(f[4]), f[2], f[3]) + return (best[1], best[2]) if best else (None, None) + + +# ---- the gateway's MAC address ---------------------------------------------- + +def parse_arp_macos(text, ip): + """`arp -n IP` -> MAC ("? (192.168.1.1) at b4:fb:e4:b5:67:55 on en0 ifscope [ethernet]").""" + m = re.search(r"\(" + re.escape(ip) + r"\) at (\S+)", text) + return norm_mac(m.group(1)) if m else None + + +def parse_neigh_linux(text, ip): + """`ip neigh show IP` -> MAC ("192.168.1.1 dev wlan0 lladdr b4:fb:... REACHABLE").""" + for line in text.splitlines(): + f = line.split() + if f and f[0] == ip and "lladdr" in f: + return norm_mac(f[f.index("lladdr") + 1]) if f.index("lladdr") + 1 < len(f) else None + return None + + +def parse_arp_windows(text, ip): + """`arp -a IP` -> MAC (" 192.168.1.1 b4-fb-e4-b5-67-55 dynamic").""" + for line in text.splitlines(): + f = line.split() + if len(f) >= 2 and f[0] == ip: + return norm_mac(f[1]) + return None + + +# ---- Wi-Fi name -------------------------------------------------------------- + +def parse_summary_macos(text): + """`ipconfig getsummary IFACE` -> (ssid, is_wifi). macOS prints "" without Location permission.""" + kind = re.search(r"^\s*InterfaceType\s*:\s*(\S+)", text, re.M) + ssid = re.search(r"^\s*SSID\s*:\s*(.+?)\s*$", text, re.M) + name = ssid.group(1) if ssid else None + if name in ("", ""): + name = None + return name, (kind.group(1).lower() == "wifi") if kind else None + + +def parse_nmcli(text): + """`nmcli -t -f active,ssid dev wifi` -> the active SSID (colons in names come escaped as \\:).""" + for line in text.splitlines(): + if line.startswith("yes:"): + return line[4:].replace("\\:", ":") or None + return None + + +def parse_netsh(text): + """`netsh wlan show interfaces` -> the connected SSID (not the BSSID line).""" + state = re.search(r"^\s*State\s*:\s*(\S+)", text, re.M) + ssid = re.search(r"^\s*SSID\s*:\s*(.+?)\s*$", text, re.M) + if not ssid or (state and state.group(1).lower() != "connected"): + return None + return ssid.group(1) + + +# ---- Tailscale ----------------------------------------------------------------- + +def tailscale_cli(): + extra = [] + if frame_host.MAC: + extra.append("/Applications/Tailscale.app/Contents/MacOS/Tailscale") + elif frame_host.WINDOWS: + for base in (os.environ.get("ProgramFiles"), os.environ.get("ProgramFiles(x86)")): + if base: + extra.append(os.path.join(base, "Tailscale", "tailscale.exe")) + return frame_host.which("tailscale", *extra) + + +def parse_tailscale(text): + """`tailscale status --json` -> {"up", "ip", "name", "tailnet", "peers": [...]}. + + Each peer: {"name", "dns" (MagicDNS name, no trailing dot), "ips", "os", "online"}. + """ + try: + data = json.loads(text) + except ValueError: + return {"up": False, "peers": []} + if not isinstance(data, dict): + return {"up": False, "peers": []} + me = data.get("Self") or {} + tailnet = (data.get("CurrentTailnet") or {}).get("Name") if isinstance(data.get("CurrentTailnet"), dict) else None + out = {"up": data.get("BackendState") == "Running", + "ip": next((ip for ip in me.get("TailscaleIPs") or [] if "." in ip), None), + "name": (me.get("DNSName") or "").rstrip(".") or None, + "tailnet": tailnet, "peers": []} + for p in (data.get("Peer") or {}).values(): + if not isinstance(p, dict): + continue + out["peers"].append({"name": p.get("HostName") or "", "dns": (p.get("DNSName") or "").rstrip("."), + "ips": [ip for ip in p.get("TailscaleIPs") or [] if isinstance(ip, str)], + "os": p.get("OS") or "", "online": bool(p.get("Online"))}) + return out + + +def tailscale_status(): + cli = tailscale_cli() + if not cli: + return {"up": False, "installed": False, "peers": []} + out = parse_tailscale(run([cli, "status", "--json"], timeout=4) or "{}") + out["installed"] = True + return out + + +TAILNET_V4 = ipaddress.ip_network("100.64.0.0/10") +TAILNET_V6 = ipaddress.ip_network("fd7a:115c:a1e0::/48") + + +def is_tailscale(host): + host = host.lower().rstrip(".") + if host.endswith(".ts.net"): + return True + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return ip in (TAILNET_V4 if ip.version == 4 else TAILNET_V6) + + +def guess_kind(host): + """What sort of address a host is: mdns, tailscale, lan or manual.""" + h = host.lower().rstrip(".") + if h.endswith(".local"): + return "mdns" + if is_tailscale(h): + return "tailscale" + try: + ip = ipaddress.ip_address(h.split("%")[0]) + if ip.is_private or ip.is_link_local: + return "lan" + except ValueError: + pass + return "manual" + + +# ---- putting it together ---------------------------------------------------------- + +def network_id(gateway, mac): + """A short, stable id for a network: its gateway's IP and MAC. None until both are known.""" + if not gateway or not mac: + return None + return "n-" + hashlib.sha1(f"{gateway}|{mac}".encode()).hexdigest()[:10] + + +def local_ip(towards="192.0.2.1"): + """This computer's address on the default route (UDP connect sends nothing).""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.connect((towards, 9)) + return s.getsockname()[0] + except OSError: + return None + + +def gateway(): + """(gateway IP, interface) of the default route.""" + if frame_host.MAC: + return parse_route_macos(run(["route", "-n", "get", "default"])) + if frame_host.WINDOWS: + return parse_route_windows(run(["route", "print", "-4", "0.0.0.0"])) + return parse_route_linux(run(["ip", "-4", "route", "show", "default"])) + + +def gateway_mac(ip): + if frame_host.MAC: + return parse_arp_macos(run(["arp", "-n", ip]), ip) + if frame_host.WINDOWS: + return parse_arp_windows(run(["arp", "-a", ip]), ip) + return parse_neigh_linux(run(["ip", "neigh", "show", ip]), ip) + + +def poke(ip): + """Make the system look up the gateway's MAC (an ARP entry can expire).""" + try: + with socket.create_connection((ip, 53), timeout=0.3): + pass + except OSError: + pass + + +def wifi(interface): + """(ssid or None, is_wifi or None) for the default route's interface.""" + if frame_host.MAC: + if interface: + ssid, is_wifi = parse_summary_macos(run(["ipconfig", "getsummary", interface])) + if ssid or is_wifi is False: + return ssid, is_wifi + m = re.search(r"Current Wi-Fi Network: (.+)", run(["networksetup", "-getairportnetwork", interface])) + return (m.group(1).strip() if m else None), is_wifi + return None, None + if frame_host.WINDOWS: + ssid = parse_netsh(run(["netsh", "wlan", "show", "interfaces"])) + return ssid, True if ssid else None + if frame_host.which("nmcli"): + ssid = parse_nmcli(run(["nmcli", "-t", "-f", "active,ssid", "dev", "wifi"])) + else: + ssid = run(["iwgetid", "-r"]).strip() or None + return ssid, True if ssid else (interface.startswith(("wl", "wlan")) if interface else None) + + +def fingerprint(): + """The cheap part, polled every few seconds: (gateway, interface, gateway MAC).""" + gw, iface = gateway() + mac = None + if gw: + mac = gateway_mac(gw) + if not mac: + poke(gw) + mac = gateway_mac(gw) + return gw, iface, mac + + +def current_network(fp=None, with_tailscale=True): + """Everything the connection status shows about this computer's network.""" + gw, iface, mac = fp or fingerprint() + ssid, is_wifi = wifi(iface) if gw else (None, None) + net = {"id": network_id(gw, mac), "gateway": gw, "gateway_mac": mac, "interface": iface, + "ssid": ssid, "wifi": is_wifi, "local_ip": local_ip(gw) if gw else None, "checked": time.time()} + if with_tailscale: + ts = tailscale_status() + net["tailscale"] = {k: ts.get(k) for k in ("up", "installed", "ip", "name", "tailnet")} + return net diff --git a/ui/index.html b/ui/index.html index 8ac1265..e5b977c 100644 --- a/ui/index.html +++ b/ui/index.html @@ -86,6 +86,58 @@ .wait { color: var(--muted); font-size: 13px; display: flex; align-items: center; gap: 8px; } .wait::before { content: ""; width: 7px; height: 7px; border-radius: 50%; background: var(--dim); flex: none; } + /* ---- connection pill, its details dialog, and the Devices tab (frame_link.py) ---- */ + .pill { height: 40px; padding: 0 12px; gap: 9px; max-width: 420px; min-width: 190px; flex: 0 1 auto; background: var(--btn); } + .pill .dot { flex: none; } + .pill .dot.wait { background: var(--warn); box-shadow: 0 0 6px rgba(217,162,58,.7); animation: pulse 1s ease-in-out infinite; } + @keyframes pulse { 50% { opacity: .35; } } + .pill .pt { display: flex; flex-direction: column; align-items: flex-start; min-width: 0; line-height: 1.2; text-align: left; } + .pill .pt b { font-weight: 500; font-size: 13px; color: var(--bright); max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .pill .pt span { font-size: 11.5px; color: var(--muted); max-width: 100%; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } + .devsel { background: var(--btn); color: var(--text); border: 0; border-radius: 3px; height: 40px; padding: 0 8px; font: inherit; font-size: 13px; max-width: 160px; } + .dlg { background: #1e2329; color: var(--text); border: 1px solid rgba(255,255,255,.1); border-radius: 4px; + padding: 22px; width: min(640px, 94vw); box-shadow: 0 20px 60px rgba(0,0,0,.6); } + .dlg::backdrop { background: rgba(0,0,0,.55); } + .dlg h2 { margin: 0 0 14px; font-size: 15px; letter-spacing: 1.5px; text-transform: uppercase; color: var(--bright); } + .dlg h3, [data-page=devices] h3 { margin: 16px 0 6px; font-size: 11px; letter-spacing: 1.3px; text-transform: uppercase; color: var(--muted); font-weight: 700; } + .dlg label, .dev-form label { display: block; font-size: 12.5px; color: var(--muted); margin-top: 10px; } + .dlg label input, .dev-form label input { margin-top: 5px; } + .facts { display: grid; grid-template-columns: max-content 1fr; gap: 4px 14px; margin: 0; font-size: 13px; } + .facts dt { color: var(--muted); } .facts dd { margin: 0; color: var(--bright); overflow-wrap: anywhere; } + .stages { list-style: none; margin: 0; padding: 0; } + .stages li { display: grid; grid-template-columns: 22px 1fr auto; gap: 2px 8px; padding: 6px 0; border-top: 1px solid rgba(255,255,255,.05); } + .stages li:first-child { border-top: 0; } + .stages .ic { width: 16px; height: 16px; border-radius: 50%; margin-top: 2px; display: grid; place-items: center; font-size: 11px; font-weight: 700; } + .stages .done .ic { background: rgba(89,191,64,.2); color: var(--green-hi); } + .stages .failed .ic { background: rgba(217,65,38,.25); color: #ff8a73; } + .stages .pending .ic { border: 1.5px solid var(--dim); } + .stages .active .ic { border: 2px solid rgba(255,255,255,.15); border-top-color: var(--blue); animation: spin .8s linear infinite; } + .stages .lb { color: var(--bright); font-size: 13.5px; } .stages .pending .lb { color: var(--muted); } + .stages .dt { grid-column: 2 / 4; color: var(--muted); font-size: 12.5px; overflow-wrap: anywhere; } + .stages .failed .dt { color: #ff8a73; } + .stages .tm { color: var(--dim); font-size: 12px; font-variant-numeric: tabular-nums; } + .probes { width: 100%; border-collapse: collapse; font-size: 12.5px; } + .probes td { padding: 5px 8px 5px 0; border-top: 1px solid rgba(255,255,255,.05); vertical-align: top; } + .probes td:first-child { color: var(--bright); min-width: 170px; overflow-wrap: anywhere; } + .res-answered, .res-ok { color: var(--green-hi); } .res-refused, .res-sshfailed, .res-wrong, .res-denied { color: #ff8a73; } + .res-timeout, .res-unresolved, .res-unreachable, .res-unpinned { color: var(--warn); } + .res-trying, .res-resolving, .res-waiting, .res-checking { color: var(--link); } + .dev-grid { display: grid; grid-template-columns: minmax(260px, 1fr) minmax(0, 2.2fr); gap: 22px; align-items: start; } + @media (max-width: 1000px) { .dev-grid { grid-template-columns: 1fr; } } + .dev-item { cursor: pointer; border-radius: 3px; padding: 10px !important; margin: 0 -10px; } + .dev-item:hover { background: rgba(255,255,255,.04); } + .dev-item.sel { background: rgba(26,159,255,.12); } + .dev-form { display: grid; grid-template-columns: 2fr 1.3fr 1fr .8fr; gap: 0 10px; align-items: end; } + .dev-form input[readonly] { color: var(--muted); } + .dev-panel select, .dlg select { background: rgba(0,0,0,.28); color: var(--text); border: 1px solid transparent; border-radius: 3px; + padding: 8px 8px; font: inherit; font-size: 13px; } + .addr .t { overflow-wrap: anywhere; } + .addr .s { white-space: normal; } + .addr-edit { display: grid; grid-template-columns: 2fr 1fr 1.4fr auto auto; gap: 8px; align-items: center; width: 100%; } + .addr-add { display: grid; grid-template-columns: 2fr 1fr 1.4fr auto; gap: 8px; margin-top: 12px; } + @media (max-width: 700px) { .dev-form, .addr-edit, .addr-add { grid-template-columns: 1fr; } } + .found { margin-top: 10px; } + /* ---- drop anywhere ---- */ .dropzone { position: fixed; inset: 0; z-index: 40; display: grid; place-items: center; pointer-events: none; background: rgba(14,20,27,.82); backdrop-filter: blur(3px); } @@ -379,9 +431,11 @@ Games2 Android3 Tools4 + Devices5
- Connecting… + + — @@ -644,6 +699,25 @@ +
+
+
+

Headsets

+
+
Loading…
+
Frame Control talks to one headset at a time. Each can be reached at several addresses; + it tries them all at once and uses the best one that answers on the network you're on.
+

This computer's network

+
Checking…
+
+ +
+
Networks are told apart by their router (its IP and hardware address), so this works on wired + networks and when your computer won't share the Wi-Fi name.
+
+
Pick a headset.
+
+