Add SteamVR and SteamOS mirror checks

scripts/check_steamos_mirror.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -lt 1 ]; then
+  echo "usage: $0 RUN_DIR" >&2
+  exit 1
+fi
+
+RUN_DIR="$1"
+BASE_URL="${STEAMOS_MIRROR_BASE_URL:-https://steamdeck-packages.steamos.cloud/archlinux-mirror}"
+REPOS="${STEAMOS_MIRROR_REPOS:-holo-3.8 holo-main jupiter-3.8 jupiter-main}"
+TERMS="${STEAMOS_MIRROR_TERMS:-fremont|deckard|steam frame|frame|roy|triton|ibex|puck|lilac|xr|qcom|snapdragon|aarch64|arm64|controller|dongle|firmware|foxnet}"
+
+OUT_DIR="$RUN_DIR/api/steamos-mirror"
+REPORT_DIR="$RUN_DIR/reports"
+EXTRACT_DIR="$RUN_DIR/extracted/steamos-mirror"
+mkdir -p "$OUT_DIR" "$REPORT_DIR" "$EXTRACT_DIR"
+
+ERROR_FILE="$REPORT_DIR/steamos-mirror-errors.txt"
+ALL_REPOS="$REPORT_DIR/steamos-mirror-repos.txt"
+INTERESTING_REPOS="$REPORT_DIR/steamos-mirror-interesting-repos.txt"
+PACKAGES="$REPORT_DIR/steamos-mirror-packages.tsv"
+KEY_LINES="$REPORT_DIR/steamos-mirror-key-lines.txt"
+
+: > "$ERROR_FILE"
+: > "$ALL_REPOS"
+: > "$INTERESTING_REPOS"
+: > "$PACKAGES"
+: > "$KEY_LINES"
+
+fetch() {
+  name="$1"
+  url="$2"
+  out="$OUT_DIR/$name"
+
+  if ! curl -A 'Mozilla/5.0' --retry 2 --retry-delay 2 --max-time 45 -fsSL "$url" > "$out" 2>/dev/null; then
+    printf '%s\n' "failed to fetch $url" >> "$ERROR_FILE"
+    rm -f "$out"
+    return 1
+  fi
+}
+
+extract_repo_links() {
+  path="$1"
+  if [ ! -f "$path" ]; then
+    return 0
+  fi
+  rg -o 'href="[^"]+/"' "$path" \
+    | sed 's/^href="//; s|/"$||; s|/$||' \
+    | awk '$0 != ".." && $0 != "Parent directory" { print }' || true
+}
+
+parse_desc_files() {
+  repo="$1"
+  dir="$2"
+  find "$dir" -name desc -print | while IFS= read -r desc; do
+    awk -v repo="$repo" '
+      /^%NAME%$/ { getline name }
+      /^%VERSION%$/ { getline version }
+      END {
+        if (name != "" && version != "") {
+          print repo "\t" name "\t" version
+        }
+      }
+    ' "$desc"
+  done
+}
+
+fetch "root.html" "$BASE_URL/" || true
+fetch "sources-root.html" "$BASE_URL/sources/" || true
+
+{
+  extract_repo_links "$OUT_DIR/root.html"
+  extract_repo_links "$OUT_DIR/sources-root.html" | sed 's|^|sources/|'
+} | sort -u > "$ALL_REPOS"
+
+rg -i "$TERMS" "$ALL_REPOS" > "$INTERESTING_REPOS" || true
+
+for repo in $REPOS; do
+  db_name="$repo.db"
+  if ! fetch "$db_name" "$BASE_URL/$repo/os/x86_64/$db_name"; then
+    continue
+  fi
+
+  repo_extract="$EXTRACT_DIR/$repo"
+  rm -rf "$repo_extract"
+  mkdir -p "$repo_extract"
+
+  if command -v bsdtar >/dev/null 2>&1; then
+    if bsdtar -xf "$OUT_DIR/$db_name" -C "$repo_extract" 2>>"$ERROR_FILE"; then
+      parse_desc_files "$repo" "$repo_extract" >> "$PACKAGES"
+    else
+      printf '%s\n' "failed to extract package database for $repo" >> "$ERROR_FILE"
+    fi
+  else
+    printf '%s\n' "missing bsdtar; cannot extract package database for $repo" >> "$ERROR_FILE"
+  fi
+done
+
+sort -u "$PACKAGES" -o "$PACKAGES"
+
+{
+  rg -i "$TERMS" "$PACKAGES" || true
+  rg -i "$TERMS" "$INTERESTING_REPOS" || true
+} > "$KEY_LINES"
+
+printf '%s\n' \
+  "Saved SteamOS mirror metadata to $OUT_DIR" \
+  "Saved repository list to $ALL_REPOS" \
+  "Saved package list to $PACKAGES" \
+  "Saved matching lines to $KEY_LINES"

scripts/check_steamvr_depots.sh

@@ -0,0 +1,119 @@
+#!/bin/sh
+set -eu
+
+if [ "$#" -lt 1 ]; then
+  echo "usage: $0 RUN_DIR" >&2
+  exit 1
+fi
+
+RUN_DIR="$1"
+OUT_DIR="$RUN_DIR/api/steamvr-depots"
+REPORT_DIR="$RUN_DIR/reports"
+mkdir -p "$OUT_DIR" "$REPORT_DIR"
+
+ERROR_FILE="$REPORT_DIR/steamvr-depots-errors.txt"
+KEY_LINES="$REPORT_DIR/steamvr-depots-key-lines.txt"
+LOCAL_HITS="$REPORT_DIR/steamvr-local-hits.txt"
+NEWS_REPORT="$REPORT_DIR/steamvr-news.tsv"
+COMMANDS="$REPORT_DIR/steamvr-depot-download-candidates.txt"
+TERMS="${STEAMVR_TERMS:-fremont|deckard|steam frame|frame|roy|triton|ibex|puck|lilac|xr|qcom|snapdragon|aarch64|arm64|controller|dongle|firmware|foxnet|activeFrame|summonOverlayKey}"
+
+: > "$ERROR_FILE"
+: > "$KEY_LINES"
+: > "$LOCAL_HITS"
+: > "$NEWS_REPORT"
+
+is_challenge_page() {
+  path="$1"
+  rg -qi "Checking your browser|Just a moment|Cloudflare|cf-chl|cf-browser-verification" "$path"
+}
+
+fetch_html() {
+  name="$1"
+  url="$2"
+  tmp="$OUT_DIR/$name.tmp"
+  out="$OUT_DIR/$name"
+
+  if curl -A 'Mozilla/5.0' --retry 2 --retry-delay 2 --max-time 45 -fsSL "$url" > "$tmp" 2>/dev/null \
+    && ! is_challenge_page "$tmp"; then
+    mv "$tmp" "$out"
+    return 0
+  fi
+
+  rm -f "$tmp" "$out"
+  printf '%s\n' "failed to fetch $url: blocked or challenge page" >> "$ERROR_FILE"
+  return 1
+}
+
+fetch_json() {
+  name="$1"
+  url="$2"
+  out="$OUT_DIR/$name"
+
+  if ! curl -A 'Mozilla/5.0' --retry 2 --retry-delay 2 --max-time 45 -fsSL "$url" > "$out" 2>/dev/null; then
+    printf '%s\n' "failed to fetch $url" >> "$ERROR_FILE"
+    rm -f "$out"
+    return 1
+  fi
+}
+
+fetch_html "steamdb-depots.html" "https://steamdb.info/app/250820/depots/" || true
+fetch_json "steamvr-news.json" "https://api.steampowered.com/ISteamNews/GetNewsForApp/v2/?appid=250820&count=40&maxlength=12000&format=json" || true
+
+if [ -f "$OUT_DIR/steamvr-news.json" ] && command -v jq >/dev/null 2>&1; then
+  jq -r '
+    .appnews.newsitems[]
+    | [
+        (try (.date | todate) catch (.date | tostring)),
+        .title,
+        ((.contents // "") | gsub("[\r\n\t]+"; " ") | .[0:360])
+      ]
+    | @tsv
+  ' "$OUT_DIR/steamvr-news.json" > "$NEWS_REPORT" 2>>"$ERROR_FILE" || true
+fi
+
+cat > "$COMMANDS" <<'EOF'
+# SteamVR AppID: 250820
+# Fill in manifest IDs from SteamDB before running Steam's console download_depot command.
+download_depot 250820 250821 <manifest_id>  # Windows OpenVR Win32
+download_depot 250820 250823 <manifest_id>  # Linux OpenVR Linux
+download_depot 250820 250824 <manifest_id>  # Windows/Linux OpenVR Content
+download_depot 250820 250827 <manifest_id>  # Windows/Linux OpenVR Content 2
+download_depot 250820 250830 <manifest_id>  # SteamVR Environments Content
+download_depot 250820 250831 <manifest_id>  # SteamVR Environments Windows
+download_depot 250820 250832 <manifest_id>  # SteamVR Environments Linux
+EOF
+
+SCAN_DIRS="${STEAMVR_SCAN_DIRS:-}"
+DEFAULT_STEAMVR_DIR="$HOME/Library/Application Support/Steam/steamapps/common/SteamVR"
+if [ -z "$SCAN_DIRS" ] && [ -d "$DEFAULT_STEAMVR_DIR" ]; then
+  SCAN_DIRS="$DEFAULT_STEAMVR_DIR"
+fi
+
+if [ -n "$SCAN_DIRS" ]; then
+  for scan_dir in $SCAN_DIRS; do
+    if [ -d "$scan_dir" ]; then
+      rg -n -i "$TERMS" "$scan_dir" >> "$LOCAL_HITS" || true
+    else
+      printf '%s\n' "missing SteamVR scan dir: $scan_dir" >> "$ERROR_FILE"
+    fi
+  done
+fi
+
+{
+  if [ -f "$OUT_DIR/steamdb-depots.html" ]; then
+    rg -n -i "$TERMS|Build ID|Time Updated|250821|250823|250824|250827|250830|public|beta|previous" "$OUT_DIR/steamdb-depots.html" || true
+  fi
+  if [ -s "$NEWS_REPORT" ]; then
+    rg -n -i "$TERMS|OpenXR|dashboard|overlay|firmware" "$NEWS_REPORT" || true
+  fi
+  if [ -s "$LOCAL_HITS" ]; then
+    cat "$LOCAL_HITS"
+  fi
+} > "$KEY_LINES"
+
+printf '%s\n' \
+  "Saved SteamVR depot metadata to $OUT_DIR" \
+  "Saved local SteamVR hits to $LOCAL_HITS" \
+  "Saved matching lines to $KEY_LINES" \
+  "Saved candidate download commands to $COMMANDS"

scripts/compare_runs.py

@@ -123,6 +123,8 @@ def build_summary(prev_dir: Path, curr_dir: Path):
     deltas.extend(compare_report_sets(prev_dir, curr_dir, "Komodo machine media", "komodo-machine-media.tsv"))
     deltas.extend(compare_report_sets(prev_dir, curr_dir, "Komodo frame media", "komodo-frame-media.tsv"))
     deltas.extend(compare_text_lines(prev_dir, curr_dir, "SteamDB", "steamdb-key-lines.txt"))
+    deltas.extend(compare_text_lines(prev_dir, curr_dir, "SteamVR depots", "steamvr-depots-key-lines.txt"))
+    deltas.extend(compare_text_lines(prev_dir, curr_dir, "SteamOS mirror", "steamos-mirror-key-lines.txt"))
     deltas.extend(compare_text_lines(prev_dir, curr_dir, "Valve endpoint", "valve-key-lines.txt"))
 
     if deltas:

scripts/draft_status_update.py

@@ -49,12 +49,16 @@ def build_draft(run_dir: Path):
     steamdb_lines = first_n_nonempty(reports / "steamdb-key-lines.txt", 8)
     valve_lines = first_n_nonempty(reports / "valve-key-lines.txt", 8)
     tracking_lines = first_n_nonempty(reports / "steamtracking-pairing-focus.txt", 8)
+    steamvr_lines = first_n_nonempty(reports / "steamvr-depots-key-lines.txt", 8)
+    steamos_lines = first_n_nonempty(reports / "steamos-mirror-key-lines.txt", 8)
     discovered_count = count_lines(reports / "discovered-visual-assets.tsv")
     retrieved_count = count_lines(reports / "retrieved-visual-assets.tsv")
     blocked_count = count_lines(reports / "blocked-visual-assets.tsv")
     manual_urls = blocked_asset_urls(reports / "blocked-visual-assets.tsv", 10)
     komodo_blocked = has_nonempty(reports / "komodo-errors.txt")
     steamdb_blocked = has_nonempty(reports / "steamdb-errors.txt")
+    steamvr_blocked = has_nonempty(reports / "steamvr-depots-errors.txt")
+    steamos_blocked = has_nonempty(reports / "steamos-mirror-errors.txt")
     valve_blocked = has_nonempty(reports / "valve-errors.txt")
 
     lines = [
@@ -105,6 +109,22 @@ def build_draft(run_dir: Path):
     else:
         lines.append("- No focused SteamTracking report found.")
 
+    lines.extend(["", "#### SteamVR Depots"])
+    if steamvr_blocked:
+        lines.append("- SteamVR depot metadata was blocked or partially unavailable in this run. See `steamvr-depots-errors.txt`.")
+    if steamvr_lines:
+        lines.extend([f"- `{line}`" for line in steamvr_lines])
+    elif not steamvr_blocked:
+        lines.append("- No SteamVR depot key-line report found.")
+
+    lines.extend(["", "#### SteamOS Package Mirror"])
+    if steamos_blocked:
+        lines.append("- SteamOS package mirror metadata was blocked or partially unavailable in this run. See `steamos-mirror-errors.txt`.")
+    if steamos_lines:
+        lines.extend([f"- `{line}`" for line in steamos_lines])
+    elif not steamos_blocked:
+        lines.append("- No SteamOS mirror key-line report found.")
+
     lines.extend(["", "#### Valve Support / CDN"])
     if valve_blocked:
         lines.append("- Some Valve endpoint fetches failed in this run. See `valve-errors.txt`.")

scripts/init_run.sh

@@ -37,6 +37,14 @@ if [ ! -f "$RUN_NOTE" ]; then
 
 -
 
+### SteamVR Depots
+
+-
+
+### SteamOS Package Mirror
+
+-
+
 ### Valve Support / CDN
 
 -

scripts/run_watch.sh

@@ -46,6 +46,8 @@ printf '%s\n' "Run note: $RUN_NOTE"
 "$SCRIPT_DIR/check_komodo.sh" "$RUN_DIR"
 "$SCRIPT_DIR/check_steamdb.sh" "$RUN_DIR"
 "$SCRIPT_DIR/check_steamtracking.sh" "$RUN_DIR" "$TRACKING_DIR"
+"$SCRIPT_DIR/check_steamvr_depots.sh" "$RUN_DIR"
+"$SCRIPT_DIR/check_steamos_mirror.sh" "$RUN_DIR"
 "$SCRIPT_DIR/check_valve_endpoints.sh" "$RUN_DIR"
 python3 "$SCRIPT_DIR/save_visual_assets.py" --run-dir "$RUN_DIR" --base-dir "$BASE_DIR"
 

scripts/write_run_summary.py

@@ -15,6 +15,13 @@ FILTER_TERMS = (
     "2026",
     "pair",
     "firmware",
+    "fremont",
+    "deckard",
+    "roy",
+    "lilac",
+    "xr",
+    "vr",
+    "dongle",
 )
 REJECT_TERMS = (
     "application_config",
@@ -102,6 +109,8 @@ def build(run_dir: Path):
         "",
         f"- Komodo blocked: `{'yes' if count(reports / 'komodo-errors.txt') else 'no'}`",
         f"- SteamDB blocked: `{'yes' if count(reports / 'steamdb-errors.txt') else 'no'}`",
+        f"- SteamVR depot metadata blocked: `{'yes' if count(reports / 'steamvr-depots-errors.txt') else 'no'}`",
+        f"- SteamOS mirror metadata blocked: `{'yes' if count(reports / 'steamos-mirror-errors.txt') else 'no'}`",
         f"- Valve blocked: `{'yes' if count(reports / 'valve-errors.txt') else 'no'}`",
         f"- Discovered visual assets: `{discovered}`",
         f"- Retrieved visual assets: `{retrieved}`",
@@ -125,6 +134,8 @@ def build(run_dir: Path):
     notable = []
     notable.extend([f"- SteamDB: `{line}`" for line in filtered_first(reports / "steamdb-key-lines.txt", 5)])
     notable.extend([f"- SteamTracking: `{line}`" for line in filtered_first(reports / "steamtracking-pairing-focus.txt", 5)])
+    notable.extend([f"- SteamVR depots: `{line}`" for line in filtered_first(reports / "steamvr-depots-key-lines.txt", 5)])
+    notable.extend([f"- SteamOS mirror: `{line}`" for line in filtered_first(reports / "steamos-mirror-key-lines.txt", 5)])
     notable.extend([f"- Valve: `{line}`" for line in filtered_first(reports / "valve-key-lines.txt", 5)])
     notable.extend([f"- Asset blocked: `{line}`" for line in filtered_first(reports / "blocked-visual-assets.tsv", 5)])