mirror of
https://github.com/seanbetts/steam-hardware-watch.git
synced 2026-10-06 02:00:19 +02:00
Add SteamDB browser fallback
This commit is contained in:
seanbetts committed
1 parent
5a049365c1
commit
b43753f4df
10 files changed
+505
-17
No files matched your search
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)"
|
||||
LOCAL_DIR="$REPO_DIR/.local"
|
||||
|
||||
PROFILE_DIR="${1:-${STEAMDB_PROFILE_DIR:-$LOCAL_DIR/playwright-steamdb-profile}}"
|
||||
PORT="${2:-${STEAMDB_CDP_PORT:-55685}}"
|
||||
URL="${STEAMDB_BOOTSTRAP_URL:-https://steamdb.info/app/4165870/}"
|
||||
ENV_FILE="${STEAMDB_ENV_FILE:-$LOCAL_DIR/steamdb-env.sh}"
|
||||
|
||||
mkdir -p "$PROFILE_DIR" "$LOCAL_DIR"
|
||||
|
||||
existing_pid="$(ps aux | awk -v profile="$PROFILE_DIR" -v port="$PORT" '
|
||||
index($0, "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") &&
|
||||
index($0, profile) &&
|
||||
index($0, "--remote-debugging-port=" port) {
|
||||
print $2
|
||||
exit
|
||||
}
|
||||
')"
|
||||
|
||||
if [ -z "$existing_pid" ]; then
|
||||
open -g -na "Google Chrome" --args \
|
||||
"--user-data-dir=$PROFILE_DIR" \
|
||||
"--remote-debugging-port=$PORT" \
|
||||
--no-first-run \
|
||||
--disable-sync \
|
||||
--new-window \
|
||||
"$URL"
|
||||
|
||||
attempts=0
|
||||
until curl -fsS "http://127.0.0.1:$PORT/json/version" >/dev/null 2>&1; do
|
||||
attempts=$((attempts + 1))
|
||||
if [ "$attempts" -ge 30 ]; then
|
||||
echo "failed to observe a Chrome CDP endpoint on port $PORT" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "SteamDB bootstrap browser already running on port $PORT (pid $existing_pid)."
|
||||
fi
|
||||
|
||||
cat > "$ENV_FILE" <<EOF
|
||||
export STEAMDB_PLAYWRIGHT_FALLBACK=1
|
||||
export STEAMDB_PROFILE_DIR="$PROFILE_DIR"
|
||||
export STEAMDB_CDP_ENDPOINT="http://127.0.0.1:$PORT"
|
||||
export STEAMDB_BROWSER_CHANNEL="chrome"
|
||||
export STEAMDB_REFERER="https://steamdb.info/app/4165870/"
|
||||
EOF
|
||||
|
||||
printf '%s\n' "SteamDB browser bootstrap ready."
|
||||
printf '%s\n' "Profile: $PROFILE_DIR"
|
||||
printf '%s\n' "CDP endpoint: http://127.0.0.1:$PORT"
|
||||
printf '%s\n' "Env file: $ENV_FILE"
|
||||
printf '%s\n' ""
|
||||
printf '%s\n' "Next steps:"
|
||||
printf '%s\n' "1. If SteamDB needs manual interaction, switch to the dedicated Chrome window when convenient."
|
||||
printf '%s\n' "2. Leave that dedicated browser running in the background."
|
||||
printf '%s\n' "3. Source the env file before a run, or let check_steamdb.sh pick it up automatically."
|
||||
@@ -9,20 +9,83 @@ fi
|
||||
RUN_DIR="$1"
|
||||
OUT_DIR="$RUN_DIR/api/steamdb"
|
||||
REPORT_DIR="$RUN_DIR/reports"
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)"
|
||||
mkdir -p "$OUT_DIR" "$REPORT_DIR"
|
||||
|
||||
ERROR_FILE="$REPORT_DIR/steamdb-errors.txt"
|
||||
: > "$ERROR_FILE"
|
||||
: > "$REPORT_DIR/steamdb-key-lines.txt"
|
||||
|
||||
load_local_env() {
|
||||
env_file="${STEAMDB_ENV_FILE:-$REPO_DIR/.local/steamdb-env.sh}"
|
||||
if [ -f "$env_file" ]; then
|
||||
# shellcheck disable=SC1090
|
||||
. "$env_file"
|
||||
fi
|
||||
}
|
||||
|
||||
maybe_set_cdp_endpoint() {
|
||||
if [ -n "${STEAMDB_CDP_ENDPOINT:-}" ] || [ -z "${STEAMDB_PROFILE_DIR:-}" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
port="$(ps aux | awk -v profile="$STEAMDB_PROFILE_DIR" '
|
||||
index($0, profile) && match($0, /--remote-debugging-port=([0-9]+)/) {
|
||||
value = substr($0, RSTART, RLENGTH)
|
||||
sub(/^--remote-debugging-port=/, "", value)
|
||||
print value
|
||||
exit
|
||||
}
|
||||
' 2>/dev/null || true)"
|
||||
|
||||
if [ -n "$port" ]; then
|
||||
STEAMDB_CDP_ENDPOINT="http://127.0.0.1:$port"
|
||||
export STEAMDB_CDP_ENDPOINT
|
||||
fi
|
||||
}
|
||||
|
||||
is_challenge_page() {
|
||||
path="$1"
|
||||
rg -qi "Checking your browser|Just a moment|Cloudflare|cf-chl|cf-browser-verification" "$path"
|
||||
}
|
||||
|
||||
load_local_env
|
||||
|
||||
STEAMDB_USER_AGENT="${STEAMDB_USER_AGENT:-Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36}"
|
||||
export STEAMDB_USER_AGENT
|
||||
|
||||
fetch_html() {
|
||||
name="$1"
|
||||
url="$2"
|
||||
if ! curl -A 'Mozilla/5.0' --retry 2 --retry-delay 2 --max-time 45 -fsSL "$url" > "$OUT_DIR/$name.html" 2>/dev/null; then
|
||||
printf '%s\n' "failed to fetch $url" >> "$ERROR_FILE"
|
||||
rm -f "$OUT_DIR/$name.html"
|
||||
return 1
|
||||
tmp="$OUT_DIR/$name.html.tmp"
|
||||
out="$OUT_DIR/$name.html"
|
||||
|
||||
if curl -A "$STEAMDB_USER_AGENT" \
|
||||
-H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8' \
|
||||
-H 'Accept-Language: en-US,en;q=0.9' \
|
||||
-H 'Upgrade-Insecure-Requests: 1' \
|
||||
--compressed \
|
||||
--retry 2 --retry-delay 2 --max-time 45 -fsSL "$url" > "$tmp" 2>/dev/null; then
|
||||
if ! is_challenge_page "$tmp"; then
|
||||
mv "$tmp" "$out"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "${STEAMDB_PLAYWRIGHT_FALLBACK:-0}" = "1" ] && command -v node >/dev/null 2>&1; then
|
||||
maybe_set_cdp_endpoint
|
||||
if node "$SCRIPT_DIR/fetch_steamdb_with_playwright.js" "$url" "$tmp" >/dev/null 2>&1 \
|
||||
&& [ -s "$tmp" ] \
|
||||
&& ! is_challenge_page "$tmp"; then
|
||||
mv "$tmp" "$out"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f "$tmp" "$out"
|
||||
printf '%s\n' "failed to fetch $url: blocked or challenge page" >> "$ERROR_FILE"
|
||||
return 1
|
||||
}
|
||||
|
||||
fetch_html "controller-app" "https://steamdb.info/app/4165870/" || true
|
||||
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)"
|
||||
LOCAL_DIR="$REPO_DIR/.local"
|
||||
|
||||
PROFILE_DIR="${1:-${STEAMDB_PROFILE_DIR:-$LOCAL_DIR/playwright-steamdb-profile}}"
|
||||
ENV_FILE="${STEAMDB_ENV_FILE:-$LOCAL_DIR/steamdb-env.sh}"
|
||||
|
||||
pids="$(ps aux | awk -v profile="$PROFILE_DIR" '
|
||||
index($0, profile) && !index($0, "awk -v profile=") {
|
||||
print $2
|
||||
}
|
||||
')"
|
||||
|
||||
if [ -n "$pids" ]; then
|
||||
for pid in $pids; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
printf '%s\n' "Closed dedicated SteamDB browser/profile processes for $PROFILE_DIR."
|
||||
else
|
||||
printf '%s\n' "No dedicated SteamDB browser/profile processes found for $PROFILE_DIR."
|
||||
fi
|
||||
|
||||
rm -f \
|
||||
"$PROFILE_DIR/SingletonLock" \
|
||||
"$PROFILE_DIR/SingletonSocket" \
|
||||
"$PROFILE_DIR/SingletonCookie"
|
||||
|
||||
if [ -f "$ENV_FILE" ]; then
|
||||
rm -f "$ENV_FILE"
|
||||
printf '%s\n' "Removed $ENV_FILE"
|
||||
fi
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require("fs");
|
||||
const { URL } = require("url");
|
||||
|
||||
function loadPlaywright() {
|
||||
const candidates = [];
|
||||
if (process.env.PLAYWRIGHT_CORE_PATH) candidates.push(process.env.PLAYWRIGHT_CORE_PATH);
|
||||
candidates.push("playwright-core");
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
return require(candidate);
|
||||
} catch (err) {
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
"Unable to load playwright-core. Set PLAYWRIGHT_CORE_PATH or install playwright-core."
|
||||
);
|
||||
}
|
||||
|
||||
function launchOptions(headless) {
|
||||
const options = { headless };
|
||||
const channel = process.env.STEAMDB_BROWSER_CHANNEL || process.env.PLAYWRIGHT_BROWSER_CHANNEL;
|
||||
if (channel) {
|
||||
options.channel = channel;
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function userAgent() {
|
||||
return process.env.STEAMDB_USER_AGENT ||
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36";
|
||||
}
|
||||
|
||||
function findPage(context, origin) {
|
||||
const pages = context.pages();
|
||||
return pages.find((candidate) => candidate.url().startsWith(origin)) || pages[0] || null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [, , targetUrl, outputPath] = process.argv;
|
||||
if (!targetUrl || !outputPath) {
|
||||
console.error("usage: fetch_steamdb_with_playwright.js URL OUTPUT_PATH");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const { chromium } = loadPlaywright();
|
||||
const headless = process.env.STEAMDB_PLAYWRIGHT_HEADLESS !== "0" &&
|
||||
process.env.PLAYWRIGHT_HEADLESS !== "0";
|
||||
const profileDir = process.env.STEAMDB_PROFILE_DIR || process.env.PLAYWRIGHT_PROFILE_DIR;
|
||||
const storageState = process.env.STEAMDB_STORAGE_STATE || process.env.PLAYWRIGHT_STORAGE_STATE;
|
||||
const referer = process.env.STEAMDB_REFERER || process.env.PLAYWRIGHT_REFERER;
|
||||
const cdpEndpoint = process.env.STEAMDB_CDP_ENDPOINT || process.env.PLAYWRIGHT_CDP_ENDPOINT;
|
||||
const settleMs = Number(process.env.STEAMDB_PLAYWRIGHT_SETTLE_MS || 8000);
|
||||
|
||||
let browser = null;
|
||||
let context = null;
|
||||
let page = null;
|
||||
let shouldCloseBrowser = false;
|
||||
let shouldCloseContext = false;
|
||||
let shouldClosePage = false;
|
||||
|
||||
if (cdpEndpoint) {
|
||||
browser = await chromium.connectOverCDP(cdpEndpoint);
|
||||
shouldCloseBrowser = true;
|
||||
context = browser.contexts()[0];
|
||||
if (!context) {
|
||||
throw new Error(`No browser context available via CDP endpoint ${cdpEndpoint}`);
|
||||
}
|
||||
const origin = new URL(targetUrl).origin + "/";
|
||||
page = findPage(context, origin);
|
||||
if (!page) {
|
||||
page = await context.newPage();
|
||||
shouldClosePage = true;
|
||||
}
|
||||
} else if (profileDir) {
|
||||
context = await chromium.launchPersistentContext(profileDir, {
|
||||
...launchOptions(headless),
|
||||
userAgent: userAgent(),
|
||||
});
|
||||
shouldCloseContext = true;
|
||||
page = context.pages()[0] || await context.newPage();
|
||||
} else {
|
||||
browser = await chromium.launch(launchOptions(headless));
|
||||
shouldCloseBrowser = true;
|
||||
context = await browser.newContext({
|
||||
userAgent: userAgent(),
|
||||
storageState: storageState || undefined,
|
||||
extraHTTPHeaders: referer ? { Referer: referer } : undefined,
|
||||
});
|
||||
shouldCloseContext = true;
|
||||
page = await context.newPage();
|
||||
}
|
||||
|
||||
try {
|
||||
await page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 45000 });
|
||||
if (settleMs > 0) {
|
||||
await page.waitForTimeout(settleMs);
|
||||
}
|
||||
fs.writeFileSync(outputPath, await page.content());
|
||||
} finally {
|
||||
if (shouldClosePage && page && !page.isClosed()) {
|
||||
await page.close();
|
||||
}
|
||||
if (shouldCloseContext && context) {
|
||||
await context.close();
|
||||
}
|
||||
if (shouldCloseBrowser && browser) {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err && err.stack ? err.stack : String(err));
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -13,14 +13,20 @@ TRACKING_DIR="${3:-/tmp/SteamTracking-master}"
|
||||
SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd)"
|
||||
KOMODO_ENV_FILE="${KOMODO_ENV_FILE:-$REPO_DIR/.local/komodo-env.sh}"
|
||||
STEAMDB_ENV_FILE="${STEAMDB_ENV_FILE:-$REPO_DIR/.local/steamdb-env.sh}"
|
||||
|
||||
cleanup() {
|
||||
if [ "${KOMODO_KEEP_BROWSER_OPEN:-0}" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
if [ -f "$KOMODO_ENV_FILE" ] && [ -x "$SCRIPT_DIR/close_komodo.sh" ]; then
|
||||
:
|
||||
elif [ -f "$KOMODO_ENV_FILE" ] && [ -x "$SCRIPT_DIR/close_komodo.sh" ]; then
|
||||
"$SCRIPT_DIR/close_komodo.sh" >/dev/null 2>&1 || true
|
||||
fi
|
||||
|
||||
if [ "${STEAMDB_KEEP_BROWSER_OPEN:-0}" = "1" ]; then
|
||||
:
|
||||
elif [ -f "$STEAMDB_ENV_FILE" ] && [ -x "$SCRIPT_DIR/close_steamdb.sh" ]; then
|
||||
"$SCRIPT_DIR/close_steamdb.sh" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
Reference in new issue
Block a user