diff --git a/scripts/check_customs_shipments.sh b/scripts/check_customs_shipments.sh index e2513be..303cae9 100755 --- a/scripts/check_customs_shipments.sh +++ b/scripts/check_customs_shipments.sh @@ -21,6 +21,16 @@ IMPORTINFO_USER_AGENT=${IMPORTINFO_USER_AGENT:-"Mozilla/5.0 (Macintosh; Intel Ma MANIFEST=$OUT_DIR/importinfo-inputs.tsv : > "$MANIFEST" +looks_like_importinfo_shipment_table() { + path=$1 + for header in "Master BOL" "House BOL" "Arrival Date" "Commodity"; do + if ! grep -qi "$header" "$path"; then + return 1 + fi + done + return 0 +} + fetch_importinfo() { slug=$1 query=$2 @@ -37,6 +47,11 @@ fetch_importinfo() { --max-time 45 \ -fsSL "$url" > "$tmp" 2>/dev/null then + if ! looks_like_importinfo_shipment_table "$tmp"; then + rm -f "$tmp" "$out" + printf 'unexpected or blocked content from %s\n' "$url" >> "$ERROR_FILE" + return 0 + fi mv "$tmp" "$out" printf '%s\t%s\t%s\n' "$query" "$url" "$out" >> "$MANIFEST" return 0 diff --git a/scripts/draft_status_update.py b/scripts/draft_status_update.py index 9513664..b579779 100755 --- a/scripts/draft_status_update.py +++ b/scripts/draft_status_update.py @@ -51,7 +51,14 @@ def build_draft(run_dir: Path): tracking_lines = first_n_nonempty(reports / "steamtracking-pairing-focus.txt", 8) steamvr_lines = first_n_nonempty(reports / "steamvr-depots-key-lines.txt", 8) steamos_lines = first_n_nonempty(reports / "steamos-mirror-key-lines.txt", 8) - customs_lines = first_n_nonempty(reports / "customs-shipments-key-lines.txt", 8) + customs_key_lines_path = reports / "customs-shipments-key-lines.txt" + customs_errors_path = reports / "customs-shipments-errors.txt" + customs_report_path = reports / "customs-shipments.md" + customs_lines = first_n_nonempty(customs_key_lines_path, 8) + customs_outputs_available = any( + path.exists() + for path in (customs_key_lines_path, customs_errors_path, customs_report_path) + ) discovered_count = count_lines(reports / "discovered-visual-assets.tsv") retrieved_count = count_lines(reports / "retrieved-visual-assets.tsv") blocked_count = count_lines(reports / "blocked-visual-assets.tsv") @@ -61,7 +68,7 @@ def build_draft(run_dir: Path): steamvr_blocked = has_nonempty(reports / "steamvr-depots-errors.txt") steamos_blocked = has_nonempty(reports / "steamos-mirror-errors.txt") valve_blocked = has_nonempty(reports / "valve-errors.txt") - customs_blocked = has_nonempty(reports / "customs-shipments-errors.txt") + customs_blocked = has_nonempty(customs_errors_path) lines = [ f"# Status Draft: {run_date}", @@ -136,11 +143,13 @@ def build_draft(run_dir: Path): lines.append("- No Valve key-line report found.") lines.extend(["", "#### Customs / Shipments"]) - if customs_blocked: + if not customs_outputs_available: + lines.append("- Customs shipment outputs were not generated or are unavailable for this run.") + elif customs_blocked: lines.append("- Customs shipment fetches failed or were partially blocked in this run. See `customs-shipments-errors.txt`.") if customs_lines: lines.extend([f"- `{line}`" for line in customs_lines]) - elif not customs_blocked: + elif customs_outputs_available and not customs_blocked: lines.append("- No relevant customs shipment rows found.") lines.extend( diff --git a/scripts/parse_importinfo_shipments.py b/scripts/parse_importinfo_shipments.py index 52b74a0..27d10e4 100755 --- a/scripts/parse_importinfo_shipments.py +++ b/scripts/parse_importinfo_shipments.py @@ -157,8 +157,29 @@ def is_relevant(record): record["notify_party"], ) ).upper() + record_text = " ".join( + ( + record["query"], + record["run_date"], + record["master_bol"], + record["house_bol"], + record["voyage"], + record["bill_type"], + record["carrier_code"], + record["imo"], + record["vessel_name"], + record["arrival_date"], + record["us_port"], + record["foreign_port"], + record["quantity"], + record["weight"], + record["type_of_service"], + record["commodity"], + ) + ).upper() commodity_text = record["commodity"].upper() - return any(term in party_text for term in PARTY_TERMS) and any( + has_valve_signal = "VALVE" in party_text or "VALVE CORPORATION" in record_text + return has_valve_signal and any(term in party_text for term in PARTY_TERMS) and any( term in commodity_text for term in PRODUCT_TERMS ) diff --git a/scripts/write_run_summary.py b/scripts/write_run_summary.py index 495ff6d..6389a44 100755 --- a/scripts/write_run_summary.py +++ b/scripts/write_run_summary.py @@ -90,6 +90,17 @@ def count_block_reason(path: Path, reason: str): return total +def customs_status(reports: Path): + key_lines = reports / "customs-shipments-key-lines.txt" + errors = reports / "customs-shipments-errors.txt" + report = reports / "customs-shipments.md" + if not any(path.exists() for path in (key_lines, errors, report)): + return "unavailable" + if count(errors): + return "blocked" + return "available" + + def build(run_dir: Path): reports = run_dir / "reports" compare = sorted(reports.glob("compare-vs-*.md")) @@ -112,7 +123,7 @@ def build(run_dir: Path): f"- SteamVR depot metadata blocked: `{'yes' if count(reports / 'steamvr-depots-errors.txt') else 'no'}`", f"- SteamOS mirror metadata blocked: `{'yes' if count(reports / 'steamos-mirror-errors.txt') else 'no'}`", f"- Valve blocked: `{'yes' if count(reports / 'valve-errors.txt') else 'no'}`", - f"- Customs shipments blocked: `{'yes' if count(reports / 'customs-shipments-errors.txt') else 'no'}`", + f"- Customs shipments status: `{customs_status(reports)}`", f"- Discovered visual assets: `{discovered}`", f"- Retrieved visual assets: `{retrieved}`", f"- Blocked visual assets: `{blocked}`", diff --git a/tests/test_customs_shipments.py b/tests/test_customs_shipments.py index 7a77cf8..5e82f80 100644 --- a/tests/test_customs_shipments.py +++ b/tests/test_customs_shipments.py @@ -197,6 +197,22 @@ class CustomsShipmentParserTests(unittest.TestCase): self.assertEqual("CEVA C/O VALVE CORPORATION", rows[0]["consignee"]) self.assertEqual("GAME CONSOLE", rows[0]["commodity"]) + def test_filters_generic_logistics_product_rows_without_valve_signal(self): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + html = tmp_path / "generic-logistics.html" + html.write_text( + IMPORTINFO_HTML.replace("CEVA C/O VALVE CORPORATION", "CEVA LOGISTICS US INC."), + encoding="utf-8", + ) + reports = tmp_path / "reports" + + result = run_parser(["--input", f"tech-front-game-console={html}"], reports) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode) + self.assertEqual([], read_rows(reports)) + def test_manifest_inputs_dedupe_duplicate_bols(self): with TemporaryDirectory() as tmp: tmp_path = Path(tmp) @@ -299,6 +315,48 @@ class CheckCustomsShipmentsShellTests(unittest.TestCase): ) self.assertIn("failed to fetch", errors) + def test_records_200_challenge_page_without_key_lines(self): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + run_dir = tmp_path / "run" + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + write_executable( + fake_bin / "curl", + """ + #!/bin/sh + cat <<'HTML' +
Please verify you are human.
+ HTML + """, + ) + env = os.environ.copy() + env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}" + + result = subprocess.run( + [str(ROOT / "scripts" / "check_customs_shipments.sh"), str(run_dir)], + cwd=ROOT, + env=env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode) + errors = (run_dir / "reports" / "customs-shipments-errors.txt").read_text( + encoding="utf-8" + ) + self.assertIn("unexpected or blocked content from", errors) + self.assertFalse( + (run_dir / "api" / "customs" / "importinfo-ceva-valve.html").exists() + ) + key_lines = ( + run_dir / "reports" / "customs-shipments-key-lines.txt" + ).read_text(encoding="utf-8") + self.assertEqual("", key_lines) + class CustomsSummaryIntegrationTests(unittest.TestCase): def test_run_summary_and_status_draft_include_customs_lines(self): @@ -348,6 +406,48 @@ class CustomsSummaryIntegrationTests(unittest.TestCase): self.assertIn("Customs / Shipments", status_draft) self.assertIn("GAME CONSOLE", status_draft) + def test_status_draft_reports_missing_customs_outputs_as_unavailable(self): + with TemporaryDirectory() as tmp: + run_dir = Path(tmp) / "2026-05-05" + reports = run_dir / "reports" + reports.mkdir(parents=True) + + draft = subprocess.run( + [ + "python3", + str(ROOT / "scripts" / "draft_status_update.py"), + "--run-dir", + str(run_dir), + ], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + summary = subprocess.run( + [ + "python3", + str(ROOT / "scripts" / "write_run_summary.py"), + "--run-dir", + str(run_dir), + ], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + self.assertEqual(0, draft.returncode) + self.assertEqual(0, summary.returncode) + status_draft = (reports / "status-draft.md").read_text(encoding="utf-8") + run_summary = (reports / "run-summary.md").read_text(encoding="utf-8") + self.assertIn("Customs / Shipments", status_draft) + self.assertIn("not generated", status_draft) + self.assertNotIn("No relevant customs shipment rows found.", status_draft) + self.assertIn("Customs shipments status: `unavailable`", run_summary) + if __name__ == "__main__": unittest.main()