Harden customs shipment edge cases

scripts/check_customs_shipments.sh

@@ -21,6 +21,16 @@ IMPORTINFO_USER_AGENT=${IMPORTINFO_USER_AGENT:-"Mozilla/5.0 (Macintosh; Intel Ma
 MANIFEST=$OUT_DIR/importinfo-inputs.tsv
 : > "$MANIFEST"
 
+looks_like_importinfo_shipment_table() {
+    path=$1
+    for header in "Master BOL" "House BOL" "Arrival Date" "Commodity"; do
+        if ! grep -qi "$header" "$path"; then
+            return 1
+        fi
+    done
+    return 0
+}
+
 fetch_importinfo() {
     slug=$1
     query=$2
@@ -37,6 +47,11 @@ fetch_importinfo() {
         --max-time 45 \
         -fsSL "$url" > "$tmp" 2>/dev/null
     then
+        if ! looks_like_importinfo_shipment_table "$tmp"; then
+            rm -f "$tmp" "$out"
+            printf 'unexpected or blocked content from %s\n' "$url" >> "$ERROR_FILE"
+            return 0
+        fi
         mv "$tmp" "$out"
         printf '%s\t%s\t%s\n' "$query" "$url" "$out" >> "$MANIFEST"
         return 0

scripts/draft_status_update.py

@@ -51,7 +51,14 @@ def build_draft(run_dir: Path):
     tracking_lines = first_n_nonempty(reports / "steamtracking-pairing-focus.txt", 8)
     steamvr_lines = first_n_nonempty(reports / "steamvr-depots-key-lines.txt", 8)
     steamos_lines = first_n_nonempty(reports / "steamos-mirror-key-lines.txt", 8)
-    customs_lines = first_n_nonempty(reports / "customs-shipments-key-lines.txt", 8)
+    customs_key_lines_path = reports / "customs-shipments-key-lines.txt"
+    customs_errors_path = reports / "customs-shipments-errors.txt"
+    customs_report_path = reports / "customs-shipments.md"
+    customs_lines = first_n_nonempty(customs_key_lines_path, 8)
+    customs_outputs_available = any(
+        path.exists()
+        for path in (customs_key_lines_path, customs_errors_path, customs_report_path)
+    )
     discovered_count = count_lines(reports / "discovered-visual-assets.tsv")
     retrieved_count = count_lines(reports / "retrieved-visual-assets.tsv")
     blocked_count = count_lines(reports / "blocked-visual-assets.tsv")
@@ -61,7 +68,7 @@ def build_draft(run_dir: Path):
     steamvr_blocked = has_nonempty(reports / "steamvr-depots-errors.txt")
     steamos_blocked = has_nonempty(reports / "steamos-mirror-errors.txt")
     valve_blocked = has_nonempty(reports / "valve-errors.txt")
-    customs_blocked = has_nonempty(reports / "customs-shipments-errors.txt")
+    customs_blocked = has_nonempty(customs_errors_path)
 
     lines = [
         f"# Status Draft: {run_date}",
@@ -136,11 +143,13 @@ def build_draft(run_dir: Path):
         lines.append("- No Valve key-line report found.")
 
     lines.extend(["", "#### Customs / Shipments"])
-    if customs_blocked:
+    if not customs_outputs_available:
+        lines.append("- Customs shipment outputs were not generated or are unavailable for this run.")
+    elif customs_blocked:
         lines.append("- Customs shipment fetches failed or were partially blocked in this run. See `customs-shipments-errors.txt`.")
     if customs_lines:
         lines.extend([f"- `{line}`" for line in customs_lines])
-    elif not customs_blocked:
+    elif customs_outputs_available and not customs_blocked:
         lines.append("- No relevant customs shipment rows found.")
 
     lines.extend(

scripts/parse_importinfo_shipments.py

@@ -157,8 +157,29 @@ def is_relevant(record):
             record["notify_party"],
         )
     ).upper()
+    record_text = " ".join(
+        (
+            record["query"],
+            record["run_date"],
+            record["master_bol"],
+            record["house_bol"],
+            record["voyage"],
+            record["bill_type"],
+            record["carrier_code"],
+            record["imo"],
+            record["vessel_name"],
+            record["arrival_date"],
+            record["us_port"],
+            record["foreign_port"],
+            record["quantity"],
+            record["weight"],
+            record["type_of_service"],
+            record["commodity"],
+        )
+    ).upper()
     commodity_text = record["commodity"].upper()
-    return any(term in party_text for term in PARTY_TERMS) and any(
+    has_valve_signal = "VALVE" in party_text or "VALVE CORPORATION" in record_text
+    return has_valve_signal and any(term in party_text for term in PARTY_TERMS) and any(
         term in commodity_text for term in PRODUCT_TERMS
     )
 

scripts/write_run_summary.py

@@ -90,6 +90,17 @@ def count_block_reason(path: Path, reason: str):
     return total
 
 
+def customs_status(reports: Path):
+    key_lines = reports / "customs-shipments-key-lines.txt"
+    errors = reports / "customs-shipments-errors.txt"
+    report = reports / "customs-shipments.md"
+    if not any(path.exists() for path in (key_lines, errors, report)):
+        return "unavailable"
+    if count(errors):
+        return "blocked"
+    return "available"
+
+
 def build(run_dir: Path):
     reports = run_dir / "reports"
     compare = sorted(reports.glob("compare-vs-*.md"))
@@ -112,7 +123,7 @@ def build(run_dir: Path):
         f"- SteamVR depot metadata blocked: `{'yes' if count(reports / 'steamvr-depots-errors.txt') else 'no'}`",
         f"- SteamOS mirror metadata blocked: `{'yes' if count(reports / 'steamos-mirror-errors.txt') else 'no'}`",
         f"- Valve blocked: `{'yes' if count(reports / 'valve-errors.txt') else 'no'}`",
-        f"- Customs shipments blocked: `{'yes' if count(reports / 'customs-shipments-errors.txt') else 'no'}`",
+        f"- Customs shipments status: `{customs_status(reports)}`",
         f"- Discovered visual assets: `{discovered}`",
         f"- Retrieved visual assets: `{retrieved}`",
         f"- Blocked visual assets: `{blocked}`",

tests/test_customs_shipments.py

@@ -197,6 +197,22 @@ class CustomsShipmentParserTests(unittest.TestCase):
             self.assertEqual("CEVA C/O VALVE CORPORATION", rows[0]["consignee"])
             self.assertEqual("GAME CONSOLE", rows[0]["commodity"])
 
+    def test_filters_generic_logistics_product_rows_without_valve_signal(self):
+        with TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            html = tmp_path / "generic-logistics.html"
+            html.write_text(
+                IMPORTINFO_HTML.replace("CEVA C/O VALVE CORPORATION", "CEVA LOGISTICS US INC."),
+                encoding="utf-8",
+            )
+            reports = tmp_path / "reports"
+
+            result = run_parser(["--input", f"tech-front-game-console={html}"], reports)
+
+            self.assertEqual("", result.stderr)
+            self.assertEqual(0, result.returncode)
+            self.assertEqual([], read_rows(reports))
+
     def test_manifest_inputs_dedupe_duplicate_bols(self):
         with TemporaryDirectory() as tmp:
             tmp_path = Path(tmp)
@@ -299,6 +315,48 @@ class CheckCustomsShipmentsShellTests(unittest.TestCase):
             )
             self.assertIn("failed to fetch", errors)
 
+    def test_records_200_challenge_page_without_key_lines(self):
+        with TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            run_dir = tmp_path / "run"
+            fake_bin = tmp_path / "bin"
+            fake_bin.mkdir()
+            write_executable(
+                fake_bin / "curl",
+                """
+                #!/bin/sh
+                cat <<'HTML'
+                <html><body><h1>Checking your browser</h1><p>Please verify you are human.</p></body></html>
+                HTML
+                """,
+            )
+            env = os.environ.copy()
+            env["PATH"] = f"{fake_bin}{os.pathsep}{env['PATH']}"
+
+            result = subprocess.run(
+                [str(ROOT / "scripts" / "check_customs_shipments.sh"), str(run_dir)],
+                cwd=ROOT,
+                env=env,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=False,
+            )
+
+            self.assertEqual("", result.stderr)
+            self.assertEqual(0, result.returncode)
+            errors = (run_dir / "reports" / "customs-shipments-errors.txt").read_text(
+                encoding="utf-8"
+            )
+            self.assertIn("unexpected or blocked content from", errors)
+            self.assertFalse(
+                (run_dir / "api" / "customs" / "importinfo-ceva-valve.html").exists()
+            )
+            key_lines = (
+                run_dir / "reports" / "customs-shipments-key-lines.txt"
+            ).read_text(encoding="utf-8")
+            self.assertEqual("", key_lines)
+
 
 class CustomsSummaryIntegrationTests(unittest.TestCase):
     def test_run_summary_and_status_draft_include_customs_lines(self):
@@ -348,6 +406,48 @@ class CustomsSummaryIntegrationTests(unittest.TestCase):
             self.assertIn("Customs / Shipments", status_draft)
             self.assertIn("GAME CONSOLE", status_draft)
 
+    def test_status_draft_reports_missing_customs_outputs_as_unavailable(self):
+        with TemporaryDirectory() as tmp:
+            run_dir = Path(tmp) / "2026-05-05"
+            reports = run_dir / "reports"
+            reports.mkdir(parents=True)
+
+            draft = subprocess.run(
+                [
+                    "python3",
+                    str(ROOT / "scripts" / "draft_status_update.py"),
+                    "--run-dir",
+                    str(run_dir),
+                ],
+                cwd=ROOT,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=False,
+            )
+            summary = subprocess.run(
+                [
+                    "python3",
+                    str(ROOT / "scripts" / "write_run_summary.py"),
+                    "--run-dir",
+                    str(run_dir),
+                ],
+                cwd=ROOT,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                check=False,
+            )
+
+            self.assertEqual(0, draft.returncode)
+            self.assertEqual(0, summary.returncode)
+            status_draft = (reports / "status-draft.md").read_text(encoding="utf-8")
+            run_summary = (reports / "run-summary.md").read_text(encoding="utf-8")
+            self.assertIn("Customs / Shipments", status_draft)
+            self.assertIn("not generated", status_draft)
+            self.assertNotIn("No relevant customs shipment rows found.", status_draft)
+            self.assertIn("Customs shipments status: `unavailable`", run_summary)
+
 
 if __name__ == "__main__":
     unittest.main()