Merge pull request #2 from bod09/fix/redux-model-revision

Repin Parakeet Redux to the current upstream revision (fixes install 404)

assets/backends/redux.json

@@ -10,13 +10,13 @@
   },
   "model": {
     "source": "https://huggingface.co/moondream/parakeet-redux",
-    "revision": "fad622f25f303105c20d70e201bcc477c88b620c",
+    "revision": "2bf128600aac4b16946f7ed8372e56117fe5e23b",
     "files": [
       {"path": "model.safetensors", "size": 177774490, "sha256": "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"},
       {"path": "config.json", "size": 12988, "sha256": "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"},
       {"path": "ternary.json", "size": 57970, "sha256": "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"},
       {"path": "tokenizer.json", "size": 1159960, "sha256": "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"},
-      {"path": "README.md", "size": 8533, "sha256": "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"}
+      {"path": "README.md", "size": 8832, "sha256": "930055e6bac7d75114e719850ffaffc8d327e73b4295518c38c81399acd63e46", "compatible": [{"size": 8533, "sha256": "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"}]}
     ]
   },
   "attribution": "Moondream/M87 Labs, Parakeet Redux, derived from NVIDIA Parakeet TDT 0.6B v3; original model card README.md retained with the files.",

docs/build.md

@@ -93,7 +93,7 @@ and target application compatibility require further headset work.
 
 ## Inference runtime
 
-Redux weights at `fad622f25f303105c20d70e201bcc477c88b620c` are CC-BY-4.0. Pinned
+Redux weights at `2bf128600aac4b16946f7ed8372e56117fe5e23b` are CC-BY-4.0. Pinned
 file sizes/hashes and attribution live in `assets/backends/redux.json`; offline
 `--list-models`/`--check-model redux --model-dir /absolute/model` (or
 `scripts/model-status.py`) verify without inference or downloads. Manifest

docs/design.md

@@ -226,9 +226,17 @@ may deliver at most once; cancellation invalidates it before any later reply.
 ## On-device Redux
 
 Use the **same** `moondream/parakeet-redux` revision from the benchmark:
-`fad622f25f303105c20d70e201bcc477c88b620c` (177,774,490-byte weight file), initially
+`2bf128600aac4b16946f7ed8372e56117fe5e23b` (177,774,490-byte weight file), initially
 moondream 2.4.0 / kestrel 0.8.0. Do not substitute dense Ultra or silently fall
-back to desktop/cloud inference.
+back to desktop/cloud inference. The benchmark ran on revision
+`fad622f25f303105c20d70e201bcc477c88b620c`, which no longer resolves after the
+upstream history was rewritten; the weights, config, ternary map and tokenizer at
+`2bf1286` are byte-identical to it (same SHA-256), only the model card README differs.
+The manifest explicitly accepts the previous card's pinned size and SHA-256 for
+existing installations, so updating FrameYap requires no model download. New
+downloads must verify against the current revision's card; unknown or modified
+cards are still rejected. The optional per-file `compatible` list contains exact
+`size`/`sha256` pairs for previously installed artifacts, never download fallbacks.
 
 The earlier benchmark used this local API (its source belongs to the originating
 repository, not this project):

docs/packaging.md

@@ -45,7 +45,7 @@ prefix. ARM64/glibc packaging is not a claim of compatibility with arbitrary Lin
 ```sh
 python3 scripts/package-release.py --stage /path/to/stage --output /existing/output \
   --version 0.1.202609241627 --arch linux-aarch64 \
-  --model-revision fad622f25f303105c20d70e201bcc477c88b620c --external-runtime
+  --model-revision 2bf128600aac4b16946f7ed8372e56117fe5e23b --external-runtime
 ```
 
 Use the actual binary's numeric `MAJOR.MINOR.YYYYMMDDHHMM` UTC version

docs/third-party.md

@@ -27,7 +27,7 @@ source-only, and Python packages are fetched from PyPI on the user's machine.
 
 | Component | Where | License |
 | --- | --- | --- |
-| Parakeet Redux weights, revision `fad622f25f303105c20d70e201bcc477c88b620c`, derived from NVIDIA Parakeet TDT 0.6B v3 by Moondream/M87 Labs; pinned sizes and hashes in `assets/backends/redux.json` | <https://huggingface.co/moondream/parakeet-redux> | CC-BY-4.0 |
+| Parakeet Redux weights, revision `2bf128600aac4b16946f7ed8372e56117fe5e23b`, derived from NVIDIA Parakeet TDT 0.6B v3 by Moondream/M87 Labs; pinned sizes and hashes in `assets/backends/redux.json` | <https://huggingface.co/moondream/parakeet-redux> | CC-BY-4.0 |
 | Kestrel, `kestrel-kernels` | <https://github.com/m87-labs/kestrel> | Local inference is free and needs no API key, per the Kestrel README. Finetuned-model inference needs an API key and is not used here. |
 | `moondream` | PyPI | See the package |
 | PyTorch (CPU build) | <https://github.com/pytorch/pytorch> | BSD-3-Clause |

docs/worker.md

@@ -57,7 +57,7 @@ device="cpu", cpu_threads=threads)` and persistent
 Install an **isolated** Python runtime with the
 moondream 2.4.0, kestrel 0.8.0 and compatible CPU dependencies; provide
 preinstalled local weights from revision
-`fad622f25f303105c20d70e201bcc477c88b620c` and pass its directory
+`2bf128600aac4b16946f7ed8372e56117fe5e23b` and pass its directory
 explicitly. The code verifies exact sizes and SHA-256 of weights/config/tokenizer
 against `assets/backends/redux.json` via the shared, offline
 `python/frameyap/model_files.py` schema/verifier before importing model libraries.

python/frameyap/model_files.py

@@ -61,6 +61,7 @@ class ModelFile:
     path: str
     size: int
     sha256: str
+    compatible: tuple[tuple[int, str], ...] = ()
 
 
 @dataclass(frozen=True)
@@ -127,13 +128,28 @@ def _parse(value):
     parsed = []
     names = set()
     for item in files:
-        _fields(item, ("path", "size", "sha256"), "model file")
+        if not isinstance(item, dict):
+            raise ManifestError("invalid model file fields")
+        _fields(item, ("path", "size", "sha256", "compatible") if "compatible" in item
+                else ("path", "size", "sha256"), "model file")
         name = _relative(item["path"], "model file path")
         size = item["size"]
         if name in names or type(size) is not int or not 0 < size <= _MAX_FILE or not isinstance(item["sha256"], str) or not _HASH.fullmatch(item["sha256"]):
             raise ManifestError("duplicate or invalid pinned model file")
         names.add(name)
-        parsed.append(ModelFile(name, size, item["sha256"]))
+        compatible = item.get("compatible", [])
+        if not isinstance(compatible, list) or len(compatible) > 8:
+            raise ManifestError("invalid compatible model files")
+        pins = []
+        for previous in compatible:
+            _fields(previous, ("size", "sha256"), "compatible model file")
+            old_size, old_hash = previous["size"], previous["sha256"]
+            if (type(old_size) is not int or not 0 < old_size <= _MAX_FILE or
+                    not isinstance(old_hash, str) or not _HASH.fullmatch(old_hash) or
+                    (old_size, old_hash) in [(size, item["sha256"]), *pins]):
+                raise ManifestError("duplicate or invalid compatible model file")
+            pins.append((old_size, old_hash))
+        parsed.append(ModelFile(name, size, item["sha256"], tuple(pins)))
     license_info = value["license"]
     _fields(license_info, ("id", "text"), "license")
     requirements = value["requirements"]
@@ -177,7 +193,11 @@ def load_backends(manifest_dir=DEFAULT_MANIFEST_DIR):
 
 
 def check_file(directory, item):
-    """Return (reason, filename); reason None means the pinned file verifies."""
+    """Verify the current pin or an explicitly compatible existing artifact.
+
+    Downloaders construct a temporary ModelFile with only the current pin so
+    newly fetched bytes must match the revision advertised to the user.
+    """
     root = Path(directory)
     if not root.is_absolute():
         return "model_dir_not_absolute", item.path
@@ -201,13 +221,14 @@ def check_file(directory, item):
             before = os.fstat(file_fd)
             if not stat.S_ISREG(before.st_mode):
                 return "unsafe_file", item.path
-            if before.st_size != item.size:
+            pins = ((item.size, item.sha256), *item.compatible)
+            if before.st_size not in {size for size, _ in pins}:
                 return "size_mismatch", item.path
             digest = hashlib.sha256()
             while chunk := os.read(file_fd, 1024 * 1024):
                 digest.update(chunk)
             after = os.fstat(file_fd)
-            if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or digest.hexdigest() != item.sha256:
+            if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or (before.st_size, digest.hexdigest()) not in pins:
                 return "hash_mismatch", item.path
             return None, None
         finally:

tests/test_backends.py

@@ -13,14 +13,17 @@ from frameyap.model_files import DEFAULT_MANIFEST_DIR, ManifestError, load_backe
 class BackendTests(unittest.TestCase):
     def test_redux_manifest_preserves_pinned_artifacts_and_launcher_contract(self):
         redux = load_backends()["redux"]
-        self.assertEqual(redux.revision, "fad622f25f303105c20d70e201bcc477c88b620c")
+        self.assertEqual(redux.revision, "2bf128600aac4b16946f7ed8372e56117fe5e23b")
         self.assertEqual({f.path: (f.size, f.sha256) for f in redux.files}, {
             "model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"),
             "config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"),
             "ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"),
             "tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"),
-            "README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),
+            "README.md": (8832, "930055e6bac7d75114e719850ffaffc8d327e73b4295518c38c81399acd63e46"),
         })
+        self.assertEqual(redux.files[-1].compatible,
+                         ((8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),))
+        self.assertTrue(all(not file.compatible for file in redux.files[:-1]))
         self.assertEqual(redux.launcher["protocol"], "frameyap-worker-v1")
         self.assertEqual(redux.launcher["arguments"], ["--model", "{model_dir}", "--threads", "{threads}", "--clip-dir", "{clip_dir}"])
         self.assertIn("CC-BY-4.0", redux.license_id)
@@ -47,6 +50,10 @@ class BackendTests(unittest.TestCase):
                 lambda f: f["model"]["files"][0].update(path="../secret"),
                 lambda f: f["model"]["files"][0].update(size=True),
                 lambda f: f["model"]["files"][0].update(sha256="A" * 64),
+                lambda f: f["model"]["files"][0].update(compatible="bad"),
+                lambda f: f["model"]["files"][0].update(compatible=[{"size": True, "sha256": "a" * 64}]),
+                lambda f: f["model"]["files"][0].update(compatible=[{"size": 3, "sha256": "a" * 64}]),
+                lambda f: f["model"]["files"][0].update(compatible=[{"size": 4, "sha256": "a" * 64, "path": "other"}]),
                 lambda f: f["launcher"]["arguments"].append("{unknown}"),
                 lambda f: f["launcher"].update(path="/bin/sh"),
                 lambda f: f["launcher"].update(protocol="not-the-wire-protocol"),

tests/test_installer.py

@@ -1206,6 +1206,57 @@ with patch.object(module, "check_host"), patch.object(module.urllib.request, "ur
             self.assertIn("mismatched", json.loads(output.getvalue())["message"])
             fetch.assert_not_called()
 
+    def test_model_card_upgrade_retains_verified_legacy_without_network(self):
+        shutil.copyfile(REPO / "python/frameyap/model_files.py",
+                        self.stage / "python/frameyap/model_files.py")
+        manifest = json.loads((REPO / "assets/backends/redux.json").read_text())
+        old, new = b"previous attributed card", b"updated attributed model card"
+        def pin(content):
+            return {"size": len(content), "sha256": hashlib.sha256(content).hexdigest()}
+        manifest["model"]["files"] = [{"path": "README.md", **pin(old)}]
+        manifests = self.stage / "assets/backends"
+        manifests.mkdir()
+        card_manifest = manifests / "redux.json"
+        card_manifest.write_text(json.dumps(manifest))
+        archive, digest = self.package("0.1.202609241530")
+        self.install("0.1.202609241530", archive, digest, "--without-model")
+        destination = self.base / "models"
+        destination.mkdir()
+        card = destination / "README.md"
+        card.write_bytes(old)
+        manifest["model"]["files"] = [{"path": "README.md", **pin(new),
+                                          "compatible": [pin(old)]}]
+        card_manifest.write_text(json.dumps(manifest))
+        archive, digest = self.package("0.1.202609241531")
+        with patch.object(installer.urllib.request, "urlopen") as fetch:
+            self.install("0.1.202609241531", archive, digest, "--without-model")
+            module, backend, _ = installer.installed_backend(self.data / "frameyap", "redux")
+            self.assertEqual(module.check_model(backend, destination)["state"], "installed_verified")
+            argv = ["--install-model", "--model-dir", str(destination), "--yes", "--json"]
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(installer.cli(argv), 0)
+            fetch.assert_not_called()
+            self.assertEqual(card.read_bytes(), old)
+            card.write_bytes(b"x" * len(old))
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(installer.cli(argv), 1)
+            fetch.assert_not_called()
+            card.unlink()
+        class Response(io.BytesIO):
+            def geturl(self):
+                return "https://huggingface.co/fixture"
+        # Fresh downloads require the current card even though the old one is
+        # permitted for already installed, independently hashed model files.
+        with patch.object(installer.urllib.request, "urlopen", return_value=Response(old)):
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(installer.cli(argv), 1)
+            self.assertFalse(card.exists())
+        with patch.object(installer.urllib.request, "urlopen", return_value=Response(new)) as fetch:
+            with contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(installer.cli(argv), 0)
+            fetch.assert_called_once()
+            self.assertEqual(card.read_bytes(), new)
+
     def test_runtime_install_is_explicit_cpu_pinned_and_updates_only_python_path(self):
         # subprocess is mocked: no venv, pip or network is touched.
         root = self.data / "frameyap"

tests/test_model_files.py

@@ -74,6 +74,29 @@ class ModelFileTests(unittest.TestCase):
             (model / "nested").symlink_to(root)
             self.assertEqual(check_model(backend, model)["reason"], "unsafe_file")
 
+    def test_compatible_card_still_requires_exact_hash_and_safe_file(self):
+        with tempfile.TemporaryDirectory() as path:
+            root = Path(path)
+            _, backend = self.fixture(root)
+            current, previous = b"current card", b"old card"
+            item = ModelFile("README.md", len(current), hashlib.sha256(current).hexdigest(),
+                             ((len(previous), hashlib.sha256(previous).hexdigest()),))
+            backend = replace(backend, files=(item,))
+            card = root / "README.md"
+            for content in (current, previous):
+                card.write_bytes(content)
+                self.assertEqual(check_model(backend, root)["state"], "installed_verified")
+            # A download temp pin must never accept an older artifact.
+            self.assertEqual(check_file(root, ModelFile(item.path, item.size, item.sha256))[0],
+                             "size_mismatch")
+            card.write_bytes(b"bad card")
+            self.assertEqual(check_model(backend, root)["reason"], "hash_mismatch")
+            card.unlink()
+            outside = root / "outside"
+            outside.write_bytes(previous)
+            card.symlink_to(outside)
+            self.assertEqual(check_model(backend, root)["reason"], "unsafe_file")
+
     def test_second_backend_status_without_runtime_changes(self):
         with tempfile.TemporaryDirectory() as path:
             root = Path(path)