feat(models): add pinned backend manifests and offline verification

python/frameyap/model_files.py

@@ -1,9 +1,245 @@
-"""Pinned public moondream/parakeet-redux artifacts; weights are never in Git."""
-REVISION = "fad622f25f303105c20d70e201bcc477c88b620c"
-FILES = {
-    "model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"),
-    "config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"),
-    "ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"),
-    "tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"),
-    "README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),
-}
+"""Strict backend manifests and offline, shared pinned-model verification.
+
+No model/runtime dependencies are imported. The caller opts in to hashing; neither
+loading a manifest nor checking local files fetches anything or starts a worker.
+The wire protocol and launcher argument contract are described in the manifest.
+"""
+
+from dataclasses import dataclass
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import stat
+
+DEFAULT_MANIFEST_DIR = Path(__file__).resolve().parents[2] / "assets" / "backends"
+_ID = re.compile(r"[a-z][a-z0-9_-]{0,47}\Z", re.ASCII)
+_HASH = re.compile(r"[0-9a-f]{64}\Z", re.ASCII)
+_MAX_MANIFEST = 65536
+_MAX_FILE = 16 * 1024 ** 3
+_ALLOWED_TOKENS = ("{model_dir}", "{clip_dir}", "{threads}")
+
+
+class ManifestError(ValueError):
+    """Invalid, unsafe or unsupported manifest (never silently skip one)."""
+
+
+def _keys(pairs):
+    result = {}
+    for key, value in pairs:
+        if key in result:
+            raise ManifestError(f"duplicate manifest field: {key}")
+        result[key] = value
+    return result
+
+
+def _fields(value, keys, label):
+    if not isinstance(value, dict) or set(value) != set(keys):
+        raise ManifestError(f"invalid {label} fields")
+
+
+def _text(value, label):
+    if not isinstance(value, str) or not value or len(value) > 1024 or any(
+        ord(c) < 32 or ord(c) == 127 for c in value
+    ):
+        raise ManifestError(f"invalid {label}")
+    return value
+
+
+def _relative(value, label):
+    _text(value, label)
+    if (value.startswith("/") or "\\" in value or any(
+        not component or component in (".", "..") for component in value.split("/")
+    )):
+        raise ManifestError(f"unsafe {label}")
+    return value
+
+
+@dataclass(frozen=True)
+class ModelFile:
+    path: str
+    size: int
+    sha256: str
+
+
+@dataclass(frozen=True)
+class Backend:
+    id: str
+    display_name: str
+    launcher: dict
+    source: str
+    revision: str
+    files: tuple[ModelFile, ...]
+    attribution: str
+    license_id: str
+    license_text: str
+    cpu: str
+    gpu: str
+
+    @property
+    def total_bytes(self):
+        return sum(file.size for file in self.files)
+
+    def description(self):
+        return {"id": self.id, "display_name": self.display_name,
+                "total_bytes": self.total_bytes, "launcher": self.launcher,
+                "source": self.source, "revision": self.revision,
+                "attribution": self.attribution,
+                "license": {"id": self.license_id, "text": self.license_text},
+                "requirements": {"cpu": self.cpu, "gpu": self.gpu}}
+
+
+def _parse(value):
+    _fields(value, ("schema", "id", "display_name", "launcher", "model",
+                    "attribution", "license", "requirements"), "manifest")
+    if type(value["schema"]) is not int or value["schema"] != 1:
+        raise ManifestError("unsupported manifest schema")
+    ident = value["id"]
+    if not isinstance(ident, str) or not _ID.fullmatch(ident):
+        raise ManifestError("invalid backend id")
+    display = _text(value["display_name"], "display_name")
+    launcher = value["launcher"]
+    _fields(launcher, ("type", "path", "arguments", "protocol"), "launcher")
+    if launcher["type"] not in ("python", "executable") or launcher["protocol"] != "frameyap-worker-v1":
+        raise ManifestError("unsupported launcher or protocol")
+    _relative(launcher["path"], "launcher path")
+    args = launcher["arguments"]
+    if not isinstance(args, list) or not 1 <= len(args) <= 32:
+        raise ManifestError("invalid launcher arguments")
+    for arg in args:
+        _text(arg, "launcher argument")
+        # Never interpolate arbitrary formatting or execute through a shell.
+        if "{" in arg or "}" in arg:
+            if arg not in _ALLOWED_TOKENS:
+                raise ManifestError("unsupported launcher argument placeholder")
+    if "{model_dir}" not in args or "{clip_dir}" not in args:
+        raise ManifestError("launcher must accept model_dir and clip_dir")
+    model = value["model"]
+    _fields(model, ("source", "revision", "files"), "model")
+    source = _text(model["source"], "model source")
+    if not source.startswith("https://"):
+        raise ManifestError("model source must be HTTPS")
+    revision = _text(model["revision"], "model revision")
+    files = model["files"]
+    if not isinstance(files, list) or not 1 <= len(files) <= 64:
+        raise ManifestError("invalid model file list")
+    parsed = []
+    names = set()
+    for item in files:
+        _fields(item, ("path", "size", "sha256"), "model file")
+        name = _relative(item["path"], "model file path")
+        size = item["size"]
+        if name in names or type(size) is not int or not 0 < size <= _MAX_FILE or not isinstance(item["sha256"], str) or not _HASH.fullmatch(item["sha256"]):
+            raise ManifestError("duplicate or invalid pinned model file")
+        names.add(name)
+        parsed.append(ModelFile(name, size, item["sha256"]))
+    license_info = value["license"]
+    _fields(license_info, ("id", "text"), "license")
+    requirements = value["requirements"]
+    _fields(requirements, ("cpu", "gpu"), "requirements")
+    return Backend(ident, display, launcher, source, revision, tuple(parsed),
+                   _text(value["attribution"], "attribution"),
+                   _text(license_info["id"], "license id"),
+                   _text(license_info["text"], "license text"),
+                   _text(requirements["cpu"], "CPU requirements"),
+                   _text(requirements["gpu"], "GPU requirements"))
+
+
+def load_backends(manifest_dir=DEFAULT_MANIFEST_DIR):
+    """Return all manifests keyed by ID; reject invalid/symlinked/duplicate entries."""
+    root = Path(manifest_dir)
+    if root.is_symlink() or not root.is_dir():
+        raise ManifestError("manifest directory missing or unsafe")
+    result = {}
+    for file in sorted(root.iterdir()):
+        if file.suffix != ".json":
+            continue
+        try:
+            fd = os.open(file, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
+            with os.fdopen(fd, "rb") as handle:
+                info = os.fstat(handle.fileno())
+                if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_MANIFEST:
+                    raise ManifestError("manifest file oversized or unsafe")
+                raw = handle.read(_MAX_MANIFEST + 1)
+            if len(raw) > _MAX_MANIFEST:
+                raise ManifestError("manifest file oversized or unsafe")
+            data = json.loads(raw.decode("utf-8"), object_pairs_hook=_keys)
+            backend = _parse(data)
+        except (OSError, UnicodeError, json.JSONDecodeError) as error:
+            raise ManifestError(f"invalid manifest: {file.name}") from error
+        if file.stem != backend.id or backend.id in result:
+            raise ManifestError("manifest filename and backend id must match")
+        result[backend.id] = backend
+    if not result:
+        raise ManifestError("no backend manifests found")
+    return result
+
+
+def check_file(directory, item):
+    """Return (reason, filename); reason None means the pinned file verifies."""
+    root = Path(directory)
+    if not root.is_absolute():
+        return "model_dir_not_absolute", item.path
+    if not root.exists() and not root.is_symlink():
+        return "directory_missing", item.path
+    if root.is_symlink() or not root.is_dir():
+        return "unsafe_model_dir", item.path
+    flags_dir = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+    try:
+        fd = os.open(root, flags_dir)
+    except OSError:
+        return "unsafe_model_dir", item.path
+    try:
+        components = item.path.split("/")
+        for component in components[:-1]:
+            new_fd = os.open(component, flags_dir, dir_fd=fd)
+            os.close(fd)
+            fd = new_fd
+        file_fd = os.open(components[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
+        try:
+            before = os.fstat(file_fd)
+            if not stat.S_ISREG(before.st_mode):
+                return "unsafe_file", item.path
+            if before.st_size != item.size:
+                return "size_mismatch", item.path
+            digest = hashlib.sha256()
+            while chunk := os.read(file_fd, 1024 * 1024):
+                digest.update(chunk)
+            after = os.fstat(file_fd)
+            if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or digest.hexdigest() != item.sha256:
+                return "hash_mismatch", item.path
+            return None, None
+        finally:
+            os.close(file_fd)
+    except FileNotFoundError:
+        # A dangling symlink must not be mistaken for an absent file.
+        try:
+            os.stat(components[-1], dir_fd=fd, follow_symlinks=False)
+        except FileNotFoundError:
+            return "missing_files", item.path
+        except OSError:
+            return "unsafe_file", item.path
+        return "unsafe_file", item.path
+    except OSError:
+        return "unsafe_file", item.path
+    finally:
+        os.close(fd)
+
+
+def check_model(backend, directory):
+    """Offline status for the *direct* pinned model directory (never a download)."""
+    status = backend.description()
+    status.update(state="installed_verified", reason=None, file=None)
+    missing = None
+    for item in backend.files:
+        reason, name = check_file(directory, item)
+        if reason in ("directory_missing", "missing_files"):
+            if missing is None:
+                missing = reason, name
+        elif reason is not None:
+            status.update(state="invalid", reason=reason, file=name)
+            return status
+    if missing:
+        status.update(state="not_installed", reason=missing[0], file=missing[1])
+    return status

python/frameyap/worker.py

@@ -8,7 +8,6 @@ by default, or captured privately with explicit --advanced-debug. This module ne
 """
 
 import argparse
-import hashlib
 import os
 from pathlib import Path
 import stat
@@ -21,9 +20,9 @@ MAX_TEXT = 4096
 MIN_SAMPLES = 3200
 MAX_SAMPLES = 320000
 try:
-    from .model_files import REVISION, FILES
+    from .model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
 except ImportError:  # direct executable script
-    from model_files import REVISION, FILES
+    from model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
 
 
 class LocalModelError(ValueError):
@@ -73,25 +72,19 @@ def private_dir(path):
         raise ValueError("clip directory must be private and owned by current user")
 
 
-def local_model(path):
-    """Require a real local directory with weight file(s), never a hub identifier."""
-    root = Path(path)
-    if not root.is_absolute() or not root.is_dir() or root.is_symlink():
-        raise LocalModelError("absolute local model directory required")
-    for name, (size, expected) in FILES.items():
-        file = root / name
-        if not file.is_file() or file.is_symlink() or file.stat().st_size != size:
-            raise LocalModelError("pinned local Redux model weights missing or incomplete")
-        digest = hashlib.sha256()
-        with file.open("rb") as stream:
-            for chunk in iter(lambda: stream.read(1024 * 1024), b""):
-                digest.update(chunk)
-        if digest.hexdigest() != expected:
-            raise LocalModelError("local Redux model does not match pinned revision")
-    return str(root)
+def local_model(path, manifest_dir=DEFAULT_MANIFEST_DIR):
+    """Check Redux against its manifest before loading any inference libraries."""
+    try:
+        backend = load_backends(manifest_dir)["redux"]
+    except (ManifestError, KeyError) as error:
+        raise LocalModelError("pinned Redux manifest missing or invalid") from error
+    result = check_model(backend, path)
+    if result["state"] != "installed_verified":
+        raise LocalModelError("pinned local Redux model weights missing, unsafe or mismatched: " + result["reason"])
+    return str(Path(path))
 
 
-def load_model(path, threads):
+def load_model(path, threads, manifest_dir=DEFAULT_MANIFEST_DIR):
     # Set before importing moondream/torch/huggingface dependencies, even when
     # invoked directly without the native adapter.
     os.environ["HF_HUB_OFFLINE"] = "1"
@@ -101,7 +94,7 @@ def load_model(path, threads):
         os.environ[key] = str(threads)
     os.environ["CUDA_VISIBLE_DEVICES"] = ""
     os.environ["TOKENIZERS_PARALLELISM"] = "false"
-    directory = local_model(path)
+    directory = local_model(path, manifest_dir)
     import torch
     torch.set_num_threads(threads)
     torch.set_num_interop_threads(1)
@@ -183,6 +176,7 @@ def run(model, directory, input_fd=0, output_fd=1, advanced_debug=False):
 def main(argv=None):
     parser = argparse.ArgumentParser(description="FrameYap local offline Redux worker")
     parser.add_argument("--model", required=True)
+    parser.add_argument("--manifest-dir", type=Path, default=DEFAULT_MANIFEST_DIR)
     parser.add_argument("--threads", type=int, default=2)
     parser.add_argument("--clip-dir", required=True)
     parser.add_argument("--advanced-debug", action="store_true",
@@ -210,7 +204,7 @@ def main(argv=None):
             send_frame(protocol_fd, b"F", b"M")
             return 1
         try:
-            model = load_model(args.model, args.threads)
+            model = load_model(args.model, args.threads, args.manifest_dir)
         except LocalModelError:
             if args.advanced_debug: traceback.print_exc(file=sys.stderr)
             send_frame(protocol_fd, b"F", b"M")