mirror of
https://github.com/baketnk/frame-yap.git
synced 2026-10-06 05:00:04 +02:00
feat(models): add pinned backend manifests and offline verification
This commit is contained in:
1 parent
07c03ea27e
commit
5cbbda3ec2
8 files changed
+589
-53
No files matched your search
@@ -0,0 +1,31 @@
|
|||||||
|
{
|
||||||
|
"schema": 1,
|
||||||
|
"id": "redux",
|
||||||
|
"display_name": "Parakeet Redux (CPU)",
|
||||||
|
"launcher": {
|
||||||
|
"type": "python",
|
||||||
|
"path": "python/frameyap/worker.py",
|
||||||
|
"arguments": ["--model", "{model_dir}", "--threads", "{threads}", "--clip-dir", "{clip_dir}"],
|
||||||
|
"protocol": "frameyap-worker-v1"
|
||||||
|
},
|
||||||
|
"model": {
|
||||||
|
"source": "https://huggingface.co/moondream/parakeet-redux",
|
||||||
|
"revision": "fad622f25f303105c20d70e201bcc477c88b620c",
|
||||||
|
"files": [
|
||||||
|
{"path": "model.safetensors", "size": 177774490, "sha256": "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"},
|
||||||
|
{"path": "config.json", "size": 12988, "sha256": "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"},
|
||||||
|
{"path": "ternary.json", "size": 57970, "sha256": "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"},
|
||||||
|
{"path": "tokenizer.json", "size": 1159960, "sha256": "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"},
|
||||||
|
{"path": "README.md", "size": 8533, "sha256": "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"attribution": "Moondream/M87 Labs, Parakeet Redux, derived from NVIDIA Parakeet TDT 0.6B v3; original model card README.md retained with the files.",
|
||||||
|
"license": {
|
||||||
|
"id": "CC-BY-4.0",
|
||||||
|
"text": "Model weights and card: Creative Commons Attribution 4.0 International. Retain attribution and the pinned README.md; runtime licenses are separate."
|
||||||
|
},
|
||||||
|
"requirements": {
|
||||||
|
"cpu": "Local ARM64 CPU inference with Python, moondream 2.4.0 and compatible CPU Torch; 2 threads by default. Runtime not included.",
|
||||||
|
"gpu": "None required; Redux worker disables CUDA."
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,9 +1,245 @@
|
|||||||
"""Pinned public moondream/parakeet-redux artifacts; weights are never in Git."""
|
"""Strict backend manifests and offline, shared pinned-model verification.
|
||||||
REVISION = "fad622f25f303105c20d70e201bcc477c88b620c"
|
|
||||||
FILES = {
|
No model/runtime dependencies are imported. The caller opts in to hashing; neither
|
||||||
"model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"),
|
loading a manifest nor checking local files fetches anything or starts a worker.
|
||||||
"config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"),
|
The wire protocol and launcher argument contract are described in the manifest.
|
||||||
"ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"),
|
"""
|
||||||
"tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"),
|
|
||||||
"README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),
|
from dataclasses import dataclass
|
||||||
}
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
|
||||||
|
DEFAULT_MANIFEST_DIR = Path(__file__).resolve().parents[2] / "assets" / "backends"
|
||||||
|
_ID = re.compile(r"[a-z][a-z0-9_-]{0,47}\Z", re.ASCII)
|
||||||
|
_HASH = re.compile(r"[0-9a-f]{64}\Z", re.ASCII)
|
||||||
|
_MAX_MANIFEST = 65536
|
||||||
|
_MAX_FILE = 16 * 1024 ** 3
|
||||||
|
_ALLOWED_TOKENS = ("{model_dir}", "{clip_dir}", "{threads}")
|
||||||
|
|
||||||
|
|
||||||
|
class ManifestError(ValueError):
|
||||||
|
"""Invalid, unsafe or unsupported manifest (never silently skip one)."""
|
||||||
|
|
||||||
|
|
||||||
|
def _keys(pairs):
|
||||||
|
result = {}
|
||||||
|
for key, value in pairs:
|
||||||
|
if key in result:
|
||||||
|
raise ManifestError(f"duplicate manifest field: {key}")
|
||||||
|
result[key] = value
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _fields(value, keys, label):
|
||||||
|
if not isinstance(value, dict) or set(value) != set(keys):
|
||||||
|
raise ManifestError(f"invalid {label} fields")
|
||||||
|
|
||||||
|
|
||||||
|
def _text(value, label):
|
||||||
|
if not isinstance(value, str) or not value or len(value) > 1024 or any(
|
||||||
|
ord(c) < 32 or ord(c) == 127 for c in value
|
||||||
|
):
|
||||||
|
raise ManifestError(f"invalid {label}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _relative(value, label):
|
||||||
|
_text(value, label)
|
||||||
|
if (value.startswith("/") or "\\" in value or any(
|
||||||
|
not component or component in (".", "..") for component in value.split("/")
|
||||||
|
)):
|
||||||
|
raise ManifestError(f"unsafe {label}")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ModelFile:
|
||||||
|
path: str
|
||||||
|
size: int
|
||||||
|
sha256: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Backend:
|
||||||
|
id: str
|
||||||
|
display_name: str
|
||||||
|
launcher: dict
|
||||||
|
source: str
|
||||||
|
revision: str
|
||||||
|
files: tuple[ModelFile, ...]
|
||||||
|
attribution: str
|
||||||
|
license_id: str
|
||||||
|
license_text: str
|
||||||
|
cpu: str
|
||||||
|
gpu: str
|
||||||
|
|
||||||
|
@property
|
||||||
|
def total_bytes(self):
|
||||||
|
return sum(file.size for file in self.files)
|
||||||
|
|
||||||
|
def description(self):
|
||||||
|
return {"id": self.id, "display_name": self.display_name,
|
||||||
|
"total_bytes": self.total_bytes, "launcher": self.launcher,
|
||||||
|
"source": self.source, "revision": self.revision,
|
||||||
|
"attribution": self.attribution,
|
||||||
|
"license": {"id": self.license_id, "text": self.license_text},
|
||||||
|
"requirements": {"cpu": self.cpu, "gpu": self.gpu}}
|
||||||
|
|
||||||
|
|
||||||
|
def _parse(value):
|
||||||
|
_fields(value, ("schema", "id", "display_name", "launcher", "model",
|
||||||
|
"attribution", "license", "requirements"), "manifest")
|
||||||
|
if type(value["schema"]) is not int or value["schema"] != 1:
|
||||||
|
raise ManifestError("unsupported manifest schema")
|
||||||
|
ident = value["id"]
|
||||||
|
if not isinstance(ident, str) or not _ID.fullmatch(ident):
|
||||||
|
raise ManifestError("invalid backend id")
|
||||||
|
display = _text(value["display_name"], "display_name")
|
||||||
|
launcher = value["launcher"]
|
||||||
|
_fields(launcher, ("type", "path", "arguments", "protocol"), "launcher")
|
||||||
|
if launcher["type"] not in ("python", "executable") or launcher["protocol"] != "frameyap-worker-v1":
|
||||||
|
raise ManifestError("unsupported launcher or protocol")
|
||||||
|
_relative(launcher["path"], "launcher path")
|
||||||
|
args = launcher["arguments"]
|
||||||
|
if not isinstance(args, list) or not 1 <= len(args) <= 32:
|
||||||
|
raise ManifestError("invalid launcher arguments")
|
||||||
|
for arg in args:
|
||||||
|
_text(arg, "launcher argument")
|
||||||
|
# Never interpolate arbitrary formatting or execute through a shell.
|
||||||
|
if "{" in arg or "}" in arg:
|
||||||
|
if arg not in _ALLOWED_TOKENS:
|
||||||
|
raise ManifestError("unsupported launcher argument placeholder")
|
||||||
|
if "{model_dir}" not in args or "{clip_dir}" not in args:
|
||||||
|
raise ManifestError("launcher must accept model_dir and clip_dir")
|
||||||
|
model = value["model"]
|
||||||
|
_fields(model, ("source", "revision", "files"), "model")
|
||||||
|
source = _text(model["source"], "model source")
|
||||||
|
if not source.startswith("https://"):
|
||||||
|
raise ManifestError("model source must be HTTPS")
|
||||||
|
revision = _text(model["revision"], "model revision")
|
||||||
|
files = model["files"]
|
||||||
|
if not isinstance(files, list) or not 1 <= len(files) <= 64:
|
||||||
|
raise ManifestError("invalid model file list")
|
||||||
|
parsed = []
|
||||||
|
names = set()
|
||||||
|
for item in files:
|
||||||
|
_fields(item, ("path", "size", "sha256"), "model file")
|
||||||
|
name = _relative(item["path"], "model file path")
|
||||||
|
size = item["size"]
|
||||||
|
if name in names or type(size) is not int or not 0 < size <= _MAX_FILE or not isinstance(item["sha256"], str) or not _HASH.fullmatch(item["sha256"]):
|
||||||
|
raise ManifestError("duplicate or invalid pinned model file")
|
||||||
|
names.add(name)
|
||||||
|
parsed.append(ModelFile(name, size, item["sha256"]))
|
||||||
|
license_info = value["license"]
|
||||||
|
_fields(license_info, ("id", "text"), "license")
|
||||||
|
requirements = value["requirements"]
|
||||||
|
_fields(requirements, ("cpu", "gpu"), "requirements")
|
||||||
|
return Backend(ident, display, launcher, source, revision, tuple(parsed),
|
||||||
|
_text(value["attribution"], "attribution"),
|
||||||
|
_text(license_info["id"], "license id"),
|
||||||
|
_text(license_info["text"], "license text"),
|
||||||
|
_text(requirements["cpu"], "CPU requirements"),
|
||||||
|
_text(requirements["gpu"], "GPU requirements"))
|
||||||
|
|
||||||
|
|
||||||
|
def load_backends(manifest_dir=DEFAULT_MANIFEST_DIR):
|
||||||
|
"""Return all manifests keyed by ID; reject invalid/symlinked/duplicate entries."""
|
||||||
|
root = Path(manifest_dir)
|
||||||
|
if root.is_symlink() or not root.is_dir():
|
||||||
|
raise ManifestError("manifest directory missing or unsafe")
|
||||||
|
result = {}
|
||||||
|
for file in sorted(root.iterdir()):
|
||||||
|
if file.suffix != ".json":
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
fd = os.open(file, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
|
||||||
|
with os.fdopen(fd, "rb") as handle:
|
||||||
|
info = os.fstat(handle.fileno())
|
||||||
|
if not stat.S_ISREG(info.st_mode) or info.st_size > _MAX_MANIFEST:
|
||||||
|
raise ManifestError("manifest file oversized or unsafe")
|
||||||
|
raw = handle.read(_MAX_MANIFEST + 1)
|
||||||
|
if len(raw) > _MAX_MANIFEST:
|
||||||
|
raise ManifestError("manifest file oversized or unsafe")
|
||||||
|
data = json.loads(raw.decode("utf-8"), object_pairs_hook=_keys)
|
||||||
|
backend = _parse(data)
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError) as error:
|
||||||
|
raise ManifestError(f"invalid manifest: {file.name}") from error
|
||||||
|
if file.stem != backend.id or backend.id in result:
|
||||||
|
raise ManifestError("manifest filename and backend id must match")
|
||||||
|
result[backend.id] = backend
|
||||||
|
if not result:
|
||||||
|
raise ManifestError("no backend manifests found")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def check_file(directory, item):
|
||||||
|
"""Return (reason, filename); reason None means the pinned file verifies."""
|
||||||
|
root = Path(directory)
|
||||||
|
if not root.is_absolute():
|
||||||
|
return "model_dir_not_absolute", item.path
|
||||||
|
if not root.exists() and not root.is_symlink():
|
||||||
|
return "directory_missing", item.path
|
||||||
|
if root.is_symlink() or not root.is_dir():
|
||||||
|
return "unsafe_model_dir", item.path
|
||||||
|
flags_dir = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
|
||||||
|
try:
|
||||||
|
fd = os.open(root, flags_dir)
|
||||||
|
except OSError:
|
||||||
|
return "unsafe_model_dir", item.path
|
||||||
|
try:
|
||||||
|
components = item.path.split("/")
|
||||||
|
for component in components[:-1]:
|
||||||
|
new_fd = os.open(component, flags_dir, dir_fd=fd)
|
||||||
|
os.close(fd)
|
||||||
|
fd = new_fd
|
||||||
|
file_fd = os.open(components[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
|
||||||
|
try:
|
||||||
|
before = os.fstat(file_fd)
|
||||||
|
if not stat.S_ISREG(before.st_mode):
|
||||||
|
return "unsafe_file", item.path
|
||||||
|
if before.st_size != item.size:
|
||||||
|
return "size_mismatch", item.path
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
while chunk := os.read(file_fd, 1024 * 1024):
|
||||||
|
digest.update(chunk)
|
||||||
|
after = os.fstat(file_fd)
|
||||||
|
if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns) or digest.hexdigest() != item.sha256:
|
||||||
|
return "hash_mismatch", item.path
|
||||||
|
return None, None
|
||||||
|
finally:
|
||||||
|
os.close(file_fd)
|
||||||
|
except FileNotFoundError:
|
||||||
|
# A dangling symlink must not be mistaken for an absent file.
|
||||||
|
try:
|
||||||
|
os.stat(components[-1], dir_fd=fd, follow_symlinks=False)
|
||||||
|
except FileNotFoundError:
|
||||||
|
return "missing_files", item.path
|
||||||
|
except OSError:
|
||||||
|
return "unsafe_file", item.path
|
||||||
|
return "unsafe_file", item.path
|
||||||
|
except OSError:
|
||||||
|
return "unsafe_file", item.path
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
|
||||||
|
|
||||||
|
def check_model(backend, directory):
|
||||||
|
"""Offline status for the *direct* pinned model directory (never a download)."""
|
||||||
|
status = backend.description()
|
||||||
|
status.update(state="installed_verified", reason=None, file=None)
|
||||||
|
missing = None
|
||||||
|
for item in backend.files:
|
||||||
|
reason, name = check_file(directory, item)
|
||||||
|
if reason in ("directory_missing", "missing_files"):
|
||||||
|
if missing is None:
|
||||||
|
missing = reason, name
|
||||||
|
elif reason is not None:
|
||||||
|
status.update(state="invalid", reason=reason, file=name)
|
||||||
|
return status
|
||||||
|
if missing:
|
||||||
|
status.update(state="not_installed", reason=missing[0], file=missing[1])
|
||||||
|
return status
|
||||||
+16
-22
@@ -8,7 +8,6 @@ by default, or captured privately with explicit --advanced-debug. This module ne
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import stat
|
import stat
|
||||||
@@ -21,9 +20,9 @@ MAX_TEXT = 4096
|
|||||||
MIN_SAMPLES = 3200
|
MIN_SAMPLES = 3200
|
||||||
MAX_SAMPLES = 320000
|
MAX_SAMPLES = 320000
|
||||||
try:
|
try:
|
||||||
from .model_files import REVISION, FILES
|
from .model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
|
||||||
except ImportError: # direct executable script
|
except ImportError: # direct executable script
|
||||||
from model_files import REVISION, FILES
|
from model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
|
||||||
|
|
||||||
|
|
||||||
class LocalModelError(ValueError):
|
class LocalModelError(ValueError):
|
||||||
@@ -73,25 +72,19 @@ def private_dir(path):
|
|||||||
raise ValueError("clip directory must be private and owned by current user")
|
raise ValueError("clip directory must be private and owned by current user")
|
||||||
|
|
||||||
|
|
||||||
def local_model(path):
|
def local_model(path, manifest_dir=DEFAULT_MANIFEST_DIR):
|
||||||
"""Require a real local directory with weight file(s), never a hub identifier."""
|
"""Check Redux against its manifest before loading any inference libraries."""
|
||||||
root = Path(path)
|
try:
|
||||||
if not root.is_absolute() or not root.is_dir() or root.is_symlink():
|
backend = load_backends(manifest_dir)["redux"]
|
||||||
raise LocalModelError("absolute local model directory required")
|
except (ManifestError, KeyError) as error:
|
||||||
for name, (size, expected) in FILES.items():
|
raise LocalModelError("pinned Redux manifest missing or invalid") from error
|
||||||
file = root / name
|
result = check_model(backend, path)
|
||||||
if not file.is_file() or file.is_symlink() or file.stat().st_size != size:
|
if result["state"] != "installed_verified":
|
||||||
raise LocalModelError("pinned local Redux model weights missing or incomplete")
|
raise LocalModelError("pinned local Redux model weights missing, unsafe or mismatched: " + result["reason"])
|
||||||
digest = hashlib.sha256()
|
return str(Path(path))
|
||||||
with file.open("rb") as stream:
|
|
||||||
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
||||||
digest.update(chunk)
|
|
||||||
if digest.hexdigest() != expected:
|
|
||||||
raise LocalModelError("local Redux model does not match pinned revision")
|
|
||||||
return str(root)
|
|
||||||
|
|
||||||
|
|
||||||
def load_model(path, threads):
|
def load_model(path, threads, manifest_dir=DEFAULT_MANIFEST_DIR):
|
||||||
# Set before importing moondream/torch/huggingface dependencies, even when
|
# Set before importing moondream/torch/huggingface dependencies, even when
|
||||||
# invoked directly without the native adapter.
|
# invoked directly without the native adapter.
|
||||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||||
@@ -101,7 +94,7 @@ def load_model(path, threads):
|
|||||||
os.environ[key] = str(threads)
|
os.environ[key] = str(threads)
|
||||||
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
os.environ["CUDA_VISIBLE_DEVICES"] = ""
|
||||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||||
directory = local_model(path)
|
directory = local_model(path, manifest_dir)
|
||||||
import torch
|
import torch
|
||||||
torch.set_num_threads(threads)
|
torch.set_num_threads(threads)
|
||||||
torch.set_num_interop_threads(1)
|
torch.set_num_interop_threads(1)
|
||||||
@@ -183,6 +176,7 @@ def run(model, directory, input_fd=0, output_fd=1, advanced_debug=False):
|
|||||||
def main(argv=None):
|
def main(argv=None):
|
||||||
parser = argparse.ArgumentParser(description="FrameYap local offline Redux worker")
|
parser = argparse.ArgumentParser(description="FrameYap local offline Redux worker")
|
||||||
parser.add_argument("--model", required=True)
|
parser.add_argument("--model", required=True)
|
||||||
|
parser.add_argument("--manifest-dir", type=Path, default=DEFAULT_MANIFEST_DIR)
|
||||||
parser.add_argument("--threads", type=int, default=2)
|
parser.add_argument("--threads", type=int, default=2)
|
||||||
parser.add_argument("--clip-dir", required=True)
|
parser.add_argument("--clip-dir", required=True)
|
||||||
parser.add_argument("--advanced-debug", action="store_true",
|
parser.add_argument("--advanced-debug", action="store_true",
|
||||||
@@ -210,7 +204,7 @@ def main(argv=None):
|
|||||||
send_frame(protocol_fd, b"F", b"M")
|
send_frame(protocol_fd, b"F", b"M")
|
||||||
return 1
|
return 1
|
||||||
try:
|
try:
|
||||||
model = load_model(args.model, args.threads)
|
model = load_model(args.model, args.threads, args.manifest_dir)
|
||||||
except LocalModelError:
|
except LocalModelError:
|
||||||
if args.advanced_debug: traceback.print_exc(file=sys.stderr)
|
if args.advanced_debug: traceback.print_exc(file=sys.stderr)
|
||||||
send_frame(protocol_fd, b"F", b"M")
|
send_frame(protocol_fd, b"F", b"M")
|
||||||
|
|||||||
+20
-20
@@ -1,7 +1,6 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Explicit setup only: fetch the pinned ~178 MB Redux weights; never used by runtime."""
|
"""Explicit setup only: fetch the pinned ~178 MB Redux weights; never used by runtime."""
|
||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
@@ -9,14 +8,7 @@ import tempfile
|
|||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python"))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python"))
|
||||||
from frameyap.model_files import FILES, REVISION
|
from frameyap.model_files import check_file, check_model, load_backends
|
||||||
|
|
||||||
|
|
||||||
def matches(path, size, digest):
|
|
||||||
if not path.is_file() or path.is_symlink() or path.stat().st_size != size:
|
|
||||||
return False
|
|
||||||
with path.open("rb") as stream:
|
|
||||||
return hashlib.file_digest(stream, "sha256").hexdigest() == digest
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -26,30 +18,38 @@ def main():
|
|||||||
dest = args.destination.expanduser().absolute()
|
dest = args.destination.expanduser().absolute()
|
||||||
if dest.is_symlink():
|
if dest.is_symlink():
|
||||||
parser.error("destination must not be a symlink")
|
parser.error("destination must not be a symlink")
|
||||||
|
backend = load_backends()["redux"]
|
||||||
dest.mkdir(mode=0o700, parents=True, exist_ok=True)
|
dest.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
for name, (size, digest) in FILES.items():
|
for item in backend.files:
|
||||||
target = dest / name
|
target = dest / item.path
|
||||||
if matches(target, size, digest):
|
reason, _ = check_file(dest, item)
|
||||||
|
if reason is None:
|
||||||
continue
|
continue
|
||||||
if target.exists() or target.is_symlink():
|
if reason != "missing_files":
|
||||||
parser.error(f"existing mismatched file: {target}; move it aside explicitly")
|
parser.error(f"existing mismatched or unsafe file: {target}; move it aside explicitly")
|
||||||
url = f"https://huggingface.co/moondream/parakeet-redux/resolve/{REVISION}/{name}"
|
# This explicit Redux-only tool does not implement a generic model downloader.
|
||||||
|
url = f"{backend.source}/resolve/{backend.revision}/{item.path}"
|
||||||
fd, temp = tempfile.mkstemp(prefix=".download-", dir=dest)
|
fd, temp = tempfile.mkstemp(prefix=".download-", dir=dest)
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "wb") as output, urllib.request.urlopen(url, timeout=60) as source:
|
with os.fdopen(fd, "wb") as output, urllib.request.urlopen(url, timeout=60) as source:
|
||||||
total = 0
|
total = 0
|
||||||
while chunk := source.read(1024 * 1024):
|
while chunk := source.read(1024 * 1024):
|
||||||
total += len(chunk)
|
total += len(chunk)
|
||||||
if total > size:
|
if total > item.size:
|
||||||
raise ValueError("download exceeded pinned size")
|
raise ValueError("download exceeded pinned size")
|
||||||
output.write(chunk)
|
output.write(chunk)
|
||||||
if not matches(Path(temp), size, digest):
|
if check_file(dest, item)[0] != "missing_files":
|
||||||
raise ValueError(f"pinned SHA-256/size mismatch: {name}")
|
parser.error(f"destination changed during download: {target}")
|
||||||
|
# Check the temporary file using the same pinned verifier before install.
|
||||||
|
if check_file(dest, type(item)(Path(temp).name, item.size, item.sha256))[0] is not None:
|
||||||
|
raise ValueError(f"pinned SHA-256/size mismatch: {item.path}")
|
||||||
os.replace(temp, target)
|
os.replace(temp, target)
|
||||||
finally:
|
finally:
|
||||||
Path(temp).unlink(missing_ok=True)
|
Path(temp).unlink(missing_ok=True)
|
||||||
print(f"Pinned Redux {REVISION} verified in {dest}")
|
if check_model(backend, dest)["state"] != "installed_verified":
|
||||||
print("Model attribution: moondream/parakeet-redux, CC-BY-4.0; see downloaded README.md.")
|
raise ValueError("pinned model verification failed")
|
||||||
|
print(f"Pinned Redux {backend.revision} verified in {dest}")
|
||||||
|
print(f"Model attribution: {backend.attribution} License: {backend.license_id}.")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Executable
+55
@@ -0,0 +1,55 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Offline backend inventory and pinned file status; never downloads or imports ASR."""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python"))
|
||||||
|
from frameyap.model_files import DEFAULT_MANIFEST_DIR, ManifestError, check_model, load_backends
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None):
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument("--list-models", action="store_true")
|
||||||
|
group.add_argument("--check-model", metavar="ID")
|
||||||
|
parser.add_argument("--manifest-dir", type=Path, default=DEFAULT_MANIFEST_DIR)
|
||||||
|
parser.add_argument("--model-dir", type=Path,
|
||||||
|
help="direct model directory for --check-model; store root containing ID/ directories for --list-models")
|
||||||
|
parser.add_argument("--json", action="store_true", help="stable schema v1 on stdout")
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
if args.check_model and args.model_dir is None:
|
||||||
|
parser.error("--check-model requires --model-dir")
|
||||||
|
try:
|
||||||
|
backends = load_backends(args.manifest_dir)
|
||||||
|
except ManifestError as error:
|
||||||
|
parser.error(str(error))
|
||||||
|
if args.check_model:
|
||||||
|
if args.check_model not in backends:
|
||||||
|
parser.error("unknown backend id")
|
||||||
|
result = check_model(backends[args.check_model], args.model_dir)
|
||||||
|
payload = {"schema": 1, "model": result}
|
||||||
|
code = 0 if result["state"] == "installed_verified" else 1
|
||||||
|
else:
|
||||||
|
models = []
|
||||||
|
for ident, backend in backends.items():
|
||||||
|
if args.model_dir is not None:
|
||||||
|
models.append(check_model(backend, args.model_dir / ident))
|
||||||
|
else:
|
||||||
|
result = backend.description()
|
||||||
|
result.update(state="unknown", reason="model_dir_unspecified", file=None)
|
||||||
|
models.append(result)
|
||||||
|
payload = {"schema": 1, "models": models}
|
||||||
|
code = 0
|
||||||
|
if args.json:
|
||||||
|
print(json.dumps(payload, ensure_ascii=False, sort_keys=True))
|
||||||
|
else:
|
||||||
|
for model in payload.get("models", [payload.get("model")]):
|
||||||
|
print(f"{model['id']} ({model['display_name']}): {model['state']}" +
|
||||||
|
(f" [{model['reason']}: {model['file']}]" if model['reason'] else ""))
|
||||||
|
return code
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
"""Manifest schema tests: adding a second backend requires no C++ worker changes."""
|
||||||
|
import copy
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "python"))
|
||||||
|
from frameyap.model_files import DEFAULT_MANIFEST_DIR, ManifestError, load_backends
|
||||||
|
|
||||||
|
|
||||||
|
class BackendTests(unittest.TestCase):
|
||||||
|
def test_redux_manifest_preserves_pinned_artifacts_and_launcher_contract(self):
|
||||||
|
redux = load_backends()["redux"]
|
||||||
|
self.assertEqual(redux.revision, "fad622f25f303105c20d70e201bcc477c88b620c")
|
||||||
|
self.assertEqual({f.path: (f.size, f.sha256) for f in redux.files}, {
|
||||||
|
"model.safetensors": (177774490, "78ec25733ee0d0c1586d1346fc86db9d0c2e436e3a8ab1d32a82d1bb8f848d21"),
|
||||||
|
"config.json": (12988, "503c653b2e3bb788adbcb04f5abdee532d958686564081baeed133ff10143f6e"),
|
||||||
|
"ternary.json": (57970, "1221c6d3ce901ffe09c089da758a8db8b76189f80cff41c5afc244fc61e2051d"),
|
||||||
|
"tokenizer.json": (1159960, "bd321b096832a3f270bd3b2a88823957920f1a5c5ada71114a26ea729d0cbe91"),
|
||||||
|
"README.md": (8533, "a8b327f983a8b8ff262ff7bead3a791fbed9350632002af8db85ab5cd84cdaa5"),
|
||||||
|
})
|
||||||
|
self.assertEqual(redux.launcher["protocol"], "frameyap-worker-v1")
|
||||||
|
self.assertEqual(redux.launcher["arguments"], ["--model", "{model_dir}", "--threads", "{threads}", "--clip-dir", "{clip_dir}"])
|
||||||
|
self.assertIn("CC-BY-4.0", redux.license_id)
|
||||||
|
|
||||||
|
def test_second_backend_and_strict_schema(self):
|
||||||
|
fixture = json.loads((DEFAULT_MANIFEST_DIR / "redux.json").read_text())
|
||||||
|
with tempfile.TemporaryDirectory() as path:
|
||||||
|
root = Path(path)
|
||||||
|
def save(value, name="other.json"):
|
||||||
|
(root / name).write_text(json.dumps(value))
|
||||||
|
fixture["id"] = "other"
|
||||||
|
fixture["display_name"] = "Independent offline test backend"
|
||||||
|
fixture["launcher"] = {"type": "executable", "path": "bin/other-worker",
|
||||||
|
"arguments": ["--model", "{model_dir}", "--clip-dir", "{clip_dir}"],
|
||||||
|
"protocol": "frameyap-worker-v1"}
|
||||||
|
fixture["model"]["files"] = [{"path": "nested/model.bin", "size": 3, "sha256": "a" * 64}]
|
||||||
|
save(fixture)
|
||||||
|
self.assertEqual(load_backends(root)["other"].files[0].path, "nested/model.bin")
|
||||||
|
(root / "redux.json").write_bytes((DEFAULT_MANIFEST_DIR / "redux.json").read_bytes())
|
||||||
|
self.assertEqual(list(load_backends(root)), ["other", "redux"])
|
||||||
|
for edit in (
|
||||||
|
lambda f: f.update(schema=2),
|
||||||
|
lambda f: f.update(unexpected="x"),
|
||||||
|
lambda f: f["model"]["files"][0].update(path="../secret"),
|
||||||
|
lambda f: f["model"]["files"][0].update(size=True),
|
||||||
|
lambda f: f["model"]["files"][0].update(sha256="A" * 64),
|
||||||
|
lambda f: f["launcher"]["arguments"].append("{unknown}"),
|
||||||
|
lambda f: f["launcher"].update(path="/bin/sh"),
|
||||||
|
lambda f: f["launcher"].update(protocol="not-the-wire-protocol"),
|
||||||
|
):
|
||||||
|
invalid = copy.deepcopy(fixture)
|
||||||
|
edit(invalid)
|
||||||
|
save(invalid)
|
||||||
|
with self.subTest(invalid=invalid), self.assertRaises(ManifestError):
|
||||||
|
load_backends(root)
|
||||||
|
save(fixture)
|
||||||
|
(root / "other.json").write_text('{"id":"other","id":"other"}')
|
||||||
|
with self.assertRaises(ManifestError):
|
||||||
|
load_backends(root)
|
||||||
|
(root / "other.json").unlink()
|
||||||
|
(root / "other.json").symlink_to(DEFAULT_MANIFEST_DIR / "redux.json")
|
||||||
|
with self.assertRaises(ManifestError):
|
||||||
|
load_backends(root)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
"""Offline hash/status and CLI tests; fixtures are bytes, not models."""
|
||||||
|
import contextlib
|
||||||
|
from dataclasses import replace
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
sys.path.insert(0, str(ROOT / "python"))
|
||||||
|
from frameyap.model_files import ModelFile, check_file, check_model, load_backends
|
||||||
|
|
||||||
|
CLI = ROOT / "scripts" / "model-status.py"
|
||||||
|
|
||||||
|
|
||||||
|
class ModelFileTests(unittest.TestCase):
|
||||||
|
def fixture(self, root):
|
||||||
|
manifests = root / "manifests"
|
||||||
|
manifests.mkdir()
|
||||||
|
backend = json.loads((ROOT / "assets/backends/redux.json").read_text())
|
||||||
|
backend["model"]["files"] = [{"path": "nested/weights.bin", "size": 7,
|
||||||
|
"sha256": hashlib.sha256(b"fixture").hexdigest()}]
|
||||||
|
(manifests / "redux.json").write_text(json.dumps(backend))
|
||||||
|
return manifests, load_backends(manifests)["redux"]
|
||||||
|
|
||||||
|
def cli(self, manifests, *arguments):
|
||||||
|
return subprocess.run([sys.executable, str(CLI), "--manifest-dir", str(manifests),
|
||||||
|
"--json", *map(str, arguments)], capture_output=True, text=True, timeout=5)
|
||||||
|
|
||||||
|
def test_status_and_cli_from_same_verifier(self):
|
||||||
|
with tempfile.TemporaryDirectory() as path:
|
||||||
|
root = Path(path)
|
||||||
|
manifests, backend = self.fixture(root)
|
||||||
|
store = root / "models"
|
||||||
|
model = store / "redux"
|
||||||
|
result = self.cli(manifests, "--list-models", "--model-dir", store)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
listed = json.loads(result.stdout)
|
||||||
|
self.assertEqual(list(listed), ["models", "schema"])
|
||||||
|
self.assertEqual(listed["models"][0]["state"], "not_installed")
|
||||||
|
self.assertEqual(listed["models"][0]["reason"], "directory_missing")
|
||||||
|
self.assertEqual(listed["models"][0]["total_bytes"], 7)
|
||||||
|
self.assertEqual(self.cli(manifests, "--check-model", "redux", "--model-dir", model).returncode, 1)
|
||||||
|
(model / "nested").mkdir(parents=True)
|
||||||
|
weight = model / "nested/weights.bin"
|
||||||
|
weight.write_bytes(b"fixture")
|
||||||
|
checked = self.cli(manifests, "--check-model", "redux", "--model-dir", model)
|
||||||
|
self.assertEqual(checked.returncode, 0, checked.stderr)
|
||||||
|
self.assertEqual(json.loads(checked.stdout), {"schema": 1, "model": check_model(backend, model)})
|
||||||
|
self.assertEqual(check_file(model, backend.files[0]), (None, None))
|
||||||
|
weight.write_bytes(b"altered")
|
||||||
|
self.assertEqual(check_model(backend, model)["reason"], "hash_mismatch")
|
||||||
|
self.assertEqual(json.loads(self.cli(manifests, "--check-model", "redux", "--model-dir", model).stdout)["model"]["state"], "invalid")
|
||||||
|
weight.write_bytes(b"short")
|
||||||
|
self.assertEqual(check_model(backend, model)["reason"], "size_mismatch")
|
||||||
|
weight.unlink()
|
||||||
|
self.assertEqual(check_model(backend, model)["state"], "not_installed")
|
||||||
|
outside = root / "outside"
|
||||||
|
outside.write_bytes(b"fixture")
|
||||||
|
weight.symlink_to(outside)
|
||||||
|
self.assertEqual(check_model(backend, model)["reason"], "unsafe_file")
|
||||||
|
weight.unlink()
|
||||||
|
weight.symlink_to(root / "absent")
|
||||||
|
self.assertEqual(check_model(backend, model)["reason"], "unsafe_file")
|
||||||
|
weight.unlink()
|
||||||
|
(model / "nested").rmdir()
|
||||||
|
(model / "nested").symlink_to(root)
|
||||||
|
self.assertEqual(check_model(backend, model)["reason"], "unsafe_file")
|
||||||
|
|
||||||
|
def test_second_backend_status_without_runtime_changes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as path:
|
||||||
|
root = Path(path)
|
||||||
|
manifests, _ = self.fixture(root)
|
||||||
|
independent = json.loads((manifests / "redux.json").read_text())
|
||||||
|
independent["id"] = "independent"
|
||||||
|
independent["display_name"] = "Independent backend"
|
||||||
|
independent["launcher"] = {"type": "executable", "path": "bin/independent-worker",
|
||||||
|
"arguments": ["--model", "{model_dir}", "--clip-dir", "{clip_dir}"],
|
||||||
|
"protocol": "frameyap-worker-v1"}
|
||||||
|
independent["model"]["files"] = [{"path": "alternate.bin", "size": 3,
|
||||||
|
"sha256": hashlib.sha256(b"abc").hexdigest()}]
|
||||||
|
(manifests / "independent.json").write_text(json.dumps(independent))
|
||||||
|
model_dir = root / "store/independent"
|
||||||
|
model_dir.mkdir(parents=True)
|
||||||
|
(model_dir / "alternate.bin").write_bytes(b"abc")
|
||||||
|
listed = self.cli(manifests, "--list-models", "--model-dir", root / "store")
|
||||||
|
self.assertEqual(listed.returncode, 0, listed.stderr)
|
||||||
|
models = json.loads(listed.stdout)["models"]
|
||||||
|
self.assertEqual([(model["id"], model["state"]) for model in models],
|
||||||
|
[("independent", "installed_verified"), ("redux", "not_installed")])
|
||||||
|
self.assertEqual(self.cli(manifests, "--check-model", "independent", "--model-dir", model_dir).returncode, 0)
|
||||||
|
|
||||||
|
def test_explicit_fetch_tool_reuses_shared_verifier_offline(self):
|
||||||
|
spec = importlib.util.spec_from_file_location("fetch_model", ROOT / "scripts/fetch-model.py")
|
||||||
|
fetch = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(fetch)
|
||||||
|
with tempfile.TemporaryDirectory() as path:
|
||||||
|
root = Path(path)
|
||||||
|
_, backend = self.fixture(root)
|
||||||
|
backend = replace(backend, files=(ModelFile("weights.bin", 7, hashlib.sha256(b"fixture").hexdigest()),))
|
||||||
|
model = root / "local-model"
|
||||||
|
# The Redux-only tool uses this small fixture manifest here; all network
|
||||||
|
# calls are mocked, never real downloads.
|
||||||
|
with patch.object(fetch, "load_backends", return_value={"redux": backend}), \
|
||||||
|
patch.object(fetch.urllib.request, "urlopen", return_value=io.BytesIO(b"fixture")) as urlopen, \
|
||||||
|
patch.object(sys, "argv", ["fetch-model.py", "--destination", str(model)]):
|
||||||
|
with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()):
|
||||||
|
fetch.main()
|
||||||
|
urlopen.assert_called_once()
|
||||||
|
self.assertEqual(check_model(backend, model)["state"], "installed_verified")
|
||||||
|
fetch.main()
|
||||||
|
urlopen.assert_called_once()
|
||||||
|
(model / "weights.bin").write_bytes(b"changed")
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
fetch.main()
|
||||||
|
urlopen.assert_called_once()
|
||||||
|
|
||||||
|
def test_missing_manifest_unknown_id_and_no_runtime_imports(self):
|
||||||
|
with tempfile.TemporaryDirectory() as path:
|
||||||
|
manifests, _ = self.fixture(Path(path))
|
||||||
|
result = self.cli(manifests, "--check-model", "unknown", "--model-dir", path)
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
self.assertEqual(result.stdout, "")
|
||||||
|
result = self.cli(Path(path) / "absent", "--list-models")
|
||||||
|
self.assertEqual(result.returncode, 2)
|
||||||
|
result = self.cli(manifests, "--list-models")
|
||||||
|
self.assertEqual(result.returncode, 0)
|
||||||
|
self.assertEqual(json.loads(result.stdout)["models"][0]["state"], "unknown")
|
||||||
|
self.assertEqual(json.loads(result.stdout)["models"][0]["reason"], "model_dir_unspecified")
|
||||||
|
# This subprocess never imports inference packages even if not installed.
|
||||||
|
result = subprocess.run([sys.executable, "-c", "import sys; import frameyap.model_files; "
|
||||||
|
"assert not {'moondream', 'torch', 'numpy'} & set(sys.modules)"],
|
||||||
|
env={**os.environ, "PYTHONPATH": str(ROOT / "python")},
|
||||||
|
capture_output=True, timeout=5)
|
||||||
|
self.assertEqual(result.returncode, 0, result.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -238,14 +238,18 @@ sys.exit(worker.main(sys.argv[1:]))
|
|||||||
|
|
||||||
def test_pinned_model_hashes_and_symlinks(self):
|
def test_pinned_model_hashes_and_symlinks(self):
|
||||||
import hashlib
|
import hashlib
|
||||||
|
from frameyap.model_files import Backend, ModelFile
|
||||||
with tempfile.TemporaryDirectory() as path:
|
with tempfile.TemporaryDirectory() as path:
|
||||||
root = Path(path)
|
root = Path(path)
|
||||||
(root / "model.safetensors").write_bytes(b"fixture")
|
(root / "model.safetensors").write_bytes(b"fixture")
|
||||||
digest = hashlib.sha256(b"fixture").hexdigest()
|
digest = hashlib.sha256(b"fixture").hexdigest()
|
||||||
with patch.object(worker, "FILES", {"model.safetensors": (7, digest)}):
|
backend = Backend("redux", "Redux", {}, "https://example.test", "pinned",
|
||||||
|
(ModelFile("model.safetensors", 7, digest),), "source",
|
||||||
|
"CC-BY-4.0", "license", "cpu", "none")
|
||||||
|
with patch.object(worker, "load_backends", return_value={"redux": backend}):
|
||||||
self.assertEqual(worker.local_model(path), path)
|
self.assertEqual(worker.local_model(path), path)
|
||||||
(root / "model.safetensors").write_bytes(b"changed")
|
(root / "model.safetensors").write_bytes(b"changed")
|
||||||
with self.assertRaisesRegex(ValueError, "pinned revision"):
|
with self.assertRaisesRegex(ValueError, "hash_mismatch"):
|
||||||
worker.local_model(path)
|
worker.local_model(path)
|
||||||
(root / "model.safetensors").unlink()
|
(root / "model.safetensors").unlink()
|
||||||
(root / "other").write_bytes(b"fixture")
|
(root / "other").write_bytes(b"fixture")
|
||||||
|
|||||||
Reference in new issue
Block a user