[codex] Add large asset guardrails

.gitattributes

@@ -1 +1,18 @@
+# Bundled Core AI resources are stored through Git LFS. Keep future model
+# weights and exported binary tensors out of normal Git blobs.
 *.mlirb filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.mlmodelc/** filter=lfs diff=lfs merge=lfs -text
+*.mlpackage/** filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.gguf filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.float16 filter=lfs diff=lfs merge=lfs -text

.github/workflows/hosted-software.yml

@@ -38,6 +38,9 @@ jobs:
       - name: Validate the versioned CI matrix
         run: python3 Scripts/ci_matrix.py validate
 
+      - name: Verify large assets stay in Git LFS
+        run: python3 Scripts/check_large_files.py
+
       - name: Compile repository Python tooling
         run: python3 -m compileall -q Scripts
 

Documentation/MODEL_ASSETS.md

@@ -0,0 +1,42 @@
+# Model Asset Policy
+
+Core AI Lab intentionally keeps a small number of bundled model resources for
+local Chatterbox and diarization workflows. Those assets are allowed only when
+their provenance and license are documented, and large binary payloads must be
+stored through Git LFS rather than normal Git blobs.
+
+## Current tracked large assets
+
+As of this audit, the only tracked files over 10 MiB in the working tree are
+Git LFS-attributed `main.mlirb` resources:
+
+| Size | Path |
+| ---: | --- |
+| 234 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboS3Gen.aimodel/main.mlirb` |
+| 233 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboT3TransformerInt4.aimodel/main.mlirb` |
+| 117 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboT3Embeddings.aimodel/main.mlirb` |
+| 42 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboVocoder.aimodel/main.mlirb` |
+| 14 MiB | `CoreAILab/Resources/Diarization/CAMPPlus192_float16_600f.aimodel/main.mlirb` |
+
+Do not rewrite repository history to remove older blobs without an owner-level
+cleanup plan. History cleanup needs coordination because it changes every clone,
+branch, tag, fork, and open pull request.
+
+## Adding or refreshing assets
+
+- Prefer documented fetch or conversion steps over committing generated weights.
+- Keep external model downloads outside the repository unless a bundled fixture
+  is explicitly required for product behavior or tests.
+- Store any intentional large binary asset through Git LFS and document its
+  source, license, and generation command in the nearest README or notice file.
+- Do not commit build products, downloaded checkpoints, local virtual
+  environments, result bundles, or ad-hoc exported model directories.
+- Run the large-file guard before pushing:
+
+  ```bash
+  python3 Scripts/check_large_files.py
+  ```
+
+The guard fails when a tracked file over 10 MiB is not covered by a Git LFS
+attribute. `.gitattributes` already routes common Core AI, Core ML, tensor, and
+checkpoint formats through Git LFS for future commits.

README.md

@@ -164,6 +164,7 @@ Detailed conversion and evidence commands live in:
 
 - [`Conversion/Chatterbox/README.md`](Conversion/Chatterbox/README.md)
 - [`Conversion/Diarization/README.md`](Conversion/Diarization/README.md)
+- [`Documentation/MODEL_ASSETS.md`](Documentation/MODEL_ASSETS.md)
 
 ## For Contributors and Agents
 

Scripts/check_large_files.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+DEFAULT_LIMIT_BYTES = 10 * 1024 * 1024
+
+
+@dataclass(frozen=True)
+class TrackedFile:
+    path: str
+    size: int
+    lfs_attributed: bool
+
+
+def run_git(repo: Path, args: list[str], *, stdin: bytes | None = None) -> bytes:
+    result = subprocess.run(
+        ["git", *args],
+        cwd=repo,
+        input=stdin,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+    if result.returncode != 0:
+        stderr = result.stderr.decode("utf-8", errors="replace").strip()
+        raise RuntimeError(f"git {' '.join(args)} failed: {stderr}")
+    return result.stdout
+
+
+def repository_root(path: Path) -> Path:
+    output = run_git(path, ["rev-parse", "--show-toplevel"])
+    return Path(output.decode("utf-8").strip())
+
+
+def tracked_paths(repo: Path) -> list[str]:
+    output = run_git(repo, ["ls-files", "-z"])
+    return [item.decode("utf-8") for item in output.split(b"\0") if item]
+
+
+def chunked(items: list[str], count: int) -> list[list[str]]:
+    return [items[index : index + count] for index in range(0, len(items), count)]
+
+
+def lfs_attributed_paths(repo: Path, paths: list[str]) -> set[str]:
+    attributed: set[str] = set()
+    for chunk in chunked(paths, 200):
+        output = run_git(repo, ["check-attr", "-z", "filter", "--", *chunk])
+        fields = [item.decode("utf-8") for item in output.split(b"\0") if item]
+        for index in range(0, len(fields), 3):
+            path, attribute, value = fields[index : index + 3]
+            if attribute == "filter" and value == "lfs":
+                attributed.add(path)
+    return attributed
+
+
+def collect_tracked_files(repo: Path) -> list[TrackedFile]:
+    paths = tracked_paths(repo)
+    lfs_paths = lfs_attributed_paths(repo, paths)
+    files: list[TrackedFile] = []
+    for path in paths:
+        absolute_path = repo / path
+        if not absolute_path.is_file():
+            continue
+        files.append(
+            TrackedFile(
+                path=path,
+                size=absolute_path.stat().st_size,
+                lfs_attributed=path in lfs_paths,
+            )
+        )
+    return files
+
+
+def format_size(size: int) -> str:
+    for unit in ("bytes", "KiB", "MiB", "GiB"):
+        if size < 1024 or unit == "GiB":
+            if unit == "bytes":
+                return f"{size} {unit}"
+            return f"{size:.1f} {unit}"
+        size /= 1024
+    return f"{size:.1f} GiB"
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Fail if tracked files above the size limit are not covered by Git LFS."
+        )
+    )
+    parser.add_argument(
+        "--repo",
+        type=Path,
+        default=Path.cwd(),
+        help="Repository path to inspect. Defaults to the current directory.",
+    )
+    parser.add_argument(
+        "--limit-bytes",
+        type=int,
+        default=DEFAULT_LIMIT_BYTES,
+        help=f"Tracked-file size limit before LFS is required. Default: {DEFAULT_LIMIT_BYTES}.",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(sys.argv[1:] if argv is None else argv)
+    repo = repository_root(args.repo.resolve())
+    tracked = collect_tracked_files(repo)
+    large_files = sorted(
+        (item for item in tracked if item.size > args.limit_bytes),
+        key=lambda item: item.size,
+        reverse=True,
+    )
+    violations = [item for item in large_files if not item.lfs_attributed]
+
+    if large_files:
+        print(f"Tracked files over {format_size(args.limit_bytes)}:")
+        for item in large_files:
+            status = "LFS" if item.lfs_attributed else "plain Git"
+            print(f"  {status:9} {format_size(item.size):>10}  {item.path}")
+    else:
+        print(f"No tracked files over {format_size(args.limit_bytes)}.")
+
+    if violations:
+        print()
+        for item in violations:
+            print(
+                "::error "
+                f"file={item.path}::Tracked file is {format_size(item.size)} "
+                "but is not covered by Git LFS attributes."
+            )
+        print(
+            "\nMove the asset outside the repository, or add an intentional Git LFS "
+            "rule before committing it."
+        )
+        return 1
+
+    print("Large-file policy passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

Scripts/tests/test_check_large_files.py

@@ -0,0 +1,109 @@
+from __future__ import annotations
+
+import contextlib
+import io
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+
+SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(SCRIPTS_DIRECTORY))
+
+import check_large_files  # noqa: E402
+
+
+class LargeFilePolicyTests(unittest.TestCase):
+    def make_repo(self) -> Path:
+        directory = Path(tempfile.mkdtemp())
+        self.addCleanup(shutil.rmtree, directory)
+        subprocess.run(["git", "init"], cwd=directory, check=True, stdout=subprocess.PIPE)
+        subprocess.run(
+            ["git", "config", "user.email", "tests@example.com"],
+            cwd=directory,
+            check=True,
+        )
+        subprocess.run(
+            ["git", "config", "user.name", "Tests"],
+            cwd=directory,
+            check=True,
+        )
+        return directory
+
+    def add_all_without_lfs_filter(self, repo: Path) -> None:
+        for path in sorted(repo.rglob("*")):
+            if path.is_dir() or ".git" in path.parts:
+                continue
+            relative_path = path.relative_to(repo).as_posix()
+            object_id = subprocess.run(
+                ["git", "hash-object", "-w", "--stdin"],
+                cwd=repo,
+                input=path.read_bytes(),
+                stdout=subprocess.PIPE,
+                check=True,
+            ).stdout.decode("utf-8").strip()
+            subprocess.run(
+                [
+                    "git",
+                    "update-index",
+                    "--add",
+                    "--cacheinfo",
+                    f"100644,{object_id},{relative_path}",
+                ],
+                cwd=repo,
+                check=True,
+            )
+
+    def run_policy(self, repo: Path, limit: int) -> tuple[int, str]:
+        stdout = io.StringIO()
+        with contextlib.redirect_stdout(stdout):
+            exit_code = check_large_files.main(
+                ["--repo", str(repo), "--limit-bytes", str(limit)]
+            )
+        return exit_code, stdout.getvalue()
+
+    def test_large_plain_git_file_fails(self) -> None:
+        repo = self.make_repo()
+        asset = repo / "Weights" / "model.bin"
+        asset.parent.mkdir()
+        asset.write_bytes(b"x" * 2048)
+        self.add_all_without_lfs_filter(repo)
+
+        exit_code, output = self.run_policy(repo, 1024)
+
+        self.assertEqual(exit_code, 1)
+        self.assertIn("plain Git", output)
+        self.assertIn("Weights/model.bin", output)
+
+    def test_large_lfs_attributed_file_passes(self) -> None:
+        repo = self.make_repo()
+        (repo / ".gitattributes").write_text(
+            "*.bin filter=lfs diff=lfs merge=lfs -text\n"
+        )
+        asset = repo / "Weights" / "model.bin"
+        asset.parent.mkdir()
+        asset.write_bytes(b"x" * 2048)
+        self.add_all_without_lfs_filter(repo)
+
+        exit_code, output = self.run_policy(repo, 1024)
+
+        self.assertEqual(exit_code, 0)
+        self.assertIn("LFS", output)
+        self.assertIn("Large-file policy passed", output)
+
+    def test_small_plain_git_file_passes(self) -> None:
+        repo = self.make_repo()
+        (repo / "README.md").write_text("small\n")
+        self.add_all_without_lfs_filter(repo)
+
+        exit_code, output = self.run_policy(repo, 1024)
+
+        self.assertEqual(exit_code, 0)
+        self.assertIn("No tracked files over", output)
+
+
+if __name__ == "__main__":
+    unittest.main()