From d93424ad20b02060add3b67855e6ccafcc220688 Mon Sep 17 00:00:00 2001 From: rudrankriyam Date: Fri, 3 Jul 2026 15:41:21 +0530 Subject: [PATCH] Add large asset guardrails --- .gitattributes | 17 +++ .github/workflows/hosted-software.yml | 3 + Documentation/MODEL_ASSETS.md | 42 +++++++ README.md | 1 + Scripts/check_large_files.py | 148 ++++++++++++++++++++++++ Scripts/tests/test_check_large_files.py | 109 +++++++++++++++++ 6 files changed, 320 insertions(+) create mode 100644 Documentation/MODEL_ASSETS.md create mode 100644 Scripts/check_large_files.py create mode 100644 Scripts/tests/test_check_large_files.py diff --git a/.gitattributes b/.gitattributes index dfb9827..fbdcf55 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,18 @@ +# Bundled Core AI resources are stored through Git LFS. Keep future model +# weights and exported binary tensors out of normal Git blobs. *.mlirb filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.mlmodelc/** filter=lfs diff=lfs merge=lfs -text +*.mlpackage/** filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.gguf filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.float16 filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/hosted-software.yml b/.github/workflows/hosted-software.yml index 55344c7..b364c1a 100644 --- a/.github/workflows/hosted-software.yml +++ b/.github/workflows/hosted-software.yml @@ -38,6 +38,9 @@ jobs: - name: Validate the versioned CI matrix run: python3 Scripts/ci_matrix.py validate + - name: Verify large assets stay in Git LFS + run: python3 Scripts/check_large_files.py + - name: Compile repository Python tooling run: python3 -m compileall -q Scripts diff --git a/Documentation/MODEL_ASSETS.md b/Documentation/MODEL_ASSETS.md new file mode 100644 index 0000000..4f43e95 --- /dev/null +++ b/Documentation/MODEL_ASSETS.md @@ -0,0 +1,42 @@ +# Model Asset Policy + +Core AI Lab intentionally keeps a small number of bundled model resources for +local Chatterbox and diarization workflows. Those assets are allowed only when +their provenance and license are documented, and large binary payloads must be +stored through Git LFS rather than normal Git blobs. + +## Current tracked large assets + +As of this audit, the only tracked files over 10 MiB in the working tree are +Git LFS-attributed `main.mlirb` resources: + +| Size | Path | +| ---: | --- | +| 234 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboS3Gen.aimodel/main.mlirb` | +| 233 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboT3TransformerInt4.aimodel/main.mlirb` | +| 117 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboT3Embeddings.aimodel/main.mlirb` | +| 42 MiB | `CoreAILab/Resources/Chatterbox/ChatterboxTurboVocoder.aimodel/main.mlirb` | +| 14 MiB | `CoreAILab/Resources/Diarization/CAMPPlus192_float16_600f.aimodel/main.mlirb` | + +Do not rewrite repository history to remove older blobs without an owner-level +cleanup plan. History cleanup needs coordination because it changes every clone, +branch, tag, fork, and open pull request. + +## Adding or refreshing assets + +- Prefer documented fetch or conversion steps over committing generated weights. +- Keep external model downloads outside the repository unless a bundled fixture + is explicitly required for product behavior or tests. +- Store any intentional large binary asset through Git LFS and document its + source, license, and generation command in the nearest README or notice file. +- Do not commit build products, downloaded checkpoints, local virtual + environments, result bundles, or ad-hoc exported model directories. +- Run the large-file guard before pushing: + + ```bash + python3 Scripts/check_large_files.py + ``` + +The guard fails when a tracked file over 10 MiB is not covered by a Git LFS +attribute. `.gitattributes` already routes common Core AI, Core ML, tensor, and +checkpoint formats through Git LFS for future commits. diff --git a/README.md b/README.md index a1b0640..8bb1870 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ Detailed conversion and evidence commands live in: - [`Conversion/Chatterbox/README.md`](Conversion/Chatterbox/README.md) - [`Conversion/Diarization/README.md`](Conversion/Diarization/README.md) +- [`Documentation/MODEL_ASSETS.md`](Documentation/MODEL_ASSETS.md) ## For Contributors and Agents diff --git a/Scripts/check_large_files.py b/Scripts/check_large_files.py new file mode 100644 index 0000000..fa96249 --- /dev/null +++ b/Scripts/check_large_files.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +import argparse +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +DEFAULT_LIMIT_BYTES = 10 * 1024 * 1024 + + +@dataclass(frozen=True) +class TrackedFile: + path: str + size: int + lfs_attributed: bool + + +def run_git(repo: Path, args: list[str], *, stdin: bytes | None = None) -> bytes: + result = subprocess.run( + ["git", *args], + cwd=repo, + input=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if result.returncode != 0: + stderr = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"git {' '.join(args)} failed: {stderr}") + return result.stdout + + +def repository_root(path: Path) -> Path: + output = run_git(path, ["rev-parse", "--show-toplevel"]) + return Path(output.decode("utf-8").strip()) + + +def tracked_paths(repo: Path) -> list[str]: + output = run_git(repo, ["ls-files", "-z"]) + return [item.decode("utf-8") for item in output.split(b"\0") if item] + + +def chunked(items: list[str], count: int) -> list[list[str]]: + return [items[index : index + count] for index in range(0, len(items), count)] + + +def lfs_attributed_paths(repo: Path, paths: list[str]) -> set[str]: + attributed: set[str] = set() + for chunk in chunked(paths, 200): + output = run_git(repo, ["check-attr", "-z", "filter", "--", *chunk]) + fields = [item.decode("utf-8") for item in output.split(b"\0") if item] + for index in range(0, len(fields), 3): + path, attribute, value = fields[index : index + 3] + if attribute == "filter" and value == "lfs": + attributed.add(path) + return attributed + + +def collect_tracked_files(repo: Path) -> list[TrackedFile]: + paths = tracked_paths(repo) + lfs_paths = lfs_attributed_paths(repo, paths) + files: list[TrackedFile] = [] + for path in paths: + absolute_path = repo / path + if not absolute_path.is_file(): + continue + files.append( + TrackedFile( + path=path, + size=absolute_path.stat().st_size, + lfs_attributed=path in lfs_paths, + ) + ) + return files + + +def format_size(size: int) -> str: + for unit in ("bytes", "KiB", "MiB", "GiB"): + if size < 1024 or unit == "GiB": + if unit == "bytes": + return f"{size} {unit}" + return f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} GiB" + + +def parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Fail if tracked files above the size limit are not covered by Git LFS." + ) + ) + parser.add_argument( + "--repo", + type=Path, + default=Path.cwd(), + help="Repository path to inspect. Defaults to the current directory.", + ) + parser.add_argument( + "--limit-bytes", + type=int, + default=DEFAULT_LIMIT_BYTES, + help=f"Tracked-file size limit before LFS is required. Default: {DEFAULT_LIMIT_BYTES}.", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(sys.argv[1:] if argv is None else argv) + repo = repository_root(args.repo.resolve()) + tracked = collect_tracked_files(repo) + large_files = sorted( + (item for item in tracked if item.size > args.limit_bytes), + key=lambda item: item.size, + reverse=True, + ) + violations = [item for item in large_files if not item.lfs_attributed] + + if large_files: + print(f"Tracked files over {format_size(args.limit_bytes)}:") + for item in large_files: + status = "LFS" if item.lfs_attributed else "plain Git" + print(f" {status:9} {format_size(item.size):>10} {item.path}") + else: + print(f"No tracked files over {format_size(args.limit_bytes)}.") + + if violations: + print() + for item in violations: + print( + "::error " + f"file={item.path}::Tracked file is {format_size(item.size)} " + "but is not covered by Git LFS attributes." + ) + print( + "\nMove the asset outside the repository, or add an intentional Git LFS " + "rule before committing it." + ) + return 1 + + print("Large-file policy passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/Scripts/tests/test_check_large_files.py b/Scripts/tests/test_check_large_files.py new file mode 100644 index 0000000..c2db388 --- /dev/null +++ b/Scripts/tests/test_check_large_files.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import contextlib +import io +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +SCRIPTS_DIRECTORY = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(SCRIPTS_DIRECTORY)) + +import check_large_files # noqa: E402 + + +class LargeFilePolicyTests(unittest.TestCase): + def make_repo(self) -> Path: + directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, directory) + subprocess.run(["git", "init"], cwd=directory, check=True, stdout=subprocess.PIPE) + subprocess.run( + ["git", "config", "user.email", "tests@example.com"], + cwd=directory, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "Tests"], + cwd=directory, + check=True, + ) + return directory + + def add_all_without_lfs_filter(self, repo: Path) -> None: + for path in sorted(repo.rglob("*")): + if path.is_dir() or ".git" in path.parts: + continue + relative_path = path.relative_to(repo).as_posix() + object_id = subprocess.run( + ["git", "hash-object", "-w", "--stdin"], + cwd=repo, + input=path.read_bytes(), + stdout=subprocess.PIPE, + check=True, + ).stdout.decode("utf-8").strip() + subprocess.run( + [ + "git", + "update-index", + "--add", + "--cacheinfo", + f"100644,{object_id},{relative_path}", + ], + cwd=repo, + check=True, + ) + + def run_policy(self, repo: Path, limit: int) -> tuple[int, str]: + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + exit_code = check_large_files.main( + ["--repo", str(repo), "--limit-bytes", str(limit)] + ) + return exit_code, stdout.getvalue() + + def test_large_plain_git_file_fails(self) -> None: + repo = self.make_repo() + asset = repo / "Weights" / "model.bin" + asset.parent.mkdir() + asset.write_bytes(b"x" * 2048) + self.add_all_without_lfs_filter(repo) + + exit_code, output = self.run_policy(repo, 1024) + + self.assertEqual(exit_code, 1) + self.assertIn("plain Git", output) + self.assertIn("Weights/model.bin", output) + + def test_large_lfs_attributed_file_passes(self) -> None: + repo = self.make_repo() + (repo / ".gitattributes").write_text( + "*.bin filter=lfs diff=lfs merge=lfs -text\n" + ) + asset = repo / "Weights" / "model.bin" + asset.parent.mkdir() + asset.write_bytes(b"x" * 2048) + self.add_all_without_lfs_filter(repo) + + exit_code, output = self.run_policy(repo, 1024) + + self.assertEqual(exit_code, 0) + self.assertIn("LFS", output) + self.assertIn("Large-file policy passed", output) + + def test_small_plain_git_file_passes(self) -> None: + repo = self.make_repo() + (repo / "README.md").write_text("small\n") + self.add_all_without_lfs_filter(repo) + + exit_code, output = self.run_policy(repo, 1024) + + self.assertEqual(exit_code, 0) + self.assertIn("No tracked files over", output) + + +if __name__ == "__main__": + unittest.main()