From 17738d181eaedf0edd0622608f6f7318f471cd17 Mon Sep 17 00:00:00 2001 From: Viet Nguyen Date: Thu, 13 Aug 2026 09:15:53 +0000 Subject: [PATCH] =?UTF-8?q?README=20sinh=20t=E1=BB=AB=20registry,=20ki?= =?UTF-8?q?=E1=BB=83m=20tra=20nh=E1=BA=B9=20m=E1=BB=97i=20l=E1=BA=A7n=20pu?= =?UTF-8?q?sh,=20m=E1=BB=99t=20release=20cho=20m=E1=BB=97i=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ba câu hỏi, ba câu trả lời có lý do. **Không chia thư mục theo loại model.** `models//`, phẳng, và bảng chỉ mục trong README nhóm theo `task` lúc hiển thị. `models/asr/…` sẽ ghi vào tên thư mục một sự thật mà manifest đã nói — rồi hai bên lệch nhau, vì một model có thể phục vụ hai nhiệm vụ, tên nhiệm vụ có thể đổi, và đổi tên thư mục làm chết mọi đường dẫn trỏ vào nó. Nhóm lại là một *cách nhìn*; đường dẫn là một *địa chỉ*. **README có phần sinh tự động.** Một README model chứa hai loại câu: sự thật về artefact — giấy phép, dung lượng, có được phát tán lại không, đo được bao nhiêu — và *vì sao model này có mặt trong sản phẩm*. Loại đầu registry đã là nguồn duy nhất, nên nó nằm giữa hai dấu mốc và được `scripts/readme.py` viết lại; văn xuôi bên ngoài không ai đụng vào. Viết tay thì nó trôi: export lại đổi digest và dung lượng, manifest được cập nhật vì app đọc nó, còn tám README lặng lẽ giữ số của hôm qua — không ai phát hiện vì không có gì đọc README. `--check` trong CI biến "README cũ" thành build đỏ. **Một release cho một model, tag `-vN`, không phải nhánh riêng.** Nhánh là chỗ cho thay đổi đang làm dở; release là chỗ cho artefact. Nhánh riêng mỗi model là hai mươi nhánh không bao giờ merge và không bao giờ chết, mỗi cái trôi dần khỏi script export trên master. Version tăng bằng tay chứ không theo ngày: export lại là một quyết định, không phải một ngày trôi qua — và tag theo ngày sẽ đè lên artefact mà manifest đã công bố đang trỏ tới. Workflow từ chối chạy nếu tag đã có. Kèm theo, một thứ sẽ đắt dần: `verify.py` mặc định chỉ hỏi `HEAD` — file còn đó không, dung lượng có đúng như manifest nói không — thay vì tải cả file về băm. Băm cả danh mục là vài GB mỗi lần push (riêng hai bản MiLMMT đã 3.3 GB) cho một câu hỏi mà câu trả lời gần như luôn là "link còn sống". Digest kiểm mỗi tuần một lần bằng `--digest`. Và xoá `models/milmmt-46/` — thư mục rỗng còn sót, trong khi hai model thật là `milmmt-46-1b` và `milmmt-46-4b`. --- .github/workflows/check.yml | 46 +++++ .github/workflows/export.yml | 28 +++- README.en.md | 44 +++++ README.md | 44 +++++ models/campplus-sv/README.md | 17 ++ models/gipformer-65m/README.md | 23 +++ models/milmmt-46-1b/README.md | 19 +++ models/milmmt-46-4b/README.md | 19 +++ models/sense-voice-small/README.md | 21 +++ models/silero-vad-v5/README.md | 23 +++ models/small100/README.md | 19 +++ models/whisper-tiny/README.md | 23 +++ scripts/__pycache__/readme.cpython-312.pyc | Bin 0 -> 10700 bytes scripts/__pycache__/verify.cpython-312.pyc | Bin 3921 -> 6176 bytes scripts/readme.py | 185 +++++++++++++++++++++ scripts/verify.py | 44 ++++- 16 files changed, 551 insertions(+), 4 deletions(-) create mode 100644 scripts/__pycache__/readme.cpython-312.pyc create mode 100644 scripts/readme.py diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 31493ec..2e44b26 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,6 +3,12 @@ name: check on: push: pull_request: + # The digests are checked once a week rather than on every push. Hashing the catalogue means + # downloading several gigabytes — the two MiLMMT builds alone are 3.3 GB — and the answer changes + # only when somebody republishes a file. Every push still asks the cheap question: is it there, + # and is it the size the manifest claims. + schedule: + - cron: "17 4 * * 1" jobs: compile: @@ -19,6 +25,46 @@ jobs: # manually or on workflow_dispatch. - run: python -m compileall -q scripts models + readme: + name: the READMEs still match the registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: Techainer/summo-registry + path: registry + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + # The numbers in each model's README are generated from that model's manifest. This is what + # makes "the README says 611 MB and the manifest says 640 MB" a failed build rather than + # something a reader finds in six months. + - run: python scripts/readme.py --registry registry --check + + verify-digests: + name: full sha256 of every redistributable file (weekly) + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: Techainer/summo-registry + path: registry + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Download and hash + run: | + status=0 + for manifest in registry/models/*.json; do + if [ "$(python -c "import json,sys; print(json.load(open('$manifest')).get('redistributable', False))")" = "True" ]; then + python scripts/verify.py "$manifest" --digest || status=1 + fi + done + exit $status + verify-published: name: verify redistributable manifests in summo-registry runs-on: ubuntu-latest diff --git a/.github/workflows/export.yml b/.github/workflows/export.yml index faf0e0a..043f679 100644 --- a/.github/workflows/export.yml +++ b/.github/workflows/export.yml @@ -12,6 +12,10 @@ on: models/.json in summo-registry. required: true type: string + version: + description: "Release version for this export, e.g. v2. Never reuse one." + required: true + type: string keep_fp32: description: Also upload the unquantised intermediate (export.py --keep-fp32) required: false @@ -87,9 +91,29 @@ jobs: run: | python export.py --out build/ ${{ inputs.keep_fp32 && '--keep-fp32' || '' }} - - name: Compute the release tag + # `-vN`, one release per model, not one branch per model. + # + # A branch is for a change in flight; a release is for an artefact. Branch-per-model would + # mean twenty branches that never merge and never die, each drifting from the export scripts + # on `master`, and a bug fixed in one export never reaching the other nineteen. Tags cost + # nothing and are what a release is addressed by: `small100-v2` is a name a manifest URL can + # contain forever. + # + # The version is bumped by hand — `v1`, `v2` — because a re-export is a decision somebody + # made, not a date that passed. Two exports on the same day are two versions; a date-based tag + # would collide and quietly overwrite an artefact a published manifest already points at. + - name: Check the release tag is free id: tag - run: echo "tag=${{ inputs.model }}-$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + run: | + tag="${{ inputs.model }}-${{ inputs.version }}" + if gh release view "$tag" >/dev/null 2>&1; then + echo "::error::release $tag already exists — bump the version rather than replacing an" \ + "artefact that a published manifest may already point at" + exit 1 + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} - name: Upload artefacts to a release uses: softprops/action-gh-release@v2 diff --git a/README.en.md b/README.en.md index 4cfbf0f..b062272 100644 --- a/README.en.md +++ b/README.en.md @@ -83,3 +83,47 @@ The scripts in this repo (export, `scripts/manifest.py`, `scripts/verify.py`, th ours, published under [Apache-2.0](LICENSE). That licence does **not** cover the models themselves — each model keeps its own licence, listed in the table above and repeated in each model's `models//README.md`. + + +**Nhận dạng giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`gipformer-65m`](models/gipformer-65m/) | 73 MB | MIT | ✅ | +| [`sense-voice-small`](models/sense-voice-small/) | 1,177 MB | FunASR Model Open Source License Agreement v1.1 | ❌ | +| [`whisper-tiny`](models/whisper-tiny/) | 256 MB | MIT | ✅ | + +**Phát hiện giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`silero-vad-v5`](models/silero-vad-v5/) | 2 MB | MIT | ✅ | + +**Nhận diện người nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`campplus-sv`](models/campplus-sv/) | 28 MB | Apache-2.0 | ✅ | + +**Dịch** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`milmmt-46-1b`](models/milmmt-46-1b/) | 806 MB | Gemma Terms of Use | ❌ | +| [`milmmt-46-4b`](models/milmmt-46-4b/) | 2,490 MB | Gemma Terms of Use | ❌ | +| [`small100`](models/small100/) | 611 MB | MIT | ✅ | + + +## Release naming, and why not a branch per model + +One release per model, tagged `-vN`: `small100-v1`, `gipformer-65m-v2`. + +A branch is for a change in flight; a release is for an artefact. A branch per model would mean +twenty branches that never merge and never die, each drifting from the export scripts on `master` — +and a bug fixed in one export would never reach the other nineteen. Tags cost nothing and are what a +manifest URL can point at forever. + +The version is bumped by hand rather than derived from the date: a re-export is a decision somebody +made, not a day that passed. Two exports on the same day are two versions, and a date-based tag +would collide and quietly replace an artefact a published manifest already points at. The workflow +refuses to run when the tag exists. diff --git a/README.md b/README.md index 611e7f6..613a5f2 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,47 @@ Script trong repo này (export, `scripts/manifest.py`, `scripts/verify.py`, các chúng tôi, phát hành theo [Apache-2.0](LICENSE). Giấy phép đó **không** áp dụng cho bản thân các model — mỗi model giữ giấy phép riêng của nó, liệt kê trong bảng ở trên và nhắc lại trong `models//README.md` của từng model. + + +**Nhận dạng giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`gipformer-65m`](models/gipformer-65m/) | 73 MB | MIT | ✅ | +| [`sense-voice-small`](models/sense-voice-small/) | 1,177 MB | FunASR Model Open Source License Agreement v1.1 | ❌ | +| [`whisper-tiny`](models/whisper-tiny/) | 256 MB | MIT | ✅ | + +**Phát hiện giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`silero-vad-v5`](models/silero-vad-v5/) | 2 MB | MIT | ✅ | + +**Nhận diện người nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`campplus-sv`](models/campplus-sv/) | 28 MB | Apache-2.0 | ✅ | + +**Dịch** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`milmmt-46-1b`](models/milmmt-46-1b/) | 806 MB | Gemma Terms of Use | ❌ | +| [`milmmt-46-4b`](models/milmmt-46-4b/) | 2,490 MB | Gemma Terms of Use | ❌ | +| [`small100`](models/small100/) | 611 MB | MIT | ✅ | + + +## Đặt tên release, và vì sao không phải nhánh riêng cho mỗi model + +Một release cho một model, tag `-vN`: `small100-v1`, `gipformer-65m-v2`. + +Nhánh là chỗ cho **một thay đổi đang làm dở**; release là chỗ cho **một artefact**. Nhánh riêng mỗi +model nghĩa là hai mươi nhánh không bao giờ merge và không bao giờ chết, mỗi cái trôi dần khỏi script +export trên `master` — và một lỗi sửa ở export này sẽ không bao giờ tới mười chín cái kia. Tag thì +miễn phí và là thứ một URL trong manifest có thể trỏ vào mãi mãi. + +Số version tăng **bằng tay**, không theo ngày: export lại là một quyết định của ai đó, không phải +một ngày trôi qua. Hai lần export trong cùng một ngày là hai version; tag theo ngày sẽ đè lên nhau +và âm thầm thay mất artefact mà một manifest đã công bố đang trỏ tới. Workflow **từ chối chạy** nếu +tag đã tồn tại. diff --git a/models/campplus-sv/README.md b/models/campplus-sv/README.md index 6d4b4c8..021dda2 100644 --- a/models/campplus-sv/README.md +++ b/models/campplus-sv/README.md @@ -48,3 +48,20 @@ latency path. `task: speaker-embed`, `mode: batch`, via `sherpa-onnx/speaker-emb The manifest has no latency (`latency_ms`) or quality (`quality`) numbers for this model, so none are listed above. + + +| | | +|---|---| +| `id` | `campplus-sv` | +| Nhiệm vụ | `speaker-embed` | +| Runtime | `sherpa-onnx/speaker-embedding` | +| Ngôn ngữ | * | +| Giấy phép | Apache-2.0 | +| Phát tán lại | được | +| Dung lượng | 28 MB (1 file) | +| RAM (idle / đỉnh) | 60 MB / 180 MB | +| RAM tối thiểu | 512 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.006 | + +*Bảng này sinh tự động từ `summo-registry/models/campplus-sv.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/gipformer-65m/README.md b/models/gipformer-65m/README.md index 8f183f8..c3dda29 100644 --- a/models/gipformer-65m/README.md +++ b/models/gipformer-65m/README.md @@ -65,3 +65,26 @@ sha256 and size for each file are in `summo-registry/models/gipformer-65m.json`. | Latency — finalize (p95) | 700 ms | | Quality — WER (Fleurs VI) | 2.41% | | Quality — CER (Fleurs VI) | 1.67% | + + +| | | +|---|---| +| `id` | `gipformer-65m` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/transducer-offline` | +| Ngôn ngữ | vi | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 73 MB (4 file) | +| RAM (idle / đỉnh) | 150 MB / 800 MB | +| RAM tối thiểu | 1024 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.024 | +| RTF · `cpu_x86_avx2_4t` | 0.06 | +| Chất lượng · `wer_fleurs_vi` | 0.0241 | +| Chất lượng · `cer_fleurs_vi` | 0.0167 | +| Độ trễ · first_partial | 200 ms | +| Độ trễ · finalize_p50 | 300 ms | +| Độ trễ · finalize_p95 | 700 ms | + +*Bảng này sinh tự động từ `summo-registry/models/gipformer-65m.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/milmmt-46-1b/README.md b/models/milmmt-46-1b/README.md index af9a43b..67e000d 100644 --- a/models/milmmt-46-1b/README.md +++ b/models/milmmt-46-1b/README.md @@ -56,3 +56,22 @@ mradermacher. Original file (the only one, since this model is never mirrored): The manifest has no RTF or quality (`quality`) numbers for this model (both are empty `{}`), and being `mode: batch` there is no "first partial" to speak of — so those rows are omitted above. + + +| | | +|---|---| +| `id` | `milmmt-46-1b` | +| Nhiệm vụ | `translate` | +| Runtime | `llama.cpp/gguf` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nb, nl, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, yue, zh, zh-Hant | +| Giấy phép | Gemma Terms of Use | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 806 MB (1 file) | +| RAM (idle / đỉnh) | 900 MB / 1200 MB | +| RAM tối thiểu | 2048 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 700 ms | +| Độ trễ · finalize_p95 | 1600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/milmmt-46-1b.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/milmmt-46-4b/README.md b/models/milmmt-46-4b/README.md index e32b16f..291c16a 100644 --- a/models/milmmt-46-4b/README.md +++ b/models/milmmt-46-4b/README.md @@ -53,3 +53,22 @@ mradermacher. Original file: The manifest has no RTF or quality (`quality`) numbers for this model, and being `mode: batch` there is no "first partial" — those rows are omitted above. + + +| | | +|---|---| +| `id` | `milmmt-46-4b` | +| Nhiệm vụ | `translate` | +| Runtime | `llama.cpp/gguf` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nb, nl, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, yue, zh, zh-Hant | +| Giấy phép | Gemma Terms of Use | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 2,490 MB (1 file) | +| RAM (idle / đỉnh) | 2700 MB / 3200 MB | +| RAM tối thiểu | 6144 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 2100 ms | +| Độ trễ · finalize_p95 | 3600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/milmmt-46-4b.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/sense-voice-small/README.md b/models/sense-voice-small/README.md index d127768..dc97c74 100644 --- a/models/sense-voice-small/README.md +++ b/models/sense-voice-small/README.md @@ -74,3 +74,24 @@ sha256 and size for each file are in `summo-registry/models/sense-voice-small.js The manifest has no quality (`quality`) numbers — its description notes accuracy has not yet been measured by `summo-bench`; the publisher's own claim is that it beats Whisper-small while running roughly five times faster, and that claim is theirs, not ours. + + +| | | +|---|---| +| `id` | `sense-voice-small` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/sense-voice` | +| Ngôn ngữ | zh, yue, ja, ko, en | +| Giấy phép | FunASR Model Open Source License Agreement v1.1 | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 1,177 MB (3 file) | +| RAM (idle / đỉnh) | 300 MB / 800 MB | +| RAM tối thiểu | 1536 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.044 | +| RTF · `cpu_x86_avx512vnni_4t` | 0.062 | +| Độ trễ · first_partial | 250 ms | +| Độ trễ · finalize_p50 | 300 ms | +| Độ trễ · finalize_p95 | 800 ms | + +*Bảng này sinh tự động từ `summo-registry/models/sense-voice-small.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/silero-vad-v5/README.md b/models/silero-vad-v5/README.md index 4afa6ae..e59e9d5 100644 --- a/models/silero-vad-v5/README.md +++ b/models/silero-vad-v5/README.md @@ -50,3 +50,26 @@ Summo's default voice activity detector. `task: vad`, `mode: live`, via `onnx/si | Quality — F1 (ten-test set) | 0.940 | | Quality — Precision (ten-test set) | 0.925 | | Quality — Recall (ten-test set) | 0.956 | + + +| | | +|---|---| +| `id` | `silero-vad-v5` | +| Nhiệm vụ | `vad` | +| Runtime | `onnx/silero-vad` | +| Ngôn ngữ | * | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 2 MB (1 file) | +| RAM (idle / đỉnh) | 12 MB / 30 MB | +| RAM tối thiểu | 128 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.0063 | +| Chất lượng · `f1_ten_testset` | 0.94 | +| Chất lượng · `precision_ten_testset` | 0.925 | +| Chất lượng · `recall_ten_testset` | 0.956 | +| Độ trễ · first_partial | 17 ms | +| Độ trễ · finalize_p50 | 91 ms | +| Độ trễ · finalize_p95 | 982 ms | + +*Bảng này sinh tự động từ `summo-registry/models/silero-vad-v5.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/small100/README.md b/models/small100/README.md index 5d8dafb..041f9ef 100644 --- a/models/small100/README.md +++ b/models/small100/README.md @@ -92,3 +92,22 @@ The manifest has no RTF or quality (`quality`) numbers for this model, and being there is no "first partial" — those rows are omitted above. Finer-grained measurements behind two of the export's design decisions (split encoder/decoder, the KV cache) live in `export.py`'s docstring. + + +| | | +|---|---| +| `id` | `small100` | +| Nhiệm vụ | `translate` | +| Runtime | `onnx/m2m100` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nl, no, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, zh | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 611 MB (4 file) | +| RAM (idle / đỉnh) | 700 MB / 1100 MB | +| RAM tối thiểu | 1536 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 244 ms | +| Độ trễ · finalize_p95 | 600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/small100.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/whisper-tiny/README.md b/models/whisper-tiny/README.md index 4874285..6b86966 100644 --- a/models/whisper-tiny/README.md +++ b/models/whisper-tiny/README.md @@ -65,3 +65,26 @@ sha256 and size for each file are in `summo-registry/models/whisper-tiny.json`. | Quality — WER (Fleurs VI) | 65.5% | Being `mode: batch`, there is no "first partial" to speak of. + + +| | | +|---|---| +| `id` | `whisper-tiny` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/whisper` | +| Ngôn ngữ | * | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 256 MB (5 file) | +| RAM (idle / đỉnh) | 200 MB / 600 MB | +| RAM tối thiểu | 1024 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.107 | +| RTF · `cpu_x86_avx2_4t` | 0.3 | +| Chất lượng · `wer_whisper_testset_en` | 0.045 | +| Chất lượng · `wer_fleurs_vi` | 0.655 | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 700 ms | +| Độ trễ · finalize_p95 | 1500 ms | + +*Bảng này sinh tự động từ `summo-registry/models/whisper-tiny.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/scripts/__pycache__/readme.cpython-312.pyc b/scripts/__pycache__/readme.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b33cfd774ad44f6bcf6b91490563b3a579b6c8c9 GIT binary patch literal 10700 zcmb_iYj70TmF}MDd5<0=Y4kt>8YBTDB!PL@Ah3u)!m%A>8%S)+#!_p#XVlU>#O)r$ z45NyYbt+PCifStDvICvXx z=N-J0FME#}aPl>L`Psp(LV!Kq*VB8fkM)so zP-{d4zbMHh+RkzzUMmqpyl@t~Aq_b^&W1xmr`^s%8js2&;Sei%i8wAxJBYyXL7{Uz z%64?QnVr`Nhm6(yEq z+fb4$Hyn=0nou|*3s}^~{(9=KSP_IA5WU!^9e>y*F$TGb5kVVYl-VE`WrqcrJddIl zharjwrkPC@EL4wA>OaO%*wd2}IV#ptWS2#kv0QWpr32pz)N@i38DFSPW7 z*g2LL;SJ+MLk^~cV#EF=6ShajdC+Gb!Fq{uPHzR#KO*OKTVhW~ zL_rQj*)c&FXQMEJKzJ^?RboSt;IKfXPPRWh9Ok1iSs4~ivg^d*H_Y6hC(h*w!Zc5W za}(2cR6sl<06+4C022>#WDH2==0I?fQP)r;g)OE}=YU27J92q-z<$qwbUeFzur$n&fmAt8y~j55maW9Xx)|7K_*he3+CDNtMyF~GAWTfi=b zgP@Q&J6sEnhjDzG&IiLGc?47##7tnS`E8`F16mB28cYi$gFN`c0q7{PpEu zhe6`dG-rqdVrUH37?ih@eGY|!ivkM?hUSbYOtc+<+&CC7tt`a0aXe21Nown~{|>Fb zhgKb22o{8s#c)XKhu_T0&^%Uia8pM|&gyr;jA5sBbacn6N=iZ0T3N28X6V1vGFO^! z83A`l^^Yvw`XI#eOSi59qkeJe_MgK$^wCemNCQ;AJo3?1nH_=R?}k>^HkNeGRfqil zG{1EFvY6{_hWg&6+wXWsJdA3D6<3XCI6hV;jf1oV(jmZZ7}8Ao(zyN zQWX8D1~M0@IJn}NgQ9rDC5AU%Hk_b5Ouru+PrcpQ6Jy#tcly3Cvd0czLjju3aDj** zAwBjB*O&a6Y6uRKGN{}Qe^MG= zaa?HSdQ7V-ogy*t)ghX%fwpehqH`5D==H#Rt!yil(bIZ<%D`#x*5SueZ%{ zEmD)FxQXnKGozqes}tW4H?2u!U2#6ZOXFi&HLYm19y!xTr>V#Aw?S_~8Xm*b#<-E+ zkYDfhmkoaeO;)F=&v_J8e^t_TvX?rwo}%6a?VU7FS{}p0rnot$KXJN*Z&|q>vn)gLTlsB%1Mh*I&lGq5_i4Uuj^?+Q z?C18&#yNV@%6F`}8?8FuVD~y>+fM>Q7!HI1CxZji;UoGmu%r{u03S?fSXwAN!0y{WK$LTdS9&DO!IsTfyK-1K0Jwrte9~dX-Yetnp zY)8=I^A?pma;5C!}*2nAforP(kfeEI*-L!#LEdaW3mN8~X z8E`#<{o5F7F@ z-TraR(9Wu+0DxIZqxGD)bnE>nJ3jK!wQ<#=9SI~MQVE@ME>iyJYKRSh7ZuftlCuEL zFk*#PznDYk_ycbQy-r)(*a(c|XZ6yy+-XB5OSjJh%2~Sg6LiA<$kOe1!R;>Hp7Ta< z+xAAFIG_;<`C|rlTg(hNFCchSGm)gh;BW-}^6~y)58uJVCnWuXvCv3SOCqB z9fnesK_^VmimLXY7#bv85PQ4wy=53t1U|s8|5-#e64|HC;rPj;?9YG74yl&Y0I5Vd zsttDp*nM9vu$5~8$PN!$(ZQe;sQ{|KNG#pH$m0CIgAHOvHYmkRaTfl?Z1Ikc4()G+ zo+5zI^mGqo3VI3>*X$ELY2d-1!oH*926!4SoASC@1-fZO-SjXqL!1R%01pEj?wl=!y~7kwyvvVDL&mdeRx`)_9c!aO^J!* z@S=Uwg01UwNz>EXdRnC-2cS#a;iX$Y&=6KCiif-r7KHtwwoVtJ7=B!5hgP1kcjQrc zWIy!p9F@YMA$EA_)`!r1=u7BkNCU#S?AM`QTDtuK2T(TJ+4co(_gIJPm01QRkLNvKpH&0NxCsya%xJ9!rBpMgK=G8l&`>Y=md{MTl_maI!{&Ny4A`W`qdV0))q z7b|vVoI9ubvgNhs12ci-{?u@$+%xsuvYE11CZ0;XmgMK&QfgXKJ&J3KV%?gx)g_-w zzLxSWtnW~oI@6reuv4)3 zTWZ^n+OM{!CvF-tn-65_58kcsyJJ$UN0Dg#m8OeLDRJJNX?RAl?tAFmkX^U=O8doj zW$V7hb^B*tP!7qF1?<7F-2}Pr-kcn>8AN_DC-U?)@PU94ySpl zcX=Jw4Zkq3yGCPkdJD$69 z=9k`o^Z$!q`No^dTi#4RuN)H;VMOT=m738->sZ!O{@rumI+r-OXkiOfwxwU4e{-SZ zu+o0y&TggcIi=>Si`M6}bsH|6n>&}vRcdzMv+h|oLbHc?D*H2a9e3+?&sW{A?OLeq%G4f2Dn0ZH zGD_X{^pW)5d0KJrnvcx;m6}8MtY1MoJ+w~fV5V;8-MXje58tnScA@s!Ozj~g)V+dE z;+rWcB`EIpdH1|!UP=d*nuGU1sz9h4ByX$CTC0~$mNN4vpdbh4PcGWJmM!oT>m18A z%-Jd1Q9A3WPV~-nEB1BDx)adb_H}yM0ku2tH^hyaVKnMA0C=w5^2sfjC(_a6K{T-x?$1b0|3k@&Q?alt0mrw;AqXc+lt4HB6!kAnKVD9mtAiG2MGrpB%A(ro)>oFrj>{g$j)(XL3&1Hw)6X4Q00LE+cP$KyR zJzEWs(LIV;z~N4S$o&1m&rXwe}T5!|k~U4F->*h2SI~7NkxB zD3^*E+SvL8F`!%hr_K_$e&tB^s7RLF}^&B32^2y7l819#ojkp~SMt~6e3 zytMuiRc@-ADVt`d{n^Hrf3N6HaB2F-rfa6O?|RwIn#{Im|B_ZJx~CaUUcpGlrq`M zr+XgM)WHhOZ~AHLhpjg)cd9bGdKc^ZGBthEec7hhq-`WA1 zI9)shY5<(U;K^FY#kgTrOD&XeDF@c`J_#W^9-1#(PGGPJHF;IiZ5r58qXzR!dIH$D zVhAwy(s5xtzxVJ9>TUfDr7D;4KZn~!w4u`Ebc-{=d}}gUonkIFX59@J2IdA*rqt<-+XE1YndU)}F0{?H zB~PR_WvW_cEYtM#9#pQ{+({$lPFYeU-JEu(<+M+!*sECgX3MM2kIjs|6PzAQ_U0S3 zrt9Vn^SkF;=ZWIzQkbqBzyTf#3BkLj8be?mFHkbdLrbG*xK|~)pj0Iw2v2RR(v3p& zU=kO*5W6cj;|;QQNsA>V(4gQPdFF=cdVVwjF>BK>Lp;=a!*)HdOj^5wBQK7nD;7Yv zf};iJjAxuxiMM0Sc@*5$>IAX9ykzt@z&S%6&b0!-G012~wKR(UU{#76M$zl7N;+pc zO@Mh@0p=MYLTr>;W%LeO8g0^Z#W(N#Lvz3wl}?pL*a{I+fMRLj)Kntx0(6R-_%dWr zi@BHM6X2G4bYpQmJ*V?5xd9w)(o2hPElO8WI{NHEp!5f3KWUMh_5N{-+@j~@Erpy% z&&3O%Q`e43Yup;QjKKjDfbr?Id6Ksm`WN6`Q92;UcD=s%CiH9)_})=0(R1-abPj;t z-!Y~7Tdq?rk#VJ7<}<(wOo3!=aoZo1EInHTE&h0kA&k?KjGjA}Lx--VcwtYiDY56D zC^4JPzy8(^6>Cdu9Qb~c#J;UT5goE1ZRo7((Gt z2ZVUxQqu=9X8245-D&R57Dj~Yewcs=G7D#~gSi8Wbh`@*(ETmkS}Nio6I!A?giR>v&zApDunnz)py8m5Ls{@IeLM z14C?=9eUjNEJIxoIc3Q)pcyMK1kkrho&#*GEH|tCmn;w@h713&EPO$;p7pAgBmQxY zNP=44L;R5-AcmLmCs8efgM8RKIH)!r=O%iK>-kj>^HF_8RVM(EL7<`aBjoKyaVJ*#H>C;JP1GRR_S#{sh zwBTsUI5tf6erBcIjmh3C$1WaAe|4d0&;7=(g~qO%`!bEa0QCRMWOSJyIx9ajQnoS; ziSN14J=dM7ZoX%2SvErD)bq;}ygso~PWJ;idst_zO7)hM{G-XMlk;a6wjNTpd_{5f zELsmgtZSI+dswsnQ_5~?o3=i1xf3JrZb)rQ@44E!;MzHVV8PWrZO&G>&ri-w&VDoN zbZH^dqse`#14_e=MdwaD-c;4!uiCs&wHYAcv;~hWmLIxr7;n6Bvp&;tIO9436dbO^ zs~JaAp1^z6$!}b0Ty(Z(tLrXwz1x+oti4~kX`ymcYRkP!&$1hs`K+D-X5PqDZu;S# zba|%v8KnY3gI3elX=Apc>U?Y_HXGN*c3v(sEK5*j=@n?;9UeD}*DdT#1t$|Nm zs$X<&E(~0Kp?t19dFEc_mSq=EU#_97&hN&)9h(y#Sj&|1Hx{jLW@{QC3TWFn-TT1l zN*qqe314FSJ?93D*=_u7PQD4(tmC(gZT3W>;hnb>XH&BOCjSqkw??lW`lYC}y{5eO z`r^PT44Mw`%Ikt+_uXUspMUxy4a+1UQ2lvB+Y8$n)jBwc3OG3EVRE5tBPc#9Gk%go zBxnc~nu0UD=#|OaP@)&(9*Efkumj(~x>F^UcJh+mki1;Ty;8^>FXV~@4O)Md>4)#~ zJk>;iCaQgK&=&z#guy|A;FfstF@}#2)?08h2E<{_q>$5CPVj+83^IX_Blysc@ECp9 zwzp;}v@M6z$IB3)jv!n+6zEYGu;Y<{&`rJv6^M8xMEbA-H2n#)j%I$pkD{A@Lsk5W za{P+2{|8n7(7IvDlr^uLGCnMCnQ~;E?34{Qt8-h{u_f!?m31|IVl&wcQ>I6iG~M#Z ux|MEvbds*3n;*5A=>w0do~B3WWkOSob2@g{*zhr9{m#*4BgL%KxcVO$B!>C` literal 0 HcmV?d00001 diff --git a/scripts/__pycache__/verify.cpython-312.pyc b/scripts/__pycache__/verify.cpython-312.pyc index 5a120c80518d047b8252cf93e3d2246ae67318eb..ad4aed925982522ab4742ef9704a6f3e0538ee21 100644 GIT binary patch delta 3160 zcmai0U2GKB6~43cv$OwRV|#7>e2D>P!+Pt4;uexZY(r>VpelwcX&SA^J9oW)NKt?CQuSpCR7lQC&z;?k z38~bzeCFPB&pG$pd%kndUEVz3AN_MA5=8L(?VGbF`dZILJGq&2y(8ZasNzqAABEI_ zD#16b230?NBP6QIlaC`rWgjAuMWzTp4wd;bp9~fM&VO`q#8*95GO*YPc;1hVf48l33sM{&5;IZ!>7~DUGrwGM4O(*ytr+$Go z3u`vEY)#jh9ZEpZP#bJuU`w0vcEAP2&~jA*yJU4mD_CiKv}jn^G=dnDlZ2M>RME;a zb*n@um^E0xTO@0OQMPfxREeGj*T(w#RBZyxjbYU+8M>*c7A8tIU-ykUkyord zVg9A@vAkk4r;b`|D@Df>+JrJ*wqaSxQq!`S^Ej*6if&F6;SiPyZo$cD-ACp`Hc!+d z9GfcaprD`_;5~Nk_$u5VoR|3B!HRD`diiaHW(DVcepgCRh0EfiuO5h5(Qb?oIw90~ z4^ig_a(lHRR=CSjQ)3sJl`7H+4+0NK!tez5)j6A~{>27&*01vHuW5%vRsZWrJJQst zd?{H$s@w(a$!`H{IxAJ-U8Dy7mmu(iP2qRZ1u^U6(5Qzk0jZ%159{Fz{3z-{Pa=ye z!Rm{!%AqsgndV34bq!jR z^_W^_k!A_bnH1-V0;piX0g$c%?f)Tf4S4&45Lyh7VA!>Ai9y2)a}}bwhQu`9-@uh- zOf>i&>}}IS7DGpY09tX5*x5W2YJkf#zXmV{Td?n?3qzR=DR@{rbolTwJYLjvfGvQ| zAXuqVf_J8O;)6q;EIX`%Unzp;8h8eH^B}C~1xQtI4@EC2WeYzBr(lM4&6sSYqM~aj z$zy3e*i71HR@JQ-U^qugI1d>{iHZx5zM8eiOEV}8EO``fhY~g6Tw}wgVMA5uJ4B2L zJ3qz}1+oDguO}NN&I7KyTFPsXehp``W}yIHgM)Qfp|-GW#SWXD&2!)eS3D~rit;v*FQ>I}NSJJ%zfF={M0PRhjq!vc6bS!nGZgxC-`rxPW#5gA{{}yXsif#EYyRiL( z$=@8mGVsyMH?|M0tlu=(J~#g1y7?b|uwg#3)Hi(fz@N5XJA8HDQeyN*%jhTT615n3 zR%=7ip7}lVsRd;z)W0P3KZr4#u<`fd^TqujYU%nMae?lYuFY?ad{N_JxN7yo1za5T z4f}o9dNvL{EneH#592>REe;ET>&Oq|>;Ay-M)7)!Z#W@dPlz!7ct5|@*Ikc?AM}q< zR+1?zXf(!17>ILG+)<@1Y#L{T1&+1>aRrEjN)v3zL-+s?(3Su*ltsi1O^mP|U8$~TF>gEsZ1O1G%oan?=CoWn!zvbqR{`4n z@ticq`Hu3=s{i-SsN5C^z{??YIlRa_zm(5&5vL|6oRa($p>Svd z&59KPn%nEp+@uP30)xdzeze2xX|gIzW~=k#!1`9B-R}JwdxNtnC|8qB{@S-9sV(g6 z&`{zPKeQTt%?W!)Q%-I5TF$z!NlRw z3B3b~U|xA@*Ffr#_k>=?iZ#i)NhuE%40;FjnR-p7B%JiQl9FZhgQlIgLw%blYwI)O zbA!(x!mHdjW<+xmQ1CAnIXJ+bkUfxME(aJ0Gd7vFJ&Z!>vq{0_t+M3`WLmQ+i;;(u zI{vzy*TOv`o~rCUxiXqccQQ+IPSNxtRHQmu85}Jch=)BOb@&MdMKkCK@Q*Qpx14{6 zM|OrEgHJ%BZ9uK>9IDR?GAnpY`%?0F@)e!uKeTsf05lmb}2gy{F!I!oGB z`5Cxtcw+9Q$6?B1XdM6owTt8K3hOxGKTja8=btEc8%1xU&}|gFjoSZ;I`2vC+>`hE jJGqv713g?@?GT5A@adTweAg#JME>b>aPPu;Z=e4H^gDJ$ delta 1095 zcmZ9LO>Epm6vxN*c zJcf92lmkV|0U=c*p^7R}Iiad2)MKyQI02_lKms8lIDob)TykP&LrGx{&-32Ks=z4z(74;6wYm|&;bkLNEo4vT5DdiCJUe_S0M^( zbi)r;WnI@1CXEX~*MVn@hUqo-a2yGH}ct4I%PGsc~{$s7}Y+AVmCv$Us zjxW@kTcv5q+(3Rgw{D;vZo(8( zTSWY;`gE;!kj+qrx^@0XJ*&?RC>&Q}bo}`JD2zRy4<;X*I3lZr<WbI zsGsf(>npxzO3u=7(l{MJoN9Ql;DAml~|5gb)RF_Aa9v3`+c`Mw6%&fXzvBc6kig z{n)3W8~E^~;_aRngsD7MXezKRO``6>!&8JAMd7{|MDU!PKC6J&dD2`iUm6+6Z2gIJ zlZ*)ewYkwE=Hj;bef9C{=Wd?*V(Igx@2U&i=DlK-e>(LBDa-8rDj!YFZK+=g><24s zeCJ?Z`2;9SAsRXfgHGPL7)P|L!`U%9b}I0Uavay~rIMts3x*89)5=ULb5a?#LQ&a& z94G#H8&Fn3Yv3{U0=57vaIB2lldxSk^7^vjEd*y&N^N@2UXoI!Aw2^xOIHb<%P diff --git a/scripts/readme.py b/scripts/readme.py new file mode 100644 index 0000000..49a44ea --- /dev/null +++ b/scripts/readme.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Regenerate the facts in every model README from the registry, and the index in the top one. + + python scripts/readme.py --registry ../summo-registry # rewrite + python scripts/readme.py --registry ../summo-registry --check # fail if stale + +## Why generate rather than write + +A model README carries two kinds of sentence. One is a *fact about the artefact* — its licence, its +size, whether it may be redistributed, what it measured at — and the registry manifest is already +the single source of that. The other is *why this model is in the product at all*, which no machine +can write and no manifest records. + +Written by hand, the first kind drifts: a re-export changes a digest and a size, the manifest is +updated because the app reads it, and eight READMEs quietly keep yesterday's numbers. Nobody notices +because nothing reads a README. + +So the facts live between markers and are rewritten from the manifest; the prose outside the markers +is never touched. `--check` in CI turns "the README is stale" into a failed build rather than +something a reader finds months later. + +## Why the models are not in folders per task + +`models//`, flat, and the index below groups them by task at render time. A `models/asr/…` +layout would encode in a directory name a fact the manifest already states — and the two would +eventually disagree, because a model can serve two tasks, tasks get renamed, and a directory rename +breaks every link to it. Grouping is a *view*; a path is an *address*. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +START = "" +END = "" + +INDEX_START = "" +INDEX_END = "" + +TASKS = { + "asr": "Nhận dạng giọng nói", + "vad": "Phát hiện giọng nói", + "speaker-embed": "Nhận diện người nói", + "translate": "Dịch", +} + + +def mb(value: int) -> str: + return f"{value / 1e6:,.0f} MB" + + +def facts_table(manifest: dict) -> str: + """The block a manifest can answer on its own.""" + files = manifest.get("files", []) + total = sum(f.get("size", 0) for f in files) + profile = manifest.get("profile", {}) + + rows = [ + ("`id`", f"`{manifest['id']}`"), + ("Nhiệm vụ", f"`{manifest['task']}`"), + ("Runtime", f"`{manifest.get('runtime', '—')}`"), + ("Ngôn ngữ", ", ".join(manifest.get("langs", [])) or "—"), + ("Giấy phép", manifest.get("license", "—")), + ( + "Phát tán lại", + "được" if manifest.get("redistributable") else "**không được** — trỏ thẳng nguồn gốc", + ), + ("Dung lượng", f"{mb(total)} ({len(files)} file)"), + ] + + ram = profile.get("rss_mb") or {} + if ram: + rows.append(("RAM (idle / đỉnh)", f"{ram.get('idle', '—')} MB / {ram.get('peak', '—')} MB")) + if profile.get("min_ram_mb"): + rows.append(("RAM tối thiểu", f"{profile['min_ram_mb']} MB")) + for key, value in (profile.get("rtf") or {}).items(): + rows.append((f"RTF · `{key}`", f"{value}")) + for key, value in (profile.get("quality") or {}).items(): + rows.append((f"Chất lượng · `{key}`", f"{value}")) + for key, value in (profile.get("latency_ms") or {}).items(): + rows.append((f"Độ trễ · {key}", f"{value} ms")) + + lines = [ + "| | |", + "|---|---|", + *(f"| {name} | {value} |" for name, value in rows), + "", + "*Bảng này sinh tự động từ `summo-registry/models/" + f"{manifest['id']}.json` bằng `scripts/readme.py` — đừng sửa tay.*", + ] + return "\n".join(lines) + + +def index_table(manifests: list[dict]) -> str: + """Every model, grouped by what it does.""" + out: list[str] = [] + for task, heading in TASKS.items(): + rows = [m for m in manifests if m.get("task") == task] + if not rows: + continue + out.append(f"**{heading}**") + out.append("") + out.append("| Model | Dung lượng | Giấy phép | Phát tán lại |") + out.append("|---|---|---|---|") + for m in sorted(rows, key=lambda x: x["id"]): + total = sum(f.get("size", 0) for f in m.get("files", [])) + mark = "✅" if m.get("redistributable") else "❌" + out.append( + f"| [`{m['id']}`](models/{m['id']}/) | {mb(total)} | {m.get('license', '—')} | {mark} |" + ) + out.append("") + return "\n".join(out).rstrip() + + +def splice(text: str, start: str, end: str, block: str) -> str: + """Replace what is between the markers, adding them at the end if they are absent.""" + if start in text and end in text: + head, rest = text.split(start, 1) + _, tail = rest.split(end, 1) + return f"{head}{start}\n{block}\n{end}{tail}" + return f"{text.rstrip()}\n\n{start}\n{block}\n{end}\n" + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--registry", + type=Path, + default=Path("../summo-registry"), + help="checkout of Techainer/summo-registry", + ) + parser.add_argument( + "--check", + action="store_true", + help="do not write; exit non-zero if anything would change", + ) + args = parser.parse_args() + + manifests = [ + json.loads(path.read_text()) + for path in sorted((args.registry / "models").glob("*.json")) + ] + if not manifests: + sys.exit(f"no manifests under {args.registry / 'models'}") + + stale: list[str] = [] + + for manifest in manifests: + readme = Path("models") / manifest["id"] / "README.md" + if not readme.exists(): + stale.append(f"{readme} is missing") + continue + current = readme.read_text() + wanted = splice(current, START, END, facts_table(manifest)) + if wanted != current: + stale.append(str(readme)) + if not args.check: + readme.write_text(wanted) + + for top in (Path("README.md"), Path("README.en.md")): + if not top.exists(): + continue + current = top.read_text() + wanted = splice(current, INDEX_START, INDEX_END, index_table(manifests)) + if wanted != current: + stale.append(str(top)) + if not args.check: + top.write_text(wanted) + + if args.check and stale: + print("These are out of date with the registry — run `python scripts/readme.py`:") + for name in stale: + print(f" {name}") + sys.exit(1) + + print(f"{len(manifests)} manifest(s); {'would rewrite' if args.check else 'rewrote'} {len(stale)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify.py b/scripts/verify.py index 774e5c2..02f1c8d 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -10,6 +10,13 @@ download does. `check.yml` runs it against every redistributable manifest in summo-registry on every push. +Two depths, because they cost differently: + +* by default, a `HEAD` per file — is it still there, and is it the size the manifest claims. Runs on + every push, answers in under a second per model. +* `--digest` downloads each file and hashes it. That is several gigabytes across the catalogue, so + it runs on a schedule rather than on every push. + Exits 0 if every file in the manifest matches, 1 otherwise. """ @@ -23,6 +30,30 @@ from pathlib import Path +def check_reachable(entry: dict, timeout: float) -> str | None: + """Ask each source for headers only: is it there, and is it the size the manifest claims? + + This is what runs on every push. Downloading every file to hash it means fetching several + gigabytes per run — the two MiLMMT builds alone are 3.3 GB — for a question that is almost + always "is the link still alive". A `HEAD` answers that in a few hundred milliseconds, and a + wrong `Content-Length` catches a file republished under the same name, which is the common way + a digest goes stale. The digests themselves are checked on a schedule by `--digest`. + """ + urls = [entry["url"], *entry.get("mirror", [])] + last_error: Exception | None = None + for url in urls: + try: + request = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(request, timeout=timeout) as response: + length = response.headers.get("Content-Length") + if length is not None and int(length) != entry["size"]: + return f"{url}: {int(length)} bytes, manifest says {entry['size']}" + return None + except Exception as error: # noqa: BLE001 — try the next source + last_error = error + return f"no source reachable ({last_error})" + + def verify_file(entry: dict, timeout: float) -> str | None: """Return None if `entry` checks out, or a description of what did not.""" urls = [entry["url"], *entry.get("mirror", [])] @@ -56,7 +87,12 @@ def main() -> None: ) parser.add_argument("manifest", type=Path, help="path to a summo-registry models/.json") parser.add_argument( - "--timeout", type=float, default=300.0, help="per-file download timeout, in seconds" + "--timeout", type=float, default=300.0, help="per-file timeout, in seconds" + ) + parser.add_argument( + "--digest", + action="store_true", + help="download every file and check its sha256, rather than only asking whether it is there", ) args = parser.parse_args() @@ -68,7 +104,11 @@ def main() -> None: failures = [] for entry in files: - error = verify_file(entry, args.timeout) + error = ( + verify_file(entry, args.timeout) + if args.digest + else check_reachable(entry, args.timeout) + ) if error: failures.append(f"{entry['name']}: {error}") print(f"FAIL {model_id}/{entry['name']}: {error}")