diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 31493ec..2e44b26 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,6 +3,12 @@ name: check on: push: pull_request: + # The digests are checked once a week rather than on every push. Hashing the catalogue means + # downloading several gigabytes — the two MiLMMT builds alone are 3.3 GB — and the answer changes + # only when somebody republishes a file. Every push still asks the cheap question: is it there, + # and is it the size the manifest claims. + schedule: + - cron: "17 4 * * 1" jobs: compile: @@ -19,6 +25,46 @@ jobs: # manually or on workflow_dispatch. - run: python -m compileall -q scripts models + readme: + name: the READMEs still match the registry + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: Techainer/summo-registry + path: registry + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + # The numbers in each model's README are generated from that model's manifest. This is what + # makes "the README says 611 MB and the manifest says 640 MB" a failed build rather than + # something a reader finds in six months. + - run: python scripts/readme.py --registry registry --check + + verify-digests: + name: full sha256 of every redistributable file (weekly) + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: Techainer/summo-registry + path: registry + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Download and hash + run: | + status=0 + for manifest in registry/models/*.json; do + if [ "$(python -c "import json,sys; print(json.load(open('$manifest')).get('redistributable', False))")" = "True" ]; then + python scripts/verify.py "$manifest" --digest || status=1 + fi + done + exit $status + verify-published: name: verify redistributable manifests in summo-registry runs-on: ubuntu-latest diff --git a/.github/workflows/export.yml b/.github/workflows/export.yml index faf0e0a..043f679 100644 --- a/.github/workflows/export.yml +++ b/.github/workflows/export.yml @@ -12,6 +12,10 @@ on: models/.json in summo-registry. required: true type: string + version: + description: "Release version for this export, e.g. v2. Never reuse one." + required: true + type: string keep_fp32: description: Also upload the unquantised intermediate (export.py --keep-fp32) required: false @@ -87,9 +91,29 @@ jobs: run: | python export.py --out build/ ${{ inputs.keep_fp32 && '--keep-fp32' || '' }} - - name: Compute the release tag + # `-vN`, one release per model, not one branch per model. + # + # A branch is for a change in flight; a release is for an artefact. Branch-per-model would + # mean twenty branches that never merge and never die, each drifting from the export scripts + # on `master`, and a bug fixed in one export never reaching the other nineteen. Tags cost + # nothing and are what a release is addressed by: `small100-v2` is a name a manifest URL can + # contain forever. + # + # The version is bumped by hand — `v1`, `v2` — because a re-export is a decision somebody + # made, not a date that passed. Two exports on the same day are two versions; a date-based tag + # would collide and quietly overwrite an artefact a published manifest already points at. + - name: Check the release tag is free id: tag - run: echo "tag=${{ inputs.model }}-$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + run: | + tag="${{ inputs.model }}-${{ inputs.version }}" + if gh release view "$tag" >/dev/null 2>&1; then + echo "::error::release $tag already exists — bump the version rather than replacing an" \ + "artefact that a published manifest may already point at" + exit 1 + fi + echo "tag=$tag" >> "$GITHUB_OUTPUT" + env: + GH_TOKEN: ${{ github.token }} - name: Upload artefacts to a release uses: softprops/action-gh-release@v2 diff --git a/README.en.md b/README.en.md index 4cfbf0f..b062272 100644 --- a/README.en.md +++ b/README.en.md @@ -83,3 +83,47 @@ The scripts in this repo (export, `scripts/manifest.py`, `scripts/verify.py`, th ours, published under [Apache-2.0](LICENSE). That licence does **not** cover the models themselves — each model keeps its own licence, listed in the table above and repeated in each model's `models//README.md`. + + +**Nhận dạng giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`gipformer-65m`](models/gipformer-65m/) | 73 MB | MIT | ✅ | +| [`sense-voice-small`](models/sense-voice-small/) | 1,177 MB | FunASR Model Open Source License Agreement v1.1 | ❌ | +| [`whisper-tiny`](models/whisper-tiny/) | 256 MB | MIT | ✅ | + +**Phát hiện giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`silero-vad-v5`](models/silero-vad-v5/) | 2 MB | MIT | ✅ | + +**Nhận diện người nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`campplus-sv`](models/campplus-sv/) | 28 MB | Apache-2.0 | ✅ | + +**Dịch** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`milmmt-46-1b`](models/milmmt-46-1b/) | 806 MB | Gemma Terms of Use | ❌ | +| [`milmmt-46-4b`](models/milmmt-46-4b/) | 2,490 MB | Gemma Terms of Use | ❌ | +| [`small100`](models/small100/) | 611 MB | MIT | ✅ | + + +## Release naming, and why not a branch per model + +One release per model, tagged `-vN`: `small100-v1`, `gipformer-65m-v2`. + +A branch is for a change in flight; a release is for an artefact. A branch per model would mean +twenty branches that never merge and never die, each drifting from the export scripts on `master` — +and a bug fixed in one export would never reach the other nineteen. Tags cost nothing and are what a +manifest URL can point at forever. + +The version is bumped by hand rather than derived from the date: a re-export is a decision somebody +made, not a day that passed. Two exports on the same day are two versions, and a date-based tag +would collide and quietly replace an artefact a published manifest already points at. The workflow +refuses to run when the tag exists. diff --git a/README.md b/README.md index 611e7f6..613a5f2 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,47 @@ Script trong repo này (export, `scripts/manifest.py`, `scripts/verify.py`, các chúng tôi, phát hành theo [Apache-2.0](LICENSE). Giấy phép đó **không** áp dụng cho bản thân các model — mỗi model giữ giấy phép riêng của nó, liệt kê trong bảng ở trên và nhắc lại trong `models//README.md` của từng model. + + +**Nhận dạng giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`gipformer-65m`](models/gipformer-65m/) | 73 MB | MIT | ✅ | +| [`sense-voice-small`](models/sense-voice-small/) | 1,177 MB | FunASR Model Open Source License Agreement v1.1 | ❌ | +| [`whisper-tiny`](models/whisper-tiny/) | 256 MB | MIT | ✅ | + +**Phát hiện giọng nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`silero-vad-v5`](models/silero-vad-v5/) | 2 MB | MIT | ✅ | + +**Nhận diện người nói** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`campplus-sv`](models/campplus-sv/) | 28 MB | Apache-2.0 | ✅ | + +**Dịch** + +| Model | Dung lượng | Giấy phép | Phát tán lại | +|---|---|---|---| +| [`milmmt-46-1b`](models/milmmt-46-1b/) | 806 MB | Gemma Terms of Use | ❌ | +| [`milmmt-46-4b`](models/milmmt-46-4b/) | 2,490 MB | Gemma Terms of Use | ❌ | +| [`small100`](models/small100/) | 611 MB | MIT | ✅ | + + +## Đặt tên release, và vì sao không phải nhánh riêng cho mỗi model + +Một release cho một model, tag `-vN`: `small100-v1`, `gipformer-65m-v2`. + +Nhánh là chỗ cho **một thay đổi đang làm dở**; release là chỗ cho **một artefact**. Nhánh riêng mỗi +model nghĩa là hai mươi nhánh không bao giờ merge và không bao giờ chết, mỗi cái trôi dần khỏi script +export trên `master` — và một lỗi sửa ở export này sẽ không bao giờ tới mười chín cái kia. Tag thì +miễn phí và là thứ một URL trong manifest có thể trỏ vào mãi mãi. + +Số version tăng **bằng tay**, không theo ngày: export lại là một quyết định của ai đó, không phải +một ngày trôi qua. Hai lần export trong cùng một ngày là hai version; tag theo ngày sẽ đè lên nhau +và âm thầm thay mất artefact mà một manifest đã công bố đang trỏ tới. Workflow **từ chối chạy** nếu +tag đã tồn tại. diff --git a/models/campplus-sv/README.md b/models/campplus-sv/README.md index 6d4b4c8..021dda2 100644 --- a/models/campplus-sv/README.md +++ b/models/campplus-sv/README.md @@ -48,3 +48,20 @@ latency path. `task: speaker-embed`, `mode: batch`, via `sherpa-onnx/speaker-emb The manifest has no latency (`latency_ms`) or quality (`quality`) numbers for this model, so none are listed above. + + +| | | +|---|---| +| `id` | `campplus-sv` | +| Nhiệm vụ | `speaker-embed` | +| Runtime | `sherpa-onnx/speaker-embedding` | +| Ngôn ngữ | * | +| Giấy phép | Apache-2.0 | +| Phát tán lại | được | +| Dung lượng | 28 MB (1 file) | +| RAM (idle / đỉnh) | 60 MB / 180 MB | +| RAM tối thiểu | 512 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.006 | + +*Bảng này sinh tự động từ `summo-registry/models/campplus-sv.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/gipformer-65m/README.md b/models/gipformer-65m/README.md index 8f183f8..c3dda29 100644 --- a/models/gipformer-65m/README.md +++ b/models/gipformer-65m/README.md @@ -65,3 +65,26 @@ sha256 and size for each file are in `summo-registry/models/gipformer-65m.json`. | Latency — finalize (p95) | 700 ms | | Quality — WER (Fleurs VI) | 2.41% | | Quality — CER (Fleurs VI) | 1.67% | + + +| | | +|---|---| +| `id` | `gipformer-65m` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/transducer-offline` | +| Ngôn ngữ | vi | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 73 MB (4 file) | +| RAM (idle / đỉnh) | 150 MB / 800 MB | +| RAM tối thiểu | 1024 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.024 | +| RTF · `cpu_x86_avx2_4t` | 0.06 | +| Chất lượng · `wer_fleurs_vi` | 0.0241 | +| Chất lượng · `cer_fleurs_vi` | 0.0167 | +| Độ trễ · first_partial | 200 ms | +| Độ trễ · finalize_p50 | 300 ms | +| Độ trễ · finalize_p95 | 700 ms | + +*Bảng này sinh tự động từ `summo-registry/models/gipformer-65m.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/milmmt-46-1b/README.md b/models/milmmt-46-1b/README.md index af9a43b..67e000d 100644 --- a/models/milmmt-46-1b/README.md +++ b/models/milmmt-46-1b/README.md @@ -56,3 +56,22 @@ mradermacher. Original file (the only one, since this model is never mirrored): The manifest has no RTF or quality (`quality`) numbers for this model (both are empty `{}`), and being `mode: batch` there is no "first partial" to speak of — so those rows are omitted above. + + +| | | +|---|---| +| `id` | `milmmt-46-1b` | +| Nhiệm vụ | `translate` | +| Runtime | `llama.cpp/gguf` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nb, nl, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, yue, zh, zh-Hant | +| Giấy phép | Gemma Terms of Use | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 806 MB (1 file) | +| RAM (idle / đỉnh) | 900 MB / 1200 MB | +| RAM tối thiểu | 2048 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 700 ms | +| Độ trễ · finalize_p95 | 1600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/milmmt-46-1b.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/milmmt-46-4b/README.md b/models/milmmt-46-4b/README.md index e32b16f..291c16a 100644 --- a/models/milmmt-46-4b/README.md +++ b/models/milmmt-46-4b/README.md @@ -53,3 +53,22 @@ mradermacher. Original file: The manifest has no RTF or quality (`quality`) numbers for this model, and being `mode: batch` there is no "first partial" — those rows are omitted above. + + +| | | +|---|---| +| `id` | `milmmt-46-4b` | +| Nhiệm vụ | `translate` | +| Runtime | `llama.cpp/gguf` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nb, nl, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, yue, zh, zh-Hant | +| Giấy phép | Gemma Terms of Use | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 2,490 MB (1 file) | +| RAM (idle / đỉnh) | 2700 MB / 3200 MB | +| RAM tối thiểu | 6144 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 2100 ms | +| Độ trễ · finalize_p95 | 3600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/milmmt-46-4b.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/sense-voice-small/README.md b/models/sense-voice-small/README.md index d127768..dc97c74 100644 --- a/models/sense-voice-small/README.md +++ b/models/sense-voice-small/README.md @@ -74,3 +74,24 @@ sha256 and size for each file are in `summo-registry/models/sense-voice-small.js The manifest has no quality (`quality`) numbers — its description notes accuracy has not yet been measured by `summo-bench`; the publisher's own claim is that it beats Whisper-small while running roughly five times faster, and that claim is theirs, not ours. + + +| | | +|---|---| +| `id` | `sense-voice-small` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/sense-voice` | +| Ngôn ngữ | zh, yue, ja, ko, en | +| Giấy phép | FunASR Model Open Source License Agreement v1.1 | +| Phát tán lại | **không được** — trỏ thẳng nguồn gốc | +| Dung lượng | 1,177 MB (3 file) | +| RAM (idle / đỉnh) | 300 MB / 800 MB | +| RAM tối thiểu | 1536 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.044 | +| RTF · `cpu_x86_avx512vnni_4t` | 0.062 | +| Độ trễ · first_partial | 250 ms | +| Độ trễ · finalize_p50 | 300 ms | +| Độ trễ · finalize_p95 | 800 ms | + +*Bảng này sinh tự động từ `summo-registry/models/sense-voice-small.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/silero-vad-v5/README.md b/models/silero-vad-v5/README.md index 4afa6ae..e59e9d5 100644 --- a/models/silero-vad-v5/README.md +++ b/models/silero-vad-v5/README.md @@ -50,3 +50,26 @@ Summo's default voice activity detector. `task: vad`, `mode: live`, via `onnx/si | Quality — F1 (ten-test set) | 0.940 | | Quality — Precision (ten-test set) | 0.925 | | Quality — Recall (ten-test set) | 0.956 | + + +| | | +|---|---| +| `id` | `silero-vad-v5` | +| Nhiệm vụ | `vad` | +| Runtime | `onnx/silero-vad` | +| Ngôn ngữ | * | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 2 MB (1 file) | +| RAM (idle / đỉnh) | 12 MB / 30 MB | +| RAM tối thiểu | 128 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.0063 | +| Chất lượng · `f1_ten_testset` | 0.94 | +| Chất lượng · `precision_ten_testset` | 0.925 | +| Chất lượng · `recall_ten_testset` | 0.956 | +| Độ trễ · first_partial | 17 ms | +| Độ trễ · finalize_p50 | 91 ms | +| Độ trễ · finalize_p95 | 982 ms | + +*Bảng này sinh tự động từ `summo-registry/models/silero-vad-v5.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/small100/README.md b/models/small100/README.md index 5d8dafb..041f9ef 100644 --- a/models/small100/README.md +++ b/models/small100/README.md @@ -92,3 +92,22 @@ The manifest has no RTF or quality (`quality`) numbers for this model, and being there is no "first partial" — those rows are omitted above. Finer-grained measurements behind two of the export's design decisions (split encoder/decoder, the KV cache) live in `export.py`'s docstring. + + +| | | +|---|---| +| `id` | `small100` | +| Nhiệm vụ | `translate` | +| Runtime | `onnx/m2m100` | +| Ngôn ngữ | ar, az, bg, bn, ca, cs, da, de, el, en, es, fa, fi, fr, he, hi, hr, hu, id, it, ja, kk, km, ko, lo, ms, my, nl, no, pl, pt, ro, ru, sk, sl, sv, ta, th, tl, tr, ur, uz, vi, zh | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 611 MB (4 file) | +| RAM (idle / đỉnh) | 700 MB / 1100 MB | +| RAM tối thiểu | 1536 MB | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 244 ms | +| Độ trễ · finalize_p95 | 600 ms | + +*Bảng này sinh tự động từ `summo-registry/models/small100.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/models/whisper-tiny/README.md b/models/whisper-tiny/README.md index 4874285..6b86966 100644 --- a/models/whisper-tiny/README.md +++ b/models/whisper-tiny/README.md @@ -65,3 +65,26 @@ sha256 and size for each file are in `summo-registry/models/whisper-tiny.json`. | Quality — WER (Fleurs VI) | 65.5% | Being `mode: batch`, there is no "first partial" to speak of. + + +| | | +|---|---| +| `id` | `whisper-tiny` | +| Nhiệm vụ | `asr` | +| Runtime | `sherpa-onnx/whisper` | +| Ngôn ngữ | * | +| Giấy phép | MIT | +| Phát tán lại | được | +| Dung lượng | 256 MB (5 file) | +| RAM (idle / đỉnh) | 200 MB / 600 MB | +| RAM tối thiểu | 1024 MB | +| RTF · `cpu_x86_avx512vnni_8t` | 0.107 | +| RTF · `cpu_x86_avx2_4t` | 0.3 | +| Chất lượng · `wer_whisper_testset_en` | 0.045 | +| Chất lượng · `wer_fleurs_vi` | 0.655 | +| Độ trễ · first_partial | 0 ms | +| Độ trễ · finalize_p50 | 700 ms | +| Độ trễ · finalize_p95 | 1500 ms | + +*Bảng này sinh tự động từ `summo-registry/models/whisper-tiny.json` bằng `scripts/readme.py` — đừng sửa tay.* + diff --git a/scripts/__pycache__/readme.cpython-312.pyc b/scripts/__pycache__/readme.cpython-312.pyc new file mode 100644 index 0000000..b33cfd7 Binary files /dev/null and b/scripts/__pycache__/readme.cpython-312.pyc differ diff --git a/scripts/__pycache__/verify.cpython-312.pyc b/scripts/__pycache__/verify.cpython-312.pyc index 5a120c8..ad4aed9 100644 Binary files a/scripts/__pycache__/verify.cpython-312.pyc and b/scripts/__pycache__/verify.cpython-312.pyc differ diff --git a/scripts/readme.py b/scripts/readme.py new file mode 100644 index 0000000..49a44ea --- /dev/null +++ b/scripts/readme.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Regenerate the facts in every model README from the registry, and the index in the top one. + + python scripts/readme.py --registry ../summo-registry # rewrite + python scripts/readme.py --registry ../summo-registry --check # fail if stale + +## Why generate rather than write + +A model README carries two kinds of sentence. One is a *fact about the artefact* — its licence, its +size, whether it may be redistributed, what it measured at — and the registry manifest is already +the single source of that. The other is *why this model is in the product at all*, which no machine +can write and no manifest records. + +Written by hand, the first kind drifts: a re-export changes a digest and a size, the manifest is +updated because the app reads it, and eight READMEs quietly keep yesterday's numbers. Nobody notices +because nothing reads a README. + +So the facts live between markers and are rewritten from the manifest; the prose outside the markers +is never touched. `--check` in CI turns "the README is stale" into a failed build rather than +something a reader finds months later. + +## Why the models are not in folders per task + +`models//`, flat, and the index below groups them by task at render time. A `models/asr/…` +layout would encode in a directory name a fact the manifest already states — and the two would +eventually disagree, because a model can serve two tasks, tasks get renamed, and a directory rename +breaks every link to it. Grouping is a *view*; a path is an *address*. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +START = "" +END = "" + +INDEX_START = "" +INDEX_END = "" + +TASKS = { + "asr": "Nhận dạng giọng nói", + "vad": "Phát hiện giọng nói", + "speaker-embed": "Nhận diện người nói", + "translate": "Dịch", +} + + +def mb(value: int) -> str: + return f"{value / 1e6:,.0f} MB" + + +def facts_table(manifest: dict) -> str: + """The block a manifest can answer on its own.""" + files = manifest.get("files", []) + total = sum(f.get("size", 0) for f in files) + profile = manifest.get("profile", {}) + + rows = [ + ("`id`", f"`{manifest['id']}`"), + ("Nhiệm vụ", f"`{manifest['task']}`"), + ("Runtime", f"`{manifest.get('runtime', '—')}`"), + ("Ngôn ngữ", ", ".join(manifest.get("langs", [])) or "—"), + ("Giấy phép", manifest.get("license", "—")), + ( + "Phát tán lại", + "được" if manifest.get("redistributable") else "**không được** — trỏ thẳng nguồn gốc", + ), + ("Dung lượng", f"{mb(total)} ({len(files)} file)"), + ] + + ram = profile.get("rss_mb") or {} + if ram: + rows.append(("RAM (idle / đỉnh)", f"{ram.get('idle', '—')} MB / {ram.get('peak', '—')} MB")) + if profile.get("min_ram_mb"): + rows.append(("RAM tối thiểu", f"{profile['min_ram_mb']} MB")) + for key, value in (profile.get("rtf") or {}).items(): + rows.append((f"RTF · `{key}`", f"{value}")) + for key, value in (profile.get("quality") or {}).items(): + rows.append((f"Chất lượng · `{key}`", f"{value}")) + for key, value in (profile.get("latency_ms") or {}).items(): + rows.append((f"Độ trễ · {key}", f"{value} ms")) + + lines = [ + "| | |", + "|---|---|", + *(f"| {name} | {value} |" for name, value in rows), + "", + "*Bảng này sinh tự động từ `summo-registry/models/" + f"{manifest['id']}.json` bằng `scripts/readme.py` — đừng sửa tay.*", + ] + return "\n".join(lines) + + +def index_table(manifests: list[dict]) -> str: + """Every model, grouped by what it does.""" + out: list[str] = [] + for task, heading in TASKS.items(): + rows = [m for m in manifests if m.get("task") == task] + if not rows: + continue + out.append(f"**{heading}**") + out.append("") + out.append("| Model | Dung lượng | Giấy phép | Phát tán lại |") + out.append("|---|---|---|---|") + for m in sorted(rows, key=lambda x: x["id"]): + total = sum(f.get("size", 0) for f in m.get("files", [])) + mark = "✅" if m.get("redistributable") else "❌" + out.append( + f"| [`{m['id']}`](models/{m['id']}/) | {mb(total)} | {m.get('license', '—')} | {mark} |" + ) + out.append("") + return "\n".join(out).rstrip() + + +def splice(text: str, start: str, end: str, block: str) -> str: + """Replace what is between the markers, adding them at the end if they are absent.""" + if start in text and end in text: + head, rest = text.split(start, 1) + _, tail = rest.split(end, 1) + return f"{head}{start}\n{block}\n{end}{tail}" + return f"{text.rstrip()}\n\n{start}\n{block}\n{end}\n" + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--registry", + type=Path, + default=Path("../summo-registry"), + help="checkout of Techainer/summo-registry", + ) + parser.add_argument( + "--check", + action="store_true", + help="do not write; exit non-zero if anything would change", + ) + args = parser.parse_args() + + manifests = [ + json.loads(path.read_text()) + for path in sorted((args.registry / "models").glob("*.json")) + ] + if not manifests: + sys.exit(f"no manifests under {args.registry / 'models'}") + + stale: list[str] = [] + + for manifest in manifests: + readme = Path("models") / manifest["id"] / "README.md" + if not readme.exists(): + stale.append(f"{readme} is missing") + continue + current = readme.read_text() + wanted = splice(current, START, END, facts_table(manifest)) + if wanted != current: + stale.append(str(readme)) + if not args.check: + readme.write_text(wanted) + + for top in (Path("README.md"), Path("README.en.md")): + if not top.exists(): + continue + current = top.read_text() + wanted = splice(current, INDEX_START, INDEX_END, index_table(manifests)) + if wanted != current: + stale.append(str(top)) + if not args.check: + top.write_text(wanted) + + if args.check and stale: + print("These are out of date with the registry — run `python scripts/readme.py`:") + for name in stale: + print(f" {name}") + sys.exit(1) + + print(f"{len(manifests)} manifest(s); {'would rewrite' if args.check else 'rewrote'} {len(stale)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify.py b/scripts/verify.py index 774e5c2..02f1c8d 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -10,6 +10,13 @@ download does. `check.yml` runs it against every redistributable manifest in summo-registry on every push. +Two depths, because they cost differently: + +* by default, a `HEAD` per file — is it still there, and is it the size the manifest claims. Runs on + every push, answers in under a second per model. +* `--digest` downloads each file and hashes it. That is several gigabytes across the catalogue, so + it runs on a schedule rather than on every push. + Exits 0 if every file in the manifest matches, 1 otherwise. """ @@ -23,6 +30,30 @@ from pathlib import Path +def check_reachable(entry: dict, timeout: float) -> str | None: + """Ask each source for headers only: is it there, and is it the size the manifest claims? + + This is what runs on every push. Downloading every file to hash it means fetching several + gigabytes per run — the two MiLMMT builds alone are 3.3 GB — for a question that is almost + always "is the link still alive". A `HEAD` answers that in a few hundred milliseconds, and a + wrong `Content-Length` catches a file republished under the same name, which is the common way + a digest goes stale. The digests themselves are checked on a schedule by `--digest`. + """ + urls = [entry["url"], *entry.get("mirror", [])] + last_error: Exception | None = None + for url in urls: + try: + request = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(request, timeout=timeout) as response: + length = response.headers.get("Content-Length") + if length is not None and int(length) != entry["size"]: + return f"{url}: {int(length)} bytes, manifest says {entry['size']}" + return None + except Exception as error: # noqa: BLE001 — try the next source + last_error = error + return f"no source reachable ({last_error})" + + def verify_file(entry: dict, timeout: float) -> str | None: """Return None if `entry` checks out, or a description of what did not.""" urls = [entry["url"], *entry.get("mirror", [])] @@ -56,7 +87,12 @@ def main() -> None: ) parser.add_argument("manifest", type=Path, help="path to a summo-registry models/.json") parser.add_argument( - "--timeout", type=float, default=300.0, help="per-file download timeout, in seconds" + "--timeout", type=float, default=300.0, help="per-file timeout, in seconds" + ) + parser.add_argument( + "--digest", + action="store_true", + help="download every file and check its sha256, rather than only asking whether it is there", ) args = parser.parse_args() @@ -68,7 +104,11 @@ def main() -> None: failures = [] for entry in files: - error = verify_file(entry, args.timeout) + error = ( + verify_file(entry, args.timeout) + if args.digest + else check_reachable(entry, args.timeout) + ) if error: failures.append(f"{entry['name']}: {error}") print(f"FAIL {model_id}/{entry['name']}: {error}")