From 31a56b2335ff1adf82417b3339176cdb75e18d54 Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Thu, 6 Aug 2026 00:45:05 +0800 Subject: [PATCH 1/9] Build platform v2 ingestion and catalog foundation --- .github/workflows/ci.yml | 14 +- .github/workflows/cloudflare-backup.yml | 7 +- .github/workflows/program-fact-refresh.yml | 49 +- .github/workflows/vercel-production-alias.yml | 45 +- docs/entity-candidate-materializer.md | 111 ++ .../migrations/0010_release_retention.sql | 102 ++ .../0014_entity_candidate_materialization.sql | 292 ++++ package-lock.json | 681 ++++++++- package.json | 13 +- quality/regression/registry.json | 63 + scripts/cloudflare/backup-preflight.ts | 124 ++ scripts/ingestion/build-pipeline-bootstrap.ts | 16 +- scripts/ingestion/build-source-import.ts | 12 +- .../ingestion/build-source-manifest-cohort.ts | 620 ++++++++ scripts/quality/inventory-untracked-assets.ts | 424 ++++++ scripts/quality/platform-data-quality.ts | 366 +++++ scripts/source-manifest-registry.ts | 447 ++++++ src/app/[locale]/programs/page.tsx | 42 +- src/app/[locale]/scholarships/page.tsx | 40 +- .../features/ProgramExplorerV2.module.css | 37 + src/components/features/ProgramExplorerV2.tsx | 158 ++ .../features/ScholarshipExplorerV2.module.css | 37 + .../features/ScholarshipExplorerV2.tsx | 148 ++ src/lib/catalog/d1-list.ts | 417 ++++++ src/lib/catalog/d1.ts | 142 ++ src/lib/catalog/index.ts | 18 + src/lib/catalog/json.ts | 152 +- src/lib/catalog/list-cursor.ts | 90 ++ src/lib/catalog/release.ts | 18 +- src/lib/catalog/runtime.ts | 15 + src/lib/catalog/shadow.ts | 144 +- src/lib/catalog/types.ts | 107 +- src/lib/program-catalog.ts | 418 ++++++ src/lib/scholarship-catalog.ts | 410 +++++ tests/unit/catalog-repository.test.ts | 2 + .../unit/cloudflare-backup-preflight.test.ts | 78 + .../unit/minimax-recapture-validator.test.ts | 137 ++ tests/unit/platform-data-quality.test.ts | 91 ++ tests/unit/program-catalog.test.ts | 58 + tests/unit/release-workflow-safety.test.ts | 37 + tests/unit/scholarship-catalog.test.ts | 111 ++ .../source-manifest-cohort-builder.test.ts | 216 +++ tests/unit/source-manifest-registry.test.ts | 137 ++ ...sparse-school-expansion-2026-08-04.test.ts | 6 +- tests/unit/untracked-asset-inventory.test.ts | 85 ++ workers/entity-materializer/src/index.ts | 86 ++ .../entity-materializer/tests/index.test.ts | 30 + workers/entity-materializer/tsconfig.json | 17 + workers/entity-materializer/wrangler.jsonc | 24 + .../src/entity-materializer-scheduler.ts | 143 ++ workers/ingestion/src/entity-materializer.ts | 1329 +++++++++++++++++ workers/ingestion/src/repository.ts | 4 +- .../tests/entity-materializer.test.ts | 423 ++++++ workers/release-builder/src/index.ts | 58 +- workers/release-builder/src/retention.ts | 183 +++ .../release-builder/tests/retention.test.ts | 199 +++ 56 files changed, 9111 insertions(+), 122 deletions(-) create mode 100644 docs/entity-candidate-materializer.md create mode 100644 infra/d1/catalog/migrations/0010_release_retention.sql create mode 100644 infra/d1/pipeline/migrations/0014_entity_candidate_materialization.sql create mode 100644 quality/regression/registry.json create mode 100644 scripts/cloudflare/backup-preflight.ts create mode 100644 scripts/ingestion/build-source-manifest-cohort.ts create mode 100644 scripts/quality/inventory-untracked-assets.ts create mode 100644 scripts/quality/platform-data-quality.ts create mode 100644 scripts/source-manifest-registry.ts create mode 100644 src/components/features/ProgramExplorerV2.module.css create mode 100644 src/components/features/ProgramExplorerV2.tsx create mode 100644 src/components/features/ScholarshipExplorerV2.module.css create mode 100644 src/components/features/ScholarshipExplorerV2.tsx create mode 100644 src/lib/catalog/d1-list.ts create mode 100644 src/lib/catalog/list-cursor.ts create mode 100644 src/lib/catalog/runtime.ts create mode 100644 src/lib/program-catalog.ts create mode 100644 src/lib/scholarship-catalog.ts create mode 100644 tests/unit/cloudflare-backup-preflight.test.ts create mode 100644 tests/unit/minimax-recapture-validator.test.ts create mode 100644 tests/unit/platform-data-quality.test.ts create mode 100644 tests/unit/program-catalog.test.ts create mode 100644 tests/unit/release-workflow-safety.test.ts create mode 100644 tests/unit/scholarship-catalog.test.ts create mode 100644 tests/unit/source-manifest-cohort-builder.test.ts create mode 100644 tests/unit/source-manifest-registry.test.ts create mode 100644 tests/unit/untracked-asset-inventory.test.ts create mode 100644 workers/entity-materializer/src/index.ts create mode 100644 workers/entity-materializer/tests/index.test.ts create mode 100644 workers/entity-materializer/tsconfig.json create mode 100644 workers/entity-materializer/wrangler.jsonc create mode 100644 workers/ingestion/src/entity-materializer-scheduler.ts create mode 100644 workers/ingestion/src/entity-materializer.ts create mode 100644 workers/ingestion/tests/entity-materializer.test.ts create mode 100644 workers/release-builder/src/retention.ts create mode 100644 workers/release-builder/tests/retention.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 36fa0e3..7777ff5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,9 @@ jobs: - name: Install dependencies run: npm ci + - name: Audit production dependencies + run: npm audit --omit=dev --audit-level=high + - name: Lint run: npm run lint @@ -54,15 +57,21 @@ jobs: - name: Validate D1 migrations and database integrity run: npm run validate:d1 - - name: Validate pilot source manifests + - name: Validate recursive source manifests run: npm run validate:manifests + - name: Run isolated extraction and prompt-injection regressions + run: npm run quality:synthetic-regression + - name: Validate scheduled maintenance capacity run: npm run validate:maintenance - name: Run ingestion Worker tests run: npm run test:ingestion + - name: Run Entity Materializer tests + run: npm run test:entity-materializer + - name: Run publisher Worker tests run: npm run test:publisher @@ -72,6 +81,9 @@ jobs: - name: Validate ingestion Worker bundle and bindings run: npm run check:worker:ingestion + - name: Validate Entity Materializer Worker bundle and bindings + run: npm run check:worker:entity-materializer + - name: Validate publisher Worker bundle and bindings run: npm run check:worker:publisher diff --git a/.github/workflows/cloudflare-backup.yml b/.github/workflows/cloudflare-backup.yml index a0a61af..fb86201 100644 --- a/.github/workflows/cloudflare-backup.yml +++ b/.github/workflows/cloudflare-backup.yml @@ -37,8 +37,7 @@ jobs: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} run: | - test -n "$CLOUDFLARE_API_TOKEN" - test -n "$CLOUDFLARE_ACCOUNT_ID" + npx tsx scripts/cloudflare/backup-preflight.ts --phase credentials - name: Export catalog and pipeline databases shell: bash @@ -76,6 +75,10 @@ jobs: cd "$RUNNER_TEMP" sha256sum catalog.sql.gz pipeline.sql.gz > backup-sha256.txt + - name: Verify backup artifacts + shell: bash + run: npx tsx scripts/cloudflare/backup-preflight.ts --phase artifacts --directory "$RUNNER_TEMP" + - name: Upload daily and monthly copies shell: bash env: diff --git a/.github/workflows/program-fact-refresh.yml b/.github/workflows/program-fact-refresh.yml index 0439050..e60e5e2 100644 --- a/.github/workflows/program-fact-refresh.yml +++ b/.github/workflows/program-fact-refresh.yml @@ -11,7 +11,7 @@ on: default: '1000' permissions: - contents: write + contents: read concurrency: group: weekly-program-fact-refresh @@ -30,6 +30,7 @@ jobs: uses: actions/checkout@v6 with: ref: main + persist-credentials: false - name: Use Node.js 24 uses: actions/setup-node@v6 @@ -67,7 +68,7 @@ jobs: --review "$RUNNER_TEMP/current-program-review.json" \ --data-dir content/data \ --output-dir content/data \ - --audit "quality/international-program-review/fact-enrichment-${{ steps.date.outputs.value }}.json" \ + --audit "$RUNNER_TEMP/fact-enrichment-${{ steps.date.outputs.value }}.json" \ --checked-at "${{ steps.date.outputs.value }}" \ --max-urls "$MAXIMUM_URLS" \ --minimum-domain-interval-ms 5000 @@ -80,16 +81,40 @@ jobs: tests/unit/content-data.test.ts \ tests/unit/publication.test.ts - - name: Commit verified changes + - name: Prepare validated refresh candidate shell: bash run: | - if git diff --quiet -- content/data quality/international-program-review; then - echo "No verified fact changes." - exit 0 + set -euo pipefail + artifact_dir="$RUNNER_TEMP/program-fact-refresh-candidate" + mkdir -p "$artifact_dir/content/data" \ + "$artifact_dir/quality/international-program-review" + + cp "$RUNNER_TEMP/current-program-review.json" \ + "$artifact_dir/current-program-review.json" + cp "$RUNNER_TEMP/fact-enrichment-${{ steps.date.outputs.value }}.json" \ + "$artifact_dir/quality/international-program-review/" + cp content/data/programs.json content/data/admission-cycles.json \ + "$artifact_dir/content/data/" + + git diff --binary -- \ + content/data/programs.json \ + content/data/admission-cycles.json \ + > "$artifact_dir/catalog-changes.patch" + git diff --stat -- \ + content/data/programs.json \ + content/data/admission-cycles.json \ + > "$artifact_dir/change-summary.txt" + + if git diff --quiet -- \ + content/data/programs.json \ + content/data/admission-cycles.json; then + echo 'No verified fact changes.' >> "$artifact_dir/change-summary.txt" fi - git config user.name "studyinchina-data-bot" - git config user.email "studyinchina-data-bot@users.noreply.github.com" - git add content/data/programs.json content/data/admission-cycles.json \ - "quality/international-program-review/fact-enrichment-${{ steps.date.outputs.value }}.json" - git commit -m "data: refresh official program facts ${{ steps.date.outputs.value }}" - git push origin HEAD:main + + - name: Upload validated refresh candidate + uses: actions/upload-artifact@v6 + with: + name: program-fact-refresh-${{ steps.date.outputs.value }} + path: ${{ runner.temp }}/program-fact-refresh-candidate + if-no-files-found: error + retention-days: 35 diff --git a/.github/workflows/vercel-production-alias.yml b/.github/workflows/vercel-production-alias.yml index 0f76260..27995aa 100644 --- a/.github/workflows/vercel-production-alias.yml +++ b/.github/workflows/vercel-production-alias.yml @@ -15,16 +15,41 @@ jobs: name: Point studyinchina.vercel.app to the successful main deployment if: >- github.event.deployment_status.state == 'success' && - github.event.deployment.environment == 'Production' && - github.event.deployment.ref == 'main' + github.event.deployment.environment == 'Production' runs-on: ubuntu-latest timeout-minutes: 10 env: + DEPLOYMENT_SHA: ${{ github.event.deployment.sha }} DEPLOYMENT_URL: ${{ github.event.deployment_status.environment_url }} VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} steps: + - name: Check out current main + uses: actions/checkout@v6 + with: + ref: main + fetch-depth: 1 + persist-credentials: false + + - name: Verify deployment commit is current main + id: main + shell: bash + run: | + set -euo pipefail + main_sha="$(git rev-parse HEAD)" + if ! [[ "${DEPLOYMENT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Unexpected deployment SHA: ${DEPLOYMENT_SHA}" >&2 + exit 1 + fi + if [[ "${DEPLOYMENT_SHA}" == "${main_sha}" ]]; then + echo 'matches=true' >> "$GITHUB_OUTPUT" + exit 0 + fi + echo 'matches=false' >> "$GITHUB_OUTPUT" + echo "::notice::Deployment ${DEPLOYMENT_SHA} is not current main ${main_sha}; the stable alias will not be changed." + - name: Detect alias credential + if: steps.main.outputs.matches == 'true' id: credential shell: bash run: | @@ -37,7 +62,9 @@ jobs: fi - name: Validate deployment URL - if: steps.credential.outputs.configured == 'true' + if: >- + steps.main.outputs.matches == 'true' && + steps.credential.outputs.configured == 'true' shell: bash run: | set -euo pipefail @@ -47,13 +74,17 @@ jobs: fi - name: Use Node.js 24 - if: steps.credential.outputs.configured == 'true' + if: >- + steps.main.outputs.matches == 'true' && + steps.credential.outputs.configured == 'true' uses: actions/setup-node@v6 with: node-version: 24 - name: Promote stable production alias - if: steps.credential.outputs.configured == 'true' + if: >- + steps.main.outputs.matches == 'true' && + steps.credential.outputs.configured == 'true' shell: bash run: | set -euo pipefail @@ -64,7 +95,9 @@ jobs: --token "${VERCEL_TOKEN}" - name: Verify public release API - if: steps.credential.outputs.configured == 'true' + if: >- + steps.main.outputs.matches == 'true' && + steps.credential.outputs.configured == 'true' shell: bash run: | set -euo pipefail diff --git a/docs/entity-candidate-materializer.md b/docs/entity-candidate-materializer.md new file mode 100644 index 0000000..de58639 --- /dev/null +++ b/docs/entity-candidate-materializer.md @@ -0,0 +1,111 @@ +# Entity candidate materializer + +The Entity Candidate Materializer closes the gap between official catalogue +discovery and the canonical Pipeline D1 model. Discovery is intentionally cheap: +it records one immutable `extracted_entity_candidates` row for every programme or +scholarship link before MiniMax enrichment. Materialization is the trust boundary +that decides whether that identity can become an applied canonical record. + +## Why this has its own mapping contract + +`promotion_field_mappings` maps an aggregate ingestion source field to one known +record. It is not suitable for directory discovery because one source can contain +hundreds of entities whose candidate fields are all named `name`, `officialUrl`, +and `degreeLevel`. + +Migration `0014_entity_candidate_materialization.sql` therefore adds: + +- `entity_materialization_decisions`, an immutable final decision per candidate; +- `entity_candidate_field_mappings`, candidate-scoped mappings that preserve the + one-source-to-many-record shape; and +- `entity_materialization_release_requests`, an immutable daily release window + that coalesces many materialized identities into one existing Release Builder + outbox event. + +The strict `materialization_release_requests` contract from migration `0011` +remains unchanged. It is for complete offline packages containing at least 1,000 +programmes and 50 scholarships plus an applied dependency batch. The incremental +entity request is deliberately separate and does not weaken that bulk-import gate. + +## Deterministic gates + +`materializeExtractedEntityCandidate` only materializes a candidate when all of +the following are true: + +1. The candidate is validated/registered, has no issues, and meets the configured + confidence threshold (980,000 ppm by default). +2. Its source manifest is valid and matches the candidate institution and source. +3. The source has an enabled binding to an active primary/secondary official + `source_documents` row. +4. The source document URL exactly matches the normalized manifest URL, the + immutable snapshot URL stays on an allowlisted HTTPS host, and the R2 key + contains the snapshot's full SHA-256 digest. +5. Snapshot time, publisher ownership, evidence URLs, stable identity, entity + key, and immutable entity digest all recompute exactly. +6. Scholarship provider ownership is explicit and registered; no other registry + owns the same provider-scoped identity. + +Low-confidence and invalid candidates are quarantined. Missing or inconsistent +source configuration is retryable instead of being misclassified as bad data. +Identity or digest +conflicts use the stronger `conflict` decision. Both paths reconcile the catalogue +item as `unparseable` with an explicit reason and create no record, claim, field, +or release request. + +## Canonical write path + +The stable record ID is the same deterministic identity used by the existing +official entity materializer: + +```text +{entityType}-{sha256(entityType + NUL + ownerId + NUL + entityKey)} +``` + +For programmes, `ownerId` is the institution. For scholarships it is the +registered provider organization when supplied, falling back to the institution +only for school/faculty awards. Government awards require an explicit provider. + +One D1 batch writes the record, domain row, localized name, immutable source fetch +projection, evidence fragments, claims, canonical fields, record version, active +registry binding, published reconciliation, decision, field mappings, and audit +row. D1 batches are the transaction boundary; a constraint or version race rolls +the entire candidate back. Reprocessing a decided candidate returns the persisted +decision without writing again. + +The release request trigger revalidates every candidate, active registry, applied +record, and decision inside the insert statement. It creates one queued +`publication_jobs` row and one `catalog.release.requested` outbox event. A unique +UTC `release_window` limits this path to one regular release per day. Candidates +that miss the day's release remain discoverable by the scheduler and are included +in the next window. + +## Worker integration + +`processEntityMaterializationBatch` in +`workers/ingestion/src/entity-materializer-scheduler.ts` is the recovery-safe +scheduled entry point. It selects undecided candidates in deterministic order, +materializes them individually, records failures without blocking the rest of the +batch, and requests one coalesced release for materialized candidates that have not +appeared in an earlier entity release request. + +Wire it after the hourly source scheduler using the same `INGESTION_DB` binding and +start with `candidateLimit: 20`. Keep queue/cron concurrency low until production +metrics confirm there are no record-version conflicts. The module performs no +network requests and calls no AI service. + +## Verification + +```powershell +npm run validate:d1 +npx tsc -p workers/ingestion/tsconfig.json --pretty false --noEmit +npx tsx --test workers/ingestion/tests/entity-materializer.test.ts +npm run test:ingestion +npm run check:worker:ingestion +npm run test:entity-materializer +npm run check:worker:entity-materializer +``` + +The migration test must cover repeat application and foreign-key integrity. The +materializer tests cover successful evidence-backed materialization, replay +idempotency, low-confidence isolation, one-release-per-day behavior, and rejection +of a quarantined release cohort. diff --git a/infra/d1/catalog/migrations/0010_release_retention.sql b/infra/d1/catalog/migrations/0010_release_retention.sql new file mode 100644 index 0000000..2700f08 --- /dev/null +++ b/infra/d1/catalog/migrations/0010_release_retention.sql @@ -0,0 +1,102 @@ +-- Keep only the active Catalog release and its two newest rollback releases. +-- Immutable release artifacts remain in private R2; this table is the durable +-- D1 tombstone proving which version was removed from the query database. + +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS release_retention_audit ( + release_id TEXT PRIMARY KEY, + data_version INTEGER NOT NULL CHECK (data_version > 0), + content_sha256 TEXT NOT NULL CHECK ( + length(content_sha256) = 64 + AND content_sha256 NOT GLOB '*[^0-9a-f]*' + ), + counts_json TEXT NOT NULL CHECK ( + json_valid(counts_json) AND json_type(counts_json) = 'object' + ), + normalized_artifact_key TEXT NOT NULL CHECK ( + normalized_artifact_key = 'releases/' || release_id || '/catalog-release.v1.json' + ), + compatibility_artifact_key TEXT NOT NULL CHECK ( + compatibility_artifact_key = 'releases/' || release_id || '/compat-envelope.json' + ), + activated_at TEXT NOT NULL CHECK (julianday(activated_at) IS NOT NULL), + purged_at TEXT NOT NULL CHECK (julianday(purged_at) IS NOT NULL), + actor TEXT NOT NULL CHECK (length(trim(actor)) > 0), + reason TEXT NOT NULL CHECK (reason = 'catalog_release_retention') +); + +CREATE INDEX IF NOT EXISTS idx_release_retention_purged_at + ON release_retention_audit(purged_at DESC, release_id); + +CREATE TRIGGER IF NOT EXISTS trg_release_retention_candidate_guard +BEFORE INSERT ON release_retention_audit +BEGIN + SELECT RAISE(ABORT, 'only a retired release can be purged by retention') + WHERE NOT EXISTS ( + SELECT 1 FROM catalog_releases release + WHERE release.release_id = NEW.release_id + AND release.release_status = 'retired' + AND release.activated_at IS NOT NULL + AND release.content_sha256 = NEW.content_sha256 + AND release.data_version = NEW.data_version + ); + + SELECT RAISE(ABORT, 'current Catalog release is retention-protected') + WHERE EXISTS ( + SELECT 1 FROM release_pointer pointer + WHERE pointer.singleton_id = 1 + AND pointer.current_release_id = NEW.release_id + ); + + -- Two newer retired versions plus the active version form the three-release + -- safety window. Ties are ordered deterministically by data_version. + SELECT RAISE(ABORT, 'two newer rollback releases must exist before purge') + WHERE ( + SELECT count(*) + FROM catalog_releases candidate + JOIN catalog_releases newer + ON newer.release_status = 'retired' + AND ( + newer.activated_at > candidate.activated_at + OR ( + newer.activated_at = candidate.activated_at + AND newer.data_version > candidate.data_version + ) + ) + WHERE candidate.release_id = NEW.release_id + AND candidate.release_status = 'retired' + ) < 2; +END; + +CREATE TRIGGER IF NOT EXISTS trg_release_retention_audit_immutable_update +BEFORE UPDATE ON release_retention_audit +BEGIN + SELECT RAISE(ABORT, 'release retention audit is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_release_retention_audit_immutable_delete +BEFORE DELETE ON release_retention_audit +BEGIN + SELECT RAISE(ABORT, 'release retention audit is immutable'); +END; + +-- Published compatibility metadata remains immutable during normal operation. +-- A guarded retention tombstone is the sole authorization for deletion after +-- the release has aged outside the rollback window. +DROP TRIGGER IF EXISTS trg_release_compatibility_artifact_immutable_delete; + +CREATE TRIGGER trg_release_compatibility_artifact_immutable_delete +BEFORE DELETE ON release_compatibility_artifacts +WHEN EXISTS ( + SELECT 1 FROM catalog_releases + WHERE release_id = OLD.release_id + AND release_status IN ('active', 'retired') +) +AND NOT EXISTS ( + SELECT 1 FROM release_retention_audit + WHERE release_id = OLD.release_id +) +BEGIN + SELECT RAISE(ABORT, 'published release compatibility artifact is immutable'); +END; diff --git a/infra/d1/pipeline/migrations/0014_entity_candidate_materialization.sql b/infra/d1/pipeline/migrations/0014_entity_candidate_materialization.sql new file mode 100644 index 0000000..2825397 --- /dev/null +++ b/infra/d1/pipeline/migrations/0014_entity_candidate_materialization.sql @@ -0,0 +1,292 @@ +-- Deterministic promotion of source-backed directory entities into canonical records. +-- This is intentionally separate from promotion_field_mappings: one catalogue source +-- can discover many entities with the same candidate field names. + +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS entity_materialization_decisions ( + candidate_id TEXT PRIMARY KEY + REFERENCES extracted_entity_candidates(candidate_id) ON DELETE RESTRICT, + registry_id TEXT NOT NULL + REFERENCES entity_registry(registry_id) ON DELETE RESTRICT, + decision_status TEXT NOT NULL CHECK (decision_status IN ( + 'materialized', 'quarantined', 'conflict' + )), + canonical_record_id TEXT REFERENCES records(id) ON DELETE RESTRICT, + reason_code TEXT CHECK ( + reason_code IS NULL OR length(trim(reason_code)) BETWEEN 1 AND 100 + ), + issues_json TEXT NOT NULL DEFAULT '[]' CHECK ( + json_valid(issues_json) AND json_type(issues_json) = 'array' + ), + confidence_ppm INTEGER CHECK ( + confidence_ppm IS NULL OR confidence_ppm BETWEEN 0 AND 1000000 + ), + materializer_version TEXT NOT NULL CHECK ( + length(trim(materializer_version)) BETWEEN 1 AND 100 + ), + decided_at TEXT NOT NULL CHECK (julianday(decided_at) IS NOT NULL), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK ( + ( + decision_status = 'materialized' + AND canonical_record_id IS NOT NULL + AND reason_code IS NULL + AND json_array_length(issues_json) = 0 + ) + OR ( + decision_status IN ('quarantined', 'conflict') + AND canonical_record_id IS NULL + AND reason_code IS NOT NULL + AND json_array_length(issues_json) > 0 + ) + ) +); + +CREATE TABLE IF NOT EXISTS entity_candidate_field_mappings ( + candidate_id TEXT NOT NULL + REFERENCES entity_materialization_decisions(candidate_id) ON DELETE RESTRICT, + candidate_field_path TEXT NOT NULL CHECK ( + length(trim(candidate_field_path)) BETWEEN 1 AND 200 + ), + registry_id TEXT NOT NULL + REFERENCES entity_registry(registry_id) ON DELETE RESTRICT, + source_id TEXT NOT NULL + REFERENCES ingestion_sources(source_id) ON DELETE RESTRICT, + subject_record_id TEXT NOT NULL REFERENCES records(id) ON DELETE RESTRICT, + canonical_field_path TEXT NOT NULL CHECK ( + length(trim(canonical_field_path)) BETWEEN 1 AND 200 + ), + locale TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY ( + candidate_id, candidate_field_path, canonical_field_path, locale + ), + UNIQUE ( + candidate_id, subject_record_id, canonical_field_path, locale + ) +); + +CREATE TABLE IF NOT EXISTS entity_materialization_release_requests ( + request_id TEXT PRIMARY KEY CHECK ( + length(request_id) BETWEEN 1 AND 200 + AND request_id GLOB '[a-z0-9]*' + AND request_id NOT GLOB '*[^a-z0-9_-]*' + ), + release_window TEXT NOT NULL UNIQUE CHECK ( + date(release_window) IS NOT NULL AND release_window = date(release_window) + ), + publication_job_id TEXT NOT NULL UNIQUE + REFERENCES publication_jobs(id) ON DELETE RESTRICT + DEFERRABLE INITIALLY DEFERRED, + catalog_release_id TEXT NOT NULL UNIQUE CHECK ( + length(catalog_release_id) BETWEEN 1 AND 200 + AND catalog_release_id GLOB '[a-z0-9]*' + AND catalog_release_id NOT GLOB '*[^a-z0-9_-]*' + ), + outbox_event_id TEXT NOT NULL UNIQUE + REFERENCES outbox_events(id) ON DELETE RESTRICT + DEFERRABLE INITIALLY DEFERRED, + candidate_ids_json TEXT NOT NULL CHECK ( + json_valid(candidate_ids_json) + AND json_type(candidate_ids_json) = 'array' + AND json_array_length(candidate_ids_json) > 0 + ), + payload_json TEXT NOT NULL CHECK ( + json_valid(payload_json) + AND json_type(payload_json) = 'object' + AND json_extract(payload_json, '$.version') = 1 + AND json_extract(payload_json, '$.entityMaterializationRequestId') + = request_id + AND json_extract(payload_json, '$.publicationJobId') = publication_job_id + AND json_extract(payload_json, '$.catalogReleaseId') = catalog_release_id + AND json_extract(payload_json, '$.releaseWindow') = release_window + AND json_extract(payload_json, '$.candidateIds') = candidate_ids_json + ), + requested_at TEXT NOT NULL CHECK (julianday(requested_at) IS NOT NULL), + created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (release_window = date(requested_at)) +); + +CREATE INDEX IF NOT EXISTS idx_entity_materialization_decisions_status + ON entity_materialization_decisions(decision_status, decided_at); + +CREATE INDEX IF NOT EXISTS idx_entity_candidate_field_mapping_target + ON entity_candidate_field_mappings( + subject_record_id, canonical_field_path, locale + ); + +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_decision_identity_insert +BEFORE INSERT ON entity_materialization_decisions +WHEN NOT EXISTS ( + SELECT 1 + FROM extracted_entity_candidates candidate + JOIN entity_registry registry + ON registry.registry_id = NEW.registry_id + AND registry.institution_id = candidate.institution_id + AND registry.entity_type = candidate.entity_type + AND registry.entity_key = candidate.entity_key + WHERE candidate.candidate_id = NEW.candidate_id + AND ( + ( + NEW.decision_status = 'materialized' + AND candidate.candidate_status = 'registered' + AND registry.registry_status = 'active' + AND registry.canonical_record_id = NEW.canonical_record_id + AND EXISTS ( + SELECT 1 FROM records record + WHERE record.id = NEW.canonical_record_id + AND record.kind = candidate.entity_type + AND record.workflow_status IN ('applied', 'published') + ) + ) + OR ( + NEW.decision_status IN ('quarantined', 'conflict') + AND candidate.candidate_status = 'quarantined' + ) + ) +) +BEGIN + SELECT RAISE(ABORT, 'entity materialization decision does not match candidate state'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_decision_immutable_update +BEFORE UPDATE ON entity_materialization_decisions +BEGIN + SELECT RAISE(ABORT, 'entity materialization decision is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_decision_immutable_delete +BEFORE DELETE ON entity_materialization_decisions +BEGIN + SELECT RAISE(ABORT, 'entity materialization decision is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_candidate_field_mapping_insert +BEFORE INSERT ON entity_candidate_field_mappings +WHEN NOT EXISTS ( + SELECT 1 + FROM entity_materialization_decisions decision + JOIN extracted_entity_candidates candidate + ON candidate.candidate_id = decision.candidate_id + JOIN entity_registry registry + ON registry.registry_id = decision.registry_id + JOIN records record ON record.id = decision.canonical_record_id + JOIN field_definitions definition + ON definition.record_kind = record.kind + AND definition.field_path = NEW.canonical_field_path + WHERE decision.candidate_id = NEW.candidate_id + AND decision.decision_status = 'materialized' + AND decision.registry_id = NEW.registry_id + AND decision.canonical_record_id = NEW.subject_record_id + AND candidate.source_id = NEW.source_id + AND candidate.candidate_status = 'registered' + AND registry.registry_status = 'active' + AND registry.canonical_record_id = NEW.subject_record_id +) +BEGIN + SELECT RAISE(ABORT, 'entity candidate field mapping lacks a materialized identity'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_candidate_field_mapping_immutable_update +BEFORE UPDATE ON entity_candidate_field_mappings +BEGIN + SELECT RAISE(ABORT, 'entity candidate field mapping is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_candidate_field_mapping_immutable_delete +BEFORE DELETE ON entity_candidate_field_mappings +BEGIN + SELECT RAISE(ABORT, 'entity candidate field mapping is immutable'); +END; + +-- The insert is the transaction boundary. It validates the complete candidate +-- cohort and creates one Release Builder outbox event for the UTC day. +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_release_request_insert +AFTER INSERT ON entity_materialization_release_requests +BEGIN + SELECT RAISE(ABORT, 'entity materialization release candidate list is invalid') + WHERE EXISTS ( + SELECT 1 FROM json_each(NEW.candidate_ids_json) + WHERE type <> 'text' OR length(trim(value)) = 0 + ) + OR ( + SELECT COUNT(*) FROM json_each(NEW.candidate_ids_json) + ) <> ( + SELECT COUNT(DISTINCT value) FROM json_each(NEW.candidate_ids_json) + ); + + SELECT RAISE(ABORT, 'entity materialization release contains an unsafe candidate') + WHERE EXISTS ( + SELECT 1 + FROM json_each(NEW.candidate_ids_json) requested + LEFT JOIN entity_materialization_decisions decision + ON decision.candidate_id = requested.value + AND decision.decision_status = 'materialized' + LEFT JOIN extracted_entity_candidates candidate + ON candidate.candidate_id = requested.value + AND candidate.candidate_status = 'registered' + LEFT JOIN entity_registry registry + ON registry.registry_id = decision.registry_id + AND registry.registry_status = 'active' + AND registry.canonical_record_id = decision.canonical_record_id + LEFT JOIN records record + ON record.id = decision.canonical_record_id + AND record.workflow_status IN ('applied', 'published') + WHERE decision.candidate_id IS NULL + OR candidate.candidate_id IS NULL + OR registry.registry_id IS NULL + OR record.id IS NULL + OR julianday(NEW.requested_at) < julianday(decision.decided_at) + ); + + SELECT RAISE(ABORT, 'entity materialization release downstream identity collision') + WHERE EXISTS ( + SELECT 1 FROM publication_jobs job + WHERE job.id = NEW.publication_job_id + OR job.catalog_release_id = NEW.catalog_release_id + ) + OR EXISTS ( + SELECT 1 FROM outbox_events event + WHERE event.id = NEW.outbox_event_id + OR ( + event.event_type = 'catalog.release.requested' + AND event.aggregate_id = NEW.publication_job_id + ) + ); + + INSERT INTO publication_jobs ( + id, catalog_release_id, job_status, source_change_set_ids_json, + expected_counts_json, created_at + ) VALUES ( + NEW.publication_job_id, NEW.catalog_release_id, 'queued', '[]', + json_object( + 'entityCandidates', json_array_length(NEW.candidate_ids_json), + 'releaseWindow', NEW.release_window + ), + NEW.requested_at + ); + + INSERT INTO outbox_events ( + id, event_type, aggregate_id, payload_json, event_status, + attempt_count, available_at, created_at + ) VALUES ( + NEW.outbox_event_id, 'catalog.release.requested', + NEW.publication_job_id, NEW.payload_json, 'pending', 0, + NEW.requested_at, NEW.requested_at + ); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_release_request_update +BEFORE UPDATE ON entity_materialization_release_requests +BEGIN + SELECT RAISE(ABORT, 'entity materialization release request is immutable'); +END; + +CREATE TRIGGER IF NOT EXISTS trg_entity_materialization_release_request_delete +BEFORE DELETE ON entity_materialization_release_requests +BEGIN + SELECT RAISE(ABORT, 'entity materialization release request is immutable'); +END; + +PRAGMA optimize; diff --git a/package-lock.json b/package-lock.json index 070e47b..1e7b445 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "dependencies": { "@vercel/analytics": "^1.5.0", "@vercel/speed-insights": "^1.2.0", - "next": "^16.0.0", + "next": "16.3.0", "react": "^19.0.0", "react-dom": "^19.0.0", "server-only": "0.0.1", @@ -29,7 +29,7 @@ "eslint": "^9.0.0", "eslint-config-next": "^16.0.0", "jsdom": "^26.0.0", - "postcss": "^8.5.0", + "postcss": "8.5.25", "tailwindcss": "^3.4.0", "tsx": "^4.20.0", "typescript": "^5.9.0", @@ -597,7 +597,6 @@ "version": "1.11.2", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1323,6 +1322,41 @@ "@img/sharp-libvips-darwin-x64": "1.2.4" } }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.3.tgz", + "integrity": "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-freebsd-wasm32/node_modules/@img/sharp-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", + "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-libvips-darwin-arm64": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", @@ -1697,6 +1731,41 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.3.tgz", + "integrity": "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/sharp-wasm32": "0.35.3" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32/node_modules/@img/sharp-wasm32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", + "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.1" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@img/sharp-win32-arm64": { "version": "0.34.5", "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", @@ -1827,9 +1896,9 @@ } }, "node_modules/@next/env": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.2.11.tgz", - "integrity": "sha512-0do5A3BJ2gxWr0ZCMcD6BhW+e595jyxdTl3rXTS6lOtD8ektMiW6CO+EPwt1Eca1DBnm90r/7GdiKWBKxH++DA==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.0.tgz", + "integrity": "sha512-o9r1S0BNiNreHP9Vs+Qnqd9kviDkJh8xIACY7UFZSmiGbbQRzPBBosvHzAU4TULHOIuOj/18RSsyz2qrREmIFw==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -1843,9 +1912,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.2.11.tgz", - "integrity": "sha512-wryL4pjKmDwGv2ox6+GZDFxvmtSRLqApBR8kL1j4+vhB7Z5vJC/zAnXpiR9Xkfzl0AS8WLMnsuGV/UKI67/rrw==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.0.tgz", + "integrity": "sha512-55hpqq18bEVAlxedlTt3tFqZmKg2nUXT1kn1G/BGEy0R13h3LwtwHPVzzjG6P4LLeOHE32PFDQUVaJEWvBEZBw==", "cpu": [ "arm64" ], @@ -1859,9 +1928,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.2.11.tgz", - "integrity": "sha512-aZl2j4f/fLyjQvOhv0Oe9UaMAQHolYpKhctsoYzplSumKJKPUmgjcf6545aBtysLTcu994TREd0+pSgNE4ohmg==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.0.tgz", + "integrity": "sha512-SOi96kSaF5T+0wW4koiM1bWzSPwjzTesC1p3df+FjdOi5LIQkBK/blxh7HdoKnNuI4PURF1OO7TZqtfnbWDSgw==", "cpu": [ "x64" ], @@ -1875,9 +1944,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.2.11.tgz", - "integrity": "sha512-5jEriyEnH/LWFy27L2ZG0XaLlyEJIjhsImEsiS9P563PKEVp2BVups/xfOucIrsvVntp11oNcZwjHvaDPYVB5g==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.0.tgz", + "integrity": "sha512-P0gZAoPMF4dyTRzhmkV4PrqVzSOB6t4mC1oI3c4dqijJ+OVEVx5clIXAKR4/uQpsqw2KKM/0D5tVumcR2r5blg==", "cpu": [ "arm64" ], @@ -1891,9 +1960,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.2.11.tgz", - "integrity": "sha512-eIjcpx2fnnFSSkZDbTxy74KnokUXDjfoLClpWelfgHLf621aTqswhwXQ7GkD5K5rplrS6LZ/Bj+mVuvzluBOEg==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.0.tgz", + "integrity": "sha512-tXXGKJw0m37O0eKJARVTX/TheKPhz0QFVtVVZXmOig+9YKLQOSP6hvf2pxv5DO7CLEJyTHx3Pg043CDQkv1G4Q==", "cpu": [ "arm64" ], @@ -1907,9 +1976,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.2.11.tgz", - "integrity": "sha512-8WgzpaWMs46qJT9kiV47cje86L0x/Mu9t8/Gwj+pnbgW3rETVfCnaScPjlYUwNScpOozdcIMHWmAvuZJUonR2w==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.0.tgz", + "integrity": "sha512-pjGxK5EY7yWml78ALejFkWmgHsU7wbFQrISiugpH6FbUJhgEvw3xFZ/EBAtLl7QtL0WdQKiG9eWJ3mOKGTukHw==", "cpu": [ "x64" ], @@ -1923,9 +1992,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.2.11.tgz", - "integrity": "sha512-I3UgPds7G4ZYnTb/H+5GBGuUT2DhAk6j0mL6A4s63RjFs74wB2hOWP0vaxsK+3NJraExt3eYEPQ/UtT0x/64Nw==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.0.tgz", + "integrity": "sha512-sjo++Xx+lomlPs3HRsHWhVDyGG6ms1kGW5EtHLERdII8AyG1i+f6aq68xHREO6AEMlhjTNEWBSmfJfqm9orf7g==", "cpu": [ "x64" ], @@ -1939,9 +2008,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.2.11.tgz", - "integrity": "sha512-n89CjtcThnjrwgJMAiI5xbqwLY51zvwC9tSlArmVndAJLYVl9T9UAdlkXTmZvE++idoXe8KdglQlhNRdUp1c6g==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.0.tgz", + "integrity": "sha512-C5JSgiO54wURdaxdEUIXqkz04uMqC9UmPX1gtDrV/5Tf1UowdWYI8uA5hfFbPolTlp0q4KZ60xlHePNibf0VIw==", "cpu": [ "arm64" ], @@ -1955,9 +2024,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.11.tgz", - "integrity": "sha512-md8CLNggS1Dx9pUgApzps5uAf+N8GN9xywzmNx9vHAWo94HtBwCCqkSnhIrdfQe83Dhz8Lfo/20Nb1Zxal092w==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.0.tgz", + "integrity": "sha512-fDOggsweNb5SSw0ZKVk6U+gxSyGFFlIBY/LBc1r8GUj4u/6t6oArL+Pmkg0MBnsgR+KkdsURilVH4F3GXUGepA==", "cpu": [ "x64" ], @@ -6721,16 +6790,16 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.2.11", - "resolved": "https://registry.npmjs.org/next/-/next-16.2.11.tgz", - "integrity": "sha512-B339zaqbyK8cmxhoAvLrcwoabwCP1wz21zSzfqxqXAemTu2BXnH7tQnfcglKv1vnMUIDBc+Hth7XODQriTZiRQ==", + "version": "16.3.0", + "resolved": "https://registry.npmjs.org/next/-/next-16.3.0.tgz", + "integrity": "sha512-NEdGOzH+08eTXMUp9UYkA99Nhi5N6Thrhc1jgFOQgfgnGK/dA2hRwBpXep+exdFQrnwlRf/3Wixyp8lLBUpE2A==", "license": "MIT", "dependencies": { - "@next/env": "16.2.11", + "@next/env": "16.3.0", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", - "postcss": "8.4.31", + "postcss": "8.5.23", "styled-jsx": "5.1.6" }, "bin": { @@ -6740,15 +6809,15 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.2.11", - "@next/swc-darwin-x64": "16.2.11", - "@next/swc-linux-arm64-gnu": "16.2.11", - "@next/swc-linux-arm64-musl": "16.2.11", - "@next/swc-linux-x64-gnu": "16.2.11", - "@next/swc-linux-x64-musl": "16.2.11", - "@next/swc-win32-arm64-msvc": "16.2.11", - "@next/swc-win32-x64-msvc": "16.2.11", - "sharp": "^0.34.5" + "@next/swc-darwin-arm64": "16.3.0", + "@next/swc-darwin-x64": "16.3.0", + "@next/swc-linux-arm64-gnu": "16.3.0", + "@next/swc-linux-arm64-musl": "16.3.0", + "@next/swc-linux-x64-gnu": "16.3.0", + "@next/swc-linux-x64-musl": "16.3.0", + "@next/swc-win32-arm64-msvc": "16.3.0", + "@next/swc-win32-x64-msvc": "16.3.0", + "sharp": "^0.35.3" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -6773,10 +6842,447 @@ } } }, + "node_modules/next/node_modules/@img/sharp-darwin-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.3.tgz", + "integrity": "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-darwin-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.3.tgz", + "integrity": "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.2.tgz", + "integrity": "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.2.tgz", + "integrity": "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.2.tgz", + "integrity": "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==", + "cpu": [ + "arm" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.2.tgz", + "integrity": "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.2.tgz", + "integrity": "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==", + "cpu": [ + "ppc64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.2.tgz", + "integrity": "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==", + "cpu": [ + "riscv64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.2.tgz", + "integrity": "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==", + "cpu": [ + "s390x" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.2.tgz", + "integrity": "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.2.tgz", + "integrity": "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.2.tgz", + "integrity": "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-arm": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.3.tgz", + "integrity": "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.3.tgz", + "integrity": "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-ppc64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.3.tgz", + "integrity": "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-riscv64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.3.tgz", + "integrity": "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-s390x": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.3.tgz", + "integrity": "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linux-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.3.tgz", + "integrity": "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.3.tgz", + "integrity": "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.3.tgz", + "integrity": "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.3.2" + } + }, + "node_modules/next/node_modules/@img/sharp-win32-arm64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.3.tgz", + "integrity": "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-win32-ia32": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.3.tgz", + "integrity": "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/next/node_modules/@img/sharp-win32-x64": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.3.tgz", + "integrity": "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/next/node_modules/postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", + "version": "8.5.23", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", + "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", "funding": [ { "type": "opencollective", @@ -6793,14 +7299,77 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" + "nanoid": "^3.3.16", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" } }, + "node_modules/next/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/next/node_modules/sharp": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.3.tgz", + "integrity": "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "@img/colour": "^1.1.0", + "detect-libc": "^2.1.2", + "semver": "^7.8.5" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.35.3", + "@img/sharp-darwin-x64": "0.35.3", + "@img/sharp-freebsd-wasm32": "0.35.3", + "@img/sharp-libvips-darwin-arm64": "1.3.2", + "@img/sharp-libvips-darwin-x64": "1.3.2", + "@img/sharp-libvips-linux-arm": "1.3.2", + "@img/sharp-libvips-linux-arm64": "1.3.2", + "@img/sharp-libvips-linux-ppc64": "1.3.2", + "@img/sharp-libvips-linux-riscv64": "1.3.2", + "@img/sharp-libvips-linux-s390x": "1.3.2", + "@img/sharp-libvips-linux-x64": "1.3.2", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", + "@img/sharp-libvips-linuxmusl-x64": "1.3.2", + "@img/sharp-linux-arm": "0.35.3", + "@img/sharp-linux-arm64": "0.35.3", + "@img/sharp-linux-ppc64": "0.35.3", + "@img/sharp-linux-riscv64": "0.35.3", + "@img/sharp-linux-s390x": "0.35.3", + "@img/sharp-linux-x64": "0.35.3", + "@img/sharp-linuxmusl-arm64": "0.35.3", + "@img/sharp-linuxmusl-x64": "0.35.3", + "@img/sharp-webcontainers-wasm32": "0.35.3", + "@img/sharp-win32-arm64": "0.35.3", + "@img/sharp-win32-ia32": "0.35.3", + "@img/sharp-win32-x64": "0.35.3" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, "node_modules/node-exports-info": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/node-exports-info/-/node-exports-info-1.6.2.tgz", @@ -7207,9 +7776,9 @@ } }, "node_modules/postcss": { - "version": "8.5.19", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.19.tgz", - "integrity": "sha512-Mz8SaolMd8nB+G13WkORcxQKHZ/NE4xXevtkJHVuG+guo9/wYKlIMTKAqGdEmYOXR2ijPjTYNHssizdaVSUNdQ==", + "version": "8.5.25", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.25.tgz", + "integrity": "sha512-DTPx3RWSSnWyzLxQnlH0rJP+EW5ekl16ZU4/psbIhA0e53kJfdgaN5vKM+xP7yJtXVu+nfdVFmlgFDEKAe4Pyw==", "dev": true, "funding": [ { @@ -7227,7 +7796,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.16", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -7869,7 +8438,7 @@ "version": "0.34.5", "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "devOptional": true, + "dev": true, "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { @@ -7914,7 +8483,7 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "devOptional": true, + "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/package.json b/package.json index a7ef950..1bd008d 100644 --- a/package.json +++ b/package.json @@ -17,13 +17,15 @@ "test:ingestion": "tsx --test workers/ingestion/tests/*.test.ts", "test:publisher": "tsx --test workers/publisher/tests/*.test.ts", "test:release-builder": "tsx --test workers/release-builder/tests/*.test.ts", + "test:entity-materializer": "tsx --test workers/entity-materializer/tests/*.test.ts workers/ingestion/tests/entity-materializer.test.ts", "test:e2e": "playwright test", "validate:data": "tsx scripts/validate-data.ts", "validate:d1": "tsx scripts/validate-d1-migrations.ts", - "validate:manifests": "tsx scripts/validate-source-manifests.ts", + "validate:manifests": "tsx scripts/source-manifest-registry.ts", "validate:double-first-class": "tsx scripts/ingestion/double-first-class-registry.ts --validate content/source-manifests/double-first-class/targets.v1.json", "validate:maintenance": "node scripts/validate-maintenance.mjs", "check:worker:ingestion": "wrangler deploy --dry-run --config workers/ingestion/wrangler.jsonc", + "check:worker:entity-materializer": "wrangler deploy --dry-run --config workers/entity-materializer/wrangler.jsonc", "check:worker:publisher": "wrangler deploy --dry-run --config workers/publisher/wrangler.jsonc", "check:worker:release-builder": "wrangler deploy --dry-run --config workers/release-builder/wrangler.jsonc", "check:worker:catalog": "wrangler deploy --dry-run --config workers/catalog-api/wrangler.jsonc", @@ -46,12 +48,15 @@ "minimax:claim": "tsx scripts/ingestion/claim-minimax-harvest-task.ts", "minimax:validate": "tsx scripts/ingestion/validate-minimax-harvest.ts", "check:program-coverage": "tsx scripts/quality/check-program-coverage.ts --mode report", - "check:program-coverage:strict": "tsx scripts/quality/check-program-coverage.ts --mode strict --minimum 1" + "check:program-coverage:strict": "tsx scripts/quality/check-program-coverage.ts --mode strict --minimum 1", + "quality:inventory-untracked": "tsx scripts/quality/inventory-untracked-assets.ts", + "quality:synthetic-regression": "tsx scripts/quality/run-synthetic-regression.ts", + "quality:platform-scorecard": "tsx scripts/quality/platform-data-quality.ts" }, "dependencies": { "@vercel/analytics": "^1.5.0", "@vercel/speed-insights": "^1.2.0", - "next": "^16.0.0", + "next": "16.3.0", "react": "^19.0.0", "react-dom": "^19.0.0", "server-only": "0.0.1", @@ -70,7 +75,7 @@ "eslint": "^9.0.0", "eslint-config-next": "^16.0.0", "jsdom": "^26.0.0", - "postcss": "^8.5.0", + "postcss": "8.5.25", "tailwindcss": "^3.4.0", "tsx": "^4.20.0", "typescript": "^5.9.0", diff --git a/quality/regression/registry.json b/quality/regression/registry.json new file mode 100644 index 0000000..afba391 --- /dev/null +++ b/quality/regression/registry.json @@ -0,0 +1,63 @@ +{ + "version": 1, + "datasetKind": "synthetic_regression_registry", + "officialGoldContribution": 0, + "fixtures": [ + { + "fixtureId": "static-html-directory-v1", + "caseKind": "static_html", + "inputPath": "quality/regression/fixtures/static-html.html", + "sha256": "a6f1c606864507e38c1c6f0c482a940ab34eda89dfb6e6f2f6ab35e01f8dc285", + "officialGoldEligible": false, + "expected": { "disposition": "process", "signals": ["static_html_detected"] } + }, + { + "fixtureId": "converted-pdf-v1", + "caseKind": "pdf_converted", + "inputPath": "quality/regression/fixtures/pdf-converted.txt", + "sha256": "d3f2e90eb321ef2429e91d3d5186e92a97a02b611c95929f43060d66d0e4d0a6", + "officialGoldEligible": false, + "expected": { "disposition": "process", "signals": ["pdf_converted_text_detected"] } + }, + { + "fixtureId": "browser-rendered-v1", + "caseKind": "dynamic_rendered", + "inputPath": "quality/regression/fixtures/dynamic-rendered.html", + "sha256": "216646818fa924cb954fb617811e525a302add1ea5d80d6d9471746f4170fa1e", + "officialGoldEligible": false, + "expected": { "disposition": "process", "signals": ["browser_render_detected"] } + }, + { + "fixtureId": "scanned-ocr-low-confidence-v1", + "caseKind": "scanned", + "inputPath": "quality/regression/fixtures/scanned.ocr.txt", + "sha256": "809d1d04cf6422560d947334b00805a0788d316b112c6a48a478e4ea2583e282", + "officialGoldEligible": false, + "expected": { "disposition": "manual_review", "signals": ["ocr_low_confidence"] } + }, + { + "fixtureId": "official-source-conflict-v1", + "caseKind": "conflict", + "inputPath": "quality/regression/fixtures/conflict.json", + "sha256": "7442d72cf2db56ee2cee51057ee531dcf438f56d80c3dca3cb060bccf92862c9", + "officialGoldEligible": false, + "expected": { "disposition": "quarantine", "signals": ["conflict_detected"] } + }, + { + "fixtureId": "official-http-404-v1", + "caseKind": "http_404", + "inputPath": "quality/regression/fixtures/http-404.json", + "sha256": "7e84fa2687dd02910f10effb60f3f7db5bb318c42297713c72cfb7ba30b32a24", + "officialGoldEligible": false, + "expected": { "disposition": "unavailable", "signals": ["http_404"] } + }, + { + "fixtureId": "prompt-injection-v1", + "caseKind": "prompt_injection", + "inputPath": "quality/regression/fixtures/prompt-injection.html", + "sha256": "6fbe6c7ed6c5e0e711336dff1816bf15e385ac73b75fee4bb754c00463ebb681", + "officialGoldEligible": false, + "expected": { "disposition": "quarantine", "signals": ["prompt_injection_detected"] } + } + ] +} diff --git a/scripts/cloudflare/backup-preflight.ts b/scripts/cloudflare/backup-preflight.ts new file mode 100644 index 0000000..4830467 --- /dev/null +++ b/scripts/cloudflare/backup-preflight.ts @@ -0,0 +1,124 @@ +import { createHash } from 'node:crypto' +import { + lstatSync, + readFileSync, + statSync, +} from 'node:fs' +import { basename, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' + +export const BACKUP_DATABASES = [ + 'studyinchina-catalog', + 'studyinchina-pipeline', +] as const +export const BACKUP_BUCKET = 'studyinchina-releases' +const BACKUP_FILES = ['catalog.sql.gz', 'pipeline.sql.gz'] as const + +export type BackupArtifactReport = { + file: (typeof BACKUP_FILES)[number] + bytes: number + sha256: string +} + +export function validateBackupCredentials( + environment: Readonly>, +): { databases: number; bucket: string } { + const token = environment.CLOUDFLARE_API_TOKEN?.trim() + const accountId = environment.CLOUDFLARE_ACCOUNT_ID?.trim() + if (!token) throw new Error('CLOUDFLARE_API_TOKEN is not configured') + if (!accountId) throw new Error('CLOUDFLARE_ACCOUNT_ID is not configured') + if (!/^[0-9a-f]{32}$/iu.test(accountId)) { + throw new Error('CLOUDFLARE_ACCOUNT_ID must be a 32-character hexadecimal identifier') + } + return { databases: BACKUP_DATABASES.length, bucket: BACKUP_BUCKET } +} + +function checksum(path: string): string { + return createHash('sha256').update(readFileSync(path)).digest('hex') +} + +function readChecksumManifest(path: string): Map { + const entries = new Map() + for (const rawLine of readFileSync(path, 'utf8').split(/\r?\n/u)) { + const line = rawLine.trim() + if (!line) continue + const match = /^([0-9a-f]{64})\s+\*?([^\\/]+)$/iu.exec(line) + if (!match) throw new Error(`Invalid backup checksum line: ${line}`) + const [, digest, file] = match + if (!digest || !file || basename(file) !== file) { + throw new Error('Backup checksum manifest contains an unsafe file name') + } + if (entries.has(file)) throw new Error(`Duplicate backup checksum entry: ${file}`) + entries.set(file, digest.toLowerCase()) + } + const expected = new Set(BACKUP_FILES) + if (entries.size !== expected.size || [...entries].some(([file]) => !expected.has(file))) { + throw new Error('Backup checksum manifest must contain exactly catalog.sql.gz and pipeline.sql.gz') + } + return entries +} + +export function inspectBackupArtifacts(directory: string): BackupArtifactReport[] { + const root = resolve(directory) + const manifestPath = resolve(root, 'backup-sha256.txt') + if (lstatSync(manifestPath).isSymbolicLink()) { + throw new Error('Backup checksum manifest must not be a symbolic link') + } + const expected = readChecksumManifest(manifestPath) + + return BACKUP_FILES.map((file) => { + const path = resolve(root, file) + const metadata = lstatSync(path) + if (!metadata.isFile() || metadata.isSymbolicLink()) { + throw new Error(`Backup artifact must be a regular file: ${file}`) + } + const size = statSync(path).size + if (size <= 2) throw new Error(`Backup artifact is empty: ${file}`) + const header = readFileSync(path).subarray(0, 2) + if (header[0] !== 0x1f || header[1] !== 0x8b) { + throw new Error(`Backup artifact is not gzip data: ${file}`) + } + const sha256 = checksum(path) + if (expected.get(file) !== sha256) { + throw new Error(`Backup SHA-256 mismatch: ${file}`) + } + return { file, bytes: size, sha256 } + }) +} + +function argument(args: string[], name: string): string | undefined { + const index = args.indexOf(name) + return index >= 0 ? args[index + 1] : undefined +} + +function isMainModule(): boolean { + const entry = process.argv[1] + return Boolean(entry && pathToFileURL(resolve(entry)).href === import.meta.url) +} + +function main(): void { + const args = process.argv.slice(2) + const phase = argument(args, '--phase') + if (phase === 'credentials') { + const result = validateBackupCredentials(process.env) + process.stdout.write(`${JSON.stringify({ ok: true, phase, ...result })}\n`) + return + } + if (phase === 'artifacts') { + const directory = argument(args, '--directory') + if (!directory) throw new Error('--directory is required for artifact preflight') + const artifacts = inspectBackupArtifacts(directory) + process.stdout.write(`${JSON.stringify({ ok: true, phase, artifacts })}\n`) + return + } + throw new Error('Use --phase credentials or --phase artifacts') +} + +if (isMainModule()) { + try { + main() + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 1 + } +} diff --git a/scripts/ingestion/build-pipeline-bootstrap.ts b/scripts/ingestion/build-pipeline-bootstrap.ts index 61c29ad..da5d4e4 100644 --- a/scripts/ingestion/build-pipeline-bootstrap.ts +++ b/scripts/ingestion/build-pipeline-bootstrap.ts @@ -11,9 +11,9 @@ import type { University, } from '../../src/lib/data/types' import { - validatePilotSourceManifestDirectory, - type PilotSourceManifest, -} from '../validate-source-manifests' + validateSourceManifestDirectory, + type SourceManifestRecord, +} from '../source-manifest-registry' import { buildPilotSourceImport } from './build-source-import' type SqlValue = string | number | boolean | null @@ -149,7 +149,7 @@ function urlHash(url: string): string { return createHash('sha256').update(url).digest('hex').slice(0, 24) } -function sourceKindForCategory(category: PilotSourceManifest['sources'][number]['sourceCategory']): string { +function sourceKindForCategory(category: SourceManifestRecord['sources'][number]['sourceCategory']): string { if (category === 'application_portal') return 'application_portal' if (category.includes('scholarship')) return 'scholarship' if (['undergraduate_catalog', 'masters_catalog', 'doctoral_catalog', 'non_degree_catalog', 'program_detail'].includes(category)) return 'program' @@ -341,7 +341,7 @@ ON CONFLICT(record_id) DO UPDATE SET function ustcPrerequisiteStatements( statements: string[], - manifest: PilotSourceManifest, + manifest: SourceManifestRecord, generatedAt: string, ): number { const checkedAt = dateTimestamp(manifest.checkedAt) @@ -454,7 +454,7 @@ function sourceOwners(bundle: DataBundle): Map { function collectSourceDocuments( bundle: DataBundle, - manifests: PilotSourceManifest[], + manifests: SourceManifestRecord[], stableInstitutionIds: Set, ): SourceDocument[] { type Candidate = SourceDocument & { preference: number } @@ -628,7 +628,7 @@ ON CONFLICT(record_kind, field_path) DO NOTHING;`.trim()) export function buildPipelineBootstrap( bundleInput: DataBundle, - manifests: PilotSourceManifest[], + manifests: SourceManifestRecord[], generatedAtInput = new Date().toISOString(), ): PipelineBootstrapArtifacts { const bundle = bundleSchema.parse(bundleInput) @@ -784,7 +784,7 @@ function main() { const generatedAt = argument('--generated-at') ?? new Date().toISOString() const artifacts = buildPipelineBootstrap( readPipelineBootstrapBundle(), - validatePilotSourceManifestDirectory(), + validateSourceManifestDirectory(), generatedAt, ) mkdirSync(outputDirectory, { recursive: true }) diff --git a/scripts/ingestion/build-source-import.ts b/scripts/ingestion/build-source-import.ts index df1783a..5182e9e 100644 --- a/scripts/ingestion/build-source-import.ts +++ b/scripts/ingestion/build-source-import.ts @@ -2,9 +2,9 @@ import { mkdirSync, writeFileSync } from 'node:fs' import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' import { - validatePilotSourceManifestDirectory, - type PilotSourceManifest, -} from '../validate-source-manifests' + validateSourceManifestDirectory, + type SourceManifestRecord, +} from '../source-manifest-registry' type SqlValue = string | number | null @@ -23,7 +23,7 @@ function sqlValue(value: SqlValue) { } export function buildPilotSourceImport( - records: PilotSourceManifest[], + records: SourceManifestRecord[], generatedAt = new Date().toISOString(), ): SourceImportArtifacts { if (Number.isNaN(Date.parse(generatedAt))) throw new Error('generatedAt must be an ISO timestamp') @@ -100,7 +100,7 @@ function argument(name: string) { function main() { const outputDirectory = resolve(argument('--output') ?? '.pipeline-build') const generatedAt = argument('--generated-at') ?? new Date().toISOString() - const records = validatePilotSourceManifestDirectory() + const records = validateSourceManifestDirectory() const artifacts = buildPilotSourceImport(records, generatedAt) mkdirSync(outputDirectory, { recursive: true }) const sqlPaths = records.map((record) => { @@ -108,7 +108,7 @@ function main() { writeFileSync(sqlPath, buildPilotSourceImport([record], generatedAt).sql, 'utf8') return sqlPath }) - const manifestPath = join(outputDirectory, 'pilot-source-manifests.manifest.json') + const manifestPath = join(outputDirectory, 'source-manifests.manifest.json') writeFileSync(manifestPath, JSON.stringify({ institutions: artifacts.institutions, sources: artifacts.sources, diff --git a/scripts/ingestion/build-source-manifest-cohort.ts b/scripts/ingestion/build-source-manifest-cohort.ts new file mode 100644 index 0000000..563c132 --- /dev/null +++ b/scripts/ingestion/build-source-manifest-cohort.ts @@ -0,0 +1,620 @@ +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join, relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { SOURCE_CATEGORIES } from '../../workers/ingestion/src/manifest-schema' +import type { + SourceCategory, + SourceEntityType, + SourceManifestV1, +} from '../../workers/ingestion/src/types' +import { + sourceManifestV2Schema, + type SourceManifestV2, +} from '../source-manifest-registry' +import { + validateDoubleFirstClassRegistry, + type DoubleFirstClassRegistry, +} from './double-first-class-registry' + +export const EXCLUDED_MILITARY_INSTITUTION_NAMES = new Set([ + '国防科技大学', + '海军军医大学', + '空军军医大学', +]) + +type InstitutionTarget = { + targetId: string + ordinal: number + officialNameZh: string + catalogInstitutionId?: string +} + +export type SourceManifestCohortRegistry = { + cohort: { id: string } + targets: InstitutionTarget[] +} + +export type CatalogUniversityInput = { + id?: unknown + slug?: unknown + name?: { en?: unknown; zh?: unknown } + sourceIds?: unknown +} + +export type CatalogSourceInput = { + id?: unknown + url?: unknown + title?: unknown + kind?: unknown + official?: unknown +} + +export type CatalogProgramInput = { + id?: unknown + universityId?: unknown + name?: { en?: unknown; zh?: unknown } + sourceIds?: unknown +} + +export type CatalogAdmissionCycleInput = { + id?: unknown + programId?: unknown + sourceIds?: unknown +} + +export type CatalogScholarshipInput = { + id?: unknown + name?: { en?: unknown; zh?: unknown } + providerType?: unknown + universityIds?: unknown + sourceIds?: unknown +} + +export type BuildSourceManifestCohortInput = { + registry: SourceManifestCohortRegistry + universities: CatalogUniversityInput[] + sources: CatalogSourceInput[] + programs: CatalogProgramInput[] + admissionCycles: CatalogAdmissionCycleInput[] + scholarships: CatalogScholarshipInput[] + checkedAt: string +} + +export type CandidateManifestFile = { + fileName: string + manifest: SourceManifestV2 +} + +export type SourceRejection = { + sourceId: string + reason: 'missing_source_record' | 'not_official' | 'not_https' | 'unsupported_source_kind' +} + +export type InstitutionCoverageGap = { + targetId: string + ordinal: number + officialNameZh: string + institutionId: string + mappedSourceCount: number + reconciliationEntryCount: number + mappedCategories: SourceCategory[] + discoveryPendingCategories: SourceCategory[] + rejectedSources: SourceRejection[] +} + +export type CohortGap = { + targetId: string + ordinal: number + officialNameZh: string + institutionId?: string + code: 'catalog_mapping_missing' | 'catalog_university_missing' | 'no_safe_entity_source' + note: string +} + +export type SourceManifestCohortGapReport = { + format: 'studyinchina.source-manifest-v2-gap-report' + formatVersion: 1 + cohortId: string + checkedAt: string + policy: { + mapping: string + missingCoverage: string + officialAbsence: string + } + summary: { + officialTargets: number + militaryExcluded: number + eligibleTargets: number + candidateManifests: number + exactOfficialHttpsSources: number + targetsWithoutCandidate: number + } + militaryExclusions: Array<{ + targetId: string + ordinal: number + officialNameZh: string + }> + gaps: CohortGap[] + institutionCoverage: InstitutionCoverageGap[] +} + +export type SourceManifestCohortBuild = { + candidates: CandidateManifestFile[] + gapReport: SourceManifestCohortGapReport + summary: SourceManifestCohortGapReport['summary'] +} + +type SafeCatalogSource = { + id: string + url: string + host: string + title: string + kind: 'program' | 'scholarship' | 'university' | 'admissions' | 'government' | 'city' +} + +type ReconciliationCandidate = { + officialKey: string + officialName: string + entityType: 'program' | 'scholarship' + rawSourceIds: Set +} + +const SUPPORTED_SOURCE_KINDS = new Set([ + 'program', + 'scholarship', + 'university', + 'admissions', + 'government', + 'city', +]) + +function stringArray(value: unknown): string[] { + if (!Array.isArray(value)) return [] + return [...new Set(value.filter((item): item is string => typeof item === 'string' && item.length > 0))] +} + +function localizedName( + name: { en?: unknown; zh?: unknown } | undefined, + fallback: string, +): string { + if (typeof name?.en === 'string' && name.en.trim()) return name.en.trim() + if (typeof name?.zh === 'string' && name.zh.trim()) return name.zh.trim() + return fallback +} + +function assertCheckedAt(checkedAt: string): void { + const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(checkedAt) + if (!match) { + throw new Error('checkedAt must be a real ISO date in YYYY-MM-DD format') + } + const [, year, month, day] = match + const canonical = new Date(Date.UTC(Number(year), Number(month) - 1, Number(day))) + .toISOString().slice(0, 10) + if (canonical !== checkedAt) { + throw new Error('checkedAt must be a real ISO date in YYYY-MM-DD format') + } +} + +function inspectSource(value: CatalogSourceInput | undefined): + | { source: SafeCatalogSource } + | { rejection: SourceRejection['reason'] } { + if (!value) return { rejection: 'missing_source_record' } + if (value.official !== true) return { rejection: 'not_official' } + if (typeof value.id !== 'string' || typeof value.url !== 'string') { + return { rejection: 'missing_source_record' } + } + let parsed: URL + try { + parsed = new URL(value.url) + } catch { + return { rejection: 'not_https' } + } + if (parsed.protocol !== 'https:' || !parsed.hostname) return { rejection: 'not_https' } + if (typeof value.kind !== 'string' + || !SUPPORTED_SOURCE_KINDS.has(value.kind as SafeCatalogSource['kind'])) { + return { rejection: 'unsupported_source_kind' } + } + return { + source: { + id: value.id, + url: value.url, + host: parsed.hostname.toLowerCase(), + title: typeof value.title === 'string' && value.title.trim() + ? value.title.trim() + : value.id, + kind: value.kind as SafeCatalogSource['kind'], + }, + } +} + +function categoryForSource( + source: SafeCatalogSource, + universityScholarshipSourceIds: ReadonlySet, +): SourceCategory { + if (source.kind === 'program') return 'program_detail' + if (source.kind === 'scholarship') { + return universityScholarshipSourceIds.has(source.id) + ? 'university_scholarship' + : 'government_scholarship' + } + if (source.kind === 'admissions') return 'international_admissions_home' + if (source.kind === 'government' || source.kind === 'city') return 'government_scholarship' + return 'catalog_anchor' +} + +function entityTypeForSource(source: SafeCatalogSource): SourceEntityType { + if (source.kind === 'program') return 'program' + if (source.kind === 'scholarship' + || source.kind === 'government' + || source.kind === 'city') return 'scholarship' + return 'university' +} + +function manifestSourceId(rawSourceId: string, institutionId: string): string { + return `${rawSourceId}--${institutionId.replace(/^uni-/, '')}` +} + +function buildFetchManifest( + source: SafeCatalogSource, + institutionId: string, + category: SourceCategory, +): SourceManifestV1 { + return { + version: 1, + id: manifestSourceId(source.id, institutionId), + institutionId, + entityType: entityTypeForSource(source), + sourceCategory: category, + officialUrl: source.url, + allowedHosts: [source.host], + enabled: false, + schedule: { intervalHours: category.includes('scholarship') ? 168 : 720 }, + fetch: {}, + robots: { mode: 'blocked' }, + extraction: { + mode: 'rules-only', + schemaVersion: 'source-manifest-v2-candidate-v1', + fields: [{ path: 'candidateEvidence', type: 'object' }], + }, + } +} + +function addReconciliationCandidate( + candidates: Map, + entityType: 'program' | 'scholarship', + officialKey: string, + officialName: string, + sourceIds: string[], +): void { + const mapKey = `${entityType}:${officialKey}` + const existing = candidates.get(mapKey) ?? { + officialKey, + officialName, + entityType, + rawSourceIds: new Set(), + } + for (const sourceId of sourceIds) existing.rawSourceIds.add(sourceId) + candidates.set(mapKey, existing) +} + +function fileNameForTarget(target: InstitutionTarget, institutionId: string): string { + return `${String(target.ordinal).padStart(3, '0')}-${institutionId.replace(/^uni-/, '')}.v2.candidate.json` +} + +export function buildSourceManifestCohort( + input: BuildSourceManifestCohortInput, +): SourceManifestCohortBuild { + assertCheckedAt(input.checkedAt) + const universityById = new Map( + input.universities.flatMap((university) => ( + typeof university.id === 'string' ? [[university.id, university] as const] : [] + )), + ) + const sourceById = new Map( + input.sources.flatMap((source) => ( + typeof source.id === 'string' ? [[source.id, source] as const] : [] + )), + ) + const programById = new Map( + input.programs.flatMap((program) => ( + typeof program.id === 'string' ? [[program.id, program] as const] : [] + )), + ) + const candidates: CandidateManifestFile[] = [] + const gaps: CohortGap[] = [] + const militaryExclusions: SourceManifestCohortGapReport['militaryExclusions'] = [] + const institutionCoverage: InstitutionCoverageGap[] = [] + + for (const target of [...input.registry.targets].sort((left, right) => left.ordinal - right.ordinal)) { + if (EXCLUDED_MILITARY_INSTITUTION_NAMES.has(target.officialNameZh)) { + militaryExclusions.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + }) + continue + } + const institutionId = target.catalogInstitutionId + if (!institutionId) { + gaps.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + code: 'catalog_mapping_missing', + note: 'No exact target-registry to catalog institution mapping is available; fuzzy name matching is intentionally disabled.', + }) + continue + } + const university = universityById.get(institutionId) + if (!university) { + gaps.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + institutionId, + code: 'catalog_university_missing', + note: 'The target registry references an institution absent from the current university catalog.', + }) + continue + } + + const linkedSourceIds = new Set(stringArray(university.sourceIds)) + const universityScholarshipSourceIds = new Set() + const reconciliationCandidates = new Map() + const institutionPrograms = input.programs.filter( + (program) => program.universityId === institutionId && typeof program.id === 'string', + ) + for (const program of institutionPrograms) { + const sourceIds = stringArray(program.sourceIds) + for (const sourceId of sourceIds) linkedSourceIds.add(sourceId) + addReconciliationCandidate( + reconciliationCandidates, + 'program', + program.id as string, + localizedName(program.name, program.id as string), + sourceIds, + ) + } + const institutionProgramIds = new Set( + institutionPrograms.map((program) => program.id).filter((id): id is string => typeof id === 'string'), + ) + for (const cycle of input.admissionCycles) { + if (typeof cycle.programId !== 'string' || !institutionProgramIds.has(cycle.programId)) continue + const sourceIds = stringArray(cycle.sourceIds) + for (const sourceId of sourceIds) linkedSourceIds.add(sourceId) + const program = programById.get(cycle.programId) + addReconciliationCandidate( + reconciliationCandidates, + 'program', + cycle.programId, + localizedName(program?.name, cycle.programId), + sourceIds, + ) + } + for (const scholarship of input.scholarships) { + if (typeof scholarship.id !== 'string' + || !stringArray(scholarship.universityIds).includes(institutionId)) continue + const sourceIds = stringArray(scholarship.sourceIds) + for (const sourceId of sourceIds) { + linkedSourceIds.add(sourceId) + if (scholarship.providerType === 'university') { + universityScholarshipSourceIds.add(sourceId) + } + } + addReconciliationCandidate( + reconciliationCandidates, + 'scholarship', + scholarship.id, + localizedName(scholarship.name, scholarship.id), + sourceIds, + ) + } + + const rejectedSources: SourceRejection[] = [] + const safeSources = [...linkedSourceIds].sort().flatMap((sourceId) => { + const inspected = inspectSource(sourceById.get(sourceId)) + if ('rejection' in inspected) { + rejectedSources.push({ sourceId, reason: inspected.rejection }) + return [] + } + return [inspected.source] + }) + const safeRawSourceIds = new Set(safeSources.map((source) => source.id)) + const manifestSources = safeSources.map((source) => buildFetchManifest( + source, + institutionId, + categoryForSource(source, universityScholarshipSourceIds), + )) + const manifestSourceIdByRawId = new Map( + manifestSources.map((source, index) => [safeSources[index]!.id, source.id]), + ) + const reconciliationEntries = [...reconciliationCandidates.values()] + .sort((left, right) => ( + left.entityType.localeCompare(right.entityType) + || left.officialKey.localeCompare(right.officialKey) + )) + .flatMap((entry) => { + const rawSourceId = [...entry.rawSourceIds].filter((id) => safeRawSourceIds.has(id)).sort()[0] + if (!rawSourceId) return [] + return [{ + sourceId: manifestSourceIdByRawId.get(rawSourceId)!, + officialKey: entry.officialKey, + officialName: entry.officialName, + entityType: entry.entityType, + status: 'pending' as const, + }] + }) + const sourcesByCategory = new Map() + for (const source of manifestSources) { + const ids = sourcesByCategory.get(source.sourceCategory) ?? [] + ids.push(source.id) + sourcesByCategory.set(source.sourceCategory, ids) + } + const coverage: SourceManifestV2['coverage'] = SOURCE_CATEGORIES.map((sourceCategory) => { + const sourceIds = sourcesByCategory.get(sourceCategory)?.sort() + if (sourceIds?.length) { + return { + sourceCategory, + status: 'parser_pending' as const, + sourceIds, + note: 'Exact official HTTPS source mapped from current catalog relationships; parser, robots, and evidence-locator review remain pending.', + } + } + return { + sourceCategory, + status: 'discovery_pending' as const, + note: 'No exact official HTTPS source with an auditable current catalog relationship is available for this category.', + } + }) + institutionCoverage.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + institutionId, + mappedSourceCount: manifestSources.length, + reconciliationEntryCount: reconciliationEntries.length, + mappedCategories: SOURCE_CATEGORIES.filter((category) => sourcesByCategory.has(category)), + discoveryPendingCategories: SOURCE_CATEGORIES.filter((category) => !sourcesByCategory.has(category)), + rejectedSources: rejectedSources.sort((left, right) => left.sourceId.localeCompare(right.sourceId)), + }) + if (manifestSources.length === 0 || reconciliationEntries.length === 0) { + gaps.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + institutionId, + code: 'no_safe_entity_source', + note: 'No program or scholarship entity has an exact, official, HTTPS source relationship suitable for a V2 candidate.', + }) + continue + } + const manifest = sourceManifestV2Schema.parse({ + version: 2, + institutionId, + catalogStatus: 'existing', + manifestStatus: 'in_progress', + checkedAt: input.checkedAt, + officialHosts: [...new Set(safeSources.map((source) => source.host))].sort(), + sources: manifestSources, + coverage, + catalogReconciliation: { + scope: 'representative_international_programs', + status: 'in_progress', + entries: reconciliationEntries, + note: 'Candidate reconciliation is generated only from exact current catalog relationships; every entry remains pending until source-level evidence is audited.', + }, + }) + candidates.push({ + fileName: fileNameForTarget(target, institutionId), + manifest, + }) + } + + const summary = { + officialTargets: input.registry.targets.length, + militaryExcluded: militaryExclusions.length, + eligibleTargets: input.registry.targets.length - militaryExclusions.length, + candidateManifests: candidates.length, + exactOfficialHttpsSources: candidates.reduce( + (total, candidate) => total + candidate.manifest.sources.length, + 0, + ), + targetsWithoutCandidate: gaps.length, + } + const gapReport: SourceManifestCohortGapReport = { + format: 'studyinchina.source-manifest-v2-gap-report', + formatVersion: 1, + cohortId: input.registry.cohort.id, + checkedAt: input.checkedAt, + policy: { + mapping: 'Only exact sourceIds already related by current university, program, admission-cycle, or scholarship records are eligible; publisher and institution names are never fuzzy-matched.', + missingCoverage: 'Unmapped categories are discovery_pending with an explicit note.', + officialAbsence: 'officially_not_provided is never inferred; it requires separate explicit official evidence.', + }, + summary, + militaryExclusions, + gaps, + institutionCoverage, + } + return { candidates, gapReport, summary } +} + +function isInside(parent: string, child: string): boolean { + const path = relative(parent, child) + return path === '' || (!path.startsWith('..') && !path.startsWith(`..\\`) && !path.startsWith('../')) +} + +export function writeSourceManifestCohort( + build: SourceManifestCohortBuild, + outputDirectory: string, +): { outputDirectory: string; manifestDirectory: string; gapReportPath: string } { + const output = resolve(outputDirectory) + const formalManifestDirectory = resolve('content/source-manifests') + if (isInside(formalManifestDirectory, output)) { + throw new Error('Candidate output must not be inside content/source-manifests') + } + const manifestDirectory = join(output, 'manifests') + mkdirSync(manifestDirectory, { recursive: true }) + for (const candidate of build.candidates) { + writeFileSync( + join(manifestDirectory, candidate.fileName), + `${JSON.stringify(candidate.manifest, null, 2)}\n`, + 'utf8', + ) + } + const gapReportPath = join(output, 'gap-report.v1.json') + writeFileSync(gapReportPath, `${JSON.stringify(build.gapReport, null, 2)}\n`, 'utf8') + return { outputDirectory: output, manifestDirectory, gapReportPath } +} + +export function dryRunSummary(build: SourceManifestCohortBuild): string { + return JSON.stringify({ mode: 'dry-run', ...build.summary }) +} + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name) + return index >= 0 ? process.argv[index + 1] : undefined +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(resolve(path), 'utf8')) as T +} + +function runCli(): void { + const checkedAt = option('--checked-at') + if (!checkedAt) { + throw new Error('Usage requires --checked-at and either --dry-run or --output ') + } + const dryRun = process.argv.includes('--dry-run') + const output = option('--output') + if (!dryRun && !output) { + throw new Error('Refusing to write without an explicit --output directory; use --dry-run for a no-write summary') + } + const registry = validateDoubleFirstClassRegistry(readJson( + option('--registry') ?? 'content/source-manifests/double-first-class/targets.v1.json', + )) + const build = buildSourceManifestCohort({ + registry: registry as DoubleFirstClassRegistry, + universities: readJson(option('--universities') ?? 'content/data/universities.json'), + sources: readJson(option('--sources') ?? 'content/data/sources.json'), + programs: readJson(option('--programs') ?? 'content/data/programs.json'), + admissionCycles: readJson(option('--cycles') ?? 'content/data/admission-cycles.json'), + scholarships: readJson(option('--scholarships') ?? 'content/data/scholarships.json'), + checkedAt, + }) + if (dryRun) { + process.stdout.write(`${dryRunSummary(build)}\n`) + return + } + const written = writeSourceManifestCohort(build, output!) + process.stdout.write(`${JSON.stringify({ mode: 'write', ...build.summary, ...written })}\n`) +} + +if (resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) { + try { + runCli() + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) + } +} diff --git a/scripts/quality/inventory-untracked-assets.ts b/scripts/quality/inventory-untracked-assets.ts new file mode 100644 index 0000000..c597f07 --- /dev/null +++ b/scripts/quality/inventory-untracked-assets.ts @@ -0,0 +1,424 @@ +import { createHash } from 'node:crypto' +import { createReadStream } from 'node:fs' +import { lstat, readlink, writeFile } from 'node:fs/promises' +import { extname, isAbsolute, relative, resolve, sep } from 'node:path' +import process from 'node:process' +import { spawnSync } from 'node:child_process' +import { pathToFileURL } from 'node:url' + +export type UntrackedContentClass = + | 'raw_evidence' + | 'structured_candidate' + | 'source_code' + | 'test' + | 'database_migration' + | 'configuration' + | 'documentation' + | 'temporary' + | 'symlink' + | 'unclassified' + +export type UntrackedSuggestedStatus = + | 'private_r2_raw_evidence' + | 'quarantine_candidate' + | 'code_test_review' + | 'temp_ignore' + +export type UntrackedClassification = { + contentClass: UntrackedContentClass + suggestedStatus: UntrackedSuggestedStatus + classificationReason: string +} + +export type UntrackedInventoryEntry = UntrackedClassification & { + path: string + extension: string + byteSize: number + sha256: string +} + +export type UntrackedInventory = { + schemaVersion: 1 + generatedAt: string + repositoryRoot: string + summary: { + totalFiles: number + totalBytes: number + byContentClass: Record + bySuggestedStatus: Record + } + files: UntrackedInventoryEntry[] +} + +type InventoryOptions = { + repositoryPath: string + excludedRepositoryPaths?: ReadonlySet + generatedAt?: string +} + +type CliOptions = { + repositoryPath: string + outputPath: string +} + +const CODE_EXTENSIONS = new Set([ + '.cjs', '.css', '.js', '.jsx', '.mjs', '.scss', '.sh', '.ps1', '.ts', '.tsx', +]) +const CONFIG_EXTENSIONS = new Set([ + '.env', '.ini', '.jsonc', '.toml', '.yaml', '.yml', +]) +const DOCUMENT_EXTENSIONS = new Set(['.md', '.mdx', '.rst']) +const RAW_EVIDENCE_EXTENSIONS = new Set([ + '.doc', '.docx', '.gif', '.htm', '.html', '.jpeg', '.jpg', '.pdf', '.png', + '.tif', '.tiff', '.webp', '.xls', '.xlsx', +]) + +function normalizeRepositoryPath(filePath: string): string { + return filePath.replaceAll('\\', '/') +} + +function pathSegments(filePath: string): string[] { + return normalizeRepositoryPath(filePath).toLowerCase().split('/').filter(Boolean) +} + +function hasSegmentMatching(segments: string[], pattern: RegExp): boolean { + return segments.some((segment) => pattern.test(segment)) +} + +export function classifyUntrackedPath(filePath: string): UntrackedClassification { + const normalized = normalizeRepositoryPath(filePath) + const lowerPath = normalized.toLowerCase() + const segments = pathSegments(normalized) + const baseName = segments.at(-1) ?? '' + const extension = extname(baseName).toLowerCase() + + const isTemporary = baseName.startsWith('.tmp') + || baseName.startsWith('~') + || baseName.endsWith('~') + || ['.bak', '.log', '.swp', '.temp', '.tmp'].includes(extension) + || hasSegmentMatching(segments, /^(?:\.cache|\.next|coverage|temp|tmp)$/u) + if (isTemporary) { + return { + contentClass: 'temporary', + suggestedStatus: 'temp_ignore', + classificationReason: 'Temporary output, cache, log, or editor artifact', + } + } + + const isTest = segments.includes('tests') + || /(?:^|\.)test\.[^.]+$/u.test(baseName) + || /(?:^|\.)spec\.[^.]+$/u.test(baseName) + if (isTest) { + return { + contentClass: 'test', + suggestedStatus: 'code_test_review', + classificationReason: 'Test asset requires normal code review', + } + } + + const isCodeArea = ['.github', 'infra', 'scripts', 'src', 'workers'].includes(segments[0] ?? '') + || lowerPath.startsWith('content/source-manifests/') + if (isCodeArea || CODE_EXTENSIONS.has(extension)) { + const contentClass: UntrackedContentClass = extension === '.sql' + ? 'database_migration' + : CONFIG_EXTENSIONS.has(extension) + ? 'configuration' + : DOCUMENT_EXTENSIONS.has(extension) + ? 'documentation' + : 'source_code' + return { + contentClass, + suggestedStatus: 'code_test_review', + classificationReason: 'Repository implementation or reviewed configuration asset', + } + } + + const hasRawEvidenceMarker = hasSegmentMatching( + segments, + /^(?:capture|captures|evidence|raw|snapshot|snapshots|screenshots?)$/u, + ) + if (hasRawEvidenceMarker || RAW_EVIDENCE_EXTENSIONS.has(extension)) { + return { + contentClass: 'raw_evidence', + suggestedStatus: 'private_r2_raw_evidence', + classificationReason: 'Raw source evidence belongs in private R2 archival storage', + } + } + + const isQuarantineCandidate = segments.includes('quality') + || segments.includes('claims') + || segments.includes('completed') + || hasSegmentMatching(segments, /^(?:minimax-(?:expansion|harvest|recapture)|pending|quarantined?.*)$/u) + || lowerPath.startsWith('content/data/') + if (isQuarantineCandidate || extension === '.json' || extension === '.jsonl') { + return { + contentClass: 'structured_candidate', + suggestedStatus: 'quarantine_candidate', + classificationReason: 'Structured data must pass validation before promotion', + } + } + + if (CONFIG_EXTENSIONS.has(extension)) { + return { + contentClass: 'configuration', + suggestedStatus: 'code_test_review', + classificationReason: 'Configuration asset requires normal code review', + } + } + + if (DOCUMENT_EXTENSIONS.has(extension)) { + return { + contentClass: 'documentation', + suggestedStatus: 'code_test_review', + classificationReason: 'Documentation asset requires normal review', + } + } + + return { + contentClass: 'unclassified', + suggestedStatus: 'quarantine_candidate', + classificationReason: 'Unknown asset type defaults to quarantine for manual review', + } +} + +function runGit(repositoryPath: string, args: string[], encoding: BufferEncoding | 'buffer'): string | Buffer { + const result = spawnSync('git', args, { + cwd: repositoryPath, + encoding: encoding === 'buffer' ? undefined : encoding, + maxBuffer: 128 * 1024 * 1024, + shell: false, + windowsHide: true, + }) + if (result.error) throw result.error + if (result.status !== 0) { + const stderr = Buffer.isBuffer(result.stderr) + ? result.stderr.toString('utf8') + : String(result.stderr ?? '') + throw new Error(`git ${args.join(' ')} failed: ${stderr.trim()}`) + } + return result.stdout ?? (encoding === 'buffer' ? Buffer.alloc(0) : '') +} + +export function resolveRepositoryRoot(repositoryPath: string): string { + return String(runGit(resolve(repositoryPath), ['rev-parse', '--show-toplevel'], 'utf8')).trim() +} + +export function listUntrackedPaths(repositoryRoot: string): string[] { + const stdout = runGit( + repositoryRoot, + ['ls-files', '--others', '--exclude-standard', '-z'], + 'buffer', + ) + return (stdout as Buffer) + .toString('utf8') + .split('\0') + .filter(Boolean) + .map(normalizeRepositoryPath) + .sort((left, right) => left.localeCompare(right, 'en')) +} + +function resolveSafeRepositoryFile(repositoryRoot: string, repositoryPath: string): string { + const absolutePath = resolve(repositoryRoot, repositoryPath) + const relativePath = relative(repositoryRoot, absolutePath) + if (relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + throw new Error(`Refusing to read a path outside the repository: ${repositoryPath}`) + } + return absolutePath +} + +async function sha256RegularFile(filePath: string): Promise { + const hash = createHash('sha256') + await new Promise((resolvePromise, reject) => { + const stream = createReadStream(filePath) + stream.on('data', (chunk: string | Buffer) => { + hash.update(chunk) + }) + stream.on('error', reject) + stream.on('end', resolvePromise) + }) + return hash.digest('hex') +} + +export async function createUntrackedInventoryEntry( + repositoryRoot: string, + repositoryPath: string, +): Promise { + const normalizedPath = normalizeRepositoryPath(repositoryPath) + const absolutePath = resolveSafeRepositoryFile(repositoryRoot, normalizedPath) + const metadata = await lstat(absolutePath) + const extension = extname(normalizedPath).toLowerCase() || '[none]' + + if (metadata.isSymbolicLink()) { + const linkTarget = await readlink(absolutePath) + return { + path: normalizedPath, + extension, + byteSize: metadata.size, + sha256: createHash('sha256').update(linkTarget, 'utf8').digest('hex'), + contentClass: 'symlink', + suggestedStatus: 'quarantine_candidate', + classificationReason: 'Symlink target text hashed without following the link', + } + } + if (!metadata.isFile()) { + throw new Error(`Untracked path is not a regular file: ${normalizedPath}`) + } + + return { + path: normalizedPath, + extension, + byteSize: metadata.size, + sha256: await sha256RegularFile(absolutePath), + ...classifyUntrackedPath(normalizedPath), + } +} + +function increment(counter: Record, key: string): void { + counter[key] = (counter[key] ?? 0) + 1 +} + +export function assembleUntrackedInventory( + repositoryRoot: string, + files: UntrackedInventoryEntry[], + generatedAt = new Date().toISOString(), +): UntrackedInventory { + const byContentClass: Record = {} + const bySuggestedStatus: Record = {} + let totalBytes = 0 + + for (const file of files) { + totalBytes += file.byteSize + increment(byContentClass, file.contentClass) + increment(bySuggestedStatus, file.suggestedStatus) + } + + return { + schemaVersion: 1, + generatedAt, + repositoryRoot, + summary: { + totalFiles: files.length, + totalBytes, + byContentClass, + bySuggestedStatus, + }, + files, + } +} + +export async function buildUntrackedInventory(options: InventoryOptions): Promise { + const repositoryRoot = resolveRepositoryRoot(options.repositoryPath) + const excludedPaths = options.excludedRepositoryPaths ?? new Set() + const paths = listUntrackedPaths(repositoryRoot).filter((filePath) => !excludedPaths.has(filePath)) + const files: UntrackedInventoryEntry[] = [] + + // Sequential hashing keeps disk pressure predictable on developer machines and CI runners. + for (const filePath of paths) { + files.push(await createUntrackedInventoryEntry(repositoryRoot, filePath)) + } + return assembleUntrackedInventory(repositoryRoot, files, options.generatedAt) +} + +function usage(): string { + return `Usage: tsx scripts/quality/inventory-untracked-assets.ts --output [--repo ] + +Creates a read-only inventory of git-untracked files. The command only writes the +explicit JSON output file; it never deletes, moves, uploads, stages, or commits assets. + +Options: + --output Required JSON report path. Its parent directory must exist. + --repo Repository or subdirectory to inspect (default: current directory). + --help Show this help. +` +} + +export function parseInventoryArgs(argv: string[]): CliOptions { + let repositoryPath = process.cwd() + let outputPath = '' + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] + const nextValue = (): string => { + index += 1 + if (index >= argv.length || argv[index].startsWith('--')) { + throw new Error(`${argument} requires a value`) + } + return argv[index] + } + + switch (argument) { + case '--output': + outputPath = nextValue() + break + case '--repo': + repositoryPath = nextValue() + break + case '--help': + case '-h': + throw new Error(usage()) + default: + throw new Error(`Unknown argument: ${argument}\n\n${usage()}`) + } + } + + if (!outputPath) throw new Error(`--output is required\n\n${usage()}`) + if (extname(outputPath).toLowerCase() !== '.json') { + throw new Error('--output must use a .json extension') + } + return { repositoryPath, outputPath } +} + +function repositoryRelativePath(repositoryRoot: string, filePath: string): string | null { + const relativePath = relative(repositoryRoot, filePath) + if (relativePath === '' || relativePath === '..' || relativePath.startsWith(`..${sep}`) || isAbsolute(relativePath)) { + return null + } + return normalizeRepositoryPath(relativePath) +} + +function assertOutputIsSafe(repositoryRoot: string, outputPath: string): string | null { + const relativeOutputPath = repositoryRelativePath(repositoryRoot, outputPath) + if (!relativeOutputPath) return null + if (relativeOutputPath === '.git' || relativeOutputPath.startsWith('.git/')) { + throw new Error('Refusing to write the report inside .git') + } + + const tracked = spawnSync('git', ['ls-files', '--error-unmatch', '--', relativeOutputPath], { + cwd: repositoryRoot, + encoding: 'utf8', + shell: false, + windowsHide: true, + }) + if (tracked.status === 0) { + throw new Error(`Refusing to overwrite a tracked repository file: ${relativeOutputPath}`) + } + return relativeOutputPath +} + +export async function runUntrackedInventoryCli(argv: string[]): Promise { + const options = parseInventoryArgs(argv) + const repositoryRoot = resolveRepositoryRoot(options.repositoryPath) + const outputPath = resolve(options.outputPath) + const relativeOutputPath = assertOutputIsSafe(repositoryRoot, outputPath) + const excludedPaths = relativeOutputPath + ? new Set([relativeOutputPath]) + : new Set() + const inventory = await buildUntrackedInventory({ + repositoryPath: repositoryRoot, + excludedRepositoryPaths: excludedPaths, + }) + + await writeFile(outputPath, `${JSON.stringify(inventory, null, 2)}\n`, { + encoding: 'utf8', + flag: 'w', + }) +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + runUntrackedInventoryCli(process.argv.slice(2)).catch((error: unknown) => { + process.stderr.write( + `Untracked asset inventory failed: ${error instanceof Error ? error.message : String(error)}\n`, + ) + process.exitCode = 2 + }) +} diff --git a/scripts/quality/platform-data-quality.ts b/scripts/quality/platform-data-quality.ts new file mode 100644 index 0000000..4ba6ae8 --- /dev/null +++ b/scripts/quality/platform-data-quality.ts @@ -0,0 +1,366 @@ +import { readFileSync } from 'node:fs' +import { writeFile } from 'node:fs/promises' +import { join, resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { bundleSchema } from '../../src/lib/data/schema' +import { getTodayDate } from '../../src/lib/data/freshness' +import { selectPublishedData } from '../../src/lib/data/publication' +import type { DataBundle } from '../../src/lib/data/types' +import { + isCatalogReconciliationComplete, + validateSourceManifestDirectory, + type SourceManifestRecord, +} from '../source-manifest-registry' + +export const FOUR_WEEK_QUALITY_THRESHOLDS = { + publicUniversities: 257, + schoolsBelowThreePrograms: 0, + currentCycleCoveragePct: 70, + durationCoveragePct: 90, + applicationUrlCoveragePct: 80, + teachingLanguageCoveragePct: 95, + requirementsCoveragePct: 50, + scholarshipUniversityCoverage: 230, + citiesWithCoordinates: 60, + manifestInstitutions: 257, + completedManifests: 257, + completedReconciliations: 257, + verifiedOverdueRecords: 0, + publishedCyclesWithoutAnyDate: 0, +} as const + +export type QualityGate = { + metric: keyof typeof FOUR_WEEK_QUALITY_THRESHOLDS + comparison: 'at_least' | 'at_most' + actual: number + target: number + passed: boolean +} + +export type PlatformDataQualityScorecard = { + schemaVersion: 1 + generatedAt: string + evaluatedForDate: string + metrics: { + publicRecords: { + universities: number + programs: number + scholarships: number + admissionCycles: number + cities: number + } + programCoverage: { + schoolsBelowThreePrograms: number + schoolIdsBelowThreePrograms: string[] + programsWithCurrentCycle: number + currentCycleCoveragePct: number + programsWithDuration: number + durationCoveragePct: number + programsWithApplicationUrl: number + applicationUrlCoveragePct: number + programsWithTeachingLanguage: number + teachingLanguageCoveragePct: number + programsWithRequirements: number + requirementsCoveragePct: number + } + scholarships: { + universitiesCovered: number + universityCoveragePct: number + recordsWithDeadline: number + deadlineCoveragePct: number + } + cities: { + withCoordinates: number + coordinateCoveragePct: number + } + sourceManifests: { + institutionsRegistered: number + publicUniversitiesRegistered: number + publicUniversityCoveragePct: number + v2Manifests: number + completedManifests: number + completedReconciliations: number + } + anomalies: { + verifiedOverdueRecords: number + staleStatusRecords: number + publishedCyclesWithoutAnyDate: number + } + } + gates: { + passed: number + failed: number + total: number + allPassed: boolean + checks: QualityGate[] + } +} + +type BuildOptions = { + today?: string + generatedAt?: string +} + +type CliOptions = { + outputPath?: string + strict: boolean + today?: string +} + +function percent(numerator: number, denominator: number): number { + if (denominator === 0) return 0 + return Math.round((numerator / denominator) * 10_000) / 100 +} + +function allAuditedRecords(bundle: DataBundle) { + return [ + ...bundle.cities, + ...bundle.universities, + ...bundle.programs, + ...bundle.admissionCycles, + ...bundle.scholarships, + ] +} + +function atLeast( + metric: keyof typeof FOUR_WEEK_QUALITY_THRESHOLDS, + actual: number, +): QualityGate { + const target = FOUR_WEEK_QUALITY_THRESHOLDS[metric] + return { metric, comparison: 'at_least', actual, target, passed: actual >= target } +} + +function atMost( + metric: keyof typeof FOUR_WEEK_QUALITY_THRESHOLDS, + actual: number, +): QualityGate { + const target = FOUR_WEEK_QUALITY_THRESHOLDS[metric] + return { metric, comparison: 'at_most', actual, target, passed: actual <= target } +} + +export function buildPlatformDataQualityScorecard( + rawBundle: DataBundle, + manifests: SourceManifestRecord[], + options: BuildOptions = {}, +): PlatformDataQualityScorecard { + const today = options.today ?? getTodayDate() + const publicBundle = selectPublishedData(rawBundle, today) + const programCounts = new Map() + for (const program of publicBundle.programs) { + programCounts.set(program.universityId, (programCounts.get(program.universityId) ?? 0) + 1) + } + + const schoolIdsBelowThreePrograms = publicBundle.universities + .filter((university) => (programCounts.get(university.id) ?? 0) < 3) + .map((university) => university.id) + .sort() + const programsWithCycles = new Set( + publicBundle.admissionCycles.map((cycle) => cycle.programId), + ) + const programsWithDuration = publicBundle.programs.filter( + (program) => program.durationMonths !== null && program.durationMonths > 0, + ).length + const programsWithApplicationUrl = publicBundle.programs.filter( + (program) => Boolean(program.applyUrl?.trim()), + ).length + const programsWithTeachingLanguage = publicBundle.programs.filter( + (program) => program.teachingLanguages.some((language) => language.trim().length > 0), + ).length + const programsWithRequirements = publicBundle.programs.filter( + (program) => program.languageRequirements.length > 0 + || Boolean(program.details?.eligibility.some((item) => Object.values(item).some(Boolean))), + ).length + + const publicUniversityIds = new Set(publicBundle.universities.map((item) => item.id)) + const scholarshipUniversityIds = new Set( + publicBundle.scholarships + .flatMap((scholarship) => scholarship.universityIds) + .filter((id) => publicUniversityIds.has(id)), + ) + const scholarshipsWithDeadline = publicBundle.scholarships.filter( + (scholarship) => scholarship.deadline !== null, + ).length + const citiesWithCoordinates = publicBundle.cities.filter( + (city) => city.coordinates !== null, + ).length + + const manifestInstitutionIds = new Set(manifests.map((manifest) => manifest.institutionId)) + const v2Manifests = manifests.filter( + (manifest): manifest is Extract => manifest.version === 2, + ) + const completedManifests = v2Manifests.filter( + (manifest) => manifest.manifestStatus === 'complete', + ).length + const completedReconciliations = manifests.filter(isCatalogReconciliationComplete).length + + const auditedRecords = allAuditedRecords(rawBundle) + const verifiedOverdueRecords = auditedRecords.filter( + (record) => record.status === 'verified' && record.reviewAfter < today, + ).length + const staleStatusRecords = auditedRecords.filter((record) => record.status === 'stale').length + const publishedCyclesWithoutAnyDate = rawBundle.admissionCycles.filter( + (cycle) => cycle.dateStatus === 'published' + && cycle.opensOn === null + && cycle.closesOn === null, + ).length + + const programTotal = publicBundle.programs.length + const universityTotal = publicBundle.universities.length + const scholarshipTotal = publicBundle.scholarships.length + const cityTotal = publicBundle.cities.length + const metrics: PlatformDataQualityScorecard['metrics'] = { + publicRecords: { + universities: universityTotal, + programs: programTotal, + scholarships: scholarshipTotal, + admissionCycles: publicBundle.admissionCycles.length, + cities: cityTotal, + }, + programCoverage: { + schoolsBelowThreePrograms: schoolIdsBelowThreePrograms.length, + schoolIdsBelowThreePrograms, + programsWithCurrentCycle: programsWithCycles.size, + currentCycleCoveragePct: percent(programsWithCycles.size, programTotal), + programsWithDuration, + durationCoveragePct: percent(programsWithDuration, programTotal), + programsWithApplicationUrl, + applicationUrlCoveragePct: percent(programsWithApplicationUrl, programTotal), + programsWithTeachingLanguage, + teachingLanguageCoveragePct: percent(programsWithTeachingLanguage, programTotal), + programsWithRequirements, + requirementsCoveragePct: percent(programsWithRequirements, programTotal), + }, + scholarships: { + universitiesCovered: scholarshipUniversityIds.size, + universityCoveragePct: percent(scholarshipUniversityIds.size, universityTotal), + recordsWithDeadline: scholarshipsWithDeadline, + deadlineCoveragePct: percent(scholarshipsWithDeadline, scholarshipTotal), + }, + cities: { + withCoordinates: citiesWithCoordinates, + coordinateCoveragePct: percent(citiesWithCoordinates, cityTotal), + }, + sourceManifests: { + institutionsRegistered: manifestInstitutionIds.size, + publicUniversitiesRegistered: [...manifestInstitutionIds] + .filter((id) => publicUniversityIds.has(id)).length, + publicUniversityCoveragePct: percent( + [...manifestInstitutionIds].filter((id) => publicUniversityIds.has(id)).length, + universityTotal, + ), + v2Manifests: v2Manifests.length, + completedManifests, + completedReconciliations, + }, + anomalies: { + verifiedOverdueRecords, + staleStatusRecords, + publishedCyclesWithoutAnyDate, + }, + } + + const checks = [ + atLeast('publicUniversities', metrics.publicRecords.universities), + atMost('schoolsBelowThreePrograms', metrics.programCoverage.schoolsBelowThreePrograms), + atLeast('currentCycleCoveragePct', metrics.programCoverage.currentCycleCoveragePct), + atLeast('durationCoveragePct', metrics.programCoverage.durationCoveragePct), + atLeast('applicationUrlCoveragePct', metrics.programCoverage.applicationUrlCoveragePct), + atLeast('teachingLanguageCoveragePct', metrics.programCoverage.teachingLanguageCoveragePct), + atLeast('requirementsCoveragePct', metrics.programCoverage.requirementsCoveragePct), + atLeast('scholarshipUniversityCoverage', metrics.scholarships.universitiesCovered), + atLeast('citiesWithCoordinates', metrics.cities.withCoordinates), + atLeast('manifestInstitutions', metrics.sourceManifests.institutionsRegistered), + atLeast('completedManifests', metrics.sourceManifests.completedManifests), + atLeast('completedReconciliations', metrics.sourceManifests.completedReconciliations), + atMost('verifiedOverdueRecords', metrics.anomalies.verifiedOverdueRecords), + atMost('publishedCyclesWithoutAnyDate', metrics.anomalies.publishedCyclesWithoutAnyDate), + ] + const passed = checks.filter((check) => check.passed).length + + return { + schemaVersion: 1, + generatedAt: options.generatedAt ?? new Date().toISOString(), + evaluatedForDate: today, + metrics, + gates: { + passed, + failed: checks.length - passed, + total: checks.length, + allPassed: passed === checks.length, + checks, + }, + } +} + +export function parsePlatformDataQualityArgs(argv: string[]): CliOptions { + const options: CliOptions = { strict: false } + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index] + if (argument === '--strict') { + options.strict = true + } else if (argument === '--output' || argument === '--today') { + const value = argv[index + 1] + if (!value || value.startsWith('--')) throw new Error(`${argument} requires a value`) + index += 1 + if (argument === '--output') options.outputPath = value + else options.today = value + } else { + throw new Error(`Unknown argument: ${argument}`) + } + } + if (options.today && !/^\d{4}-\d{2}-\d{2}$/.test(options.today)) { + throw new Error('--today must use YYYY-MM-DD') + } + return options +} + +function loadRawCatalog(): DataBundle { + const read = (name: string): unknown => JSON.parse( + readFileSync(join(process.cwd(), 'content', 'data', `${name}.json`), 'utf8'), + ) + return bundleSchema.parse({ + sources: read('sources'), + cities: read('cities'), + universities: read('universities'), + programs: read('programs'), + admissionCycles: read('admission-cycles'), + scholarships: read('scholarships'), + }) +} + +export function conciseScorecardSummary(report: PlatformDataQualityScorecard): string { + const { publicRecords, programCoverage, scholarships, cities, sourceManifests } = report.metrics + return [ + `Data quality: ${publicRecords.universities} universities / ${publicRecords.programs} programs / ${publicRecords.scholarships} scholarships`, + `current cycles ${programCoverage.currentCycleCoveragePct}%`, + `scholarship schools ${scholarships.universitiesCovered}`, + `city coordinates ${cities.withCoordinates}/${publicRecords.cities}`, + `manifests ${sourceManifests.institutionsRegistered} (${sourceManifests.completedReconciliations} reconciled)`, + `gates ${report.gates.passed}/${report.gates.total}`, + ].join(' | ') +} + +export async function runPlatformDataQualityCli(argv: string[]): Promise { + const options = parsePlatformDataQualityArgs(argv) + const report = buildPlatformDataQualityScorecard( + loadRawCatalog(), + validateSourceManifestDirectory(), + { today: options.today }, + ) + if (options.outputPath) { + await writeFile(resolve(options.outputPath), `${JSON.stringify(report, null, 2)}\n`, 'utf8') + } else { + process.stdout.write(`${conciseScorecardSummary(report)}\n`) + } + return options.strict && !report.gates.allPassed ? 1 : 0 +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + runPlatformDataQualityCli(process.argv.slice(2)).then( + (exitCode) => { process.exitCode = exitCode }, + (error: unknown) => { + process.stderr.write(`Platform data-quality report failed: ${error instanceof Error ? error.message : String(error)}\n`) + process.exitCode = 2 + }, + ) +} diff --git a/scripts/source-manifest-registry.ts b/scripts/source-manifest-registry.ts new file mode 100644 index 0000000..ba9fb74 --- /dev/null +++ b/scripts/source-manifest-registry.ts @@ -0,0 +1,447 @@ +import { readdirSync, readFileSync } from 'node:fs' +import { basename, join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import { z } from 'zod' +import { + SOURCE_CATEGORIES, + sourceManifestSchema, +} from '../workers/ingestion/src/manifest-schema' +import { + normalizeAllowedHost, + validateManifest, +} from '../workers/ingestion/src/security' +import type { SourceCategory, SourceManifestV1 } from '../workers/ingestion/src/types' +import { + INSTITUTION_HOST_ALLOWLISTS, + pilotSourceManifestSchema, + type PilotSourceManifest, +} from './validate-source-manifests' + +export const CATALOG_RECONCILIATION_STATUSES = [ + 'published', + 'individual_application_unavailable', + 'discontinued', + 'not_announced', + 'source_unavailable', + 'pending', +] as const + +export type CatalogReconciliationStatus = + (typeof CATALOG_RECONCILIATION_STATUSES)[number] + +const coverageStatusSchema = z.enum([ + 'registered', + 'parser_pending', + 'source_unavailable', + 'discovery_pending', + 'officially_not_provided', +]) + +const coverageSchema = z.object({ + sourceCategory: z.enum(SOURCE_CATEGORIES), + status: coverageStatusSchema, + sourceIds: z.array(z.string().min(1)).optional(), + note: z.string().min(1).optional(), +}).strict() + +const checkedAtSchema = z.string().regex(/^\d{4}-\d{2}-\d{2}$/).refine( + (value) => !Number.isNaN(Date.parse(`${value}T00:00:00Z`)), + { message: 'checkedAt must be a real ISO calendar date' }, +) + +const reconciliationEntrySchema = z.object({ + sourceId: z.string().min(1), + officialKey: z.string().min(1), + officialName: z.string().min(1), + entityType: z.enum(['program', 'scholarship']), + status: z.enum(CATALOG_RECONCILIATION_STATUSES), + recordId: z.string().min(1).optional(), + note: z.string().min(1).optional(), +}).strict() + +export const sourceManifestV2Schema = z.object({ + version: z.literal(2), + institutionId: z.string().min(1), + catalogStatus: z.enum(['existing', 'planned_addition']), + manifestStatus: z.enum(['complete', 'in_progress']), + checkedAt: checkedAtSchema, + officialHosts: z.array(z.string().min(1)).min(1), + // Individual fetch manifests remain V1 because this is the format consumed + // by the ingestion Worker. V2 describes the institution-level contract. + sources: z.array(sourceManifestSchema).min(1), + coverage: z.array(coverageSchema).length(SOURCE_CATEGORIES.length), + catalogReconciliation: z.object({ + scope: z.enum([ + 'full_official_catalog', + 'representative_international_programs', + 'limited_official_catalog', + ]), + status: z.enum(['complete', 'in_progress']), + entries: z.array(reconciliationEntrySchema).min(1), + note: z.string().min(1).optional(), + }).strict(), +}).strict() + +export type SourceManifestV2 = z.infer +export type SourceManifestRecord = PilotSourceManifest | SourceManifestV2 + +export type LoadedSourceManifest = { + filePath: string + value: unknown +} + +function manifestFiles(directory: string): string[] { + return readdirSync(directory, { withFileTypes: true }) + .flatMap((entry) => { + const path = join(directory, entry.name) + if (entry.isDirectory()) return manifestFiles(path) + if (!entry.isFile() || !entry.name.endsWith('.json')) return [] + // Target registries describe cohorts, not fetchable institution manifests. + if (/^targets(?:\.|-).*\.json$/i.test(entry.name)) return [] + return [path] + }) + .sort((left, right) => left.localeCompare(right)) +} + +export function loadSourceManifestFiles( + directory = join(process.cwd(), 'content', 'source-manifests'), +): LoadedSourceManifest[] { + return manifestFiles(directory).map((filePath) => ({ + filePath, + value: JSON.parse(readFileSync(filePath, 'utf8')) as unknown, + })) +} + +function errorMessage(filePath: string, message: string): string { + return `${basename(filePath)}: ${message}` +} + +function approvedHosts(record: SourceManifestRecord): Set { + if (record.version === 2) { + return new Set(record.officialHosts.map(normalizeAllowedHost)) + } + const legacy = INSTITUTION_HOST_ALLOWLISTS[ + record.institutionId as keyof typeof INSTITUTION_HOST_ALLOWLISTS + ] + if (legacy) return new Set(legacy.map(normalizeAllowedHost)) + return new Set( + record.sources.flatMap((source) => [ + ...source.allowedHosts, + ...(source.allowedRedirectHosts ?? []), + ]).map(normalizeAllowedHost), + ) +} + +function validateCoverage( + record: SourceManifestRecord, + filePath: string, + errors: string[], +): void { + const sourcesById = new Map(record.sources.map((source) => [source.id, source])) + const coverageByCategory = new Map() + + for (const coverage of record.coverage) { + if (coverageByCategory.has(coverage.sourceCategory)) { + errors.push(errorMessage(filePath, `duplicate coverage category ${coverage.sourceCategory}`)) + } + coverageByCategory.set(coverage.sourceCategory, coverage) + const hasKnownSource = [ + 'registered', + 'parser_pending', + 'source_unavailable', + ].includes(coverage.status) + + if (hasKnownSource) { + if (!coverage.sourceIds?.length) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} ${coverage.status} coverage requires sourceIds`, + )) + continue + } + if (coverage.status === 'registered' && coverage.note !== undefined) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} registered coverage must not include a note`, + )) + } + if (coverage.status !== 'registered' && !coverage.note) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} ${coverage.status} coverage requires a note`, + )) + } + const seen = new Set() + for (const sourceId of coverage.sourceIds) { + if (seen.has(sourceId)) { + errors.push(errorMessage(filePath, `${coverage.sourceCategory} repeats ${sourceId}`)) + } + seen.add(sourceId) + const source = sourcesById.get(sourceId) + if (!source) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} references unknown source ${sourceId}`, + )) + } else if (source.sourceCategory !== coverage.sourceCategory) { + errors.push(errorMessage( + filePath, + `${sourceId} is ${source.sourceCategory}, not ${coverage.sourceCategory}`, + )) + } else if (coverage.status === 'registered' && !source.enabled) { + errors.push(errorMessage( + filePath, + `${sourceId} is disabled and cannot claim registered coverage`, + )) + } else if (coverage.status !== 'registered' && source.enabled) { + errors.push(errorMessage( + filePath, + `${sourceId} must be disabled while coverage is ${coverage.status}`, + )) + } + } + } else { + if (coverage.sourceIds !== undefined) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} missing coverage must omit sourceIds`, + )) + } + if (!coverage.note) { + errors.push(errorMessage( + filePath, + `${coverage.sourceCategory} missing coverage requires a note`, + )) + } + } + } + + for (const category of SOURCE_CATEGORIES) { + const coverage = coverageByCategory.get(category) + if (!coverage) { + errors.push(errorMessage(filePath, `missing coverage category ${category}`)) + continue + } + const actual = record.sources + .filter((source) => source.sourceCategory === category) + .map((source) => source.id) + .sort() + const declared = [...(coverage.sourceIds ?? [])].sort() + if (actual.join('|') !== declared.join('|')) { + errors.push(errorMessage( + filePath, + `${category} coverage must reference every and only source in that category`, + )) + } + } +} + +export function isCatalogReconciliationComplete( + record: SourceManifestRecord, +): boolean { + if (record.version !== 2) return false + return record.manifestStatus === 'complete' + && record.catalogReconciliation.status === 'complete' + && !record.catalogReconciliation.entries.some((entry) => entry.status === 'pending') +} + +function validateReconciliation( + record: SourceManifestV2, + filePath: string, + errors: string[], +): void { + const sources = new Set(record.sources.map((source) => source.id)) + const officialKeys = new Set() + for (const entry of record.catalogReconciliation.entries) { + if (officialKeys.has(entry.officialKey)) { + errors.push(errorMessage( + filePath, + `catalog reconciliation repeats officialKey ${entry.officialKey}`, + )) + } + officialKeys.add(entry.officialKey) + if (!sources.has(entry.sourceId)) { + errors.push(errorMessage( + filePath, + `catalog reconciliation references unknown source ${entry.sourceId}`, + )) + } + if (entry.status === 'published' && !entry.recordId) { + errors.push(errorMessage( + filePath, + `${entry.officialKey} published reconciliation requires recordId`, + )) + } + if (entry.status !== 'published' && entry.status !== 'pending' && !entry.note) { + errors.push(errorMessage( + filePath, + `${entry.officialKey} ${entry.status} reconciliation requires a note`, + )) + } + } + if (record.catalogReconciliation.status === 'complete' + && record.catalogReconciliation.entries.some((entry) => entry.status === 'pending')) { + errors.push(errorMessage( + filePath, + 'complete catalog reconciliation cannot contain pending entries', + )) + } + if (record.manifestStatus === 'complete') { + if (record.coverage.some((coverage) => coverage.status === 'discovery_pending')) { + errors.push(errorMessage( + filePath, + 'complete manifest cannot contain discovery_pending coverage', + )) + } + if (record.catalogReconciliation.status !== 'complete') { + errors.push(errorMessage( + filePath, + 'complete manifest requires complete catalog reconciliation', + )) + } + } +} + +export function validateSourceManifests( + inputs: LoadedSourceManifest[], + catalogPath = join(process.cwd(), 'content', 'data', 'universities.json'), +): SourceManifestRecord[] { + const errors: string[] = [] + const parsedRecords: Array<{ + filePath: string + record: SourceManifestRecord + }> = [] + + for (const input of inputs) { + const version = typeof input.value === 'object' + && input.value !== null + && 'version' in input.value + ? input.value.version + : undefined + const parsed = version === 2 + ? sourceManifestV2Schema.safeParse(input.value) + : pilotSourceManifestSchema.safeParse(input.value) + if (!parsed.success) { + for (const issue of parsed.error.issues) { + errors.push(errorMessage( + input.filePath, + `${issue.path.join('.') || ''}: ${issue.message}`, + )) + } + continue + } + parsedRecords.push({ + filePath: input.filePath, + record: parsed.data as SourceManifestRecord, + }) + } + + const records = parsedRecords.map(({ record }) => record) + if (records.length === 0) errors.push('No institution source manifests were found') + const institutionIds = new Map() + const sourceIds = new Map() + for (const { filePath, record } of parsedRecords) { + const previousInstitution = institutionIds.get(record.institutionId) + if (previousInstitution) { + errors.push(errorMessage( + filePath, + `duplicate institutionId ${record.institutionId}; first seen in ${previousInstitution}`, + )) + } else { + institutionIds.set(record.institutionId, basename(filePath)) + } + + let hosts: Set + try { + hosts = approvedHosts(record) + } catch (error) { + errors.push(errorMessage( + filePath, + error instanceof Error ? error.message : String(error), + )) + hosts = new Set() + } + + for (const source of record.sources) { + if (source.institutionId !== record.institutionId) { + errors.push(errorMessage(filePath, `${source.id} has a mismatched institutionId`)) + } + const previousSource = sourceIds.get(source.id) + if (previousSource) { + errors.push(errorMessage( + filePath, + `duplicate source id ${source.id}; first seen in ${previousSource}`, + )) + } else { + sourceIds.set(source.id, basename(filePath)) + } + try { + validateManifest(source as SourceManifestV1) + } catch (error) { + errors.push(errorMessage( + filePath, + `${source.id}: ${error instanceof Error ? error.message : String(error)}`, + )) + } + try { + const sourceHost = normalizeAllowedHost(new URL(source.officialUrl).hostname) + if (!hosts.has(sourceHost)) { + errors.push(errorMessage(filePath, `${source.id} uses undeclared official host ${sourceHost}`)) + } + for (const host of [...source.allowedHosts, ...(source.allowedRedirectHosts ?? [])]) { + const normalized = normalizeAllowedHost(host) + if (!hosts.has(normalized)) { + errors.push(errorMessage(filePath, `${source.id} allowlists undeclared host ${normalized}`)) + } + } + } catch (error) { + errors.push(errorMessage( + filePath, + `${source.id}: ${error instanceof Error ? error.message : String(error)}`, + )) + } + } + validateCoverage(record, filePath, errors) + if (record.version === 2) validateReconciliation(record, filePath, errors) + } + + const universities = JSON.parse(readFileSync(catalogPath, 'utf8')) as Array<{ id?: unknown }> + const catalogIds = new Set( + universities.map((university) => university.id) + .filter((id): id is string => typeof id === 'string'), + ) + for (const record of records) { + if (record.catalogStatus === 'existing' && !catalogIds.has(record.institutionId)) { + errors.push(`${record.institutionId}: existing institution is absent from universities.json`) + } + if (record.catalogStatus === 'planned_addition' && catalogIds.has(record.institutionId)) { + errors.push(`${record.institutionId}: planned institution already exists in universities.json`) + } + } + + if (errors.length > 0) { + throw new Error(`Source manifest validation failed:\n${errors.join('\n')}`) + } + return records +} + +export function validateSourceManifestDirectory( + directory?: string, +): SourceManifestRecord[] { + return validateSourceManifests(loadSourceManifestFiles(directory)) +} + +const invokedPath = process.argv[1] ? resolve(process.argv[1]) : '' +if (invokedPath === fileURLToPath(import.meta.url)) { + try { + const records = validateSourceManifestDirectory() + const sources = records.reduce((total, record) => total + record.sources.length, 0) + const completed = records.filter(isCatalogReconciliationComplete).length + console.log( + `Validated ${records.length} institution manifests, ${sources} official sources, and ${completed} complete catalog reconciliations.`, + ) + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) + } +} diff --git a/src/app/[locale]/programs/page.tsx b/src/app/[locale]/programs/page.tsx index 4b0be0e..6e6b2ff 100644 --- a/src/app/[locale]/programs/page.tsx +++ b/src/app/[locale]/programs/page.tsx @@ -1,16 +1,42 @@ import { notFound } from 'next/navigation' -import { ProgramExplorer } from '@/components/features/ProgramExplorer' +import { ProgramExplorerV2 } from '@/components/features/ProgramExplorerV2' import { PageHero } from '@/components/ui' import { getMessages } from '@/i18n/messages' import { getTodayDate } from '@/lib/data/freshness' -import { normalizeProgramField } from '@/lib/data/fields' -import { getCatalogData } from '@/lib/data/load' +import { getCatalogRepository } from '@/lib/catalog' +import { + parseProgramCatalogFilters, + queryProgramCatalogRepository, + type ProgramCatalogSearchParams, +} from '@/lib/program-catalog' import { pageMetadata, requireLocale } from '@/lib/site' export async function generateMetadata({ params }: { params: Promise<{ locale: string }> }) { const locale = requireLocale((await params).locale) || 'en'; const m = getMessages(locale); return pageMetadata(locale, m.programs.title, m.programs.intro, 'programs') } -export default async function ProgramsPage({ params, searchParams }: { params: Promise<{ locale: string }>; searchParams: Promise<{ discipline?: string | string[] }> }) { - const locale = requireLocale((await params).locale); if (!locale) notFound(); const messages = getMessages(locale); const data = await getCatalogData() - const requestedDiscipline = (await searchParams).discipline - const initialDiscipline = typeof requestedDiscipline === 'string' ? normalizeProgramField(requestedDiscipline) ?? '' : '' - return <>{messages.common.authoritativeNotice}} />
{data.programs.length === 0 ?
{messages.programs.verificationNote}
: null}
+export default async function ProgramsPage({ + params, + searchParams, +}: { + params: Promise<{ locale: string }> + searchParams: Promise +}) { + const locale = requireLocale((await params).locale) + if (!locale) notFound() + const messages = getMessages(locale) + const today = getTodayDate() + const filters = parseProgramCatalogFilters(await searchParams) + const result = await queryProgramCatalogRepository(getCatalogRepository(), filters, today) + + return <> + {messages.common.authoritativeNotice}} + /> +
+ {result.total === 0 ?
{messages.programs.verificationNote}
: null} + +
+ } diff --git a/src/app/[locale]/scholarships/page.tsx b/src/app/[locale]/scholarships/page.tsx index fe3562f..b52f585 100644 --- a/src/app/[locale]/scholarships/page.tsx +++ b/src/app/[locale]/scholarships/page.tsx @@ -1,12 +1,42 @@ import { notFound } from 'next/navigation' -import { ScholarshipCard } from '@/components/features/ScholarshipCard' +import { ScholarshipExplorerV2 } from '@/components/features/ScholarshipExplorerV2' import { PageHero } from '@/components/ui' import { getMessages } from '@/i18n/messages' -import { getCatalogData } from '@/lib/data/load' +import { getTodayDate } from '@/lib/data/freshness' +import { getCatalogRepository } from '@/lib/catalog' +import { + parseScholarshipCatalogFilters, + queryScholarshipCatalogRepository, + type ScholarshipCatalogSearchParams, +} from '@/lib/scholarship-catalog' import { pageMetadata, requireLocale } from '@/lib/site' export async function generateMetadata({ params }: { params: Promise<{ locale: string }> }) { const locale = requireLocale((await params).locale) || 'en'; const m = getMessages(locale); return pageMetadata(locale, m.scholarships.title, m.scholarships.intro, 'scholarships') } -export default async function ScholarshipsPage({ params }: { params: Promise<{ locale: string }> }) { - const locale = requireLocale((await params).locale); if (!locale) notFound(); const messages = getMessages(locale); const data = await getCatalogData() - return <>{messages.common.authoritativeNotice}} />
{messages.scholarships.catalogueNotice}
{data.scholarships.map((scholarship) => )}
+export default async function ScholarshipsPage({ + params, + searchParams, +}: { + params: Promise<{ locale: string }> + searchParams: Promise +}) { + const locale = requireLocale((await params).locale) + if (!locale) notFound() + const messages = getMessages(locale) + const today = getTodayDate() + const filters = parseScholarshipCatalogFilters(await searchParams) + const result = await queryScholarshipCatalogRepository(getCatalogRepository(), filters, today) + + return <> + {messages.common.authoritativeNotice}} + /> +
+
{messages.scholarships.catalogueNotice}
+ +
+ } diff --git a/src/components/features/ProgramExplorerV2.module.css b/src/components/features/ProgramExplorerV2.module.css new file mode 100644 index 0000000..c5e6fdf --- /dev/null +++ b/src/components/features/ProgramExplorerV2.module.css @@ -0,0 +1,37 @@ +.panel { + grid-template-columns: repeat(auto-fit, minmax(145px, 1fr)); + position: static; +} + +.search { + grid-column: span 2; +} + +.actions { + display: flex; + align-items: center; + gap: .55rem; +} + +.pagination { + display: grid; + grid-template-columns: minmax(7rem, 1fr) auto minmax(7rem, 1fr); + align-items: center; + gap: 1rem; + margin-top: 1.5rem; +} + +.pagination > :last-child { + justify-self: end; +} + +@media (max-width: 560px) { + .search { + grid-column: auto; + } + + .actions { + align-items: stretch; + flex-direction: column; + } +} diff --git a/src/components/features/ProgramExplorerV2.tsx b/src/components/features/ProgramExplorerV2.tsx new file mode 100644 index 0000000..8d838d5 --- /dev/null +++ b/src/components/features/ProgramExplorerV2.tsx @@ -0,0 +1,158 @@ +import { Button, LinkButton } from '@/components/ui' +import type { LaunchLocale } from '@/i18n/config' +import type { Messages } from '@/i18n/messages' +import { programFieldTaxonomy } from '@/lib/data/fields' +import { localize } from '@/lib/data/format' +import { degreeLabels, languageLabel } from '@/lib/data/labels' +import { + programCatalogHref, + type ProgramCatalogResult, +} from '@/lib/program-catalog' +import { ProgramCard } from './RecordCards' +import styles from './ProgramExplorerV2.module.css' + +const labels: Record = { + en: { apply: 'Apply filters', defaultOrder: 'Default order', next: 'Next', pagination: 'Program catalogue pages', previous: 'Previous', sortBy: 'Sort by' }, + zh: { apply: '应用筛选', defaultOrder: '默认顺序', next: '下一页', pagination: '项目目录分页', previous: '上一页', sortBy: '排序方式' }, + ru: { apply: 'Применить фильтры', defaultOrder: 'По умолчанию', next: 'Далее', pagination: 'Страницы каталога программ', previous: 'Назад', sortBy: 'Сортировка' }, + de: { apply: 'Filter anwenden', defaultOrder: 'Standardreihenfolge', next: 'Weiter', pagination: 'Studiengangseiten', previous: 'Zurück', sortBy: 'Sortieren nach' }, + fr: { apply: 'Appliquer les filtres', defaultOrder: 'Ordre par défaut', next: 'Suivant', pagination: 'Pages du catalogue des programmes', previous: 'Précédent', sortBy: 'Trier par' }, + es: { apply: 'Aplicar filtros', defaultOrder: 'Orden predeterminado', next: 'Siguiente', pagination: 'Páginas del catálogo de programas', previous: 'Anterior', sortBy: 'Ordenar por' }, +} + +export function ProgramExplorerV2({ + result, + locale, + messages, + today, +}: { + result: ProgramCatalogResult + locale: LaunchLocale + messages: Messages + today: string +}) { + const text = labels[locale] + const filters = result.filters + + return <> +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + {messages.common.clear} +
+
+ +

{result.total} {messages.programs.results}

+ {result.items.length ? ( +
+ {result.items.map(({ program, university, cycle }) => ( + + ))} +
+ ) :
{messages.programs.noResults}
} + + {result.pageCount > 1 ? ( + + ) : null} + +} diff --git a/src/components/features/ScholarshipExplorerV2.module.css b/src/components/features/ScholarshipExplorerV2.module.css new file mode 100644 index 0000000..8c43564 --- /dev/null +++ b/src/components/features/ScholarshipExplorerV2.module.css @@ -0,0 +1,37 @@ +.panel { + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + position: static; +} + +.search { + grid-column: span 2; +} + +.actions { + display: flex; + align-items: center; + gap: .55rem; +} + +.pagination { + display: grid; + grid-template-columns: minmax(7rem, 1fr) auto minmax(7rem, 1fr); + align-items: center; + gap: 1rem; + margin-top: 1.5rem; +} + +.pagination > :last-child { + justify-self: end; +} + +@media (max-width: 560px) { + .search { + grid-column: auto; + } + + .actions { + align-items: stretch; + flex-direction: column; + } +} diff --git a/src/components/features/ScholarshipExplorerV2.tsx b/src/components/features/ScholarshipExplorerV2.tsx new file mode 100644 index 0000000..cc9c77a --- /dev/null +++ b/src/components/features/ScholarshipExplorerV2.tsx @@ -0,0 +1,148 @@ +import { Button, LinkButton } from '@/components/ui' +import type { LaunchLocale } from '@/i18n/config' +import type { Messages } from '@/i18n/messages' +import { localize } from '@/lib/data/format' +import { degreeLabels } from '@/lib/data/labels' +import { + scholarshipCatalogHref, + type ScholarshipCatalogResult, +} from '@/lib/scholarship-catalog' +import { ScholarshipCard } from './ScholarshipCard' +import styles from './ScholarshipExplorerV2.module.css' + +type ExplorerLabels = { + apply: string + deadline: string + deadlineAnnounced: string + deadlineClosed: string + deadlineFuture: string + deadlineNext30: string + deadlineNext90: string + deadlineUnknown: string + defaultOrder: string + degree: string + funding: string + fundingAccommodation: string + fundingFullTuition: string + fundingInsurance: string + fundingPartialTuition: string + fundingStipend: string + next: string + noResults: string + pagination: string + previous: string + results: string + scholarshipName: string + school: string + searchPlaceholder: string + sortBy: string + stipendHigh: string +} + +const labels: Record = { + en: { apply: 'Apply filters', deadline: 'Deadline', deadlineAnnounced: 'Deadline announced', deadlineClosed: 'Recently closed', deadlineFuture: 'Future deadline', deadlineNext30: 'Next 30 days', deadlineNext90: 'Next 90 days', deadlineUnknown: 'Not announced', defaultOrder: 'Default order', degree: 'Linked degree level', funding: 'Funding', fundingAccommodation: 'Accommodation covered', fundingFullTuition: 'Full tuition', fundingInsurance: 'Insurance covered', fundingPartialTuition: 'Partial tuition', fundingStipend: 'Monthly stipend', next: 'Next', noResults: 'No scholarships match these filters.', pagination: 'Scholarship catalogue pages', previous: 'Previous', results: 'scholarships', scholarshipName: 'Scholarship A–Z', school: 'University', searchPlaceholder: 'Search scholarship or university', sortBy: 'Sort by', stipendHigh: 'Highest stipend' }, + zh: { apply: '应用筛选', deadline: '截止日期', deadlineAnnounced: '已公布截止日期', deadlineClosed: '近期已截止', deadlineFuture: '未来截止', deadlineNext30: '未来 30 天', deadlineNext90: '未来 90 天', deadlineUnknown: '尚未公布', defaultOrder: '默认顺序', degree: '关联学历层次', funding: '资助类型', fundingAccommodation: '包含住宿', fundingFullTuition: '全额学费', fundingInsurance: '包含保险', fundingPartialTuition: '部分学费', fundingStipend: '每月生活费', next: '下一页', noResults: '没有奖学金符合这些筛选条件。', pagination: '奖学金目录分页', previous: '上一页', results: '个奖学金', scholarshipName: '奖学金名称 A–Z', school: '大学', searchPlaceholder: '搜索奖学金或大学', sortBy: '排序方式', stipendHigh: '生活费从高到低' }, + ru: { apply: 'Применить фильтры', deadline: 'Срок подачи', deadlineAnnounced: 'Срок объявлен', deadlineClosed: 'Недавно закрытые', deadlineFuture: 'Будущий срок', deadlineNext30: 'Следующие 30 дней', deadlineNext90: 'Следующие 90 дней', deadlineUnknown: 'Не объявлено', defaultOrder: 'По умолчанию', degree: 'Связанный уровень', funding: 'Финансирование', fundingAccommodation: 'Проживание', fundingFullTuition: 'Полная оплата обучения', fundingInsurance: 'Страховка', fundingPartialTuition: 'Частичная оплата', fundingStipend: 'Ежемесячная выплата', next: 'Далее', noResults: 'Нет стипендий по выбранным фильтрам.', pagination: 'Страницы каталога стипендий', previous: 'Назад', results: 'стипендий', scholarshipName: 'Название A–Я', school: 'Университет', searchPlaceholder: 'Найти стипендию или вуз', sortBy: 'Сортировка', stipendHigh: 'Наибольшая выплата' }, + de: { apply: 'Filter anwenden', deadline: 'Bewerbungsfrist', deadlineAnnounced: 'Frist veröffentlicht', deadlineClosed: 'Kürzlich geschlossen', deadlineFuture: 'Künftige Frist', deadlineNext30: 'Nächste 30 Tage', deadlineNext90: 'Nächste 90 Tage', deadlineUnknown: 'Nicht bekannt gegeben', defaultOrder: 'Standardreihenfolge', degree: 'Verknüpfter Abschluss', funding: 'Förderung', fundingAccommodation: 'Unterkunft abgedeckt', fundingFullTuition: 'Volle Studiengebühren', fundingInsurance: 'Versicherung abgedeckt', fundingPartialTuition: 'Teilweise Studiengebühren', fundingStipend: 'Monatlicher Zuschuss', next: 'Weiter', noResults: 'Keine Stipendien entsprechen diesen Filtern.', pagination: 'Seiten des Stipendienkatalogs', previous: 'Zurück', results: 'Stipendien', scholarshipName: 'Stipendium A–Z', school: 'Universität', searchPlaceholder: 'Stipendium oder Universität suchen', sortBy: 'Sortieren nach', stipendHigh: 'Höchster Zuschuss' }, + fr: { apply: 'Appliquer les filtres', deadline: 'Date limite', deadlineAnnounced: 'Date limite publiée', deadlineClosed: 'Récemment clôturées', deadlineFuture: 'Date limite future', deadlineNext30: '30 prochains jours', deadlineNext90: '90 prochains jours', deadlineUnknown: 'Non annoncée', defaultOrder: 'Ordre par défaut', degree: 'Niveau associé', funding: 'Financement', fundingAccommodation: 'Hébergement couvert', fundingFullTuition: 'Frais complets', fundingInsurance: 'Assurance couverte', fundingPartialTuition: 'Frais partiels', fundingStipend: 'Allocation mensuelle', next: 'Suivant', noResults: 'Aucune bourse ne correspond à ces filtres.', pagination: 'Pages du catalogue des bourses', previous: 'Précédent', results: 'bourses', scholarshipName: 'Bourse A–Z', school: 'Université', searchPlaceholder: 'Rechercher une bourse ou université', sortBy: 'Trier par', stipendHigh: 'Allocation la plus élevée' }, + es: { apply: 'Aplicar filtros', deadline: 'Fecha límite', deadlineAnnounced: 'Fecha publicada', deadlineClosed: 'Cerradas recientemente', deadlineFuture: 'Fecha futura', deadlineNext30: 'Próximos 30 días', deadlineNext90: 'Próximos 90 días', deadlineUnknown: 'No anunciada', defaultOrder: 'Orden predeterminado', degree: 'Nivel vinculado', funding: 'Financiación', fundingAccommodation: 'Alojamiento cubierto', fundingFullTuition: 'Matrícula completa', fundingInsurance: 'Seguro cubierto', fundingPartialTuition: 'Matrícula parcial', fundingStipend: 'Estipendio mensual', next: 'Siguiente', noResults: 'Ninguna beca coincide con estos filtros.', pagination: 'Páginas del catálogo de becas', previous: 'Anterior', results: 'becas', scholarshipName: 'Beca A–Z', school: 'Universidad', searchPlaceholder: 'Buscar beca o universidad', sortBy: 'Ordenar por', stipendHigh: 'Mayor estipendio' }, +} + +export function ScholarshipExplorerV2({ + result, + locale, + messages, +}: { + result: ScholarshipCatalogResult + locale: LaunchLocale + messages: Messages +}) { + const text = labels[locale] + const filters = result.filters + + return <> +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + {messages.common.clear} +
+
+ +

{result.total} {text.results}

+ {result.items.length ? ( +
+ {result.items.map(({ scholarship }) => ( + + ))} +
+ ) :
{text.noResults}
} + + {result.pageCount > 1 ? ( + + ) : null} + +} diff --git a/src/lib/catalog/d1-list.ts b/src/lib/catalog/d1-list.ts new file mode 100644 index 0000000..90660bb --- /dev/null +++ b/src/lib/catalog/d1-list.ts @@ -0,0 +1,417 @@ +import { + admissionCycleSchema, + programSchema, + scholarshipSchema, + universitySchema, +} from '@/lib/data/schema' +import type { + AdmissionCycle, + DegreeLevel, + LocalizedText, + Program, + Scholarship, + University, +} from '@/lib/data/types' +import { parseCatalogReleaseInfo } from './release' +import { + CatalogRepositoryError, + type CatalogListOption, + type CatalogProgramListItem, + type CatalogProgramListPage, + type CatalogScholarshipCurrentCycle, + type CatalogScholarshipListItem, + type CatalogScholarshipListPage, +} from './types' + +type UnknownRecord = Record + +function isObject(value: unknown): value is UnknownRecord { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} + +function invalid(message: string, cause?: unknown): never { + throw new CatalogRepositoryError('INVALID_LIST_RESPONSE', message, { cause }) +} + +function stringValue(value: unknown): string | null { + return typeof value === 'string' && value.length > 0 ? value : null +} + +function localized(value: unknown, fallback: string): LocalizedText { + if (!isObject(value)) return { en: fallback } + const entries = Object.entries(value).filter((entry): entry is [string, string] => ( + typeof entry[1] === 'string' && entry[1].length > 0 + )) + return entries.length > 0 ? Object.fromEntries(entries) : { en: fallback } +} + +function sourceIds(value: UnknownRecord): string[] { + if (Array.isArray(value.sourceIds)) { + const ids = value.sourceIds.filter((item): item is string => typeof item === 'string' && item.length > 0) + if (ids.length > 0) return ids + } + if (Array.isArray(value.sources)) { + const ids = value.sources.flatMap((item) => { + if (!isObject(item)) return [] + const id = stringValue(item.id) + return id ? [id] : [] + }) + if (ids.length > 0) return ids + } + return ['remote-catalog-source'] +} + +function dateOnly(value: unknown, fallback: string): string { + const candidate = stringValue(value)?.slice(0, 10) + return candidate && /^\d{4}-\d{2}-\d{2}$/u.test(candidate) ? candidate : fallback +} + +function fieldAudit(value: UnknownRecord, today: string) { + const fieldMeta = isObject(value.fieldMeta) ? value.fieldMeta : {} + const name = isObject(fieldMeta.name) ? fieldMeta.name : {} + return { + sourceIds: sourceIds(value), + verifiedAt: dateOnly(value.verifiedAt ?? name.verifiedAt ?? name.checkedAt, today), + reviewAfter: dateOnly(value.reviewAfter ?? name.reviewAfter, today), + status: value.status === 'stale' ? 'stale' as const : 'verified' as const, + } +} + +function durationMonths(value: unknown, unit: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) return null + const factor = unit === 'academic_years' ? 12 : unit === 'semesters' ? 6 : unit === 'months' ? 1 : null + if (factor === null) return null + return Math.min(120, Math.max(1, Math.round(value * factor))) +} + +function degreeLevel(value: unknown, programType?: unknown): DegreeLevel { + if (value === 'bachelor' || value === 'master' || value === 'doctorate') return value + if (programType === 'language') return 'language' + if (programType === 'foundation') return 'foundation' + return 'other' +} + +function teachingLanguage(value: string): string { + const normalized = value.toLocaleLowerCase() + if (normalized === 'zh' || normalized === 'zho' || normalized === 'chinese') return 'Chinese' + if (normalized === 'en' || normalized === 'eng' || normalized === 'english') return 'English' + if (normalized === 'bilingual') return 'Bilingual' + return value +} + +function safeHttps(value: unknown): string | null { + const candidate = stringValue(value) + if (!candidate) return null + try { + return new URL(candidate).protocol === 'https:' ? candidate : null + } catch { + return null + } +} + +function normalizeUniversity( + value: unknown, + programUrl: string, + today: string, +): University { + const parsed = universitySchema.safeParse(value) + if (parsed.success) return parsed.data + if (!isObject(value)) invalid('Program list item is missing its institution relationship.') + const id = stringValue(value.id) + const slug = stringValue(value.slug) + if (!id || !slug || !/^[a-z0-9-]+$/u.test(slug)) invalid('Program institution identity is invalid.') + return universitySchema.parse({ + ...fieldAudit(value, today), + id, + slug, + name: localized(value.name, slug), + cityId: 'remote-catalog-city', + region: null, + officialUrl: safeHttps(value.officialUrl) ?? programUrl, + admissionsUrl: safeHttps(value.admissionsUrl), + summary: null, + featured: false, + }) +} + +function normalizeProgramCycle( + value: unknown, + program: Program, + today: string, +): AdmissionCycle | null { + if (value === null || value === undefined) return null + const parsed = admissionCycleSchema.safeParse(value) + if (parsed.success) return parsed.data + if (!isObject(value)) return null + const attributes = isObject(value.attributes) ? value.attributes : value + const application = isObject(attributes.application) ? attributes.application : attributes + const tuition = isObject(attributes.tuition) ? attributes.tuition : null + const academicYear = stringValue(attributes.academicYear) + if (!academicYear || !/^\d{4}-\d{4}$/u.test(academicYear)) return null + const state = stringValue(application.state) + const intakeValue = stringValue(attributes.intake) + const intake = intakeValue === 'spring' || intakeValue === 'autumn' ? intakeValue : 'other' + const exponent = typeof tuition?.currencyExponent === 'number' ? tuition.currencyExponent : 0 + const amountMinor = typeof tuition?.amountMinimumMinor === 'number' ? tuition.amountMinimumMinor : null + const tuitionCny = amountMinor === null ? null : amountMinor / (10 ** exponent) + const candidate = { + ...fieldAudit(value, today), + id: stringValue(value.id) ?? `remote:${program.id}:${academicYear}:${intake}`, + programId: program.id, + academicYear, + intake, + opensOn: stringValue(application.opensOn), + closesOn: stringValue(application.closesOn), + dateStatus: state === 'rolling' + ? 'rolling' + : state === 'not-announced' + ? 'not-announced' + : 'published', + tuitionCny, + tuitionPeriod: stringValue(tuition?.period) === 'academic_year' + ? 'academic-year' + : stringValue(tuition?.period), + tuitionStatus: tuitionCny === null ? null : 'confirmed', + applicationFeeCny: null, + } + const normalized = admissionCycleSchema.safeParse(candidate) + return normalized.success ? normalized.data : null +} + +function normalizeProgram(value: unknown, today: string): CatalogProgramListItem { + if (!isObject(value)) invalid('Catalog API returned an invalid program list item.') + if (isObject(value.program)) { + const parsedProgram = programSchema.safeParse(value.program) + if (!parsedProgram.success) invalid('Catalog API returned an invalid normalized program.', parsedProgram.error) + return { + program: parsedProgram.data, + university: normalizeUniversity(value.university, parsedProgram.data.programUrl, today), + currentCycle: normalizeProgramCycle(value.currentCycle, parsedProgram.data, today), + } + } + + const attributes = isObject(value.attributes) ? value.attributes : value + const relationships = isObject(value.relationships) ? value.relationships : {} + const relatedUniversity = isObject(relationships.institution) + ? relationships.institution + : isObject(value.university) + ? value.university + : null + const id = stringValue(value.id) + const slug = stringValue(value.slug) + const officialUrl = safeHttps(attributes.officialUrl ?? value.programUrl) + if (!id || !slug || !officialUrl || !relatedUniversity) { + invalid('Catalog API program identity is incomplete.') + } + const duration = isObject(attributes.duration) ? attributes.duration : {} + const languages = Array.isArray(attributes.teachingLanguageCodes) + ? attributes.teachingLanguageCodes + : Array.isArray(value.teachingLanguages) + ? value.teachingLanguages + : [] + const minimumDuration = durationMonths( + duration.minimum ?? value.durationMonths, + duration.unit ?? (value.durationMonths === undefined ? undefined : 'months'), + ) + const maximumDuration = durationMonths( + duration.maximum ?? value.durationMonthsMax, + duration.unit ?? (value.durationMonthsMax === undefined ? undefined : 'months'), + ) + const university = normalizeUniversity(relatedUniversity, officialUrl, today) + const program = programSchema.parse({ + ...fieldAudit(value, today), + id, + slug, + universityId: university.id, + name: localized(attributes.name ?? value.name, slug), + degreeLevel: degreeLevel(attributes.degreeLevel ?? value.degreeLevel, attributes.programType ?? value.programType), + discipline: stringValue(value.discipline) ?? 'other', + teachingLanguages: languages.filter((item): item is string => typeof item === 'string').map(teachingLanguage), + durationMonths: minimumDuration, + durationMonthsMax: maximumDuration, + programUrl: officialUrl, + applyUrl: safeHttps(attributes.applyUrl ?? value.applyUrl), + languageRequirements: Array.isArray(value.languageRequirements) ? value.languageRequirements : [], + verificationScope: 'identity', + }) + return { + program, + university, + currentCycle: normalizeProgramCycle(value.currentCycle, program, today), + } +} + +function scholarshipCycle( + scholarship: Scholarship, + today: string, +): CatalogScholarshipCurrentCycle { + const deadline = scholarship.deadline + const todayValue = Date.parse(`${today}T00:00:00.000Z`) + const deadlineValue = deadline ? Date.parse(`${deadline}T00:00:00.000Z`) : Number.NaN + const daysRemaining = Number.isNaN(todayValue) || Number.isNaN(deadlineValue) + ? null + : Math.ceil((deadlineValue - todayValue) / 86_400_000) + return { + id: `legacy:${scholarship.id}`, + scholarshipId: scholarship.id, + academicYear: null, + opensOn: null, + closesOn: deadline, + deadline, + deadlineState: deadline === null ? 'not-announced' : deadline < today ? 'closed' : 'future', + daysRemaining, + legacy: true, + } +} + +function providerType(value: unknown): Scholarship['providerType'] { + if (value === 'csc' || value === 'university' || value === 'province' || value === 'city') return value + if (value === 'government') return 'csc' + return 'other' +} + +function normalizeScholarship(value: unknown, today: string): CatalogScholarshipListItem { + if (!isObject(value)) invalid('Catalog API returned an invalid scholarship list item.') + if (isObject(value.scholarship)) { + const parsedScholarship = scholarshipSchema.safeParse(value.scholarship) + if (!parsedScholarship.success) invalid('Catalog API returned an invalid normalized scholarship.', parsedScholarship.error) + const universities = Array.isArray(value.universities) + ? value.universities.flatMap((item) => { + const parsed = universitySchema.safeParse(item) + return parsed.success ? [parsed.data] : [] + }) + : [] + const programs = Array.isArray(value.programs) + ? value.programs.flatMap((item) => { + const parsed = programSchema.safeParse(item) + return parsed.success ? [parsed.data] : [] + }) + : [] + return { + scholarship: parsedScholarship.data, + universities, + programs, + currentCycle: isObject(value.currentCycle) + ? { ...scholarshipCycle(parsedScholarship.data, today), ...value.currentCycle } + : scholarshipCycle(parsedScholarship.data, today), + } + } + + const attributes = isObject(value.attributes) ? value.attributes : value + const scope = isObject(attributes.scope) ? attributes.scope : {} + const coverage = isObject(attributes.coverage) ? attributes.coverage : isObject(value.coverage) ? value.coverage : {} + const id = stringValue(value.id) + const slug = stringValue(value.slug) + const officialUrl = safeHttps(attributes.officialUrl ?? value.applicationUrl) + if (!id || !slug || !officialUrl) invalid('Catalog API scholarship identity is incomplete.') + const universityIds = Array.isArray(scope.institutionIds) + ? scope.institutionIds.filter((item): item is string => typeof item === 'string') + : Array.isArray(value.universityIds) + ? value.universityIds.filter((item): item is string => typeof item === 'string') + : [] + const programIds = Array.isArray(scope.programIds) + ? scope.programIds.filter((item): item is string => typeof item === 'string') + : Array.isArray(value.programIds) + ? value.programIds.filter((item): item is string => typeof item === 'string') + : [] + const relationships = isObject(value.relationships) ? value.relationships : {} + const provider = isObject(relationships.provider) ? relationships.provider : {} + const scholarship = scholarshipSchema.parse({ + ...fieldAudit(value, today), + id, + slug, + name: localized(attributes.name ?? value.name, slug), + providerType: providerType(value.providerType ?? attributes.schemeType ?? provider.organizationType), + universityIds, + programIds, + coverage: { + tuition: coverage.tuition === 'full' || coverage.tuition === 'partial' || coverage.tuition === 'none' + ? coverage.tuition + : 'unknown', + accommodation: coverage.accommodation === 'full' || coverage.accommodation === 'partial' || coverage.accommodation === 'none' + ? coverage.accommodation + : 'unknown', + insurance: typeof coverage.insurance === 'boolean' ? coverage.insurance : 'unknown', + stipendCnyPerMonth: typeof coverage.stipendCnyPerMonth === 'number' + ? coverage.stipendCnyPerMonth + : null, + }, + deadline: stringValue(attributes.deadline ?? value.deadline), + applicationUrl: safeHttps(value.applicationUrl ?? officialUrl), + summary: attributes.summary === null || value.summary === null + ? null + : localized(attributes.summary ?? value.summary, slug), + }) + return { + scholarship, + universities: [], + programs: [], + currentCycle: scholarshipCycle(scholarship, today), + } +} + +function option(value: unknown): CatalogListOption | null { + if (!isObject(value)) return null + const optionValue = stringValue(value.value ?? value.slug ?? value.id) + if (!optionValue) return null + return { value: optionValue, name: localized(value.name, optionValue) } +} + +function envelopeParts(payload: unknown): { + rows: unknown[] + meta: UnknownRecord + facets: UnknownRecord +} { + if (!isObject(payload)) invalid('Catalog API list response must be an object.') + const meta = isObject(payload.meta) ? payload.meta : {} + if (Array.isArray(payload.data)) return { rows: payload.data, meta, facets: isObject(meta.facets) ? meta.facets : {} } + if (isObject(payload.data) && Array.isArray(payload.data.items)) { + return { + rows: payload.data.items, + meta, + facets: isObject(payload.data.facets) ? payload.data.facets : isObject(meta.facets) ? meta.facets : {}, + } + } + invalid('Catalog API list response data must be an array.') +} + +function pageMeta(meta: UnknownRecord) { + const nextCursor = meta.nextCursor === null ? null : stringValue(meta.nextCursor) + const total = Number.isInteger(meta.total) && (meta.total as number) >= 0 ? meta.total as number : null + const release = meta.release === undefined ? null : parseCatalogReleaseInfo(meta.release) + return { nextCursor, total, release } +} + +export function parseD1ProgramList(payload: unknown, today: string): CatalogProgramListPage { + const { rows, meta, facets } = envelopeParts(payload) + const items = rows.map((row) => normalizeProgram(row, today)) + const universities = Array.isArray(facets.universities) + ? facets.universities.flatMap((item) => option(item) ?? []) + : items.map(({ university }) => ({ value: university.slug, name: university.name })) + const cities = Array.isArray(facets.cities) + ? facets.cities.flatMap((item) => option(item) ?? []) + : [] + return { + items, + ...pageMeta(meta), + facets: { + universities: [...new Map(universities.map((item) => [item.value, item])).values()], + cities: [...new Map(cities.map((item) => [item.value, item])).values()], + }, + } +} + +export function parseD1ScholarshipList(payload: unknown, today: string): CatalogScholarshipListPage { + const { rows, meta, facets } = envelopeParts(payload) + const items = rows.map((row) => normalizeScholarship(row, today)) + const universities = Array.isArray(facets.universities) + ? facets.universities.flatMap((item) => option(item) ?? []) + : items.flatMap((item) => item.universities.map(({ slug, name }) => ({ value: slug, name }))) + return { + items, + ...pageMeta(meta), + facets: { + universities: [...new Map(universities.map((item) => [item.value, item])).values()], + }, + } +} diff --git a/src/lib/catalog/d1.ts b/src/lib/catalog/d1.ts index f73ad8f..6fb3d65 100644 --- a/src/lib/catalog/d1.ts +++ b/src/lib/catalog/d1.ts @@ -1,11 +1,19 @@ import { bundleSchema } from '@/lib/data/schema' import type { DataBundle } from '@/lib/data/types' +import { getTodayDate } from '@/lib/data/freshness' +import { parseD1ProgramList, parseD1ScholarshipList } from './d1-list' import { deriveCatalogRelease, parseCatalogRelease } from './release' import { + CATALOG_LIST_DEFAULT_LIMIT, + CATALOG_LIST_MAX_LIMIT, CatalogRepositoryError, type CatalogFetch, + type CatalogProgramListPage, + type CatalogProgramListQuery, type CatalogRelease, type CatalogRepository, + type CatalogScholarshipListPage, + type CatalogScholarshipListQuery, } from './types' type CatalogSnapshot = { @@ -94,10 +102,33 @@ function parseApiPayload(value: unknown): { bundleValue: unknown; releaseValue?: return { bundleValue: value } } +function listLimit(value: number | undefined): number { + if (value === undefined) return CATALOG_LIST_DEFAULT_LIMIT + if (!Number.isSafeInteger(value) || value < 1) { + throw new CatalogRepositoryError( + 'INVALID_LIST_LIMIT', + 'Catalog list limit must be a positive integer.', + ) + } + return Math.min(value, CATALOG_LIST_MAX_LIMIT) +} + +function addParam(url: URL, name: string, value: string | number | undefined): void { + if (value !== undefined && value !== '') url.searchParams.set(name, String(value)) +} + +function tuitionRange(value: string | undefined): { minimum?: number; maximum?: number } { + if (value === 'under-20000') return { maximum: 20_000 } + if (value === '20000-40000') return { minimum: 20_001, maximum: 40_000 } + if (value === 'over-40000') return { minimum: 40_001 } + return {} +} + export class D1CatalogRepository implements CatalogRepository { readonly mode = 'd1' as const private readonly apiUrl: string private readonly apiToken: string | undefined + private readonly parsedApiUrl: URL private readonly fetcher: CatalogFetch private readonly cacheTtlMs: number private readonly timeoutMs: number @@ -125,6 +156,7 @@ export class D1CatalogRepository implements CatalogRepository { if (parsedApiUrl.username || parsedApiUrl.password) { throw new CatalogRepositoryError('INVALID_API_URL', 'CATALOG_API_URL must not contain credentials.') } + this.parsedApiUrl = parsedApiUrl this.apiToken = options.apiToken?.trim() || undefined if (this.apiToken) { const expectedHost = options.apiTokenHost?.trim().toLowerCase() @@ -161,6 +193,116 @@ export class D1CatalogRepository implements CatalogRepository { return (await this.getSnapshot()).release } + async listPrograms( + query: CatalogProgramListQuery = {}, + ): Promise { + const url = this.publicEndpoint('programs') + const range = tuitionRange(query.tuition) + addParam(url, 'q', query.q) + addParam(url, 'institution', query.institution) + addParam(url, 'city', query.city) + addParam(url, 'type', query.type) + addParam(url, 'degree', query.degree) + addParam(url, 'discipline', query.discipline) + addParam(url, 'language', query.language) + addParam(url, 'academicYear', query.academicYear) + addParam(url, 'intake', query.intake) + addParam(url, 'tuition', query.tuition) + addParam(url, 'tuitionMin', query.tuitionMin ?? range.minimum) + addParam(url, 'tuitionMax', query.tuitionMax ?? range.maximum) + addParam(url, 'applicationState', query.applicationState) + addParam(url, 'scholarship', query.scholarship) + addParam(url, 'sort', query.sort) + addParam(url, 'cursor', query.cursor) + addParam(url, 'limit', listLimit(query.limit)) + return parseD1ProgramList( + await this.fetchListPayload(url), + query.today ?? getTodayDate(), + ) + } + + async listScholarships( + query: CatalogScholarshipListQuery = {}, + ): Promise { + const url = this.publicEndpoint('scholarships') + addParam(url, 'q', query.q) + addParam(url, 'provider', query.provider) + addParam(url, 'institution', query.institution) + addParam(url, 'program', query.program) + addParam(url, 'degree', query.degree) + addParam(url, 'funding', query.funding) + addParam(url, 'deadline', query.deadline) + addParam(url, 'sort', query.sort) + addParam(url, 'cursor', query.cursor) + addParam(url, 'limit', listLimit(query.limit)) + return parseD1ScholarshipList( + await this.fetchListPayload(url), + query.today ?? getTodayDate(), + ) + } + + private publicEndpoint(resource: 'programs' | 'scholarships'): URL { + const url = new URL(this.parsedApiUrl) + url.search = '' + url.hash = '' + const path = url.pathname.replace(/\/+$/u, '') + if (path.endsWith('/internal/v1/catalog-bundle')) { + url.pathname = path.slice(0, -'/internal/v1/catalog-bundle'.length) + `/api/v1/${resource}` + } else if (/\/api\/v1(?:\/[^/]+)?$/u.test(path)) { + url.pathname = path.replace(/\/api\/v1(?:\/[^/]+)?$/u, `/api/v1/${resource}`) + } else { + url.pathname = `${path}/api/v1/${resource}`.replace(/^\/\//u, '/') + } + return url + } + + private async fetchListPayload(url: URL): Promise { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), this.timeoutMs) + let response: Response + try { + response = await this.fetcher(url.toString(), { + method: 'GET', + headers: { accept: 'application/json' }, + redirect: 'error', + signal: controller.signal, + }) + } catch (error) { + clearTimeout(timeout) + throw new CatalogRepositoryError( + controller.signal.aborted ? 'CATALOG_API_TIMEOUT' : 'CATALOG_API_UNAVAILABLE', + controller.signal.aborted + ? `Catalog API request exceeded ${this.timeoutMs}ms.` + : `Catalog API request failed for ${url.origin}.`, + { cause: error }, + ) + } + + try { + if (!response.ok) { + throw new CatalogRepositoryError( + 'CATALOG_API_HTTP_ERROR', + `Catalog API returned HTTP ${response.status} ${response.statusText}`.trim(), + ) + } + try { + const bytes = await readBoundedResponse(response, this.maxResponseBytes) + return JSON.parse(new TextDecoder().decode(bytes)) as unknown + } catch (error) { + if (error instanceof CatalogRepositoryError) throw error + throw new CatalogRepositoryError( + controller.signal.aborted ? 'CATALOG_API_TIMEOUT' : 'INVALID_API_RESPONSE', + controller.signal.aborted + ? `Catalog API request exceeded ${this.timeoutMs}ms.` + : 'Catalog API did not return valid JSON.', + { cause: error }, + ) + } + } finally { + clearTimeout(timeout) + } + } + private getSnapshot(): Promise { const now = this.now() if (this.cached && this.cached.expiresAt >= now) return Promise.resolve(this.cached.snapshot) diff --git a/src/lib/catalog/index.ts b/src/lib/catalog/index.ts index c9e3537..067a294 100644 --- a/src/lib/catalog/index.ts +++ b/src/lib/catalog/index.ts @@ -1,4 +1,8 @@ export { createCatalogRepository, type CreateCatalogRepositoryOptions } from './repository' +export { + getCatalogRepository, + resetCatalogRepositoryRuntimeForTests, +} from './runtime' export { JsonCatalogRepository, createJsonCatalogRepository, @@ -24,14 +28,28 @@ export { deriveCatalogRelease, getCatalogRecordCounts, parseCatalogRelease, + parseCatalogReleaseInfo, } from './release' export { CATALOG_COLLECTIONS, + CATALOG_LIST_DEFAULT_LIMIT, + CATALOG_LIST_MAX_LIMIT, CatalogRepositoryError, type CatalogBackendMode, type CatalogBundleLoader, type CatalogCollection, type CatalogFetch, + type CatalogListOption, + type CatalogListPage, + type CatalogProgramListFacets, + type CatalogProgramListItem, + type CatalogProgramListPage, + type CatalogProgramListQuery, + type CatalogScholarshipCurrentCycle, + type CatalogScholarshipListFacets, + type CatalogScholarshipListItem, + type CatalogScholarshipListPage, + type CatalogScholarshipListQuery, type CatalogRecordCounts, type CatalogRelease, type CatalogRepository, diff --git a/src/lib/catalog/json.ts b/src/lib/catalog/json.ts index 664af44..6b19e21 100644 --- a/src/lib/catalog/json.ts +++ b/src/lib/catalog/json.ts @@ -2,8 +2,33 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { bundleSchema } from '@/lib/data/schema' import type { DataBundle } from '@/lib/data/types' +import { getTodayDate } from '@/lib/data/freshness' +import { selectPublishedData } from '@/lib/data/publication' +import { + parseProgramCatalogFilters, + queryProgramCatalog, +} from '@/lib/program-catalog' +import { + parseScholarshipCatalogFilters, + queryScholarshipCatalog, +} from '@/lib/scholarship-catalog' +import { + decodeJsonListCursor, + encodeJsonListCursor, +} from './list-cursor' import { deriveCatalogRelease } from './release' -import type { CatalogBundleLoader, CatalogRelease, CatalogRepository } from './types' +import { + CATALOG_LIST_DEFAULT_LIMIT, + CATALOG_LIST_MAX_LIMIT, + CatalogRepositoryError, + type CatalogBundleLoader, + type CatalogProgramListPage, + type CatalogProgramListQuery, + type CatalogRelease, + type CatalogRepository, + type CatalogScholarshipListPage, + type CatalogScholarshipListQuery, +} from './types' const JSON_FILES = { sources: 'sources', @@ -23,6 +48,40 @@ export function readJsonCatalogBundle(dataDirectory = join(process.cwd(), 'conte ) } +function listLimit(value: number | undefined): number { + if (value === undefined) return CATALOG_LIST_DEFAULT_LIMIT + if (!Number.isSafeInteger(value) || value < 1) { + throw new CatalogRepositoryError( + 'INVALID_LIST_LIMIT', + 'Catalog list limit must be a positive integer.', + ) + } + return Math.min(value, CATALOG_LIST_MAX_LIMIT) +} + +function queryFingerprint( + resource: 'programs' | 'scholarships', + query: CatalogProgramListQuery | CatalogScholarshipListQuery, +): string { + const entries = Object.entries(query) + .filter(([key, value]) => ( + key !== 'cursor' + && key !== 'limit' + && key !== 'today' + && value !== undefined + )) + .sort(([left], [right]) => left.localeCompare(right)) + return JSON.stringify([resource, entries]) +} + +function requestedPage( + resource: 'programs' | 'scholarships', + query: CatalogProgramListQuery | CatalogScholarshipListQuery, +): number { + return query.cursor + ? decodeJsonListCursor(query.cursor, resource, queryFingerprint(resource, query)) + : 1 +} export class JsonCatalogRepository implements CatalogRepository { readonly mode = 'json' as const private bundlePromise: Promise | undefined @@ -46,8 +105,97 @@ export class JsonCatalogRepository implements CatalogRepository { async getRelease(): Promise { return deriveCatalogRelease(await this.getBundle()) } -} + async listPrograms(query: CatalogProgramListQuery = {}): Promise { + const today = query.today ?? getTodayDate() + const data = selectPublishedData(await this.getBundle(), today) + const limit = listLimit(query.limit) + const page = requestedPage('programs', query) + const degree = query.degree + ?? (query.type === 'language' || query.type === 'foundation' ? query.type : undefined) + const filters = { + ...parseProgramCatalogFilters({ + q: query.q, + institution: query.institution, + city: query.city, + degree, + discipline: query.discipline, + language: query.language, + intake: query.intake, + tuition: query.tuition, + applicationState: query.applicationState, + sort: query.sort, + }), + page, + } + const result = queryProgramCatalog(data, filters, today, limit) + if (page > 1 && result.page !== page) { + throw new CatalogRepositoryError('INVALID_LIST_CURSOR', 'Catalog list cursor is outside the result set.') + } + const nextCursor = page < result.pageCount + ? encodeJsonListCursor('programs', queryFingerprint('programs', query), page + 1) + : null + return { + items: result.items.map(({ program, university, cycle }) => ({ + program, + university, + currentCycle: cycle ?? null, + })), + nextCursor, + total: result.total, + facets: { + universities: result.universityOptions, + cities: result.cityOptions, + }, + release: deriveCatalogRelease(data), + } + } + + async listScholarships( + query: CatalogScholarshipListQuery = {}, + ): Promise { + const today = query.today ?? getTodayDate() + const published = selectPublishedData(await this.getBundle(), today) + const data = query.provider || query.program + ? { + ...published, + scholarships: published.scholarships.filter((scholarship) => ( + (!query.provider || scholarship.providerType === query.provider) + && (!query.program || scholarship.programIds.includes(query.program)) + )), + } + : published + const limit = listLimit(query.limit) + const page = requestedPage('scholarships', query) + const filters = { + ...parseScholarshipCatalogFilters({ + q: query.q, + institution: query.institution, + degree: query.degree, + funding: query.funding, + deadline: query.deadline, + sort: query.sort, + }), + page, + } + const result = queryScholarshipCatalog(data, filters, today, limit) + if (page > 1 && result.page !== page) { + throw new CatalogRepositoryError('INVALID_LIST_CURSOR', 'Catalog list cursor is outside the result set.') + } + const nextCursor = page < result.pageCount + ? encodeJsonListCursor('scholarships', queryFingerprint('scholarships', query), page + 1) + : null + + return { + items: result.items, + nextCursor, + total: result.total, + facets: { universities: result.universityOptions }, + release: deriveCatalogRelease(published), + } + } + +} export function createJsonCatalogRepository(loader?: CatalogBundleLoader): CatalogRepository { return new JsonCatalogRepository(loader) } diff --git a/src/lib/catalog/list-cursor.ts b/src/lib/catalog/list-cursor.ts new file mode 100644 index 0000000..0ecf6c4 --- /dev/null +++ b/src/lib/catalog/list-cursor.ts @@ -0,0 +1,90 @@ +import { Buffer } from 'node:buffer' +import { CatalogRepositoryError } from './types' + +type JsonListCursor = { + v: 1 + backend: 'json' + resource: 'programs' | 'scholarships' + fingerprint: string + page: number +} + +type ShadowListCursor = { + v: 1 + backend: 'shadow' + resource: 'programs' | 'scholarships' + primary: string | null + shadow: string | null +} + +type ListCursor = JsonListCursor | ShadowListCursor + +const MAX_CURSOR_LENGTH = 1_024 + +function fail(): never { + throw new CatalogRepositoryError('INVALID_LIST_CURSOR', 'Catalog list cursor is invalid.') +} + +function encode(value: ListCursor): string { + return Buffer.from(JSON.stringify(value), 'utf8').toString('base64url') +} + +function decode(value: string): ListCursor { + if (!value || value.length > MAX_CURSOR_LENGTH) fail() + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')) as Partial + if (parsed.v !== 1 || (parsed.backend !== 'json' && parsed.backend !== 'shadow')) fail() + return parsed as ListCursor + } catch (error) { + if (error instanceof CatalogRepositoryError) throw error + return fail() + } +} + +export function encodeJsonListCursor( + resource: JsonListCursor['resource'], + fingerprint: string, + page: number, +): string { + if (!Number.isSafeInteger(page) || page < 1) fail() + return encode({ v: 1, backend: 'json', resource, fingerprint, page }) +} + +export function decodeJsonListCursor( + value: string, + resource: JsonListCursor['resource'], + fingerprint: string, +): number { + const parsed = decode(value) + if ( + parsed.backend !== 'json' + || parsed.resource !== resource + || parsed.fingerprint !== fingerprint + || !Number.isSafeInteger(parsed.page) + || parsed.page < 1 + ) fail() + return parsed.page +} + +export function encodeShadowListCursor( + resource: ShadowListCursor['resource'], + primary: string | null, + shadow: string | null, +): string { + if (!primary) fail() + return encode({ v: 1, backend: 'shadow', resource, primary, shadow }) +} + +export function decodeShadowListCursor( + value: string, + resource: ShadowListCursor['resource'], +): { primary: string | null; shadow: string | null } { + const parsed = decode(value) + if ( + parsed.backend !== 'shadow' + || parsed.resource !== resource + || (parsed.primary !== null && typeof parsed.primary !== 'string') + || (parsed.shadow !== null && typeof parsed.shadow !== 'string') + ) fail() + return { primary: parsed.primary, shadow: parsed.shadow } +} diff --git a/src/lib/catalog/release.ts b/src/lib/catalog/release.ts index 1c3513a..823943e 100644 --- a/src/lib/catalog/release.ts +++ b/src/lib/catalog/release.ts @@ -30,7 +30,7 @@ function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value) } -export function parseCatalogRelease(value: unknown, bundle: DataBundle): CatalogRelease { +export function parseCatalogReleaseInfo(value: unknown): CatalogRelease { if (!isObject(value)) { throw new CatalogRepositoryError('INVALID_RELEASE', 'Catalog API release metadata is missing.') } @@ -49,7 +49,6 @@ export function parseCatalogRelease(value: unknown, bundle: DataBundle): Catalog throw new CatalogRepositoryError('INVALID_RELEASE', 'Catalog release recordCounts is missing.') } - const actualCounts = getCatalogRecordCounts(bundle) const parsedCounts = {} as CatalogRecordCounts for (const collection of CATALOG_COLLECTIONS) { const count = recordCounts[collection] @@ -59,14 +58,23 @@ export function parseCatalogRelease(value: unknown, bundle: DataBundle): Catalog `Catalog release count for ${collection} must be a non-negative integer.`, ) } + parsedCounts[collection] = count as number + } + + return { id, dataDate, generatedAt, recordCounts: parsedCounts } +} + +export function parseCatalogRelease(value: unknown, bundle: DataBundle): CatalogRelease { + const release = parseCatalogReleaseInfo(value) + const actualCounts = getCatalogRecordCounts(bundle) + for (const collection of CATALOG_COLLECTIONS) { + const count = release.recordCounts[collection] if (count !== actualCounts[collection]) { throw new CatalogRepositoryError( 'RELEASE_COUNT_MISMATCH', `Catalog release count for ${collection} is ${String(count)}; bundle contains ${actualCounts[collection]}.`, ) } - parsedCounts[collection] = count as number } - - return { id, dataDate, generatedAt, recordCounts: parsedCounts } + return release } diff --git a/src/lib/catalog/runtime.ts b/src/lib/catalog/runtime.ts new file mode 100644 index 0000000..876854d --- /dev/null +++ b/src/lib/catalog/runtime.ts @@ -0,0 +1,15 @@ +import 'server-only' +import { createCatalogRepository } from './repository' +import type { CatalogRepository } from './types' + +let repository: CatalogRepository | undefined + +/** Shared server-side repository; JSON parsing and remote clients are reused per process. */ +export function getCatalogRepository(): CatalogRepository { + repository ??= createCatalogRepository() + return repository +} + +export function resetCatalogRepositoryRuntimeForTests(): void { + repository = undefined +} diff --git a/src/lib/catalog/shadow.ts b/src/lib/catalog/shadow.ts index a520159..8ab3004 100644 --- a/src/lib/catalog/shadow.ts +++ b/src/lib/catalog/shadow.ts @@ -3,11 +3,23 @@ import { CATALOG_COLLECTIONS, type CatalogBackendMode, type CatalogCollection, + type CatalogProgramListPage, + type CatalogProgramListQuery, type CatalogRelease, type CatalogRepository, + type CatalogScholarshipListPage, + type CatalogScholarshipListQuery, } from './types' - -export type CatalogShadowOperation = 'getBundle' | 'getRelease' +import { + decodeShadowListCursor, + encodeShadowListCursor, +} from './list-cursor' + +export type CatalogShadowOperation = + | 'getBundle' + | 'getRelease' + | 'listPrograms' + | 'listScholarships' export type CatalogShadowStatus = 'match' | 'different' | 'shadow-error' export type CatalogShadowDifferenceKind = 'missing-in-shadow' | 'extra-in-shadow' | 'value-mismatch' export type CatalogShadowScope = CatalogCollection | 'release' @@ -238,6 +250,65 @@ function compareRelease( return collector } +function comparableListPage( + page: CatalogProgramListPage | CatalogScholarshipListPage, +): unknown { + const facets = Object.fromEntries( + Object.entries(page.facets).map(([name, values]) => [ + name, + [...values].sort((left, right) => left.value.localeCompare(right.value)), + ]), + ) + return { + items: page.items, + total: page.total, + facets, + } +} + +function compareListPage( + scope: 'programs' | 'scholarships', + primary: CatalogProgramListPage | CatalogScholarshipListPage, + shadow: CatalogProgramListPage | CatalogScholarshipListPage, + maxDifferences: number, +): DifferenceCollector { + const collector = new DifferenceCollector(maxDifferences) + compareValue( + collector, + scope, + 'page', + '', + comparableListPage(primary), + comparableListPage(shadow), + ) + return collector +} + +function cursorInputs( + cursor: string | undefined, + resource: 'programs' | 'scholarships', +): { primary?: string; shadow?: string } { + if (!cursor) return {} + try { + const decoded = decodeShadowListCursor(cursor, resource) + return { + ...(decoded.primary ? { primary: decoded.primary } : {}), + ...(decoded.shadow ? { shadow: decoded.shadow } : {}), + } + } catch { + // URLs created before shadow mode remain usable by the primary backend. + return { primary: cursor } + } +} + +function combinedCursor( + resource: 'programs' | 'scholarships', + primary: string | null, + shadow: string | null, +): string | null { + return primary ? encodeShadowListCursor(resource, primary, shadow) : null +} + function serializeError(error: unknown): { name: string; message: string } { if (error instanceof Error) return { name: error.name, message: error.message } return { name: 'Error', message: String(error) } @@ -302,6 +373,75 @@ export class ShadowCatalogRepository implements CatalogRepository { return primaryResult.value } + async listPrograms( + query: CatalogProgramListQuery = {}, + ): Promise { + const cursors = cursorInputs(query.cursor, 'programs') + const [primaryResult, shadowResult] = await Promise.allSettled([ + this.primary.listPrograms({ ...query, cursor: cursors.primary }), + this.shadow.listPrograms({ ...query, cursor: cursors.shadow }), + ]) + if (primaryResult.status === 'rejected') throw primaryResult.reason + + if (shadowResult.status === 'rejected') { + await this.recordShadowError('listPrograms', shadowResult.reason) + return { + ...primaryResult.value, + nextCursor: combinedCursor('programs', primaryResult.value.nextCursor, null), + } + } + + await this.recordComparison( + 'listPrograms', + compareListPage('programs', primaryResult.value, shadowResult.value, this.maxDifferences), + ) + return { + ...primaryResult.value, + nextCursor: combinedCursor( + 'programs', + primaryResult.value.nextCursor, + shadowResult.value.nextCursor, + ), + } + } + + async listScholarships( + query: CatalogScholarshipListQuery = {}, + ): Promise { + const cursors = cursorInputs(query.cursor, 'scholarships') + const [primaryResult, shadowResult] = await Promise.allSettled([ + this.primary.listScholarships({ ...query, cursor: cursors.primary }), + this.shadow.listScholarships({ ...query, cursor: cursors.shadow }), + ]) + if (primaryResult.status === 'rejected') throw primaryResult.reason + + if (shadowResult.status === 'rejected') { + await this.recordShadowError('listScholarships', shadowResult.reason) + return { + ...primaryResult.value, + nextCursor: combinedCursor('scholarships', primaryResult.value.nextCursor, null), + } + } + + await this.recordComparison( + 'listScholarships', + compareListPage( + 'scholarships', + primaryResult.value, + shadowResult.value, + this.maxDifferences, + ), + ) + return { + ...primaryResult.value, + nextCursor: combinedCursor( + 'scholarships', + primaryResult.value.nextCursor, + shadowResult.value.nextCursor, + ), + } + } + private async recordComparison( operation: CatalogShadowOperation, collector: DifferenceCollector, diff --git a/src/lib/catalog/types.ts b/src/lib/catalog/types.ts index 6cff514..686f5cc 100644 --- a/src/lib/catalog/types.ts +++ b/src/lib/catalog/types.ts @@ -1,4 +1,11 @@ -import type { DataBundle } from '@/lib/data/types' +import type { + AdmissionCycle, + DataBundle, + LocalizedText, + Program, + Scholarship, + University, +} from '@/lib/data/types' export const CATALOG_COLLECTIONS = [ 'sources', @@ -14,6 +21,102 @@ export type CatalogBackendMode = 'json' | 'd1' | 'shadow' export type CatalogRecordCounts = Record +export const CATALOG_LIST_DEFAULT_LIMIT = 24 +export const CATALOG_LIST_MAX_LIMIT = 100 + +export type CatalogListOption = { + value: string + name: LocalizedText +} + +export type CatalogListPage = { + items: T[] + nextCursor: string | null + /** Exact for JSON and APIs that expose a count; null for cursor-only APIs. */ + total: number | null + facets: Facets + release: CatalogRelease | null +} + +export type CatalogProgramListQuery = { + q?: string + institution?: string + city?: string + type?: string + degree?: string + discipline?: string + language?: string + academicYear?: string + intake?: string + tuition?: string + tuitionMin?: number + tuitionMax?: number + applicationState?: string + scholarship?: string + sort?: string + cursor?: string + limit?: number + today?: string +} + +export type CatalogProgramListItem = { + program: Program + university: University + currentCycle: AdmissionCycle | null +} + +export type CatalogProgramListFacets = { + universities: CatalogListOption[] + cities: CatalogListOption[] +} + +export type CatalogProgramListPage = CatalogListPage< + CatalogProgramListItem, + CatalogProgramListFacets +> + +export type CatalogScholarshipListQuery = { + q?: string + provider?: string + institution?: string + program?: string + degree?: string + funding?: string + deadline?: string + sort?: string + cursor?: string + limit?: number + today?: string +} + +export type CatalogScholarshipCurrentCycle = { + id: string + scholarshipId: string + academicYear: string | null + opensOn: string | null + closesOn: string | null + deadline: string | null + deadlineState: 'future' | 'closed' | 'not-announced' + daysRemaining: number | null + legacy: boolean +} + +export type CatalogScholarshipListItem = { + scholarship: Scholarship + universities: University[] + programs: Program[] + currentCycle: CatalogScholarshipCurrentCycle +} + +export type CatalogScholarshipListFacets = { + universities: CatalogListOption[] +} + +export type CatalogScholarshipListPage = CatalogListPage< + CatalogScholarshipListItem, + CatalogScholarshipListFacets +> + export type CatalogRelease = { id: string dataDate: string @@ -25,6 +128,8 @@ export interface CatalogRepository { readonly mode: CatalogBackendMode getBundle(): Promise getRelease(): Promise + listPrograms(query?: CatalogProgramListQuery): Promise + listScholarships(query?: CatalogScholarshipListQuery): Promise } export type CatalogBundleLoader = () => unknown | Promise diff --git a/src/lib/program-catalog.ts b/src/lib/program-catalog.ts new file mode 100644 index 0000000..ffcd027 --- /dev/null +++ b/src/lib/program-catalog.ts @@ -0,0 +1,418 @@ +import { getApplicationState, selectAdmissionCycle } from '@/lib/data/admission' +import { classifyProgramField, normalizeProgramField, programSearchKeywords } from '@/lib/data/fields' +import type { + AdmissionCycle, + DataBundle, + DegreeLevel, + LocalizedText, + Program, + University, +} from '@/lib/data/types' +import type { CatalogProgramListQuery, CatalogRepository } from '@/lib/catalog/types' + +export const PROGRAM_CATALOG_PAGE_SIZE = 24 + +const DEGREE_LEVELS = new Set([ + 'bachelor', + 'master', + 'doctorate', + 'language', + 'foundation', + 'other', +]) +const TEACHING_LANGUAGES = new Set(['Chinese', 'English', 'Bilingual']) +const APPLICATION_STATES = new Set(['open', 'upcoming', 'closed', 'not-announced']) +const INTAKES = new Set(['spring', 'autumn', 'other']) +const TUITION_FILTERS = new Set([ + 'known', + 'unknown', + 'under-20000', + '20000-40000', + 'over-40000', +]) +const SORT_ORDERS = new Set([ + 'default', + 'name', + 'deadline', + 'tuition-asc', + 'tuition-desc', +]) + +export type ProgramCatalogSearchParams = Record +export type ProgramCatalogFilters = { + query: string + degree: string + discipline: string + language: string + institution: string + city: string + intake: string + tuition: string + applicationState: string + sort: string + page: number + cursor: string + cursorHistory: string[] + nextCursor?: string +} + +export type ProgramCatalogOption = { + value: string + name: LocalizedText +} + +export type ProgramCatalogItem = { + program: Program + university: University + cycle: AdmissionCycle | undefined +} + +export type ProgramCatalogResult = { + items: ProgramCatalogItem[] + filters: ProgramCatalogFilters + total: number + totalExact: boolean + page: number + pageCount: number + pageSize: number + universityOptions: ProgramCatalogOption[] + cityOptions: ProgramCatalogOption[] +} + +function first(value: string | string[] | undefined): string { + return typeof value === 'string' ? value : '' +} + +function bounded(value: string | string[] | undefined, maxLength = 160): string { + return first(value).trim().slice(0, maxLength) +} + +function cursorValue(value: string | string[] | undefined): string { + const cursor = first(value).trim() + return cursor.length <= 1_024 ? cursor : '' +} + +function cursorHistory(value: string | string[] | undefined): string[] { + const history = first(value).trim() + if (!history || history.length > 8_192) return [] + const entries = history.split(',').slice(-50) + return entries.every((entry) => entry === '~' || (entry.length > 0 && entry.length <= 1_024)) + ? entries + : [] +} + +function allowed(value: string, values: ReadonlySet): string { + return values.has(value) ? value : '' +} + +export function parseProgramCatalogFilters( + params: ProgramCatalogSearchParams, +): ProgramCatalogFilters { + const requestedDiscipline = bounded(params.discipline) + const requestedPage = Number.parseInt(first(params.page), 10) + const applicationState = bounded(params.applicationState) + || bounded(params.dateStatus) + + return { + query: bounded(params.q), + degree: allowed(bounded(params.degree), DEGREE_LEVELS), + discipline: requestedDiscipline + ? normalizeProgramField(requestedDiscipline) ?? '' + : '', + language: allowed(bounded(params.language), TEACHING_LANGUAGES), + institution: bounded(params.institution), + city: bounded(params.city), + intake: allowed(bounded(params.intake), INTAKES), + tuition: allowed(bounded(params.tuition), TUITION_FILTERS), + applicationState: allowed(applicationState, APPLICATION_STATES), + sort: allowed(bounded(params.sort), SORT_ORDERS) || 'default', + cursor: cursorValue(params.cursor), + cursorHistory: cursorHistory(params.cursorHistory), + page: Number.isSafeInteger(requestedPage) && requestedPage > 0 + ? requestedPage + : 1, + } +} + +function searchable(value: unknown): string { + if (typeof value === 'string') return value + if (Array.isArray(value)) return value.map(searchable).join(' ') + if (value && typeof value === 'object') return Object.values(value).map(searchable).join(' ') + return '' +} + +function includesQuery(values: unknown[], query: string): boolean { + const normalized = query.trim().toLocaleLowerCase() + return !normalized || searchable(values).toLocaleLowerCase().includes(normalized) +} + +function matchesApplicationState( + cycle: AdmissionCycle | undefined, + expected: string, + today: string, +): boolean { + if (!expected) return true + const state = getApplicationState(cycle, today) + return expected === 'open' + ? state === 'open' || state === 'rolling' + : state === expected +} + +function matchesTuition(cycle: AdmissionCycle | undefined, expected: string): boolean { + if (!expected) return true + const tuition = cycle?.tuitionCny + if (expected === 'known') return tuition !== null && tuition !== undefined + if (expected === 'unknown') return tuition === null || tuition === undefined + if (tuition === null || tuition === undefined) return false + if (expected === 'under-20000') return tuition <= 20_000 + if (expected === '20000-40000') return tuition > 20_000 && tuition <= 40_000 + return tuition > 40_000 +} + +function localizedSortValue(value: LocalizedText): string { + return value.en || value.zh || Object.values(value).find(Boolean) || '' +} + +function compareNullableNumbers( + left: number | null | undefined, + right: number | null | undefined, + direction: 'asc' | 'desc', +): number { + const leftKnown = left !== null && left !== undefined + const rightKnown = right !== null && right !== undefined + if (leftKnown !== rightKnown) return leftKnown ? -1 : 1 + if (!leftKnown || !rightKnown || left === right) return 0 + return direction === 'asc' ? left - right : right - left +} + +function sortEntries( + entries: ProgramCatalogItem[], + sort: string, +): ProgramCatalogItem[] { + if (sort === 'default') return entries + return entries.sort((left, right) => { + if (sort === 'name') { + return localizedSortValue(left.program.name) + .localeCompare(localizedSortValue(right.program.name), 'en') + } + if (sort === 'deadline') { + return (left.cycle?.closesOn || '9999-12-31') + .localeCompare(right.cycle?.closesOn || '9999-12-31') + } + return compareNullableNumbers( + left.cycle?.tuitionCny, + right.cycle?.tuitionCny, + sort === 'tuition-desc' ? 'desc' : 'asc', + ) + }) +} + +function catalogOptions(data: DataBundle) { + const universityIds = new Set(data.programs.map((program) => program.universityId)) + const universities = data.universities.filter((university) => universityIds.has(university.id)) + const cityIds = new Set(universities.map((university) => university.cityId)) + return { + universityOptions: universities.map(({ slug, name }) => ({ value: slug, name })), + cityOptions: data.cities + .filter((city) => cityIds.has(city.id)) + .map(({ slug, name }) => ({ value: slug, name })), + } +} + +/** + * Server-side catalogue query used by the web page today and designed to map + * directly to the D1 list endpoint later. Only the returned page is handed to + * the explorer component; the full catalogue never crosses a client boundary. + */ +export function queryProgramCatalog( + data: DataBundle, + filters: ProgramCatalogFilters, + today: string, + pageSize = PROGRAM_CATALOG_PAGE_SIZE, +): ProgramCatalogResult { + const universitiesById = new Map(data.universities.map((item) => [item.id, item])) + const citiesById = new Map(data.cities.map((item) => [item.id, item])) + const cyclesByProgram = new Map() + for (const cycle of data.admissionCycles) { + const cycles = cyclesByProgram.get(cycle.programId) || [] + cycles.push(cycle) + cyclesByProgram.set(cycle.programId, cycles) + } + + const matching = data.programs.flatMap((program): ProgramCatalogItem[] => { + const university = universitiesById.get(program.universityId) + if (!university) return [] + const city = citiesById.get(university.cityId) + const programCycles = cyclesByProgram.get(program.id) || [] + const cycle = selectAdmissionCycle(programCycles, program.id, today) + + const matches = includesQuery([ + program.name, + program.discipline, + programSearchKeywords(program), + program.teachingLanguages, + university.name, + city?.name, + ], filters.query) + && (!filters.degree || program.degreeLevel === filters.degree) + && (!filters.discipline || classifyProgramField(program) === filters.discipline) + && (!filters.language || program.teachingLanguages.includes(filters.language)) + && (!filters.institution + || university.slug === filters.institution + || university.id === filters.institution) + && (!filters.city || city?.slug === filters.city || city?.id === filters.city) + && (!filters.intake || cycle?.intake === filters.intake) + && matchesApplicationState(cycle, filters.applicationState, today) + && matchesTuition(cycle, filters.tuition) + + return matches ? [{ program, university, cycle }] : [] + }) + + sortEntries(matching, filters.sort) + const normalizedPageSize = Math.min(100, Math.max(1, pageSize)) + const pageCount = Math.ceil(matching.length / normalizedPageSize) + const page = pageCount === 0 ? 1 : Math.min(filters.page, pageCount) + const offset = (page - 1) * normalizedPageSize + + return { + items: matching.slice(offset, offset + normalizedPageSize), + filters: { ...filters, page }, + total: matching.length, + totalExact: true, + page, + pageCount, + pageSize: normalizedPageSize, + ...catalogOptions(data), + } +} + +function repositoryProgramQuery( + filters: ProgramCatalogFilters, + today: string, + cursor: string | undefined, +): CatalogProgramListQuery { + return { + q: filters.query || undefined, + degree: filters.degree || undefined, + discipline: filters.discipline || undefined, + language: filters.language || undefined, + institution: filters.institution || undefined, + city: filters.city || undefined, + intake: filters.intake || undefined, + tuition: filters.tuition || undefined, + applicationState: filters.applicationState || undefined, + sort: filters.sort === 'default' ? undefined : filters.sort, + cursor, + limit: PROGRAM_CATALOG_PAGE_SIZE, + today, + } +} + +function repositoryProgramResult( + pageResult: Awaited>, + filters: ProgramCatalogFilters, + page: number, + cursor: string | undefined, + history: string[], +): ProgramCatalogResult { + const lowerBound = (page - 1) * PROGRAM_CATALOG_PAGE_SIZE + + pageResult.items.length + + (pageResult.nextCursor ? 1 : 0) + const total = pageResult.total ?? lowerBound + const pageCount = pageResult.total === null + ? page + (pageResult.nextCursor ? 1 : 0) + : Math.ceil(pageResult.total / PROGRAM_CATALOG_PAGE_SIZE) + + return { + items: pageResult.items.map(({ program, university, currentCycle }) => ({ + program, + university, + cycle: currentCycle ?? undefined, + })), + filters: { + ...filters, + page, + cursor: cursor ?? '', + cursorHistory: history, + nextCursor: pageResult.nextCursor ?? undefined, + }, + total, + totalExact: pageResult.total !== null, + page, + pageCount, + pageSize: PROGRAM_CATALOG_PAGE_SIZE, + universityOptions: pageResult.facets.universities, + cityOptions: pageResult.facets.cities, + } +} + +/** + * Repository-backed request path. Cursor links make normal navigation one + * bounded backend request; old page-number URLs are replayed only for + * compatibility and immediately emit cursor-based links. + */ +export async function queryProgramCatalogRepository( + repository: CatalogRepository, + filters: ProgramCatalogFilters, + today: string, +): Promise { + let cursor = filters.cursor || undefined + const history = [...filters.cursorHistory] + let page = cursor ? filters.page : 1 + + while (!filters.cursor && page < filters.page) { + const preceding = await repository.listPrograms( + repositoryProgramQuery(filters, today, cursor), + ) + if (!preceding.nextCursor) { + return repositoryProgramResult(preceding, filters, page, cursor, history) + } + history.push(cursor ?? '~') + cursor = preceding.nextCursor + page += 1 + } + + const result = await repository.listPrograms( + repositoryProgramQuery(filters, today, cursor), + ) + return repositoryProgramResult(result, filters, page, cursor, history) +} + +export function programCatalogHref( + locale: string, + filters: ProgramCatalogFilters, + page = filters.page, +): string { + const params = new URLSearchParams() + let targetCursor = filters.cursor + let targetHistory = [...filters.cursorHistory] + if (page === filters.page + 1 && filters.nextCursor) { + targetHistory.push(filters.cursor || '~') + targetCursor = filters.nextCursor + } else if (page === filters.page - 1) { + const previous = targetHistory.pop() + targetCursor = previous && previous !== '~' ? previous : '' + } else if (page !== filters.page) { + targetCursor = '' + targetHistory = [] + } + + const values: Array<[string, string]> = [ + ['q', filters.query], + ['degree', filters.degree], + ['discipline', filters.discipline], + ['language', filters.language], + ['institution', filters.institution], + ['city', filters.city], + ['intake', filters.intake], + ['tuition', filters.tuition], + ['applicationState', filters.applicationState], + ['sort', filters.sort === 'default' ? '' : filters.sort], + ] + for (const [key, value] of values) if (value) params.set(key, value) + if (targetCursor) params.set('cursor', targetCursor) + if (targetHistory.length > 0) params.set('cursorHistory', targetHistory.join(',')) + if (page > 1) params.set('page', String(page)) + const query = params.toString() + return `/${locale}/programs${query ? `?${query}` : ''}` +} diff --git a/src/lib/scholarship-catalog.ts b/src/lib/scholarship-catalog.ts new file mode 100644 index 0000000..554c669 --- /dev/null +++ b/src/lib/scholarship-catalog.ts @@ -0,0 +1,410 @@ +import type { + DataBundle, + DegreeLevel, + LocalizedText, + Program, + Scholarship, + University, +} from '@/lib/data/types' +import type { CatalogRepository, CatalogScholarshipListQuery } from '@/lib/catalog/types' + +export const SCHOLARSHIP_CATALOG_PAGE_SIZE = 24 + +const DEGREE_LEVELS = new Set([ + 'bachelor', + 'master', + 'doctorate', + 'language', + 'foundation', + 'other', +]) +const FUNDING_FILTERS = new Set([ + 'full-tuition', + 'partial-tuition', + 'stipend', + 'accommodation', + 'insurance', +]) +const DEADLINE_FILTERS = new Set([ + 'future', + 'next-30-days', + 'next-90-days', + 'announced', + 'not-announced', + 'closed', +]) +const SORT_ORDERS = new Set([ + 'default', + 'name', + 'deadline', + 'stipend-desc', +]) + +export type ScholarshipCatalogSearchParams = Record +export type ScholarshipCatalogFilters = { + query: string + institution: string + degree: string + funding: string + deadline: string + sort: string + page: number + cursor: string + cursorHistory: string[] + nextCursor?: string +} + +export type ScholarshipCatalogOption = { + value: string + name: LocalizedText +} + +export type ScholarshipCatalogCycle = { + id: string + scholarshipId: string + academicYear: string | null + opensOn: string | null + closesOn: string | null + deadline: string | null + deadlineState: 'future' | 'closed' | 'not-announced' + daysRemaining: number | null + legacy: boolean +} + +export type ScholarshipCatalogItem = { + scholarship: Scholarship + universities: University[] + programs: Program[] + currentCycle: ScholarshipCatalogCycle +} + +export type ScholarshipCatalogResult = { + items: ScholarshipCatalogItem[] + filters: ScholarshipCatalogFilters + total: number + totalExact: boolean + page: number + pageCount: number + pageSize: number + universityOptions: ScholarshipCatalogOption[] +} + +function first(value: string | string[] | undefined): string { + return typeof value === 'string' ? value : '' +} + +function bounded(value: string | string[] | undefined, maxLength = 160): string { + return first(value).trim().slice(0, maxLength) +} + +function cursorValue(value: string | string[] | undefined): string { + const cursor = first(value).trim() + return cursor.length <= 1_024 ? cursor : '' +} + +function cursorHistory(value: string | string[] | undefined): string[] { + const history = first(value).trim() + if (!history || history.length > 8_192) return [] + const entries = history.split(',').slice(-50) + return entries.every((entry) => entry === '~' || (entry.length > 0 && entry.length <= 1_024)) + ? entries + : [] +} + +function allowed(value: string, values: ReadonlySet): string { + return values.has(value) ? value : '' +} + +export function parseScholarshipCatalogFilters( + params: ScholarshipCatalogSearchParams, +): ScholarshipCatalogFilters { + const requestedPage = Number.parseInt(first(params.page), 10) + + return { + query: bounded(params.q), + institution: bounded(params.institution), + degree: allowed(bounded(params.degree), DEGREE_LEVELS), + funding: allowed(bounded(params.funding), FUNDING_FILTERS), + deadline: allowed(bounded(params.deadline), DEADLINE_FILTERS), + sort: allowed(bounded(params.sort), SORT_ORDERS) || 'default', + page: Number.isSafeInteger(requestedPage) && requestedPage > 0 + ? requestedPage + : 1, + cursor: cursorValue(params.cursor), + cursorHistory: cursorHistory(params.cursorHistory), + } +} + +function searchable(value: unknown): string { + if (typeof value === 'string') return value + if (Array.isArray(value)) return value.map(searchable).join(' ') + if (value && typeof value === 'object') return Object.values(value).map(searchable).join(' ') + return '' +} + +function includesQuery(values: unknown[], query: string): boolean { + const normalized = query.trim().toLocaleLowerCase() + return !normalized || searchable(values).toLocaleLowerCase().includes(normalized) +} + +function utcDateValue(date: string): number | null { + const value = Date.parse(`${date}T00:00:00.000Z`) + return Number.isNaN(value) ? null : value +} + +function addDays(date: string, days: number): string { + const value = utcDateValue(date) + if (value === null) return date + return new Date(value + days * 86_400_000).toISOString().slice(0, 10) +} + +/** + * Compatibility cycle for the legacy scholarship shape. It deliberately does + * not infer an opening date or academic year that the source data never stated. + */ +export function selectScholarshipCurrentCycle( + scholarship: Scholarship, + today: string, +): ScholarshipCatalogCycle { + const deadline = scholarship.deadline + const todayValue = utcDateValue(today) + const deadlineValue = deadline ? utcDateValue(deadline) : null + const daysRemaining = todayValue === null || deadlineValue === null + ? null + : Math.ceil((deadlineValue - todayValue) / 86_400_000) + + return { + id: `legacy:${scholarship.id}`, + scholarshipId: scholarship.id, + academicYear: null, + opensOn: null, + closesOn: deadline, + deadline, + deadlineState: deadline === null + ? 'not-announced' + : deadline < today + ? 'closed' + : 'future', + daysRemaining, + legacy: true, + } +} + +function matchesFunding(scholarship: Scholarship, expected: string): boolean { + if (!expected) return true + if (expected === 'full-tuition') return scholarship.coverage.tuition === 'full' + if (expected === 'partial-tuition') return scholarship.coverage.tuition === 'partial' + if (expected === 'stipend') return (scholarship.coverage.stipendCnyPerMonth || 0) > 0 + if (expected === 'accommodation') { + return scholarship.coverage.accommodation === 'full' + || scholarship.coverage.accommodation === 'partial' + } + return scholarship.coverage.insurance === true +} + +function matchesDeadline( + cycle: ScholarshipCatalogCycle, + expected: string, + today: string, +): boolean { + if (!expected) return true + if (expected === 'not-announced') return cycle.deadline === null + if (expected === 'announced') return cycle.deadline !== null + if (expected === 'closed') return cycle.deadlineState === 'closed' + if (cycle.deadline === null || cycle.deadline < today) return false + if (expected === 'future') return true + const cutoff = addDays(today, expected === 'next-30-days' ? 30 : 90) + return cycle.deadline <= cutoff +} + +function localizedSortValue(value: LocalizedText): string { + return value.en || value.zh || Object.values(value).find(Boolean) || '' +} + +function deadlineSortKey(item: ScholarshipCatalogItem, today: string): string { + const deadline = item.currentCycle.deadline + if (deadline && deadline >= today) return `0:${deadline}` + if (deadline) return `1:${deadline}` + return '2:9999-12-31' +} + +function sortEntries( + entries: ScholarshipCatalogItem[], + sort: string, + today: string, +): ScholarshipCatalogItem[] { + if (sort === 'default') return entries + return entries.sort((left, right) => { + let order = 0 + if (sort === 'name') { + order = localizedSortValue(left.scholarship.name) + .localeCompare(localizedSortValue(right.scholarship.name), 'en') + } else if (sort === 'deadline') { + order = deadlineSortKey(left, today).localeCompare(deadlineSortKey(right, today)) + } else { + order = (right.scholarship.coverage.stipendCnyPerMonth || -1) + - (left.scholarship.coverage.stipendCnyPerMonth || -1) + } + return order || left.scholarship.slug.localeCompare(right.scholarship.slug, 'en') + }) +} + +function catalogOptions(data: DataBundle): ScholarshipCatalogOption[] { + const universityIds = new Set(data.scholarships.flatMap((item) => item.universityIds)) + return data.universities + .filter((university) => universityIds.has(university.id)) + .map(({ slug, name }) => ({ value: slug, name })) +} + +/** + * Server-side scholarship query shaped to map directly to the future D1 list + * endpoint. Only this bounded page is handed to the render component. + */ +export function queryScholarshipCatalog( + data: DataBundle, + filters: ScholarshipCatalogFilters, + today: string, + pageSize = SCHOLARSHIP_CATALOG_PAGE_SIZE, +): ScholarshipCatalogResult { + const universitiesById = new Map(data.universities.map((item) => [item.id, item])) + const programsById = new Map(data.programs.map((item) => [item.id, item])) + + const matching = data.scholarships.flatMap((scholarship): ScholarshipCatalogItem[] => { + const universities = scholarship.universityIds.flatMap((id) => { + const university = universitiesById.get(id) + return university ? [university] : [] + }) + const programs = scholarship.programIds.flatMap((id) => { + const program = programsById.get(id) + return program ? [program] : [] + }) + const currentCycle = selectScholarshipCurrentCycle(scholarship, today) + + const matches = includesQuery([ + scholarship.name, + scholarship.summary, + scholarship.providerType, + universities.map((item) => item.name), + programs.map((item) => item.name), + ], filters.query) + && (!filters.institution || universities.some((item) => ( + item.slug === filters.institution || item.id === filters.institution + ))) + && (!filters.degree || programs.some((item) => item.degreeLevel === filters.degree)) + && matchesFunding(scholarship, filters.funding) + && matchesDeadline(currentCycle, filters.deadline, today) + + return matches ? [{ scholarship, universities, programs, currentCycle }] : [] + }) + + sortEntries(matching, filters.sort, today) + const normalizedPageSize = Math.min(100, Math.max(1, pageSize)) + const pageCount = Math.ceil(matching.length / normalizedPageSize) + const page = pageCount === 0 ? 1 : Math.min(filters.page, pageCount) + const offset = (page - 1) * normalizedPageSize + + return { + items: matching.slice(offset, offset + normalizedPageSize), + filters: { ...filters, page }, + total: matching.length, + totalExact: true, + page, + pageCount, + pageSize: normalizedPageSize, + universityOptions: catalogOptions(data), + } +} + +function repositoryScholarshipQuery( + filters: ScholarshipCatalogFilters, + today: string, + cursor: string | undefined, +): CatalogScholarshipListQuery { + return { + q: filters.query || undefined, + institution: filters.institution || undefined, + degree: filters.degree || undefined, + funding: filters.funding || undefined, + deadline: filters.deadline || undefined, + sort: filters.sort === 'default' ? undefined : filters.sort, + cursor, + limit: SCHOLARSHIP_CATALOG_PAGE_SIZE, + today, + } +} + +function repositoryScholarshipResult( + pageResult: Awaited>, + filters: ScholarshipCatalogFilters, + page: number, + cursor: string | undefined, + history: string[], +): ScholarshipCatalogResult { + const lowerBound = (page - 1) * SCHOLARSHIP_CATALOG_PAGE_SIZE + + pageResult.items.length + + (pageResult.nextCursor ? 1 : 0) + const total = pageResult.total ?? lowerBound + return { + items: pageResult.items, + filters: { + ...filters, + page, + cursor: cursor ?? '', + cursorHistory: history, + nextCursor: pageResult.nextCursor ?? undefined, + }, + total, + totalExact: pageResult.total !== null, + page, + pageCount: pageResult.total === null + ? page + (pageResult.nextCursor ? 1 : 0) + : Math.ceil(pageResult.total / SCHOLARSHIP_CATALOG_PAGE_SIZE), + pageSize: SCHOLARSHIP_CATALOG_PAGE_SIZE, + universityOptions: pageResult.facets.universities, + } +} + +export async function queryScholarshipCatalogRepository( + repository: CatalogRepository, + filters: ScholarshipCatalogFilters, + today: string, +): Promise { + let cursor = filters.cursor || undefined + const history = [...filters.cursorHistory] + let page = cursor ? filters.page : 1 + while (!filters.cursor && page < filters.page) { + const preceding = await repository.listScholarships( + repositoryScholarshipQuery(filters, today, cursor), + ) + if (!preceding.nextCursor) { + return repositoryScholarshipResult(preceding, filters, page, cursor, history) + } + history.push(cursor ?? '~') + cursor = preceding.nextCursor + page += 1 + } + const result = await repository.listScholarships( + repositoryScholarshipQuery(filters, today, cursor), + ) + return repositoryScholarshipResult(result, filters, page, cursor, history) +} + +export function scholarshipCatalogHref( + locale: string, + filters: ScholarshipCatalogFilters, + page = filters.page, +): string { + const params = new URLSearchParams() + const values: Array<[string, string]> = [ + ['q', filters.query], + ['institution', filters.institution], + ['degree', filters.degree], + ['funding', filters.funding], + ['deadline', filters.deadline], + ['sort', filters.sort === 'default' ? '' : filters.sort], + ] + for (const [key, value] of values) if (value) params.set(key, value) + if (page > 1) params.set('page', String(page)) + const query = params.toString() + return `/${locale}/scholarships${query ? `?${query}` : ''}` +} diff --git a/tests/unit/catalog-repository.test.ts b/tests/unit/catalog-repository.test.ts index 3922a36..d1d27a5 100644 --- a/tests/unit/catalog-repository.test.ts +++ b/tests/unit/catalog-repository.test.ts @@ -229,6 +229,8 @@ describe('CatalogRepository', () => { mode: 'd1', getBundle: async () => { throw new Error('shadow unavailable') }, getRelease: async () => { throw new Error('shadow unavailable') }, + listPrograms: async () => { throw new Error('shadow unavailable') }, + listScholarships: async () => { throw new Error('shadow unavailable') }, } const repository = createShadowCatalogRepository({ primary, shadow: failingShadow }) diff --git a/tests/unit/cloudflare-backup-preflight.test.ts b/tests/unit/cloudflare-backup-preflight.test.ts new file mode 100644 index 0000000..e0273be --- /dev/null +++ b/tests/unit/cloudflare-backup-preflight.test.ts @@ -0,0 +1,78 @@ +import { createHash } from 'node:crypto' +import { gzipSync } from 'node:zlib' +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + inspectBackupArtifacts, + validateBackupCredentials, +} from '../../scripts/cloudflare/backup-preflight' + +function digest(value: Buffer): string { + return createHash('sha256').update(value).digest('hex') +} + +function backupFixture(): string { + const directory = mkdtempSync(join(tmpdir(), 'studyinchina-backup-preflight-')) + const catalog = gzipSync('INSERT INTO catalog_releases VALUES (1);') + const pipeline = gzipSync('INSERT INTO ingestion_jobs VALUES (1);') + writeFileSync(join(directory, 'catalog.sql.gz'), catalog) + writeFileSync(join(directory, 'pipeline.sql.gz'), pipeline) + writeFileSync( + join(directory, 'backup-sha256.txt'), + `${digest(catalog)} catalog.sql.gz\n${digest(pipeline)} pipeline.sql.gz\n`, + ) + return directory +} + +describe('Cloudflare backup preflight', () => { + it('validates credential presence without returning secret values', () => { + const token = 'secret-token-that-must-not-be-printed' + const result = validateBackupCredentials({ + CLOUDFLARE_API_TOKEN: token, + CLOUDFLARE_ACCOUNT_ID: '78969c65bfdd892bb12c116869ea91cf', + }) + expect(result).toEqual({ databases: 2, bucket: 'studyinchina-releases' }) + expect(JSON.stringify(result)).not.toContain(token) + expect(() => validateBackupCredentials({})).toThrow(/API_TOKEN/u) + expect(() => validateBackupCredentials({ + CLOUDFLARE_API_TOKEN: token, + CLOUDFLARE_ACCOUNT_ID: 'invalid', + })).toThrow(/32-character hexadecimal/u) + }) + + it('cryptographically verifies both non-empty gzip artifacts', () => { + const directory = backupFixture() + const result = inspectBackupArtifacts(directory) + expect(result.map(({ file }) => file)).toEqual(['catalog.sql.gz', 'pipeline.sql.gz']) + expect(result.every(({ bytes, sha256 }) => bytes > 2 && sha256.length === 64)).toBe(true) + + writeFileSync(join(directory, 'catalog.sql.gz'), Buffer.from('not-gzip')) + expect(() => inspectBackupArtifacts(directory)).toThrow(/not gzip data/u) + }) + + it('fails closed when an archive changes after checksum creation', () => { + const directory = backupFixture() + const path = join(directory, 'pipeline.sql.gz') + const archive = readFileSync(path) + archive[archive.length - 1] ^= 1 + writeFileSync(path, archive) + expect(() => inspectBackupArtifacts(directory)).toThrow(/SHA-256 mismatch/u) + }) + + it('runs preflight before export and artifact verification before upload', () => { + const workflow = readFileSync( + join(process.cwd(), '.github', 'workflows', 'cloudflare-backup.yml'), + 'utf8', + ) + const credentialPreflight = workflow.indexOf('--phase credentials') + const exportStep = workflow.indexOf('Export catalog and pipeline databases') + const artifactPreflight = workflow.indexOf('--phase artifacts') + const uploadStep = workflow.indexOf('Upload daily and monthly copies') + expect(credentialPreflight).toBeGreaterThan(0) + expect(exportStep).toBeGreaterThan(credentialPreflight) + expect(artifactPreflight).toBeGreaterThan(exportStep) + expect(uploadStep).toBeGreaterThan(artifactPreflight) + }) +}) diff --git a/tests/unit/minimax-recapture-validator.test.ts b/tests/unit/minimax-recapture-validator.test.ts new file mode 100644 index 0000000..595e914 --- /dev/null +++ b/tests/unit/minimax-recapture-validator.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from 'vitest' + +import { + validateMiniMaxRecapture, +} from '../../scripts/ingestion/validate-minimax-recapture' + +const task = { + id: 'minimax-v3-programs-001', + kind: 'programs' as const, + schools: [{ institutionRef: 'uni-example' }], + outputJsonPath: 'quality/minimax-recapture/inbox/minimax-v3-programs-001.json', +} + +function evidence(value: unknown, quote: string) { + return { + status: 'known', + value, + officialUrl: 'https://international.example.edu.cn/admissions/programs/example', + sourceTitle: '2027 International Student Admission Guide', + checkedAt: '2026-07-27', + quote, + locator: 'Program table, row 1', + } +} + +function validProgram() { + return { + institutionId: 'uni-example', + programKey: 'uni-example:program:computer-science', + programUrl: 'https://international.example.edu.cn/admissions/programs/computer-science', + rawSnapshotHash: 'sha256:official-snapshot', + internationalEligibility: evidence( + true, + 'Applicants must be non-Chinese citizens holding a valid foreign passport.', + ), + individualApplication: evidence( + true, + 'Applicants shall submit an online application through the university portal.', + ), + durationMonths: evidence(48, 'The standard duration of study is four years.'), + cycles: [{ + academicYear: '2027-2028', + intake: 'autumn', + publicationEligibility: 'open', + tuitionCny: evidence(30000, 'Tuition fee: RMB 30,000 per academic year.'), + closesOn: evidence('2027-06-30', 'Application deadline: June 30, 2027.'), + }], + } +} + +function validHarvest() { + return { + format: 'studyinchina.minimax-official-harvest', + formatVersion: 1, + batchId: task.id, + checkedAt: '2026-07-27', + collector: { + agent: 'MiniMax Coding Plan', + model: 'MiniMax-M2.7', + officialSourcesOnly: true, + }, + scope: { schoolIds: ['uni-example'] }, + programs: [validProgram()], + scholarships: [], + reconciliation: [{ institutionId: 'uni-example', categories: {} }], + sourceFailures: [], + } +} + +describe('MiniMax v3 recapture validator', () => { + it('accepts a source-backed program with complete core facts', () => { + const result = validateMiniMaxRecapture(task, validHarvest()) + + expect(result.publishablePrograms).toBe(1) + expect(result.durationCoverageRate).toBe(1) + expect(result.tuitionCoverageRate).toBe(1) + expect(result.futureDeadlineCoverageRate).toBe(1) + expect(result.missingSnapshotEvidenceCount).toBe(0) + }) + + it('rejects generated eligibility evidence templates', () => { + const harvest = validHarvest() + harvest.programs[0]!.internationalEligibility.quote = + 'The official program page identifies this program as open to non-Chinese citizens.' + + expect(() => validateMiniMaxRecapture(task, harvest)) + .toThrow(/generated evidence template/u) + }) + + it('rejects known evidence without a raw snapshot reference', () => { + const harvest = validHarvest() + Reflect.deleteProperty(harvest.programs[0]!, 'rawSnapshotHash') + + expect(() => validateMiniMaxRecapture(task, harvest)) + .toThrow(/rawSnapshotPath or rawSnapshotHash/u) + }) + + it('rejects an empty scholarship task even when failures are documented', () => { + const scholarshipTask = { + ...task, + id: 'minimax-v3-scholarships-001', + kind: 'scholarships' as const, + } + const harvest = { + ...validHarvest(), + batchId: scholarshipTask.id, + programs: [], + sourceFailures: [{ + institutionId: 'uni-example', + category: 'scholarships', + discoveryAttempts: [{}, {}, {}], + }], + } + + expect(() => validateMiniMaxRecapture(scholarshipTask, harvest)) + .toThrow(/must contain at least one verified scholarship/u) + }) + + it('enforces the duration coverage threshold', () => { + const harvest = validHarvest() + const second = validProgram() + second.programKey = 'uni-example:program:data-science' + second.durationMonths = { + status: 'source_unavailable', + value: null, + officialUrl: '', + sourceTitle: '', + checkedAt: '2026-07-27', + quote: '', + locator: '', + } + harvest.programs.push(second) + + expect(() => validateMiniMaxRecapture(task, harvest)) + .toThrow(/durationCoverageRate 0.50 < 0.60/u) + }) +}) diff --git a/tests/unit/platform-data-quality.test.ts b/tests/unit/platform-data-quality.test.ts new file mode 100644 index 0000000..a753d6d --- /dev/null +++ b/tests/unit/platform-data-quality.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from 'vitest' +import { + buildPlatformDataQualityScorecard, + conciseScorecardSummary, + parsePlatformDataQualityArgs, +} from '../../scripts/quality/platform-data-quality' +import type { DataBundle } from '@/lib/data/types' +import type { SourceManifestRecord } from '../../scripts/source-manifest-registry' + +const audit = { + sourceIds: ['source-1'], + verifiedAt: '2026-08-01', + reviewAfter: '2026-12-31', + status: 'verified' as const, +} + +function fixture(): DataBundle { + return { + sources: [{ + id: 'source-1', url: 'https://example.edu', title: 'Official', publisher: 'Example', + kind: 'university', language: 'en', official: true, accessedAt: '2026-08-01', + }], + cities: [ + { ...audit, id: 'city-1', slug: 'one', name: { en: 'One' }, province: null, region: 'east', coordinates: { lat: 1, lng: 2 }, overview: null, climate: null, foodHighlights: [], sights: [] }, + { ...audit, id: 'city-2', slug: 'two', name: { en: 'Two' }, province: null, region: 'east', coordinates: null, overview: null, climate: null, foodHighlights: [], sights: [] }, + ], + universities: [ + { ...audit, id: 'uni-1', slug: 'one', name: { en: 'One' }, cityId: 'city-1', region: 'east', officialUrl: 'https://one.edu', admissionsUrl: null, summary: null, featured: false }, + { ...audit, id: 'uni-2', slug: 'two', name: { en: 'Two' }, cityId: 'city-2', region: 'east', officialUrl: 'https://two.edu', admissionsUrl: null, summary: null, featured: false }, + ], + programs: [ + { ...audit, id: 'program-1', slug: 'one', universityId: 'uni-1', name: { en: 'One' }, degreeLevel: 'master', discipline: 'engineering', teachingLanguages: ['English'], durationMonths: 24, programUrl: 'https://one.edu/p', applyUrl: 'https://one.edu/apply', languageRequirements: [{ test: 'IELTS', minimum: '6.0' }] }, + { ...audit, id: 'program-2', slug: 'two', universityId: 'uni-1', name: { en: 'Two' }, degreeLevel: 'master', discipline: 'science', teachingLanguages: [], durationMonths: null, programUrl: 'https://one.edu/p2', applyUrl: null, languageRequirements: [] }, + { ...audit, id: 'program-3', slug: 'three', universityId: 'uni-2', name: { en: 'Three' }, degreeLevel: 'bachelor', discipline: 'business', teachingLanguages: ['Chinese'], durationMonths: null, programUrl: 'https://two.edu/p', applyUrl: null, languageRequirements: [] }, + ], + admissionCycles: [{ ...audit, id: 'cycle-1', programId: 'program-1', academicYear: '2026-2027', intake: 'autumn', opensOn: '2026-01-01', closesOn: '2026-10-01', dateStatus: 'published', tuitionCny: null, applicationFeeCny: null }], + scholarships: [{ ...audit, id: 'scholarship-1', slug: 'one', name: { en: 'One' }, providerType: 'university', universityIds: ['uni-1'], programIds: [], coverage: { tuition: 'full', accommodation: 'unknown', insurance: 'unknown', stipendCnyPerMonth: null }, deadline: '2026-10-01', applicationUrl: 'https://one.edu/s', summary: null }], + } +} + +describe('platform data-quality scorecard', () => { + it('calculates public coverage and gaps without mutating the catalog', () => { + const data = fixture() + const snapshot = JSON.stringify(data) + const report = buildPlatformDataQualityScorecard(data, [] as SourceManifestRecord[], { + today: '2026-08-06', + generatedAt: '2026-08-06T00:00:00.000Z', + }) + + expect(report.metrics.publicRecords).toMatchObject({ universities: 2, programs: 3, scholarships: 1 }) + expect(report.metrics.programCoverage).toMatchObject({ + schoolsBelowThreePrograms: 2, + programsWithCurrentCycle: 1, + currentCycleCoveragePct: 33.33, + durationCoveragePct: 33.33, + applicationUrlCoveragePct: 33.33, + teachingLanguageCoveragePct: 66.67, + requirementsCoveragePct: 33.33, + }) + expect(report.metrics.scholarships).toMatchObject({ universitiesCovered: 1, recordsWithDeadline: 1 }) + expect(report.metrics.cities).toEqual({ withCoordinates: 1, coordinateCoveragePct: 50 }) + expect(report.gates.allPassed).toBe(false) + expect(JSON.stringify(data)).toBe(snapshot) + }) + + it('reports overdue verified data and published cycles with no dates', () => { + const data = fixture() + data.universities[0].reviewAfter = '2026-08-05' + data.admissionCycles.push({ + ...data.admissionCycles[0], + id: 'cycle-no-dates', + programId: 'program-2', + opensOn: null, + closesOn: null, + }) + const report = buildPlatformDataQualityScorecard(data, [], { today: '2026-08-06' }) + + expect(report.metrics.anomalies.verifiedOverdueRecords).toBe(1) + expect(report.metrics.anomalies.publishedCyclesWithoutAnyDate).toBe(1) + }) + + it('parses strict and explicit output options and keeps console output concise', () => { + expect(parsePlatformDataQualityArgs(['--strict', '--output', 'quality.json', '--today', '2026-08-06'])) + .toEqual({ strict: true, outputPath: 'quality.json', today: '2026-08-06' }) + expect(() => parsePlatformDataQualityArgs(['--today', '06-08-2026'])).toThrow(/YYYY-MM-DD/) + + const report = buildPlatformDataQualityScorecard(fixture(), [], { today: '2026-08-06' }) + expect(conciseScorecardSummary(report)).toContain('gates ') + expect(conciseScorecardSummary(report).split('\n')).toHaveLength(1) + }) +}) diff --git a/tests/unit/program-catalog.test.ts b/tests/unit/program-catalog.test.ts new file mode 100644 index 0000000..5553a9c --- /dev/null +++ b/tests/unit/program-catalog.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest' +import admissionCycles from '../../content/data/admission-cycles.json' +import cities from '../../content/data/cities.json' +import programs from '../../content/data/programs.json' +import scholarships from '../../content/data/scholarships.json' +import sources from '../../content/data/sources.json' +import universities from '../../content/data/universities.json' +import type { DataBundle } from '@/lib/data/types' +import { + parseProgramCatalogFilters, + programCatalogHref, + queryProgramCatalog, +} from '@/lib/program-catalog' + +const data = { + admissionCycles, + cities, + programs, + scholarships, + sources, + universities, +} as DataBundle + +describe('server-side program catalogue', () => { + it('normalizes filters and returns only the requested 24-record page', () => { + const filters = parseProgramCatalogFilters({ discipline: 'engineering', page: '1' }) + const result = queryProgramCatalog(data, filters, '2026-08-05') + + expect(filters.discipline).toBe('engineering-technology') + expect(result.total).toBeGreaterThan(24) + expect(result.items).toHaveLength(24) + expect(result.items.every(({ program }) => program.id && program.universityId)).toBe(true) + }) + + it('preserves shareable filters while changing pages', () => { + const filters = parseProgramCatalogFilters({ + q: 'Chinese', + degree: 'master', + city: 'beijing', + sort: 'deadline', + }) + const href = programCatalogHref('en', filters, 2) + + expect(href).toContain('/en/programs?') + expect(href).toContain('q=Chinese') + expect(href).toContain('degree=master') + expect(href).toContain('city=beijing') + expect(href).toContain('sort=deadline') + expect(href).toContain('page=2') + }) + + it('rejects unsupported filter values instead of passing them to queries', () => { + const filters = parseProgramCatalogFilters({ degree: 'invalid', sort: 'drop-table' }) + + expect(filters.degree).toBe('') + expect(filters.sort).toBe('default') + }) +}) diff --git a/tests/unit/release-workflow-safety.test.ts b/tests/unit/release-workflow-safety.test.ts new file mode 100644 index 0000000..7736395 --- /dev/null +++ b/tests/unit/release-workflow-safety.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +import { describe, expect, it } from 'vitest' + +function readWorkflow(name: string): string { + return readFileSync(resolve(process.cwd(), '.github/workflows', name), 'utf8') +} + +describe('production release workflow safety', () => { + it('promotes only a successful Production deployment whose SHA is current main', () => { + const workflow = readWorkflow('vercel-production-alias.yml') + + expect(workflow).toContain("github.event.deployment_status.state == 'success'") + expect(workflow).toContain("github.event.deployment.environment == 'Production'") + expect(workflow).toContain('DEPLOYMENT_SHA: ${{ github.event.deployment.sha }}') + expect(workflow).toContain('main_sha="$(git rev-parse HEAD)"') + expect(workflow).toContain("steps.main.outputs.matches == 'true'") + expect(workflow).not.toContain("github.event.deployment.ref == 'main'") + + const comparison = workflow.indexOf('Verify deployment commit is current main') + const promotion = workflow.indexOf('Promote stable production alias') + expect(comparison).toBeGreaterThan(-1) + expect(promotion).toBeGreaterThan(comparison) + }) + + it('keeps fact refresh read-only and publishes an audit artifact', () => { + const workflow = readWorkflow('program-fact-refresh.yml') + + expect(workflow).toMatch(/permissions:\s+contents: read/u) + expect(workflow).toContain('persist-credentials: false') + expect(workflow).toContain('actions/upload-artifact@v6') + expect(workflow).toContain('catalog-changes.patch') + expect(workflow).not.toMatch(/\bgit\s+(?:commit|push)\b/u) + expect(workflow).not.toMatch(/contents:\s+write/u) + }) +}) diff --git a/tests/unit/scholarship-catalog.test.ts b/tests/unit/scholarship-catalog.test.ts new file mode 100644 index 0000000..8c66ebc --- /dev/null +++ b/tests/unit/scholarship-catalog.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest' +import admissionCycles from '../../content/data/admission-cycles.json' +import cities from '../../content/data/cities.json' +import programs from '../../content/data/programs.json' +import scholarships from '../../content/data/scholarships.json' +import sources from '../../content/data/sources.json' +import universities from '../../content/data/universities.json' +import type { DataBundle, Scholarship } from '@/lib/data/types' +import { + parseScholarshipCatalogFilters, + queryScholarshipCatalog, + scholarshipCatalogHref, + selectScholarshipCurrentCycle, +} from '@/lib/scholarship-catalog' + +const data = { + admissionCycles, + cities, + programs, + scholarships, + sources, + universities, +} as DataBundle + +describe('server-side scholarship catalogue', () => { + it('returns only one 24-record page and a deterministic legacy current cycle', () => { + const filters = parseScholarshipCatalogFilters({ page: '1' }) + const result = queryScholarshipCatalog(data, filters, '2026-08-05') + + expect(result.total).toBeGreaterThan(24) + expect(result.items).toHaveLength(24) + expect(result.items.every(({ currentCycle, scholarship }) => ( + currentCycle.id === `legacy:${scholarship.id}` + && currentCycle.scholarshipId === scholarship.id + ))).toBe(true) + }) + + it('caps internal page-size overrides at 100 records', () => { + const filters = parseScholarshipCatalogFilters({}) + const result = queryScholarshipCatalog(data, filters, '2026-08-05', 1_000) + + expect(result.pageSize).toBe(100) + expect(result.items).toHaveLength(100) + }) + + it('filters funding and linked degree levels using explicit structured facts', () => { + const filters = parseScholarshipCatalogFilters({ + degree: 'master', + funding: 'full-tuition', + }) + const result = queryScholarshipCatalog(data, filters, '2026-08-05', 100) + + expect(result.total).toBeGreaterThan(0) + expect(result.items.every(({ scholarship, programs: linkedPrograms }) => ( + scholarship.coverage.tuition === 'full' + && linkedPrograms.some((program) => program.degreeLevel === 'master') + ))).toBe(true) + }) + + it('does not infer missing scholarship cycle facts', () => { + const scholarship = { + ...data.scholarships[0], + deadline: null, + } as Scholarship + const cycle = selectScholarshipCurrentCycle(scholarship, '2026-08-05') + + expect(cycle).toMatchObject({ + academicYear: null, + opensOn: null, + closesOn: null, + deadlineState: 'not-announced', + daysRemaining: null, + legacy: true, + }) + }) + + it('preserves shareable filters while changing pages', () => { + const filters = parseScholarshipCatalogFilters({ + q: 'government', + institution: 'peking-university', + degree: 'doctorate', + funding: 'stipend', + deadline: 'future', + sort: 'deadline', + }) + const href = scholarshipCatalogHref('en', filters, 2) + + expect(href).toContain('/en/scholarships?') + expect(href).toContain('q=government') + expect(href).toContain('institution=peking-university') + expect(href).toContain('degree=doctorate') + expect(href).toContain('funding=stipend') + expect(href).toContain('deadline=future') + expect(href).toContain('sort=deadline') + expect(href).toContain('page=2') + }) + + it('rejects unsupported filter values instead of passing them to queries', () => { + const filters = parseScholarshipCatalogFilters({ + degree: 'invalid', + funding: 'drop-table', + deadline: 'someday', + sort: 'random', + }) + + expect(filters.degree).toBe('') + expect(filters.funding).toBe('') + expect(filters.deadline).toBe('') + expect(filters.sort).toBe('default') + }) +}) diff --git a/tests/unit/source-manifest-cohort-builder.test.ts b/tests/unit/source-manifest-cohort-builder.test.ts new file mode 100644 index 0000000..494b717 --- /dev/null +++ b/tests/unit/source-manifest-cohort-builder.test.ts @@ -0,0 +1,216 @@ +import { describe, expect, it } from 'vitest' +import { SOURCE_CATEGORIES } from '../../workers/ingestion/src/manifest-schema' +import { + buildSourceManifestCohort, + dryRunSummary, + type BuildSourceManifestCohortInput, +} from '../../scripts/ingestion/build-source-manifest-cohort' + +function fixture(): BuildSourceManifestCohortInput { + return { + checkedAt: '2026-08-06', + registry: { + cohort: { id: 'test-double-first-class' }, + targets: [ + { + targetId: 'target-001', + ordinal: 1, + officialNameZh: '测试大学', + catalogInstitutionId: 'uni-test-university', + }, + { + targetId: 'target-002', + ordinal: 2, + officialNameZh: '缺口大学', + catalogInstitutionId: 'uni-gap-university', + }, + { + targetId: 'target-003', + ordinal: 3, + officialNameZh: '未映射大学', + }, + { + targetId: 'target-004', + ordinal: 4, + officialNameZh: '国防科技大学', + }, + ], + }, + universities: [ + { + id: 'uni-test-university', + slug: 'test-university', + name: { en: 'Test University', zh: '测试大学' }, + sourceIds: ['src-test-home'], + }, + { + id: 'uni-gap-university', + slug: 'gap-university', + name: { en: 'Gap University', zh: '缺口大学' }, + sourceIds: [], + }, + ], + sources: [ + { + id: 'src-test-home', + url: 'https://international.test.edu.cn/', + title: 'Official international home', + kind: 'university', + official: true, + }, + { + id: 'src-test-program', + url: 'https://international.test.edu.cn/programs/masters', + title: 'Official international programme', + kind: 'program', + official: true, + }, + { + id: 'src-test-scholarship', + url: 'https://international.test.edu.cn/scholarships/president', + title: 'Official university scholarship', + kind: 'scholarship', + official: true, + }, + { + id: 'src-http-program', + url: 'http://international.test.edu.cn/programs/legacy', + title: 'Insecure legacy page', + kind: 'program', + official: true, + }, + { + id: 'src-unofficial-program', + url: 'https://aggregator.example/programs/test', + title: 'Third-party page', + kind: 'program', + official: false, + }, + ], + programs: [ + { + id: 'program-test-master', + universityId: 'uni-test-university', + name: { en: 'Verified Master Programme' }, + sourceIds: [ + 'src-test-program', + 'src-http-program', + 'src-unofficial-program', + ], + }, + { + id: 'program-gap-master', + universityId: 'uni-gap-university', + name: { en: 'Unresolved Programme' }, + sourceIds: ['src-missing'], + }, + ], + admissionCycles: [], + scholarships: [{ + id: 'scholarship-test-president', + name: { en: 'Test University President Scholarship' }, + providerType: 'university', + universityIds: ['uni-test-university'], + sourceIds: ['src-test-scholarship'], + }], + } +} + +describe('SourceManifestV2 cohort candidate builder', () => { + it('maps only exact official HTTPS relationships and leaves every candidate pending', () => { + const build = buildSourceManifestCohort(fixture()) + + expect(build.candidates).toHaveLength(1) + const candidate = build.candidates[0]! + expect(candidate.fileName).toBe('001-test-university.v2.candidate.json') + expect(candidate.manifest.manifestStatus).toBe('in_progress') + expect(candidate.manifest.catalogReconciliation.status).toBe('in_progress') + expect(candidate.manifest.catalogReconciliation.entries).toHaveLength(2) + expect(candidate.manifest.catalogReconciliation.entries.every( + (entry) => entry.status === 'pending' && entry.recordId === undefined, + )).toBe(true) + expect(candidate.manifest.sources.map((source) => source.officialUrl)).toEqual([ + 'https://international.test.edu.cn/', + 'https://international.test.edu.cn/programs/masters', + 'https://international.test.edu.cn/scholarships/president', + ]) + expect(candidate.manifest.sources.every( + (source) => source.enabled === false && source.robots.mode === 'blocked', + )).toBe(true) + + const mappedCoverage = candidate.manifest.coverage.filter( + (coverage) => coverage.status === 'parser_pending', + ) + expect(mappedCoverage.map((coverage) => coverage.sourceCategory)).toEqual([ + 'university_scholarship', + 'program_detail', + 'catalog_anchor', + ]) + const missingCoverage = candidate.manifest.coverage.filter( + (coverage) => coverage.status === 'discovery_pending', + ) + expect(missingCoverage).toHaveLength(SOURCE_CATEGORIES.length - mappedCoverage.length) + expect(candidate.manifest.coverage.some( + (coverage) => coverage.status === 'officially_not_provided', + )).toBe(false) + }) + + it('reports military, mapping, source-quality, and no-safe-entity gaps explicitly', () => { + const build = buildSourceManifestCohort(fixture()) + + expect(build.summary).toEqual({ + officialTargets: 4, + militaryExcluded: 1, + eligibleTargets: 3, + candidateManifests: 1, + exactOfficialHttpsSources: 3, + targetsWithoutCandidate: 2, + }) + expect(build.gapReport.militaryExclusions.map((target) => target.officialNameZh)).toEqual([ + '国防科技大学', + ]) + expect(build.gapReport.gaps.map((gap) => gap.code)).toEqual([ + 'no_safe_entity_source', + 'catalog_mapping_missing', + ]) + const testCoverage = build.gapReport.institutionCoverage.find( + (coverage) => coverage.institutionId === 'uni-test-university', + ) + expect(testCoverage?.rejectedSources).toEqual([ + { sourceId: 'src-http-program', reason: 'not_https' }, + { sourceId: 'src-unofficial-program', reason: 'not_official' }, + ]) + const gapCoverage = build.gapReport.institutionCoverage.find( + (coverage) => coverage.institutionId === 'uni-gap-university', + ) + expect(gapCoverage?.rejectedSources).toEqual([ + { sourceId: 'src-missing', reason: 'missing_source_record' }, + ]) + }) + + it('is deterministic regardless of input order and exposes a no-write dry-run summary', () => { + const input = fixture() + const expected = buildSourceManifestCohort(input) + const reordered = buildSourceManifestCohort({ + ...input, + registry: { ...input.registry, targets: [...input.registry.targets].reverse() }, + universities: [...input.universities].reverse(), + sources: [...input.sources].reverse(), + programs: [...input.programs].reverse(), + scholarships: [...input.scholarships].reverse(), + }) + + expect(reordered).toEqual(expected) + expect(JSON.parse(dryRunSummary(expected))).toEqual({ + mode: 'dry-run', + ...expected.summary, + }) + }) + + it('rejects a non-calendar checkedAt value before building candidates', () => { + const input = fixture() + input.checkedAt = '2026-02-30' + + expect(() => buildSourceManifestCohort(input)).toThrow(/checkedAt/) + }) +}) diff --git a/tests/unit/source-manifest-registry.test.ts b/tests/unit/source-manifest-registry.test.ts new file mode 100644 index 0000000..6d350c2 --- /dev/null +++ b/tests/unit/source-manifest-registry.test.ts @@ -0,0 +1,137 @@ +import { + mkdtempSync, + mkdirSync, + rmSync, + writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + isCatalogReconciliationComplete, + loadSourceManifestFiles, + validateSourceManifests, + type LoadedSourceManifest, + type SourceManifestV2, +} from '../../scripts/source-manifest-registry' +import { + loadPilotSourceManifestFiles, + type PilotSourceManifest, +} from '../../scripts/validate-source-manifests' + +const temporaryDirectories: string[] = [] + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function legacyInputs(): LoadedSourceManifest[] { + return loadPilotSourceManifestFiles().map((input) => ({ + filePath: input.filePath, + value: structuredClone(input.value), + })) +} + +function v2Fixture(): SourceManifestV2 { + const legacy = structuredClone(legacyInputs()[0]!.value) as PilotSourceManifest + const officialHosts = [ + ...new Set( + legacy.sources.flatMap((source) => [ + ...source.allowedHosts, + ...(source.allowedRedirectHosts ?? []), + ]), + ), + ] + return { + ...legacy, + version: 2, + manifestStatus: 'in_progress', + officialHosts, + catalogReconciliation: { + scope: 'full_official_catalog', + status: 'in_progress', + entries: [{ + sourceId: legacy.sources[0]!.id, + officialKey: 'official-program-001', + officialName: 'Official international programme', + entityType: 'program', + status: 'pending', + }], + }, + } +} + +describe('recursive source manifest registry', () => { + it('discovers nested school manifests and skips target registries', () => { + const directory = mkdtempSync(join(tmpdir(), 'source-manifest-registry-')) + temporaryDirectories.push(directory) + const nested = join(directory, 'cohort', 'schools') + mkdirSync(nested, { recursive: true }) + writeFileSync( + join(directory, 'cohort', 'targets.v1.json'), + JSON.stringify({ format: 'studyinchina.institution-target-registry', targets: [] }), + ) + writeFileSync( + join(nested, 'school.json'), + JSON.stringify(legacyInputs()[0]!.value), + ) + + const files = loadSourceManifestFiles(directory) + + expect(files).toHaveLength(1) + expect(files[0]!.filePath.endsWith('school.json')).toBe(true) + }) + + it('does not require the old exact ten-school pilot set', () => { + const records = validateSourceManifests(legacyInputs().slice(0, 3)) + + expect(records).toHaveLength(3) + expect(records.every((record) => record.version === 1)).toBe(true) + expect(records.every((record) => !isCatalogReconciliationComplete(record))).toBe(true) + }) + + it('still rejects institution and source identities reused across manifests', () => { + const inputs = legacyInputs().slice(0, 2) + const first = inputs[0]!.value as PilotSourceManifest + const second = inputs[1]!.value as PilotSourceManifest + second.institutionId = first.institutionId + for (const source of second.sources) source.institutionId = first.institutionId + + expect(() => validateSourceManifests(inputs)).toThrow(/duplicate institutionId/) + }) + + it('requires explicit, non-pending reconciliation before V2 is complete', () => { + const incomplete = v2Fixture() + incomplete.manifestStatus = 'complete' + incomplete.catalogReconciliation.status = 'complete' + + expect(() => validateSourceManifests([{ + filePath: 'v2-incomplete.json', + value: incomplete, + }])).toThrow(/complete catalog reconciliation cannot contain pending entries/) + + const complete = v2Fixture() + complete.manifestStatus = 'complete' + complete.catalogReconciliation.status = 'complete' + complete.catalogReconciliation.entries[0] = { + ...complete.catalogReconciliation.entries[0]!, + status: 'published', + recordId: 'prog-official-001', + } + complete.coverage = complete.coverage.map((entry) => ( + entry.status === 'discovery_pending' + ? { ...entry, status: 'officially_not_provided' as const } + : entry + )) + + const [validated] = validateSourceManifests([{ + filePath: 'v2-complete.json', + value: complete, + }]) + + expect(validated).toBeDefined() + expect(isCatalogReconciliationComplete(validated!)).toBe(true) + }) +}) diff --git a/tests/unit/sparse-school-expansion-2026-08-04.test.ts b/tests/unit/sparse-school-expansion-2026-08-04.test.ts index f938d29..6c76c16 100644 --- a/tests/unit/sparse-school-expansion-2026-08-04.test.ts +++ b/tests/unit/sparse-school-expansion-2026-08-04.test.ts @@ -429,7 +429,11 @@ describe('sparse-school and regional university expansion on 2026-08-04', () => ...data.scholarships, ] const overdueVerifiedRecords = formalRecords.filter( - (record) => record.status === 'verified' && record.reviewAfter < TODAY, + (record): record is (typeof formalRecords)[number] & { status: 'verified'; reviewAfter: string } => ( + 'status' in record + && record.status === 'verified' + && record.reviewAfter < TODAY + ), ) expect(overdueVerifiedRecords.map((record) => ({ diff --git a/tests/unit/untracked-asset-inventory.test.ts b/tests/unit/untracked-asset-inventory.test.ts new file mode 100644 index 0000000..453ac8a --- /dev/null +++ b/tests/unit/untracked-asset-inventory.test.ts @@ -0,0 +1,85 @@ +import { createHash } from 'node:crypto' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + assembleUntrackedInventory, + classifyUntrackedPath, + createUntrackedInventoryEntry, + parseInventoryArgs, +} from '../../scripts/quality/inventory-untracked-assets' + +const temporaryDirectories: string[] = [] + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { force: true, recursive: true }) + } +}) + +describe('untracked asset inventory', () => { + it.each([ + ['quality/raw/school-page.pdf', 'raw_evidence', 'private_r2_raw_evidence'], + ['quality/minimax-harvest/completed/result.json', 'structured_candidate', 'quarantine_candidate'], + ['scripts/ingestion/new-adapter.ts', 'source_code', 'code_test_review'], + ['tests/unit/new-adapter.test.ts', 'test', 'code_test_review'], + ['infra/d1/catalog/migrations/0010_release.sql', 'database_migration', 'code_test_review'], + ['.tmp-second-wave-coverage.json', 'temporary', 'temp_ignore'], + ])('classifies %s without mutating it', (filePath, contentClass, suggestedStatus) => { + expect(classifyUntrackedPath(filePath)).toMatchObject({ contentClass, suggestedStatus }) + }) + + it('computes the byte size and SHA-256 of a regular file', async () => { + const root = mkdtempSync(join(tmpdir(), 'studyinchina-untracked-')) + temporaryDirectories.push(root) + mkdirSync(join(root, 'quality', 'raw'), { recursive: true }) + const contents = Buffer.from('official source snapshot\n', 'utf8') + writeFileSync(join(root, 'quality', 'raw', 'page.html'), contents) + + const entry = await createUntrackedInventoryEntry(root, 'quality/raw/page.html') + + expect(entry.byteSize).toBe(contents.byteLength) + expect(entry.sha256).toBe(createHash('sha256').update(contents).digest('hex')) + expect(entry.suggestedStatus).toBe('private_r2_raw_evidence') + }) + + it('assembles deterministic totals for review and archival queues', () => { + const report = assembleUntrackedInventory('C:/repo', [ + { + path: 'raw/page.pdf', + extension: '.pdf', + byteSize: 10, + sha256: 'a'.repeat(64), + contentClass: 'raw_evidence', + suggestedStatus: 'private_r2_raw_evidence', + classificationReason: 'raw', + }, + { + path: 'scripts/new.ts', + extension: '.ts', + byteSize: 5, + sha256: 'b'.repeat(64), + contentClass: 'source_code', + suggestedStatus: 'code_test_review', + classificationReason: 'code', + }, + ], '2026-08-05T00:00:00.000Z') + + expect(report.summary).toEqual({ + totalFiles: 2, + totalBytes: 15, + byContentClass: { raw_evidence: 1, source_code: 1 }, + bySuggestedStatus: { private_r2_raw_evidence: 1, code_test_review: 1 }, + }) + }) + + it('requires an explicit JSON output path', () => { + expect(() => parseInventoryArgs([])).toThrow('--output is required') + expect(() => parseInventoryArgs(['--output', 'report.txt'])).toThrow('.json extension') + expect(parseInventoryArgs(['--repo', 'fixture', '--output', 'audit.json'])).toEqual({ + repositoryPath: 'fixture', + outputPath: 'audit.json', + }) + }) +}) diff --git a/workers/entity-materializer/src/index.ts b/workers/entity-materializer/src/index.ts new file mode 100644 index 0000000..e12fbfa --- /dev/null +++ b/workers/entity-materializer/src/index.ts @@ -0,0 +1,86 @@ +import { processEntityMaterializationBatch } from '../../ingestion/src/entity-materializer-scheduler' +import type { + D1Database, + ScheduledControllerLike, +} from '../../ingestion/src/types' + +const SERVICE_VERSION = '1.0.0' +export const DAILY_RELEASE_CRON = '17 19 * * *' + +export interface EntityMaterializerEnv { + PIPELINE_DB: D1Database + MATERIALIZATION_BATCH_LIMIT?: string + RELEASE_CANDIDATE_LIMIT?: string +} + +function boundedInteger( + value: string | undefined, + fallback: number, + maximum: number, +): number { + if (value === undefined) return fallback + const parsed = Number.parseInt(value, 10) + return Number.isInteger(parsed) && parsed >= 1 && parsed <= maximum + ? parsed + : fallback +} + +export function shouldRequestDailyRelease(cron: string): boolean { + return cron.trim() === DAILY_RELEASE_CRON +} + +export async function scheduleEntityMaterialization( + controller: ScheduledControllerLike, + environment: EntityMaterializerEnv, +): Promise { + await processEntityMaterializationBatch(environment.PIPELINE_DB, { + candidateLimit: boundedInteger( + environment.MATERIALIZATION_BATCH_LIMIT, + 20, + 100, + ), + releaseCandidateLimit: boundedInteger( + environment.RELEASE_CANDIDATE_LIMIT, + 500, + 1_000, + ), + now: new Date(controller.scheduledTime).toISOString(), + requestRelease: shouldRequestDailyRelease(controller.cron), + }) +} + +export function handleFetch(request: Request): Response { + const url = new URL(request.url) + if (request.method === 'GET' && url.pathname === '/health') { + return Response.json( + { + ok: true, + service: 'studyinchina-entity-materializer', + version: SERVICE_VERSION, + }, + { + headers: { + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + }, + }, + ) + } + return Response.json( + { ok: false, error: 'not_found' }, + { + status: 404, + headers: { + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + }, + }, + ) +} + +const worker = { + fetch: handleFetch, + scheduled: scheduleEntityMaterialization, +} + +export default worker diff --git a/workers/entity-materializer/tests/index.test.ts b/workers/entity-materializer/tests/index.test.ts new file mode 100644 index 0000000..57ee45b --- /dev/null +++ b/workers/entity-materializer/tests/index.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert/strict' +import test from 'node:test' + +import { + DAILY_RELEASE_CRON, + handleFetch, + shouldRequestDailyRelease, +} from '../src/index' + +test('only the dedicated daily cron can request a release', () => { + assert.equal(shouldRequestDailyRelease('47 * * * *'), false) + assert.equal(shouldRequestDailyRelease(DAILY_RELEASE_CRON), true) +}) + +test('health response exposes no database details', async () => { + const response = handleFetch(new Request('https://worker.example/health')) + assert.equal(response.status, 200) + assert.deepEqual(await response.json(), { + ok: true, + service: 'studyinchina-entity-materializer', + version: '1.0.0', + }) + assert.equal(response.headers.get('cache-control'), 'no-store') +}) + +test('unknown routes are rejected', async () => { + const response = handleFetch(new Request('https://worker.example/private')) + assert.equal(response.status, 404) + assert.deepEqual(await response.json(), { ok: false, error: 'not_found' }) +}) diff --git a/workers/entity-materializer/tsconfig.json b/workers/entity-materializer/tsconfig.json new file mode 100644 index 0000000..0d5161e --- /dev/null +++ b/workers/entity-materializer/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": [ + "src/**/*.ts", + "tests/**/*.ts", + "../ingestion/src/**/*.ts" + ] +} diff --git a/workers/entity-materializer/wrangler.jsonc b/workers/entity-materializer/wrangler.jsonc new file mode 100644 index 0000000..66a5091 --- /dev/null +++ b/workers/entity-materializer/wrangler.jsonc @@ -0,0 +1,24 @@ +{ + "$schema": "../../node_modules/wrangler/config-schema.json", + "name": "studyinchina-entity-materializer", + "main": "./src/index.ts", + "compatibility_date": "2026-07-20", + "workers_dev": true, + "preview_urls": false, + "observability": { "enabled": true }, + "triggers": { + "crons": ["47 * * * *", "17 19 * * *"] + }, + "vars": { + "MATERIALIZATION_BATCH_LIMIT": "20", + "RELEASE_CANDIDATE_LIMIT": "500" + }, + "d1_databases": [ + { + "binding": "PIPELINE_DB", + "database_name": "studyinchina-pipeline", + "database_id": "bb68098a-1429-44ad-869a-de9c3a5e478c", + "migrations_dir": "../../infra/d1/pipeline/migrations" + } + ] +} diff --git a/workers/ingestion/src/entity-materializer-scheduler.ts b/workers/ingestion/src/entity-materializer-scheduler.ts new file mode 100644 index 0000000..3c33daa --- /dev/null +++ b/workers/ingestion/src/entity-materializer-scheduler.ts @@ -0,0 +1,143 @@ +import { + materializeExtractedEntityCandidate, + requestEntityMaterializationRelease, + type EntityMaterializationResult, + type EntityReleaseRequestResult, +} from './entity-materializer' +import type { D1Database } from './types' + +type CandidateIdRow = { candidate_id: string } + +export type EntityMaterializationBatchResult = { + attempted: number + materialized: number + alreadyMaterialized: number + quarantined: number + conflicts: number + pending: number + failures: Array<{ candidateId: string; message: string }> + release: EntityReleaseRequestResult | null + results: EntityMaterializationResult[] +} + +function boundedLimit(value: number | undefined, fallback: number, maximum: number): number { + if (value === undefined) return fallback + if (!Number.isInteger(value) || value < 1 || value > maximum) { + throw new Error(`limit must be an integer from 1 to ${maximum}`) + } + return value +} + +async function candidateIds( + database: D1Database, + sql: string, + limit: number, +): Promise { + const result = await database.prepare(sql).bind(limit).all() + if (!result.success) { + throw new Error(`entity materialization queue query failed: ${result.error ?? 'unknown D1 error'}`) + } + return (result.results ?? []).map((row) => row.candidate_id) +} + +export async function listPendingEntityMaterializationCandidates( + database: D1Database, + limit = 20, +): Promise { + return candidateIds( + database, + `SELECT candidate.candidate_id + FROM extracted_entity_candidates candidate + JOIN entity_registry registry + ON registry.institution_id = candidate.institution_id + AND registry.entity_type = candidate.entity_type + AND registry.entity_key = candidate.entity_key + JOIN catalog_reconciliation_items reconciliation + ON reconciliation.candidate_id = candidate.candidate_id + WHERE candidate.candidate_status IN ('validated', 'registered', 'quarantined') + AND NOT EXISTS ( + SELECT 1 FROM entity_materialization_decisions decision + WHERE decision.candidate_id = candidate.candidate_id + ) + ORDER BY candidate.created_at, candidate.candidate_id + LIMIT ?1`, + boundedLimit(limit, 20, 100), + ) +} + +export async function listUnreleasedMaterializedEntityCandidates( + database: D1Database, + limit = 500, +): Promise { + return candidateIds( + database, + `SELECT decision.candidate_id + FROM entity_materialization_decisions decision + WHERE decision.decision_status = 'materialized' + AND NOT EXISTS ( + SELECT 1 + FROM entity_materialization_release_requests request, + json_each(request.candidate_ids_json) requested + WHERE requested.value = decision.candidate_id + ) + ORDER BY decision.decided_at, decision.candidate_id + LIMIT ?1`, + boundedLimit(limit, 500, 1_000), + ) +} + +export async function processEntityMaterializationBatch( + database: D1Database, + options: { + candidateLimit?: number + releaseCandidateLimit?: number + now?: string + requestRelease?: boolean + } = {}, +): Promise { + const now = options.now ?? new Date().toISOString() + const ids = await listPendingEntityMaterializationCandidates( + database, + options.candidateLimit, + ) + const results: EntityMaterializationResult[] = [] + const failures: Array<{ candidateId: string; message: string }> = [] + for (const candidateId of ids) { + try { + results.push(await materializeExtractedEntityCandidate(database, candidateId, { + decidedAt: now, + })) + } catch (error) { + failures.push({ + candidateId, + message: (error instanceof Error ? error.message : String(error)).slice(0, 1_000), + }) + } + } + + let release: EntityReleaseRequestResult | null = null + if (options.requestRelease !== false) { + const releaseCandidates = await listUnreleasedMaterializedEntityCandidates( + database, + options.releaseCandidateLimit, + ) + if (releaseCandidates.length > 0) { + release = await requestEntityMaterializationRelease(database, releaseCandidates, now) + } + } + + const count = (status: EntityMaterializationResult['status']) => ( + results.filter((result) => result.status === status).length + ) + return { + attempted: ids.length, + materialized: count('materialized'), + alreadyMaterialized: count('already-materialized'), + quarantined: count('quarantined'), + conflicts: count('conflict'), + pending: count('pending'), + failures, + release, + results, + } +} diff --git a/workers/ingestion/src/entity-materializer.ts b/workers/ingestion/src/entity-materializer.ts new file mode 100644 index 0000000..64006f0 --- /dev/null +++ b/workers/ingestion/src/entity-materializer.ts @@ -0,0 +1,1329 @@ +import { sha256Hex, stableJson } from './hash' +import { assertSafeSourceUrl, validateManifest } from './security' +import type { + D1Database, + D1PreparedStatement, + D1Result, + SourceCategory, + SourceManifestV1, +} from './types' + +const MATERIALIZER_VERSION = 'entity-candidate-materializer/v1' +const DEFAULT_MINIMUM_CONFIDENCE_PPM = 980_000 +const DEFAULT_REVIEW_DAYS = 30 +const MAX_EVIDENCE_QUOTE_LENGTH = 1_000 + +class RetryableSourceConfigurationError extends Error { + constructor(message: string) { + super(message) + this.name = 'RetryableSourceConfigurationError' + } +} + +type EntityKind = 'program' | 'scholarship' +type DegreeLevel = 'bachelor' | 'master' | 'doctorate' +type ProgramType = + | 'degree' + | 'language' + | 'foundation' + | 'exchange' + | 'visiting' + | 'short_term' + | 'other' +type SchemeType = + | 'government' + | 'university' + | 'province' + | 'city' + | 'foundation' + | 'other' +type LocatorType = + | 'css' + | 'xpath' + | 'json_pointer' + | 'pdf_page' + | 'pdf_region' + | 'text_offset' + | 'manual' + +type CandidateRow = { + candidate_id: string + institution_id: string + entity_type: EntityKind + entity_key: string + source_id: string + snapshot_id: string + candidate_status: string + facts_json: string + evidence_json: string + issues_json: string + entity_sha256: string + confidence_ppm: number | null + created_at: string + processed_at: string | null + registered_at: string | null + registry_id: string + identity_sha256: string + registry_status: string + canonical_record_id: string | null + reconciliation_id: string + reconciliation_disposition: string + manifest_json: string + r2_key: string + raw_sha256: string + content_type: string + byte_length: number + final_url: string + fetched_at: string + source_document_id: string | null + source_document_url: string | null + source_document_official: number | null + source_document_active: number | null + source_document_authority: string | null + source_document_publisher: string | null +} + +type ExistingDecisionRow = { + decision_status: 'materialized' | 'quarantined' | 'conflict' + registry_id: string + canonical_record_id: string | null + reason_code: string | null + issues_json: string +} + +type ExistingRecordRow = { + id: string + public_id: string + kind: string + row_version: number + workflow_status: string +} + +type Evidence = { + fieldPath: string + quote: string + locator: string | null + officialUrl: string +} + +type NormalizedCandidate = { + row: CandidateRow + manifest: SourceManifestV1 + name: string + nameLocale: 'en' | 'zh' + officialUrl: string + checkedAt: string + reviewAfter: string + programType: ProgramType | null + degreeLevel: DegreeLevel | null + schemeType: SchemeType | null + providerOrganizationId: string | null + evidence: Evidence[] + recordId: string + slug: string +} + +type CanonicalFact = { + candidateFieldPath: string + fieldPath: string + locale: string + valueType: 'localized_string' | 'url' | 'string' + value: string + riskClass: 'medium' | 'high' | 'critical' + requiredForPublish: 0 | 1 + validationProfile: string +} + +export type EntityMaterializationOptions = { + minimumConfidencePpm?: number + reviewDays?: number + decidedAt?: string +} + +export type EntityMaterializationResult = { + candidateId: string + registryId: string + status: + | 'materialized' + | 'already-materialized' + | 'quarantined' + | 'conflict' + | 'pending' + recordId: string | null + mappedFields: number + reasonCode?: string + issues?: string[] +} + +export type EntityReleaseRequestResult = { + status: 'requested' | 'already-requested' + requestId: string + releaseWindow: string + publicationJobId: string + catalogReleaseId: string + outboxEventId: string + candidateIds: string[] +} + +function ensureSuccess(result: D1Result, operation: string): void { + if (!result.success) { + throw new Error(`${operation} failed: ${result.error ?? 'unknown D1 error'}`) + } +} + +function ensureBatch(results: D1Result[], operation: string): void { + for (const result of results) ensureSuccess(result, operation) +} + +function statement( + database: D1Database, + sql: string, + ...values: unknown[] +): D1PreparedStatement { + return database.prepare(sql).bind(...values) +} + +function parseObject(value: string, label: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + throw new Error(`${label} is not valid JSON`) + } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`${label} must be an object`) + } + return parsed as Record +} + +function parseArray(value: string, label: string): unknown[] { + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + throw new Error(`${label} is not valid JSON`) + } + if (!Array.isArray(parsed)) throw new Error(`${label} must be an array`) + return parsed +} + +function isoTimestamp(value: unknown, label: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`${label} must be an ISO timestamp`) + } + const date = new Date(value) + if (Number.isNaN(date.getTime())) throw new Error(`${label} must be an ISO timestamp`) + return date.toISOString() +} + +function addReviewDays(checkedAt: string, reviewDays: number): string { + const date = new Date(checkedAt) + date.setUTCHours(0, 0, 0, 0) + date.setUTCDate(date.getUTCDate() + reviewDays) + return date.toISOString().slice(0, 10) +} + +function normalizedNameKey(value: string): string { + return value + .normalize('NFKC') + .toLocaleLowerCase('en-US') + .replace(/[^\p{L}\p{N}]+/gu, '-') + .replace(/^-+|-+$/gu, '') + .slice(0, 380) +} + +function nameLocale(value: string): 'en' | 'zh' { + const han = value.match(/[\p{Script=Han}]/gu)?.length ?? 0 + const letters = value.match(/[\p{L}]/gu)?.length ?? 0 + return han > 0 && han * 2 >= Math.max(letters, 1) ? 'zh' : 'en' +} + +function programIdentity( + rawDegreeLevel: unknown, + sourceCategory: SourceCategory, +): { programType: ProgramType; degreeLevel: DegreeLevel | null } { + if (rawDegreeLevel === 'bachelor' + || rawDegreeLevel === 'master' + || rawDegreeLevel === 'doctorate') { + return { programType: 'degree', degreeLevel: rawDegreeLevel } + } + if (rawDegreeLevel === 'language') { + return { programType: 'language', degreeLevel: null } + } + if (rawDegreeLevel === 'foundation') { + return { programType: 'foundation', degreeLevel: null } + } + if (rawDegreeLevel !== null && rawDegreeLevel !== undefined) { + throw new Error(`unsupported degreeLevel ${String(rawDegreeLevel)}`) + } + const categoryLevels: Partial> = { + undergraduate_catalog: 'bachelor', + masters_catalog: 'master', + doctoral_catalog: 'doctorate', + } + const degreeLevel = categoryLevels[sourceCategory] ?? null + return degreeLevel + ? { programType: 'degree', degreeLevel } + : { programType: 'other', degreeLevel: null } +} + +function scholarshipScheme(sourceCategory: SourceCategory): SchemeType { + if (sourceCategory === 'government_scholarship') return 'government' + if ( + sourceCategory === 'university_scholarship' + || sourceCategory === 'faculty_scholarship' + ) return 'university' + return 'other' +} + +function locatorType(locator: string | null): LocatorType { + if (!locator) return 'manual' + if (locator.startsWith('css:')) return 'css' + if (locator.startsWith('xpath:')) return 'xpath' + if (locator.startsWith('json:') || locator.startsWith('/')) return 'json_pointer' + if (/^page\s+\d+/iu.test(locator) || locator.startsWith('pdf:page=')) { + return 'pdf_page' + } + return 'manual' +} + +function normalizedEvidence(value: unknown, allowedHosts: string[]): Evidence[] { + if (!Array.isArray(value) || value.length === 0) { + throw new Error('evidence_json must contain at least one evidence item') + } + const evidence = value.map((item, index) => { + if (!item || typeof item !== 'object' || Array.isArray(item)) { + throw new Error(`evidence_json[${index}] must be an object`) + } + const record = item as Record + if (typeof record.fieldPath !== 'string' || !record.fieldPath.trim()) { + throw new Error(`evidence_json[${index}].fieldPath is required`) + } + if ( + typeof record.quote !== 'string' + || !record.quote.trim() + || record.quote.length > MAX_EVIDENCE_QUOTE_LENGTH + ) { + throw new Error(`evidence_json[${index}].quote is invalid`) + } + if (record.locator !== null && typeof record.locator !== 'string') { + throw new Error(`evidence_json[${index}].locator must be a string or null`) + } + if (typeof record.officialUrl !== 'string') { + throw new Error(`evidence_json[${index}].officialUrl is required`) + } + const url = assertSafeSourceUrl(record.officialUrl, allowedHosts) + url.hash = '' + return { + fieldPath: record.fieldPath.trim(), + quote: record.quote.trim(), + locator: record.locator as string | null, + officialUrl: url.href, + } + }) + if (!evidence.some((item) => item.fieldPath === 'name')) { + throw new Error('entity name lacks field-level evidence') + } + return evidence +} + +function canonicalFacts(candidate: NormalizedCandidate): CanonicalFact[] { + const facts: CanonicalFact[] = [ + { + candidateFieldPath: 'name', + fieldPath: 'localized.name', + locale: candidate.nameLocale, + valueType: 'localized_string', + value: candidate.name, + riskClass: 'medium', + requiredForPublish: 1, + validationProfile: 'non-empty-text', + }, + { + candidateFieldPath: 'officialUrl', + fieldPath: 'official_url', + locale: '', + valueType: 'url', + value: candidate.officialUrl, + riskClass: candidate.row.entity_type === 'scholarship' ? 'critical' : 'high', + requiredForPublish: 1, + validationProfile: 'official-https-url', + }, + ] + if (candidate.row.entity_type === 'program') { + facts.push({ + candidateFieldPath: 'degreeLevel', + fieldPath: 'program_type', + locale: '', + valueType: 'string', + value: candidate.programType!, + riskClass: 'high', + requiredForPublish: 1, + validationProfile: 'program-type', + }) + if (candidate.degreeLevel) { + facts.push({ + candidateFieldPath: 'degreeLevel', + fieldPath: 'degree_level', + locale: '', + valueType: 'string', + value: candidate.degreeLevel, + riskClass: 'high', + requiredForPublish: 0, + validationProfile: 'degree-level', + }) + } + } + return facts +} + +async function loadCandidate( + database: D1Database, + candidateId: string, +): Promise { + return database.prepare( + `SELECT candidate.candidate_id, candidate.institution_id, + candidate.entity_type, candidate.entity_key, candidate.source_id, + candidate.snapshot_id, candidate.candidate_status, + candidate.facts_json, candidate.evidence_json, + candidate.issues_json, candidate.entity_sha256, + candidate.confidence_ppm, candidate.created_at, + candidate.processed_at, candidate.registered_at, + registry.registry_id, registry.identity_sha256, + registry.registry_status, registry.canonical_record_id, + reconciliation.reconciliation_id, + reconciliation.disposition AS reconciliation_disposition, + source.manifest_json, + snapshot.r2_key, snapshot.raw_sha256, snapshot.content_type, + snapshot.byte_length, snapshot.final_url, snapshot.fetched_at, + binding.source_document_id, + document.canonical_url AS source_document_url, + document.official AS source_document_official, + document.active AS source_document_active, + document.authority_level AS source_document_authority, + document.publisher_organization_id AS source_document_publisher + FROM extracted_entity_candidates candidate + JOIN entity_registry registry + ON registry.institution_id = candidate.institution_id + AND registry.entity_type = candidate.entity_type + AND registry.entity_key = candidate.entity_key + JOIN catalog_reconciliation_items reconciliation + ON reconciliation.candidate_id = candidate.candidate_id + JOIN ingestion_sources source ON source.source_id = candidate.source_id + JOIN ingestion_snapshots snapshot + ON snapshot.snapshot_id = candidate.snapshot_id + AND snapshot.source_id = candidate.source_id + LEFT JOIN promotion_source_bindings binding + ON binding.source_id = candidate.source_id AND binding.enabled = 1 + LEFT JOIN source_documents document + ON document.id = binding.source_document_id + WHERE candidate.candidate_id = ?1`, + ).bind(candidateId).first() +} + +async function loadDecision( + database: D1Database, + candidateId: string, +): Promise { + return database.prepare( + `SELECT decision_status, registry_id, canonical_record_id, + reason_code, issues_json + FROM entity_materialization_decisions WHERE candidate_id = ?1`, + ).bind(candidateId).first() +} + +function resultFromDecision( + candidateId: string, + decision: ExistingDecisionRow, +): EntityMaterializationResult { + const issues = parseArray(decision.issues_json, 'decision.issues_json').map(String) + return { + candidateId, + registryId: decision.registry_id, + status: decision.decision_status === 'materialized' + ? 'already-materialized' + : decision.decision_status, + recordId: decision.canonical_record_id, + mappedFields: 0, + ...(decision.reason_code ? { reasonCode: decision.reason_code } : {}), + ...(issues.length > 0 ? { issues } : {}), + } +} + +async function normalizeCandidate( + row: CandidateRow, + reviewDays: number, +): Promise { + let manifest: SourceManifestV1 + try { + manifest = validateManifest(JSON.parse(row.manifest_json) as SourceManifestV1) + } catch (error) { + throw new RetryableSourceConfigurationError( + `source manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, + ) + } + if (manifest.id !== row.source_id || manifest.institutionId !== row.institution_id) { + throw new RetryableSourceConfigurationError( + 'candidate does not match its source manifest identity', + ) + } + if ( + row.source_document_id === null + || row.source_document_url === null + || row.source_document_official !== 1 + || row.source_document_active !== 1 + || !['primary_official', 'secondary_official'].includes( + row.source_document_authority ?? '', + ) + ) { + throw new RetryableSourceConfigurationError( + 'candidate source has no enabled official source binding', + ) + } + + const allowedHosts = [ + ...manifest.allowedHosts, + ...(manifest.allowedRedirectHosts ?? []), + ] + try { + const manifestUrl = assertSafeSourceUrl(manifest.officialUrl, manifest.allowedHosts) + const documentUrl = assertSafeSourceUrl(row.source_document_url, allowedHosts) + manifestUrl.hash = '' + documentUrl.hash = '' + if (manifestUrl.href !== documentUrl.href) { + throw new Error('source document URL differs from the source manifest URL') + } + assertSafeSourceUrl(row.final_url, allowedHosts) + } catch (error) { + throw new RetryableSourceConfigurationError( + `source URL binding is invalid: ${error instanceof Error ? error.message : String(error)}`, + ) + } + if (!row.r2_key.includes(row.raw_sha256)) { + throw new RetryableSourceConfigurationError( + 'snapshot R2 key is not bound to its raw SHA-256', + ) + } + + const facts = parseObject(row.facts_json, 'facts_json') + if (typeof facts.name !== 'string' || facts.name.trim().length < 2) { + throw new Error('facts_json.name must be a non-empty entity name') + } + const name = facts.name.normalize('NFKC').replace(/\s+/gu, ' ').trim() + if (name.length > 240) throw new Error('facts_json.name exceeds 240 characters') + if (typeof facts.officialUrl !== 'string') { + throw new Error('facts_json.officialUrl is required') + } + const officialUrlValue = assertSafeSourceUrl(facts.officialUrl, allowedHosts) + officialUrlValue.hash = '' + const officialUrl = officialUrlValue.href + const checkedAt = isoTimestamp(facts.checkedAt, 'facts_json.checkedAt') + if (checkedAt !== isoTimestamp(row.fetched_at, 'snapshot.fetched_at')) { + throw new Error('candidate checkedAt differs from its immutable snapshot timestamp') + } + if (facts.sourceCategory !== manifest.sourceCategory) { + throw new Error('facts_json.sourceCategory does not match the source manifest') + } + const evidence = normalizedEvidence(parseArray(row.evidence_json, 'evidence_json'), allowedHosts) + if (evidence.some((item) => item.officialUrl !== officialUrl)) { + throw new Error('evidence official URL differs from the candidate official URL') + } + + let providerOrganizationId: string | null = null + if (row.entity_type === 'scholarship') { + if (manifest.sourceCategory === 'government_scholarship') { + if ( + typeof facts.providerOrganizationId !== 'string' + || !/^[a-z0-9][a-z0-9_-]{0,199}$/u.test(facts.providerOrganizationId) + ) { + throw new Error( + 'government scholarship requires an explicit providerOrganizationId', + ) + } + providerOrganizationId = facts.providerOrganizationId + } else { + providerOrganizationId = row.institution_id + } + } + const allowedPublishers = new Set([ + row.institution_id, + ...(providerOrganizationId ? [providerOrganizationId] : []), + ]) + if (!row.source_document_publisher || !allowedPublishers.has(row.source_document_publisher)) { + throw new RetryableSourceConfigurationError( + 'official source publisher does not own the candidate source', + ) + } + + const rawDegreeLevel = facts.degreeLevel + const identity = { + institutionId: row.institution_id, + entityType: row.entity_type, + degreeLevel: rawDegreeLevel ?? null, + normalizedName: name.toLocaleLowerCase('en-US'), + } + const identitySha256 = await sha256Hex(stableJson(identity)) + const expectedEntityKey = [ + rawDegreeLevel ?? 'all', + normalizedNameKey(name), + identitySha256.slice(0, 16), + ].join(':') + if (identitySha256 !== row.identity_sha256 || expectedEntityKey !== row.entity_key) { + throw new Error('candidate identity conflicts with its stable registry identity') + } + const entitySha256 = await sha256Hex(stableJson({ facts, evidence })) + if (entitySha256 !== row.entity_sha256) { + throw new Error('candidate payload digest does not match its immutable entity digest') + } + + const ownerIdentity = [ + row.entity_type, providerOrganizationId ?? row.institution_id, row.entity_key, + ].join('\u0000') + const recordHash = await sha256Hex(ownerIdentity) + const program = row.entity_type === 'program' + ? programIdentity(rawDegreeLevel, manifest.sourceCategory) + : null + return { + row, + manifest, + name, + nameLocale: nameLocale(name), + officialUrl, + checkedAt, + reviewAfter: addReviewDays(checkedAt, reviewDays), + programType: program?.programType ?? null, + degreeLevel: program?.degreeLevel ?? null, + schemeType: row.entity_type === 'scholarship' + ? scholarshipScheme(manifest.sourceCategory) + : null, + providerOrganizationId, + evidence, + recordId: `${row.entity_type}-${recordHash}`, + slug: `${row.entity_type}-${recordHash.slice(0, 24)}`, + } +} + +async function findIdentityConflict( + database: D1Database, + candidate: NormalizedCandidate, +): Promise { + const result = await database.prepare( + `SELECT registry_id + FROM entity_registry + WHERE institution_id = ?1 AND entity_type = ?2 + AND identity_sha256 = ?3 AND registry_id <> ?4 + ORDER BY registry_id LIMIT 1`, + ).bind( + candidate.row.institution_id, + candidate.row.entity_type, + candidate.row.identity_sha256, + candidate.row.registry_id, + ).all<{ registry_id: string }>() + ensureSuccess(result, 'load entity identity conflicts') + return result.results?.[0]?.registry_id ?? null +} + +async function requireProviderOrganization( + database: D1Database, + candidate: NormalizedCandidate, +): Promise { + if (!candidate.providerOrganizationId) return + const provider = await database.prepare( + `SELECT organization.record_id + FROM organizations organization + JOIN records record ON record.id = organization.record_id + WHERE organization.record_id = ?1 + AND record.kind = 'organization' + AND record.workflow_status IN ('applied', 'published')`, + ).bind(candidate.providerOrganizationId).first<{ record_id: string }>() + if (!provider) { + throw new RetryableSourceConfigurationError( + `provider organization ${candidate.providerOrganizationId} is not registered`, + ) + } +} + +async function loadExistingRecord( + database: D1Database, + recordId: string, +): Promise { + return database.prepare( + `SELECT id, public_id, kind, row_version, workflow_status + FROM records WHERE id = ?1`, + ).bind(recordId).first() +} + +function isolationReason(error: unknown): { code: string; issue: string; conflict: boolean } { + const issue = error instanceof Error ? error.message : String(error) + const conflict = /conflict|differs|digest|identity/iu.test(issue) + return { + code: conflict ? 'entity_identity_conflict' : 'entity_candidate_invalid', + issue, + conflict, + } +} + +async function isolateCandidate( + database: D1Database, + row: CandidateRow, + code: string, + issues: string[], + conflict: boolean, + decidedAt: string, +): Promise { + const normalizedIssues = [...new Set(issues.map((issue) => issue.trim()).filter(Boolean))] + if (normalizedIssues.length === 0) normalizedIssues.push(code) + const statements = [ + statement( + database, + `UPDATE extracted_entity_candidates + SET candidate_status = 'quarantined', + issues_json = ?2, + processed_at = COALESCE(processed_at, ?3), + registered_at = NULL + WHERE candidate_id = ?1 + AND candidate_status IN ('extracted', 'validated', 'registered', 'quarantined')`, + row.candidate_id, + stableJson(normalizedIssues), + decidedAt, + ), + statement( + database, + `UPDATE catalog_reconciliation_items + SET disposition = 'unparseable', reason_code = ?2, + reason_detail = ?3, reconciled_at = ?4, + updated_at = ?4 + WHERE reconciliation_id = ?1 AND disposition = 'pending'`, + row.reconciliation_id, + code, + normalizedIssues.join('; ').slice(0, 2_000), + decidedAt, + ), + statement( + database, + `INSERT OR IGNORE INTO entity_materialization_decisions ( + candidate_id, registry_id, decision_status, canonical_record_id, + reason_code, issues_json, confidence_ppm, materializer_version, + decided_at, created_at + ) VALUES (?1, ?2, ?3, NULL, ?4, ?5, ?6, ?7, ?8, ?8)`, + row.candidate_id, + row.registry_id, + conflict ? 'conflict' : 'quarantined', + code, + stableJson(normalizedIssues), + row.confidence_ppm, + MATERIALIZER_VERSION, + decidedAt, + ), + ] + try { + ensureBatch(await database.batch(statements), 'isolate entity candidate') + } catch (error) { + const existing = await loadDecision(database, row.candidate_id) + if (existing) return resultFromDecision(row.candidate_id, existing) + throw error + } + return { + candidateId: row.candidate_id, + registryId: row.registry_id, + status: conflict ? 'conflict' : 'quarantined', + recordId: null, + mappedFields: 0, + reasonCode: code, + issues: normalizedIssues, + } +} + +async function materializationStatements( + database: D1Database, + candidate: NormalizedCandidate, + existingRecord: ExistingRecordRow | null, + decidedAt: string, +): Promise { + const { row } = candidate + const facts = canonicalFacts(candidate) + const statements: D1PreparedStatement[] = [] + for (const fact of facts) { + statements.push(statement( + database, + `INSERT INTO field_definitions ( + record_kind, field_path, value_type, risk_class, + required_for_publish, max_age_days, validation_profile + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(record_kind, field_path) DO NOTHING`, + row.entity_type, + fact.fieldPath, + fact.valueType, + fact.riskClass, + fact.requiredForPublish, + DEFAULT_REVIEW_DAYS, + fact.validationProfile, + )) + } + + const fetchId = `entity-fetch-${await sha256Hex(stableJson({ + sourceDocumentId: row.source_document_id, + snapshotId: row.snapshot_id, + }))}` + statements.push(statement( + database, + `INSERT OR IGNORE INTO source_fetches ( + id, source_id, status, requested_at, completed_at, http_status, + content_type, content_length, sha256, artifact_uri, + parser_key, parser_version, metadata_json + ) VALUES ( + ?1, ?2, 'succeeded', ?3, ?3, 200, ?4, ?5, ?6, ?7, + 'entity-candidate-materializer', '1', ?8 + )`, + fetchId, + row.source_document_id, + row.fetched_at, + row.content_type, + row.byte_length, + row.raw_sha256, + `r2://studyinchina-source-snapshots/${row.r2_key}`, + stableJson({ + ingestionSourceId: row.source_id, + ingestionSnapshotId: row.snapshot_id, + finalUrl: row.final_url, + }), + )) + + const fragmentIds: string[] = [] + for (const evidence of candidate.evidence) { + const locator = evidence.locator ?? `entity:${evidence.fieldPath}` + const fragmentId = `fragment-${await sha256Hex(stableJson({ + fetchId, + locator, + quote: evidence.quote, + }))}` + fragmentIds.push(fragmentId) + statements.push(statement( + database, + `INSERT OR IGNORE INTO source_fragments ( + id, fetch_id, locator_type, locator, page_number, + text_excerpt, sha256, created_at + ) VALUES (?1, ?2, ?3, ?4, NULL, ?5, ?6, ?7)`, + fragmentId, + fetchId, + locatorType(evidence.locator), + locator, + evidence.quote, + await sha256Hex(evidence.quote), + row.fetched_at, + )) + } + + const nextVersion = existingRecord ? existingRecord.row_version + 1 : 1 + if (existingRecord) { + statements.push(statement( + database, + `UPDATE records + SET slug = ?2, review_after = ?3, + workflow_status = CASE + WHEN workflow_status = 'published' THEN 'published' + ELSE 'applied' + END, + row_version = row_version + 1, updated_at = ?4 + WHERE id = ?1 AND row_version = ?5 AND kind = ?6`, + candidate.recordId, + candidate.slug, + candidate.reviewAfter, + decidedAt, + existingRecord.row_version, + row.entity_type, + )) + } else { + statements.push(statement( + database, + `INSERT INTO records ( + id, public_id, kind, slug, workflow_status, review_after, + row_version, created_at, updated_at + ) VALUES (?1, ?1, ?2, ?3, 'applied', ?4, 1, ?5, ?5)`, + candidate.recordId, + row.entity_type, + candidate.slug, + candidate.reviewAfter, + decidedAt, + )) + } + statements.push(statement( + database, + `INSERT OR IGNORE INTO record_slugs ( + record_id, slug, valid_from, valid_to, is_current + ) VALUES (?1, ?2, ?3, NULL, 1)`, + candidate.recordId, + candidate.slug, + decidedAt, + )) + + if (row.entity_type === 'program') { + statements.push(statement( + database, + `INSERT INTO programs ( + record_id, institution_id, academic_unit_id, parent_program_id, + program_type, degree_level, credential_type, attendance_mode, + delivery_mode, duration_min, duration_max, duration_unit, + official_url + ) VALUES ( + ?1, ?2, NULL, NULL, ?3, ?4, NULL, 'full_time', + 'on_campus', NULL, NULL, NULL, ?5 + ) + ON CONFLICT(record_id) DO UPDATE SET + institution_id = excluded.institution_id, + program_type = excluded.program_type, + degree_level = excluded.degree_level, + official_url = excluded.official_url`, + candidate.recordId, + row.institution_id, + candidate.programType, + candidate.degreeLevel, + candidate.officialUrl, + )) + } else { + statements.push(statement( + database, + `INSERT INTO scholarships ( + record_id, provider_organization_id, scheme_type, official_url + ) VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(record_id) DO UPDATE SET + provider_organization_id = excluded.provider_organization_id, + scheme_type = excluded.scheme_type, + official_url = excluded.official_url`, + candidate.recordId, + candidate.providerOrganizationId, + candidate.schemeType, + candidate.officialUrl, + )) + } + statements.push(statement( + database, + `INSERT INTO localized_content ( + record_id, locale, field_name, text_value, + translation_status, source_locale, updated_at + ) VALUES (?1, ?2, 'name', ?3, 'published', ?2, ?4) + ON CONFLICT(record_id, locale, field_name) DO UPDATE SET + text_value = excluded.text_value, + translation_status = 'published', + source_locale = excluded.source_locale, + updated_at = excluded.updated_at`, + candidate.recordId, + candidate.nameLocale, + candidate.name, + decidedAt, + )) + + for (const fact of facts) { + const claimId = `claim-${await sha256Hex(stableJson({ + candidateId: row.candidate_id, + fieldPath: fact.fieldPath, + locale: fact.locale, + value: fact.value, + }))}` + statements.push(statement( + database, + `INSERT OR IGNORE INTO claims ( + id, subject_record_id, field_path, locale, value_type, + raw_value_text, normalized_value_json, confidence, + extraction_method, extractor_version, claim_status, + provenance_precision, discovered_at, decided_at + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, + 'selector', ?9, 'candidate', 'field', ?10, NULL + )`, + claimId, + candidate.recordId, + fact.fieldPath, + fact.locale, + fact.valueType, + fact.value, + stableJson(fact.value), + Number(row.confidence_ppm) / 1_000_000, + MATERIALIZER_VERSION, + candidate.checkedAt, + )) + for (const fragmentId of [...new Set(fragmentIds)].sort()) { + statements.push(statement( + database, + `INSERT OR IGNORE INTO claim_evidence ( + claim_id, fragment_id, evidence_role + ) VALUES (?1, ?2, 'primary')`, + claimId, + fragmentId, + )) + } + statements.push( + statement( + database, + `UPDATE claims SET claim_status = 'validated' + WHERE id = ?1 AND claim_status = 'candidate'`, + claimId, + ), + statement( + database, + `UPDATE claims SET claim_status = 'accepted', decided_at = ?2 + WHERE id = ?1 AND claim_status = 'validated'`, + claimId, + decidedAt, + ), + statement( + database, + `INSERT INTO canonical_fields ( + subject_record_id, field_path, locale, field_status, + claim_id, value_json, verified_at, review_after, updated_at + ) VALUES (?1, ?2, ?3, 'accepted', ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(subject_record_id, field_path, locale) DO UPDATE SET + field_status = 'accepted', claim_id = excluded.claim_id, + value_json = excluded.value_json, verified_at = excluded.verified_at, + review_after = excluded.review_after, updated_at = excluded.updated_at`, + candidate.recordId, + fact.fieldPath, + fact.locale, + claimId, + stableJson(fact.value), + candidate.checkedAt, + candidate.reviewAfter, + decidedAt, + ), + statement( + database, + `UPDATE claims SET claim_status = 'superseded', decided_at = ?5 + WHERE subject_record_id = ?1 AND field_path = ?2 AND locale = ?3 + AND claim_status = 'accepted' AND id <> ?4`, + candidate.recordId, + fact.fieldPath, + fact.locale, + claimId, + decidedAt, + ), + ) + } + + statements.push( + statement( + database, + `INSERT INTO record_versions ( + id, record_id, version, snapshot_json, change_set_id, + changed_by, change_reason, changed_at + ) VALUES (?1, ?2, ?3, ?4, NULL, ?5, ?6, ?7)`, + `record-version-${await sha256Hex(stableJson({ + candidateId: row.candidate_id, + recordId: candidate.recordId, + version: nextVersion, + }))}`, + candidate.recordId, + nextVersion, + stableJson({ + name: candidate.name, + officialUrl: candidate.officialUrl, + programType: candidate.programType, + degreeLevel: candidate.degreeLevel, + schemeType: candidate.schemeType, + providerOrganizationId: candidate.providerOrganizationId, + }), + MATERIALIZER_VERSION, + 'source-backed entity candidate materialization', + decidedAt, + ), + statement( + database, + `UPDATE extracted_entity_candidates + SET candidate_status = 'registered', + processed_at = COALESCE(processed_at, ?2), + registered_at = COALESCE(registered_at, ?2), + issues_json = '[]' + WHERE candidate_id = ?1 + AND candidate_status IN ('validated', 'registered')`, + row.candidate_id, + decidedAt, + ), + statement( + database, + `UPDATE entity_registry + SET registry_status = 'active', canonical_record_id = ?2, + updated_at = ?3 + WHERE registry_id = ?1 + AND registry_status IN ('pending', 'active') + AND (canonical_record_id IS NULL OR canonical_record_id = ?2)`, + row.registry_id, + candidate.recordId, + decidedAt, + ), + statement( + database, + `UPDATE catalog_reconciliation_items + SET disposition = 'published', entity_key = ?2, + candidate_id = ?3, registry_id = ?4, + reason_code = NULL, reason_detail = NULL, + reconciled_at = ?5, updated_at = ?5 + WHERE reconciliation_id = ?1 AND disposition IN ('pending', 'published')`, + row.reconciliation_id, + row.entity_key, + row.candidate_id, + row.registry_id, + decidedAt, + ), + statement( + database, + `INSERT OR IGNORE INTO entity_materialization_decisions ( + candidate_id, registry_id, decision_status, canonical_record_id, + reason_code, issues_json, confidence_ppm, materializer_version, + decided_at, created_at + ) VALUES (?1, ?2, 'materialized', ?3, NULL, '[]', ?4, ?5, ?6, ?6)`, + row.candidate_id, + row.registry_id, + candidate.recordId, + row.confidence_ppm, + MATERIALIZER_VERSION, + decidedAt, + ), + ) + for (const fact of facts) { + statements.push(statement( + database, + `INSERT OR IGNORE INTO entity_candidate_field_mappings ( + candidate_id, candidate_field_path, registry_id, source_id, + subject_record_id, canonical_field_path, locale, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)`, + row.candidate_id, + fact.candidateFieldPath, + row.registry_id, + row.source_id, + candidate.recordId, + fact.fieldPath, + fact.locale, + decidedAt, + )) + } + statements.push(statement( + database, + `INSERT OR IGNORE INTO audit_log ( + id, occurred_at, actor_type, actor_id, action, + subject_type, subject_id, after_json, correlation_id, detail + ) VALUES (?1, ?2, 'worker', ?3, 'entity_materialized', + ?4, ?5, ?6, ?7, ?8)`, + `audit-${await sha256Hex(`entity-materialized\u0000${row.candidate_id}`)}`, + decidedAt, + MATERIALIZER_VERSION, + row.entity_type, + candidate.recordId, + stableJson({ + candidateId: row.candidate_id, + registryId: row.registry_id, + mappedFields: facts.length, + }), + row.candidate_id, + 'Official directory entity passed deterministic materialization gates', + )) + return statements +} + +export async function materializeExtractedEntityCandidate( + database: D1Database, + candidateId: string, + options: EntityMaterializationOptions = {}, +): Promise { + const existingDecision = await loadDecision(database, candidateId) + if (existingDecision) return resultFromDecision(candidateId, existingDecision) + + const row = await loadCandidate(database, candidateId) + if (!row) throw new Error(`entity candidate ${candidateId} does not exist or lacks registry/reconciliation state`) + if (row.candidate_status === 'extracted') { + return { + candidateId, + registryId: row.registry_id, + status: 'pending', + recordId: null, + mappedFields: 0, + reasonCode: 'candidate_not_validated', + } + } + const decidedAt = isoTimestamp(options.decidedAt ?? new Date().toISOString(), 'decidedAt') + const minimumConfidence = options.minimumConfidencePpm + ?? DEFAULT_MINIMUM_CONFIDENCE_PPM + if (!Number.isInteger(minimumConfidence) || minimumConfidence < 0 || minimumConfidence > 1_000_000) { + throw new Error('minimumConfidencePpm must be an integer from 0 to 1000000') + } + const reviewDays = options.reviewDays ?? DEFAULT_REVIEW_DAYS + if (!Number.isInteger(reviewDays) || reviewDays < 1 || reviewDays > 365) { + throw new Error('reviewDays must be an integer from 1 to 365') + } + const persistedIssues = parseArray(row.issues_json, 'issues_json').map(String) + if ( + row.candidate_status === 'quarantined' + || row.confidence_ppm === null + || row.confidence_ppm < minimumConfidence + || persistedIssues.length > 0 + ) { + const issues = [ + ...persistedIssues, + ...(row.confidence_ppm === null + ? ['candidate confidence is missing'] + : row.confidence_ppm < minimumConfidence + ? [`candidate confidence ${row.confidence_ppm} is below ${minimumConfidence}`] + : []), + ] + return isolateCandidate( + database, + row, + row.confidence_ppm === null || row.confidence_ppm < minimumConfidence + ? 'entity_confidence_below_threshold' + : 'entity_candidate_has_issues', + issues, + false, + decidedAt, + ) + } + if (!['validated', 'registered'].includes(row.candidate_status)) { + throw new Error(`entity candidate ${candidateId} has terminal status ${row.candidate_status}`) + } + + let candidate: NormalizedCandidate + try { + candidate = await normalizeCandidate(row, reviewDays) + } catch (error) { + if (error instanceof RetryableSourceConfigurationError) throw error + const isolation = isolationReason(error) + return isolateCandidate( + database, + row, + isolation.code, + [isolation.issue], + isolation.conflict, + decidedAt, + ) + } + await requireProviderOrganization(database, candidate) + const conflictingRegistryId = await findIdentityConflict(database, candidate) + if (conflictingRegistryId) { + return isolateCandidate( + database, + row, + 'entity_identity_conflict', + [ + `entity identity conflicts with registry ${conflictingRegistryId}`, + ], + true, + decidedAt, + ) + } + + const existingRecord = await loadExistingRecord(database, candidate.recordId) + if (existingRecord && ( + existingRecord.public_id !== candidate.recordId + || existingRecord.kind !== row.entity_type + || (row.canonical_record_id !== null && row.canonical_record_id !== candidate.recordId) + )) { + return isolateCandidate( + database, + row, + 'canonical_record_identity_conflict', + ['deterministic canonical record identity collides with an incompatible record'], + true, + decidedAt, + ) + } + + const statements = await materializationStatements( + database, + candidate, + existingRecord, + decidedAt, + ) + try { + ensureBatch(await database.batch(statements), 'materialize entity candidate') + } catch (error) { + const decision = await loadDecision(database, candidateId) + if (decision) return resultFromDecision(candidateId, decision) + throw error + } + return { + candidateId, + registryId: row.registry_id, + status: 'materialized', + recordId: candidate.recordId, + mappedFields: canonicalFacts(candidate).length, + } +} + +export async function requestEntityMaterializationRelease( + database: D1Database, + candidateIds: string[], + requestedAt = new Date().toISOString(), +): Promise { + const normalizedRequestedAt = isoTimestamp(requestedAt, 'requestedAt') + const releaseWindow = normalizedRequestedAt.slice(0, 10) + const normalizedCandidateIds = [...new Set(candidateIds.map((value) => value.trim()))] + .filter(Boolean) + .sort((left, right) => left.localeCompare(right, 'en')) + if (normalizedCandidateIds.length === 0) { + throw new Error('candidateIds must contain at least one materialized candidate') + } + const digest = await sha256Hex(stableJson({ + releaseWindow, + candidateIds: normalizedCandidateIds, + })) + const requestId = `entity-materialization-${digest.slice(0, 40)}` + const publicationJobId = `entity-publication-${releaseWindow}-${digest.slice(0, 24)}` + const catalogReleaseId = `catalog-entity-${releaseWindow}-${digest.slice(0, 24)}` + const outboxEventId = `entity-release-event-${releaseWindow}-${digest.slice(0, 24)}` + const candidateIdsJson = stableJson(normalizedCandidateIds) + const payloadJson = stableJson({ + version: 1, + entityMaterializationRequestId: requestId, + publicationJobId, + catalogReleaseId, + releaseWindow, + candidateIds: normalizedCandidateIds, + }) + const result = await statement( + database, + `INSERT OR IGNORE INTO entity_materialization_release_requests ( + request_id, release_window, publication_job_id, catalog_release_id, + outbox_event_id, candidate_ids_json, payload_json, + requested_at, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8)`, + requestId, + releaseWindow, + publicationJobId, + catalogReleaseId, + outboxEventId, + candidateIdsJson, + payloadJson, + normalizedRequestedAt, + ).run() + ensureSuccess(result, 'request entity materialization release') + if (Number(result.meta?.changes ?? 0) > 0) { + return { + status: 'requested', + requestId, + releaseWindow, + publicationJobId, + catalogReleaseId, + outboxEventId, + candidateIds: normalizedCandidateIds, + } + } + const existing = await database.prepare( + `SELECT request_id, publication_job_id, catalog_release_id, + outbox_event_id, candidate_ids_json + FROM entity_materialization_release_requests + WHERE release_window = ?1`, + ).bind(releaseWindow).first<{ + request_id: string + publication_job_id: string + catalog_release_id: string + outbox_event_id: string + candidate_ids_json: string + }>() + if (!existing) throw new Error('entity materialization release insert was ignored without an existing window') + return { + status: 'already-requested', + requestId: existing.request_id, + releaseWindow, + publicationJobId: existing.publication_job_id, + catalogReleaseId: existing.catalog_release_id, + outboxEventId: existing.outbox_event_id, + candidateIds: parseArray(existing.candidate_ids_json, 'candidate_ids_json').map(String), + } +} diff --git a/workers/ingestion/src/repository.ts b/workers/ingestion/src/repository.ts index 42d3549..b40ade0 100644 --- a/workers/ingestion/src/repository.ts +++ b/workers/ingestion/src/repository.ts @@ -126,8 +126,8 @@ function entityPersistenceStatements( snapshot_id, source_discovery_id, ingestion_job_id, extractor, candidate_status, facts_json, evidence_json, issues_json, entity_sha256, confidence_ppm, created_at, processed_at, registered_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'registered', - ?10, ?11, '[]', ?12, 1000000, ?13, ?13, ?13)`, + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'validated', + ?10, ?11, '[]', ?12, 1000000, ?13, ?13, NULL)`, ).bind( candidate.candidateId, candidate.institutionId, diff --git a/workers/ingestion/tests/entity-materializer.test.ts b/workers/ingestion/tests/entity-materializer.test.ts new file mode 100644 index 0000000..bed04fe --- /dev/null +++ b/workers/ingestion/tests/entity-materializer.test.ts @@ -0,0 +1,423 @@ +import assert from 'node:assert/strict' +import { readFileSync, readdirSync } from 'node:fs' +import { resolve } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import test from 'node:test' + +import { + materializeExtractedEntityCandidate, + requestEntityMaterializationRelease, +} from '../src/entity-materializer' +import { sha256Hex } from '../src/hash' +import { buildOfficialEntityExtraction } from '../src/pipeline' +import { persistSnapshotEntityExtraction } from '../src/repository' +import type { + D1Database, + D1PreparedStatement, + D1Result, + SnapshotRecord, + SourceManifestV1, +} from '../src/types' +import { sourceManifest } from './fixtures' + +type SqlValue = string | number | bigint | Uint8Array | null + +function sqliteValues(values: unknown[]): SqlValue[] { + return values.map((value) => { + if ( + value === null + || typeof value === 'string' + || typeof value === 'number' + || typeof value === 'bigint' + || value instanceof Uint8Array + ) return value + throw new TypeError(`Unsupported SQLite bind value: ${typeof value}`) + }) +} + +class SqliteStatement implements D1PreparedStatement { + private values: SqlValue[] = [] + + constructor( + private readonly database: DatabaseSync, + private readonly query: string, + ) {} + + bind(...values: unknown[]): D1PreparedStatement { + this.values = sqliteValues(values) + return this + } + + async first(): Promise { + return (this.database.prepare(this.query).get(...this.values) as T | undefined) ?? null + } + + async all(): Promise> { + return { + success: true, + results: this.database.prepare(this.query).all(...this.values) as T[], + } + } + + async run(): Promise> { + return this.runSync() as D1Result + } + + runSync(): D1Result { + const result = this.database.prepare(this.query).run(...this.values) + return { success: true, meta: { changes: Number(result.changes) } } + } +} + +class SqliteD1 implements D1Database { + constructor(readonly database: DatabaseSync) {} + + prepare(query: string): D1PreparedStatement { + return new SqliteStatement(this.database, query) + } + + async batch(statements: D1PreparedStatement[]): Promise>> { + this.database.exec('BEGIN IMMEDIATE') + try { + const results = statements.map((statement) => { + if (!(statement instanceof SqliteStatement)) { + throw new TypeError('Unexpected prepared statement implementation') + } + return statement.runSync() as D1Result + }) + this.database.exec('COMMIT') + return results + } catch (error) { + this.database.exec('ROLLBACK') + throw error + } + } +} + +const checkedAt = '2026-08-05T00:00:00.000Z' + +function pipelineDatabase(): DatabaseSync { + const database = new DatabaseSync(':memory:') + database.exec('PRAGMA foreign_keys = ON') + const directory = resolve('infra/d1/pipeline/migrations') + for (const fileName of readdirSync(directory) + .filter((value) => value.endsWith('.sql')) + .sort((left, right) => left.localeCompare(right, 'en'))) { + database.exec(readFileSync(resolve(directory, fileName), 'utf8')) + } + return database +} + +function seedInstitution(database: DatabaseSync, institutionId: string): void { + database.prepare( + `INSERT INTO records (id, public_id, kind, slug, workflow_status) + VALUES ('city-example', 'city-example', 'location', 'city-example', 'applied'), + (?, ?, 'organization', ?, 'applied')`, + ).run(institutionId, institutionId, institutionId) + database.exec( + `INSERT INTO locations (record_id, location_type, country_code) + VALUES ('city-example', 'city', 'CN')`, + ) + database.prepare( + `INSERT INTO organizations (record_id, organization_type, official_url) + VALUES (?, 'university', 'https://admissions.example.edu.cn')`, + ).run(institutionId) + database.prepare( + `INSERT INTO institutions ( + record_id, city_id, institution_type, admissions_url + ) VALUES ( + ?, 'city-example', 'comprehensive', 'https://admissions.example.edu.cn' + )`, + ).run(institutionId) +} + +function manifestFor( + sourceId: string, + governmentScholarship = false, +): SourceManifestV1 { + return sourceManifest({ + id: sourceId, + institutionId: 'uni-example', + entityType: governmentScholarship ? 'scholarship' : 'program', + sourceCategory: governmentScholarship ? 'government_scholarship' : 'undergraduate_catalog', + officialUrl: `https://admissions.example.edu.cn/catalog/${sourceId}.html`, + allowedHosts: ['admissions.example.edu.cn'], + }) +} + +async function seedCandidate( + database: DatabaseSync, + suffix: string, + governmentScholarship = false, +): Promise { + const manifest = manifestFor(`example-catalog-${suffix}`, governmentScholarship) + const sourceId = manifest.id + const snapshotId = `snapshot-${suffix}` + const jobId = `job-${suffix}` + const rawSha256 = await sha256Hex(`raw-${suffix}`) + const canonicalSha256 = await sha256Hex(`canonical-${suffix}`) + const snapshot: SnapshotRecord = { + snapshotId, + sourceId, + r2Key: `snapshots/${rawSha256}.html`, + rawSha256, + canonicalSha256, + contentType: 'text/html; charset=utf-8', + byteLength: 512, + finalUrl: manifest.officialUrl, + fetchedAt: checkedAt, + etag: null, + lastModified: null, + } + database.prepare( + `INSERT INTO ingestion_sources ( + source_id, manifest_json, enabled, created_at, updated_at + ) VALUES (?, ?, 1, ?, ?)`, + ).run(sourceId, JSON.stringify(manifest), checkedAt, checkedAt) + database.prepare( + `INSERT INTO ingestion_jobs ( + job_id, source_id, status, reason, scheduled_at, + created_at, updated_at + ) VALUES (?, ?, 'running', 'manual', ?, ?, ?)`, + ).run(jobId, sourceId, checkedAt, checkedAt, checkedAt) + + const extraction = await buildOfficialEntityExtraction( + manifest, + snapshotId, + jobId, + manifest.officialUrl, + governmentScholarship + ? ` + + + +
ScholarshipDetails
Chinese Government Scholarship ${suffix}Details
` + : ` + + + +
ProgramDetails
Bachelor of Engineering in Computer Science ${suffix}Details
`, + snapshot.contentType, + checkedAt, + ) + assert.ok(extraction) + assert.equal(extraction.candidates.length, 1) + await persistSnapshotEntityExtraction( + { INGESTION_DB: new SqliteD1(database) }, + { snapshot, entityExtraction: extraction }, + ) + + const sourceDocumentId = `source-document-${suffix}` + database.prepare( + `INSERT INTO source_documents ( + id, public_id, canonical_url, publisher_organization_id, + source_kind, authority_level, official, language_code, + active, robots_policy, created_at, updated_at + ) VALUES ( + ?, ?, ?, 'uni-example', ?, 'primary_official', + 1, 'en', 1, 'enforce', ?, ? + )`, + ).run( + sourceDocumentId, + sourceDocumentId, + manifest.officialUrl, + governmentScholarship ? 'scholarship' : 'program', + checkedAt, + checkedAt, + ) + database.prepare( + `INSERT INTO promotion_source_bindings ( + source_id, source_document_id, enabled, created_at, updated_at + ) VALUES (?, ?, 1, ?, ?)`, + ).run(sourceId, sourceDocumentId, checkedAt, checkedAt) + return extraction.candidates[0].candidateId +} + +function count(database: DatabaseSync, table: string): number { + return Number((database.prepare( + `SELECT COUNT(*) AS count FROM ${table}`, + ).get() as { count: number }).count) +} + +function plainRow>(value: unknown): T { + return { ...(value as T) } +} + +test('materializes a validated directory entity with evidence and remains idempotent', async () => { + const database = pipelineDatabase() + try { + seedInstitution(database, 'uni-example') + const candidateId = await seedCandidate(database, 'computer-science') + const d1 = new SqliteD1(database) + + const result = await materializeExtractedEntityCandidate(d1, candidateId, { + decidedAt: '2026-08-05T01:00:00.000Z', + }) + assert.equal(result.status, 'materialized') + assert.ok(result.recordId) + assert.equal(result.mappedFields, 4) + assert.deepEqual( + plainRow(database.prepare( + `SELECT candidate_status, registered_at + FROM extracted_entity_candidates WHERE candidate_id = ?`, + ).get(candidateId)), + { + candidate_status: 'registered', + registered_at: '2026-08-05T01:00:00.000Z', + }, + ) + assert.deepEqual( + plainRow(database.prepare( + `SELECT registry_status, canonical_record_id + FROM entity_registry`, + ).get()), + { registry_status: 'active', canonical_record_id: result.recordId }, + ) + assert.equal( + (database.prepare( + `SELECT disposition FROM catalog_reconciliation_items`, + ).get() as { disposition: string }).disposition, + 'published', + ) + assert.equal(count(database, 'programs'), 1) + assert.equal(count(database, 'canonical_fields'), 4) + assert.equal(count(database, 'entity_candidate_field_mappings'), 4) + assert.equal(count(database, 'entity_materialization_decisions'), 1) + assert.equal(count(database, 'record_versions'), 1) + + const before = { + claims: count(database, 'claims'), + fields: count(database, 'canonical_fields'), + mappings: count(database, 'entity_candidate_field_mappings'), + versions: count(database, 'record_versions'), + } + const repeated = await materializeExtractedEntityCandidate(d1, candidateId) + assert.equal(repeated.status, 'already-materialized') + assert.deepEqual({ + claims: count(database, 'claims'), + fields: count(database, 'canonical_fields'), + mappings: count(database, 'entity_candidate_field_mappings'), + versions: count(database, 'record_versions'), + }, before) + } finally { + database.close() + } +}) + +test('quarantines low-confidence entities without leaking canonical records', async () => { + const database = pipelineDatabase() + try { + seedInstitution(database, 'uni-example') + const candidateId = await seedCandidate(database, 'low-confidence') + database.prepare( + `UPDATE extracted_entity_candidates SET confidence_ppm = 700000 + WHERE candidate_id = ?`, + ).run(candidateId) + + const result = await materializeExtractedEntityCandidate( + new SqliteD1(database), + candidateId, + { decidedAt: '2026-08-05T01:00:00.000Z' }, + ) + assert.equal(result.status, 'quarantined') + assert.equal(result.reasonCode, 'entity_confidence_below_threshold') + assert.equal(count(database, 'programs'), 0) + assert.equal(count(database, 'canonical_fields'), 0) + assert.deepEqual( + plainRow(database.prepare( + `SELECT candidate_status FROM extracted_entity_candidates + WHERE candidate_id = ?`, + ).get(candidateId)), + { candidate_status: 'quarantined' }, + ) + assert.deepEqual( + plainRow(database.prepare( + `SELECT disposition, reason_code FROM catalog_reconciliation_items`, + ).get()), + { + disposition: 'unparseable', + reason_code: 'entity_confidence_below_threshold', + }, + ) + } finally { + database.close() + } +}) + +test('quarantines a government scholarship when its provider is not explicit', async () => { + const database = pipelineDatabase() + try { + seedInstitution(database, 'uni-example') + const candidateId = await seedCandidate(database, 'government-award', true) + + const result = await materializeExtractedEntityCandidate( + new SqliteD1(database), + candidateId, + { decidedAt: '2026-08-05T01:00:00.000Z' }, + ) + + assert.equal(result.status, 'quarantined') + assert.equal(result.reasonCode, 'entity_candidate_invalid') + assert.match(result.issues?.join(' ') ?? '', /providerOrganizationId/u) + assert.equal(count(database, 'scholarships'), 0) + assert.equal(count(database, 'canonical_fields'), 0) + } finally { + database.close() + } +}) + +test('creates one release request per UTC day and rejects unsafe cohorts', async () => { + const database = pipelineDatabase() + try { + seedInstitution(database, 'uni-example') + const safeCandidateId = await seedCandidate(database, 'safe-release') + const unsafeCandidateId = await seedCandidate(database, 'unsafe-release') + const d1 = new SqliteD1(database) + await materializeExtractedEntityCandidate(d1, safeCandidateId, { + decidedAt: '2026-08-05T01:00:00.000Z', + }) + database.prepare( + `UPDATE extracted_entity_candidates SET confidence_ppm = 100000 + WHERE candidate_id = ?`, + ).run(unsafeCandidateId) + await materializeExtractedEntityCandidate(d1, unsafeCandidateId, { + decidedAt: '2026-08-05T01:00:00.000Z', + }) + + const requested = await requestEntityMaterializationRelease( + d1, + [safeCandidateId], + '2026-08-05T02:00:00.000Z', + ) + assert.equal(requested.status, 'requested') + const repeated = await requestEntityMaterializationRelease( + d1, + [safeCandidateId], + '2026-08-05T03:00:00.000Z', + ) + assert.equal(repeated.status, 'already-requested') + assert.equal(repeated.requestId, requested.requestId) + assert.equal(count(database, 'entity_materialization_release_requests'), 1) + assert.equal(count(database, 'publication_jobs'), 1) + assert.equal(count(database, 'outbox_events'), 1) + assert.deepEqual( + plainRow(database.prepare( + `SELECT event_type, event_status FROM outbox_events`, + ).get()), + { event_type: 'catalog.release.requested', event_status: 'pending' }, + ) + + await assert.rejects( + requestEntityMaterializationRelease( + d1, + [unsafeCandidateId], + '2026-08-06T02:00:00.000Z', + ), + /unsafe candidate/iu, + ) + assert.equal(count(database, 'entity_materialization_release_requests'), 1) + assert.equal(count(database, 'publication_jobs'), 1) + } finally { + database.close() + } +}) diff --git a/workers/release-builder/src/index.ts b/workers/release-builder/src/index.ts index 3c9e7c6..b58e515 100644 --- a/workers/release-builder/src/index.ts +++ b/workers/release-builder/src/index.ts @@ -6,6 +6,10 @@ import { stableJson, } from './artifact' import { buildArtifactFromPipeline } from './snapshot' +import { + enforceCatalogReleaseRetention, + readCatalogReadiness, +} from './retention' import { buildCompatibilityArtifact, ensureImmutableCompatibilityArtifact, @@ -369,6 +373,7 @@ async function importArtifact( compatibility, now.toISOString(), ) + await enforceCatalogReleaseRetention(environment.CATALOG_DB, now) return 'already-published' } @@ -460,6 +465,7 @@ async function importArtifact( if (current?.current_release_id !== releaseId) { throw new ReleaseValidationError('release_activation_failed', 'catalog release pointer did not switch') } + await enforceCatalogReleaseRetention(environment.CATALOG_DB, now) return 'published' } @@ -679,7 +685,54 @@ export async function handleQueue( } } -export function handleFetch(request: Request): Response { +async function readinessResponse(environment?: ReleaseBuilderEnv): Promise { + const headers = { 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' } + if (!environment) { + return Response.json({ + ok: false, + service: 'studyinchina-release-builder', + version: SERVICE_VERSION, + checks: { + database: false, + activeRelease: false, + pointerConsistency: false, + retention: false, + }, + }, { status: 503, headers }) + } + try { + const checks = await readCatalogReadiness(environment.CATALOG_DB) + const ok = Object.values(checks).every(Boolean) + return Response.json({ + ok, + service: 'studyinchina-release-builder', + version: SERVICE_VERSION, + checks: { database: true, ...checks }, + }, { status: ok ? 200 : 503, headers }) + } catch { + return Response.json({ + ok: false, + service: 'studyinchina-release-builder', + version: SERVICE_VERSION, + checks: { + database: false, + activeRelease: false, + pointerConsistency: false, + retention: false, + }, + }, { status: 503, headers }) + } +} + +export function handleFetch(request: Request): Response +export function handleFetch( + request: Request, + environment: ReleaseBuilderEnv, +): Response | Promise +export function handleFetch( + request: Request, + environment?: ReleaseBuilderEnv, +): Response | Promise { const url = new URL(request.url) if (request.method === 'GET' && url.pathname === '/health') { return Response.json( @@ -687,6 +740,9 @@ export function handleFetch(request: Request): Response { { headers: { 'Cache-Control': 'no-store', 'X-Content-Type-Options': 'nosniff' } }, ) } + if (request.method === 'GET' && url.pathname === '/ready') { + return readinessResponse(environment) + } return Response.json({ ok: false, error: 'not_found' }, { status: 404 }) } diff --git a/workers/release-builder/src/retention.ts b/workers/release-builder/src/retention.ts new file mode 100644 index 0000000..a97bc54 --- /dev/null +++ b/workers/release-builder/src/retention.ts @@ -0,0 +1,183 @@ +import type { + D1Database, + D1PreparedStatement, + D1Result, +} from './types' +import { RELEASE_TABLES } from './types' + +export const CATALOG_RELEASE_LIMIT = 3 +export const CATALOG_ROLLBACK_LIMIT = CATALOG_RELEASE_LIMIT - 1 + +type RetiredRelease = { + release_id: string +} + +export type CatalogReadiness = { + activeRelease: boolean + pointerConsistency: boolean + retention: boolean +} + +type CatalogReadinessRow = { + active_releases: number + valid_pointers: number + rollback_releases: number + purgeable_releases: number +} + +function statement( + database: D1Database, + sql: string, + ...values: unknown[] +): D1PreparedStatement { + return database.prepare(sql).bind(...values) +} + +function ensureBatch(results: D1Result[], label: string): void { + const failure = results.find((result) => !result.success) + if (failure) throw new Error(`${label}: ${failure.error ?? 'D1 batch failed'}`) +} + +async function all( + database: D1Database, + sql: string, + ...values: unknown[] +): Promise { + const result = await statement(database, sql, ...values).all() + if (!result.success) throw new Error(result.error ?? 'D1 query failed') + return result.results ?? [] +} + +async function first( + database: D1Database, + sql: string, + ...values: unknown[] +): Promise { + return statement(database, sql, ...values).first() +} + +export async function readCatalogReadiness( + database: D1Database, +): Promise { + const row = await first(database, ` + SELECT + (SELECT count(*) FROM catalog_releases WHERE release_status = 'active') + AS active_releases, + (SELECT count(*) + FROM release_pointer pointer + JOIN catalog_releases release + ON release.release_id = pointer.current_release_id + AND release.release_status = 'active' + WHERE pointer.singleton_id = 1) + AS valid_pointers, + (SELECT count(*) FROM catalog_releases WHERE release_status = 'retired') + AS rollback_releases, + (SELECT count(*) + FROM catalog_releases candidate + WHERE candidate.release_status = 'retired' + AND ( + SELECT count(*) + FROM catalog_releases newer + WHERE newer.release_status = 'retired' + AND ( + newer.activated_at > candidate.activated_at + OR ( + newer.activated_at = candidate.activated_at + AND newer.data_version > candidate.data_version + ) + ) + ) >= ?1) + AS purgeable_releases + `, CATALOG_ROLLBACK_LIMIT) + + return { + activeRelease: Number(row?.active_releases ?? 0) === 1, + pointerConsistency: Number(row?.valid_pointers ?? 0) === 1, + retention: Number(row?.rollback_releases ?? 0) <= CATALOG_ROLLBACK_LIMIT + && Number(row?.purgeable_releases ?? 0) === 0, + } +} + +export async function enforceCatalogReleaseRetention( + database: D1Database, + now = new Date(), +): Promise<{ purged: number }> { + const candidates = await all(database, ` + SELECT release_id + FROM catalog_releases + WHERE release_status = 'retired' + ORDER BY activated_at DESC, data_version DESC, release_id DESC + LIMIT -1 OFFSET ?1 + `, CATALOG_ROLLBACK_LIMIT) + + for (const candidate of candidates) { + const releaseId = candidate.release_id + const purgedAt = now.toISOString() + const statements = [ + statement(database, ` + INSERT INTO release_retention_audit ( + release_id, data_version, content_sha256, counts_json, + normalized_artifact_key, compatibility_artifact_key, + activated_at, purged_at, actor, reason + ) + SELECT + release.release_id, + release.data_version, + release.content_sha256, + release.counts_json, + 'releases/' || release.release_id || '/catalog-release.v1.json', + compatibility.artifact_key, + release.activated_at, + ?2, + 'release-builder-worker', + 'catalog_release_retention' + FROM catalog_releases release + JOIN release_compatibility_artifacts compatibility + ON compatibility.release_id = release.release_id + WHERE release.release_id = ?1 + AND release.release_status = 'retired' + `, releaseId, purgedAt), + statement( + database, + 'UPDATE release_activation_requests SET previous_release_id = NULL WHERE previous_release_id = ?1', + releaseId, + ), + statement(database, 'DELETE FROM release_audit_log WHERE release_id = ?1', releaseId), + statement(database, 'DELETE FROM release_activation_requests WHERE release_id = ?1', releaseId), + ...[...RELEASE_TABLES].reverse().map((table) => statement( + database, + `DELETE FROM "${table}" WHERE release_id = ?1`, + releaseId, + )), + statement( + database, + 'DELETE FROM release_compatibility_artifacts WHERE release_id = ?1', + releaseId, + ), + statement(database, ` + DELETE FROM catalog_releases + WHERE release_id = ?1 + AND release_status = 'retired' + AND NOT EXISTS ( + SELECT 1 FROM release_pointer + WHERE singleton_id = 1 AND current_release_id = ?1 + ) + `, releaseId), + ] + ensureBatch(await database.batch(statements), `purge Catalog release ${releaseId}`) + + const verification = await first<{ release_exists: number; audit_exists: number }>( + database, + `SELECT + EXISTS(SELECT 1 FROM catalog_releases WHERE release_id = ?1) AS release_exists, + EXISTS(SELECT 1 FROM release_retention_audit WHERE release_id = ?1) AS audit_exists`, + releaseId, + ) + if (Number(verification?.release_exists ?? 1) !== 0 + || Number(verification?.audit_exists ?? 0) !== 1) { + throw new Error(`Catalog release retention verification failed for ${releaseId}`) + } + } + + return { purged: candidates.length } +} diff --git a/workers/release-builder/tests/retention.test.ts b/workers/release-builder/tests/retention.test.ts new file mode 100644 index 0000000..49e58bc --- /dev/null +++ b/workers/release-builder/tests/retention.test.ts @@ -0,0 +1,199 @@ +import assert from 'node:assert/strict' +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' +import test from 'node:test' +import { SqliteD1Database } from '../../../scripts/catalog/build-pipeline-release' +import { handleFetch } from '../src/index' +import { + enforceCatalogReleaseRetention, + readCatalogReadiness, +} from '../src/retention' +import type { ReleaseBuilderEnv } from '../src/types' + +const ZERO_COUNTS = JSON.stringify({ + sources: 0, + cities: 0, + universities: 0, + programs: 0, + admissionCycles: 0, + scholarships: 0, +}) + +function applyCatalogMigrations(database: DatabaseSync): void { + const directory = join(process.cwd(), 'infra', 'd1', 'catalog', 'migrations') + for (const file of readdirSync(directory).filter((name) => name.endsWith('.sql')).sort()) { + database.exec(readFileSync(join(directory, file), 'utf8')) + } +} + +function activateRelease(database: DatabaseSync, version: number): void { + const releaseId = `release-${version}` + const timestamp = `2026-07-${String(version).padStart(2, '0')}T00:00:00.000Z` + const hash = String(version).repeat(64) + database.prepare(` + INSERT INTO catalog_releases ( + release_id, data_version, schema_version, release_status, + data_date, generated_at, source_pipeline_run_id, content_sha256, + counts_json, created_at, validated_at + ) VALUES (?, ?, 1, 'ready', date(?), ?, ?, ?, ?, ?, ?) + `).run( + releaseId, + version, + timestamp, + timestamp, + `pipeline-${version}`, + hash, + ZERO_COUNTS, + timestamp, + timestamp, + ) + database.prepare(` + INSERT INTO release_compatibility_artifacts ( + release_id, artifact_format, artifact_key, content_sha256, + byte_length, created_at + ) VALUES (?, 'studyinchina.frontend.bundle.v1', ?, ?, 2, ?) + `).run( + releaseId, + `releases/${releaseId}/compat-envelope.json`, + hash, + timestamp, + ) + database.prepare(` + INSERT INTO release_activation_requests ( + request_id, release_id, expected_content_sha256, + expected_counts_json, actor, requested_at + ) VALUES (?, ?, ?, ?, 'retention-test', ?) + `).run(`activate-${releaseId}`, releaseId, hash, ZERO_COUNTS, timestamp) +} + +test('Catalog retention keeps current plus two rollback releases with immutable R2 tombstones', async () => { + const sqlite = new DatabaseSync(':memory:') + sqlite.exec('PRAGMA foreign_keys = ON') + applyCatalogMigrations(sqlite) + for (let version = 1; version <= 5; version += 1) activateRelease(sqlite, version) + const database = new SqliteD1Database(sqlite) + + assert.deepEqual(await readCatalogReadiness(database), { + activeRelease: true, + pointerConsistency: true, + retention: false, + }) + + const protectedRelease = sqlite.prepare(` + SELECT release_id, data_version, content_sha256, counts_json, activated_at + FROM catalog_releases WHERE release_id = 'release-3' + `).get() as Record + assert.throws(() => sqlite.prepare(` + INSERT INTO release_retention_audit ( + release_id, data_version, content_sha256, counts_json, + normalized_artifact_key, compatibility_artifact_key, + activated_at, purged_at, actor, reason + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'test', 'catalog_release_retention') + `).run( + protectedRelease.release_id, + protectedRelease.data_version, + protectedRelease.content_sha256, + protectedRelease.counts_json, + 'releases/release-3/catalog-release.v1.json', + 'releases/release-3/compat-envelope.json', + protectedRelease.activated_at, + '2026-08-05T00:00:00.000Z', + ), /two newer rollback releases must exist/u) + + assert.deepEqual( + await enforceCatalogReleaseRetention(database, new Date('2026-08-05T00:00:00.000Z')), + { purged: 2 }, + ) + assert.deepEqual( + sqlite.prepare(` + SELECT release_id, release_status + FROM catalog_releases + ORDER BY data_version + `).all().map((row) => ({ ...row })), + [ + { release_id: 'release-3', release_status: 'retired' }, + { release_id: 'release-4', release_status: 'retired' }, + { release_id: 'release-5', release_status: 'active' }, + ], + ) + assert.deepEqual( + sqlite.prepare(` + SELECT release_id, normalized_artifact_key, compatibility_artifact_key + FROM release_retention_audit ORDER BY data_version + `).all().map((row) => ({ ...row })), + [ + { + release_id: 'release-1', + normalized_artifact_key: 'releases/release-1/catalog-release.v1.json', + compatibility_artifact_key: 'releases/release-1/compat-envelope.json', + }, + { + release_id: 'release-2', + normalized_artifact_key: 'releases/release-2/catalog-release.v1.json', + compatibility_artifact_key: 'releases/release-2/compat-envelope.json', + }, + ], + ) + assert.deepEqual(await readCatalogReadiness(database), { + activeRelease: true, + pointerConsistency: true, + retention: true, + }) + assert.deepEqual(sqlite.prepare('PRAGMA foreign_key_check').all(), []) + sqlite.close() +}) + +function readinessEnvironment(row: Record | null): ReleaseBuilderEnv { + const statement = { + bind() { return this }, + first: async () => row, + all: async () => ({ success: true, results: [] }), + run: async () => ({ success: true, meta: { changes: 0 } }), + } + return { + CATALOG_DB: { + prepare: () => statement, + batch: async () => [], + }, + } as unknown as ReleaseBuilderEnv +} + +test('readiness endpoint exposes aggregate checks only and fails closed', async () => { + const ready = await handleFetch( + new Request('https://worker.example/ready'), + readinessEnvironment({ + active_releases: 1, + valid_pointers: 1, + rollback_releases: 2, + purgeable_releases: 0, + }), + ) + assert.equal(ready.status, 200) + assert.deepEqual(await ready.json(), { + ok: true, + service: 'studyinchina-release-builder', + version: '1.0.0', + checks: { + database: true, + activeRelease: true, + pointerConsistency: true, + retention: true, + }, + }) + + const notReady = await handleFetch( + new Request('https://worker.example/ready'), + readinessEnvironment({ + active_releases: 1, + valid_pointers: 1, + rollback_releases: 3, + purgeable_releases: 1, + }), + ) + assert.equal(notReady.status, 503) + const body = await notReady.json() as Record + assert.equal(body.ok, false) + assert.equal(JSON.stringify(body).includes('release-1'), false) + assert.equal(notReady.headers.get('cache-control'), 'no-store') +}) From feeca4a068ada26951464a43d39b6f37a681c1d5 Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Thu, 6 Aug 2026 00:58:51 +0800 Subject: [PATCH 2/9] Stabilize catalog SQL integration test --- tests/unit/catalog-sql-api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index e0c3842..936a8d2 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -385,7 +385,7 @@ describe('Catalog D1 normalized v1 API', () => { } finally { database.exec('ROLLBACK') } - }) + }, 15_000) it('rejects oversized limits and cursors bound to another resource', async () => { const oversized = await worker.fetch( From 05962f3ca2ea9d4f6d604803bf29b7d0eccdabca Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Thu, 6 Aug 2026 17:45:48 +0800 Subject: [PATCH 3/9] Scale catalog API and manifest onboarding --- .gitattributes | 1 + .github/workflows/data-health.yml | 17 + .../source-manifest-cohort-candidates.yml | 62 ++ docs/source-manifest-cohort-candidates.md | 70 ++ package.json | 2 + quality/regression/registry.json | 14 +- .../ingestion/build-source-manifest-cohort.ts | 595 ++++++++++++++-- src/app/[locale]/cities/[slug]/page.tsx | 10 +- src/app/[locale]/programs/[slug]/page.tsx | 8 +- src/app/[locale]/scholarships/[slug]/page.tsx | 11 +- src/app/[locale]/universities/[slug]/page.tsx | 10 +- src/lib/data/detail-prebuild.ts | 109 +++ tests/unit/catalog-sql-api.test.ts | 67 ++ ...health-platform-scorecard-workflow.test.ts | 28 + tests/unit/detail-prebuild.test.ts | 201 ++++++ .../source-manifest-cohort-builder.test.ts | 165 ++++- .../source-manifest-cohort-workflow.test.ts | 30 + workers/catalog-api/src/index.ts | 30 + workers/catalog-api/src/sql-api.ts | 649 +++++++++++++++++- workers/catalog-api/src/sql-cursor.ts | 41 +- workers/catalog-api/src/sql-types.ts | 19 + 21 files changed, 2040 insertions(+), 99 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/source-manifest-cohort-candidates.yml create mode 100644 docs/source-manifest-cohort-candidates.md create mode 100644 src/lib/data/detail-prebuild.ts create mode 100644 tests/unit/data-health-platform-scorecard-workflow.test.ts create mode 100644 tests/unit/detail-prebuild.test.ts create mode 100644 tests/unit/source-manifest-cohort-workflow.test.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b6e9582 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +quality/regression/fixtures/** text eol=lf diff --git a/.github/workflows/data-health.yml b/.github/workflows/data-health.yml index 656a163..94407e1 100644 --- a/.github/workflows/data-health.yml +++ b/.github/workflows/data-health.yml @@ -41,6 +41,10 @@ jobs: uses: actions/setup-node@v6 with: node-version: 24 + cache: npm + + - name: Install dependencies + run: npm ci --ignore-scripts - name: Resolve audit mode id: mode @@ -86,6 +90,13 @@ jobs: fi node scripts/data-health.mjs "${args[@]}" + - name: Build platform quality scorecard + id: platform_scorecard + continue-on-error: true + run: >- + npm run quality:platform-scorecard -- + --output "$RUNNER_TEMP/platform-data-quality.json" + - name: Add report to workflow summary if: always() shell: bash @@ -106,6 +117,7 @@ jobs: ${{ runner.temp }}/link-health.md ${{ runner.temp }}/data-health.json ${{ runner.temp }}/data-health.md + ${{ runner.temp }}/platform-data-quality.json if-no-files-found: warn retention-days: 30 @@ -181,6 +193,7 @@ jobs: env: DATA_HEALTH_OUTCOME: ${{ steps.data_health.outcome }} LINK_CHECK_OUTCOME: ${{ steps.link_check.outcome }} + PLATFORM_SCORECARD_OUTCOME: ${{ steps.platform_scorecard.outcome }} run: | failed=0 if [[ "$DATA_HEALTH_OUTCOME" == "failure" ]]; then @@ -191,4 +204,8 @@ jobs: echo "One or more links returned a confirmed 404 or 410." failed=1 fi + if [[ "$PLATFORM_SCORECARD_OUTCOME" == "failure" ]]; then + echo "The platform quality scorecard could not be generated." + failed=1 + fi exit "$failed" diff --git a/.github/workflows/source-manifest-cohort-candidates.yml b/.github/workflows/source-manifest-cohort-candidates.yml new file mode 100644 index 0000000..66ec571 --- /dev/null +++ b/.github/workflows/source-manifest-cohort-candidates.yml @@ -0,0 +1,62 @@ +name: Build Source Manifest Candidate Cohort + +on: + workflow_dispatch: + inputs: + checked_at: + description: Evidence check date in YYYY-MM-DD format + required: true + type: string + +permissions: + contents: read + +concurrency: + group: source-manifest-cohort-candidates + cancel-in-progress: false + +jobs: + build-candidate-artifact: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + ARTIFACT_DIRECTORY: ${{ runner.temp }}/source-manifest-cohort-candidates + CHECKED_AT: ${{ inputs.checked_at }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Validate locked official target registry + run: npm run validate:double-first-class + + - name: Build review-only candidate artifact + shell: bash + run: | + set -euo pipefail + npm run pipeline:build-source-manifest-candidates -- \ + --checked-at "$CHECKED_AT" \ + --artifact-output "$ARTIFACT_DIRECTORY" + + - name: Verify checksums and publication safety + shell: bash + run: | + set -euo pipefail + npm run pipeline:verify-source-manifest-candidates -- "$ARTIFACT_DIRECTORY" + git diff --exit-code -- content/source-manifests + + - name: Upload review-only candidate bundle + uses: actions/upload-artifact@v6 + with: + name: source-manifest-candidates-${{ github.run_id }} + path: ${{ env.ARTIFACT_DIRECTORY }} + if-no-files-found: error + retention-days: 14 diff --git a/docs/source-manifest-cohort-candidates.md b/docs/source-manifest-cohort-candidates.md new file mode 100644 index 0000000..2109eee --- /dev/null +++ b/docs/source-manifest-cohort-candidates.md @@ -0,0 +1,70 @@ +# Source Manifest cohort candidate workflow + +## Purpose + +This workflow turns the locked Ministry of Education Double First-Class target +registry and the current catalog JSON into a review queue for +`SourceManifestV2`. It does not discover new URLs, fetch websites, or publish +manifests. + +The separation is intentional: + +`official target registry + current catalog relationships -> candidate artifact -> evidence review -> formal manifest` + +The generator excludes the three military institutions, accepts only exact +catalog relationships to sources already marked official and using HTTPS, and +never fuzzy-matches a university or invents a missing source. + +## Local use + +Inspect current coverage without writing: + +```powershell +npm run pipeline:build-source-manifest-candidates -- --checked-at 2026-08-06 --dry-run +``` + +Create a new candidate artifact in an explicit empty directory: + +```powershell +npm run pipeline:build-source-manifest-candidates -- --checked-at 2026-08-06 --artifact-output C:\tmp\source-manifest-candidates +``` + +Verify a downloaded or locally generated artifact: + +```powershell +npm run pipeline:verify-source-manifest-candidates -- C:\tmp\source-manifest-candidates +``` + +The write command refuses a destination inside `content/source-manifests`, a +symbolic-link destination, and any non-empty destination. This prevents a stale +candidate from being mixed into a new run or mistaken for a production +manifest. + +## Artifact contract + +Every bundle contains: + +- `manifests/*.v2.candidate.json`: disabled, review-only candidates; +- `gap-report.v1.json`: per-institution missing mappings, rejected sources, + and uncovered source categories; +- `artifact-manifest.v1.json`: exact SHA-256 and byte length for all six + locked inputs and every generated JSON file; +- `SHA256SUMS`: independent checksums for the artifact manifest, gap report, + and every candidate. + +Verification rejects added, missing, changed, duplicated, or unsafe paths. It +also reparses every candidate through the V2 schema and confirms that every +source remains disabled, robots-blocked, and pending review. + +## GitHub workflow + +`Build Source Manifest Candidate Cohort` is a manual, read-only workflow. The +operator supplies an explicit evidence check date. It builds only under +`runner.temp`, validates the official target registry, and validates the +relevant current-catalog relationships while building each disabled candidate. +It then verifies the completed bundle, proves that +`content/source-manifests` did not change, and uploads the result as a +short-lived review artifact. + +Promotion remains a separate evidence-review action. Candidate artifacts must +never be copied wholesale into the formal manifest directory. diff --git a/package.json b/package.json index 1bd008d..c02c24d 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,8 @@ "catalog:build-pipeline": "tsx scripts/catalog/build-pipeline-release.ts", "pipeline:build-double-first-class-targets": "tsx scripts/ingestion/build-double-first-class-target-import.ts", "pipeline:build-source-cohorts": "tsx scripts/ingestion/build-official-source-cohort-import.ts", + "pipeline:build-source-manifest-candidates": "tsx scripts/ingestion/build-source-manifest-cohort.ts", + "pipeline:verify-source-manifest-candidates": "tsx scripts/ingestion/build-source-manifest-cohort.ts --verify-artifact", "pipeline:build-sources": "tsx scripts/ingestion/build-source-import.ts", "benchmark:catalog": "tsx scripts/catalog/benchmark-catalog.ts", "benchmark:catalog:smoke": "tsx scripts/catalog/benchmark-catalog.ts --institutions 25 --programs 1000 --cycles 3000 --iterations 50 --warmup 10 --output .benchmark/catalog-performance-smoke.json", diff --git a/quality/regression/registry.json b/quality/regression/registry.json index afba391..d5b742b 100644 --- a/quality/regression/registry.json +++ b/quality/regression/registry.json @@ -7,7 +7,7 @@ "fixtureId": "static-html-directory-v1", "caseKind": "static_html", "inputPath": "quality/regression/fixtures/static-html.html", - "sha256": "a6f1c606864507e38c1c6f0c482a940ab34eda89dfb6e6f2f6ab35e01f8dc285", + "sha256": "8d3f49e8236016da517f894beb336b2dcf4d52f46d2bd1e39b20da8cd5edf7ca", "officialGoldEligible": false, "expected": { "disposition": "process", "signals": ["static_html_detected"] } }, @@ -15,7 +15,7 @@ "fixtureId": "converted-pdf-v1", "caseKind": "pdf_converted", "inputPath": "quality/regression/fixtures/pdf-converted.txt", - "sha256": "d3f2e90eb321ef2429e91d3d5186e92a97a02b611c95929f43060d66d0e4d0a6", + "sha256": "0ec95bedf23d313a2b94537cd0c01fd32ded2cee141a1574cbad3b4a6b40ca5f", "officialGoldEligible": false, "expected": { "disposition": "process", "signals": ["pdf_converted_text_detected"] } }, @@ -23,7 +23,7 @@ "fixtureId": "browser-rendered-v1", "caseKind": "dynamic_rendered", "inputPath": "quality/regression/fixtures/dynamic-rendered.html", - "sha256": "216646818fa924cb954fb617811e525a302add1ea5d80d6d9471746f4170fa1e", + "sha256": "88dca97deb237ae61e8008f7bd50b8d5aecc9263f05a7d281a0c1ceeb6afc778", "officialGoldEligible": false, "expected": { "disposition": "process", "signals": ["browser_render_detected"] } }, @@ -31,7 +31,7 @@ "fixtureId": "scanned-ocr-low-confidence-v1", "caseKind": "scanned", "inputPath": "quality/regression/fixtures/scanned.ocr.txt", - "sha256": "809d1d04cf6422560d947334b00805a0788d316b112c6a48a478e4ea2583e282", + "sha256": "675bd5aa3e442d7e9066324222b1035273621931ed386133bd6934959c6dd144", "officialGoldEligible": false, "expected": { "disposition": "manual_review", "signals": ["ocr_low_confidence"] } }, @@ -39,7 +39,7 @@ "fixtureId": "official-source-conflict-v1", "caseKind": "conflict", "inputPath": "quality/regression/fixtures/conflict.json", - "sha256": "7442d72cf2db56ee2cee51057ee531dcf438f56d80c3dca3cb060bccf92862c9", + "sha256": "2e3870b94c27360402051985b6ccb3aaaf449dc2df9c3cbded50ead228b30eef", "officialGoldEligible": false, "expected": { "disposition": "quarantine", "signals": ["conflict_detected"] } }, @@ -47,7 +47,7 @@ "fixtureId": "official-http-404-v1", "caseKind": "http_404", "inputPath": "quality/regression/fixtures/http-404.json", - "sha256": "7e84fa2687dd02910f10effb60f3f7db5bb318c42297713c72cfb7ba30b32a24", + "sha256": "48b525c2f1fa66959abab3b4eefb3afc5167e7fc97c51e8fa3e531979a9d3f1f", "officialGoldEligible": false, "expected": { "disposition": "unavailable", "signals": ["http_404"] } }, @@ -55,7 +55,7 @@ "fixtureId": "prompt-injection-v1", "caseKind": "prompt_injection", "inputPath": "quality/regression/fixtures/prompt-injection.html", - "sha256": "6fbe6c7ed6c5e0e711336dff1816bf15e385ac73b75fee4bb754c00463ebb681", + "sha256": "633ed7bba1ef721da4af59c2482659c0d5e72f275601a872041a07560f6a9e4d", "officialGoldEligible": false, "expected": { "disposition": "quarantine", "signals": ["prompt_injection_detected"] } } diff --git a/scripts/ingestion/build-source-manifest-cohort.ts b/scripts/ingestion/build-source-manifest-cohort.ts index 563c132..2a61b70 100644 --- a/scripts/ingestion/build-source-manifest-cohort.ts +++ b/scripts/ingestion/build-source-manifest-cohort.ts @@ -1,5 +1,14 @@ -import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' -import { join, relative, resolve } from 'node:path' +import { createHash } from 'node:crypto' +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + readdirSync, + realpathSync, + writeFileSync, +} from 'node:fs' +import { dirname, isAbsolute, join, posix, relative, resolve, sep } from 'node:path' import { fileURLToPath } from 'node:url' import { SOURCE_CATEGORIES } from '../../workers/ingestion/src/manifest-schema' import type { @@ -15,6 +24,81 @@ import { validateDoubleFirstClassRegistry, type DoubleFirstClassRegistry, } from './double-first-class-registry' +export const CURRENT_SOURCE_MANIFEST_COHORT_INPUTS = { + registry: 'content/source-manifests/double-first-class/targets.v1.json', + universities: 'content/data/universities.json', + sources: 'content/data/sources.json', + programs: 'content/data/programs.json', + admissionCycles: 'content/data/admission-cycles.json', + scholarships: 'content/data/scholarships.json', +} as const + +export type SourceManifestCohortInputName = + keyof typeof CURRENT_SOURCE_MANIFEST_COHORT_INPUTS + +const SOURCE_MANIFEST_COHORT_INPUT_ORDER: SourceManifestCohortInputName[] = [ + 'registry', + 'universities', + 'sources', + 'programs', + 'admissionCycles', + 'scholarships', +] + +export type SourceManifestCohortInputFingerprint = { + name: SourceManifestCohortInputName + repositoryPath: string + sha256: string + byteLength: number +} + +export type SourceManifestCohortArtifactFile = { + path: string + mediaType: 'application/json' + sha256: string + byteLength: number +} + +export type SourceManifestCohortArtifactManifest = { + format: 'studyinchina.source-manifest-v2-candidate-bundle' + formatVersion: 1 + cohortId: string + checkedAt: string + disposition: 'candidate_only' + summary: SourceManifestCohortGapReport['summary'] + policy: { + sourceOfTruth: string + publication: string + network: string + } + inputs: SourceManifestCohortInputFingerprint[] + files: SourceManifestCohortArtifactFile[] +} + +export type SourceManifestCohortArtifactWrite = { + outputDirectory: string + manifestDirectory: string + gapReportPath: string + artifactManifestPath: string + checksumPath: string + verifiedFiles: number +} + +export type SourceManifestCohortArtifactVerification = { + outputDirectory: string + cohortId: string + checkedAt: string + candidateManifests: number + exactOfficialHttpsSources: number + verifiedFiles: number +} + +type SourceManifestCohortCli = + | { mode: 'dry-run'; checkedAt: string } + | { mode: 'write-artifact'; checkedAt: string; artifactOutput: string } + | { mode: 'verify-artifact'; artifactOutput: string } + + export const EXCLUDED_MILITARY_INSTITUTION_NAMES = new Set([ '国防科技大学', @@ -539,75 +623,486 @@ export function buildSourceManifestCohort( return { candidates, gapReport, summary } } +function sha256(value: string | Buffer): string { + return createHash('sha256').update(value).digest('hex') +} + +function serializeJson(value: unknown): string { + return JSON.stringify(value, null, 2) + '\n' +} + +function readFingerprintedJson( + repositoryRoot: string, + name: SourceManifestCohortInputName, +): { value: T; fingerprint: SourceManifestCohortInputFingerprint } { + const repositoryPath = CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name] + const bytes = readFileSync(resolve(repositoryRoot, repositoryPath)) + return { + value: JSON.parse(bytes.toString('utf8')) as T, + fingerprint: { + name, + repositoryPath, + sha256: sha256(bytes), + byteLength: bytes.byteLength, + }, + } +} + +export function buildCurrentSourceManifestCohort( + checkedAt: string, + repositoryRoot = resolve('.'), +): { + build: SourceManifestCohortBuild + inputFingerprints: SourceManifestCohortInputFingerprint[] +} { + const registry = readFingerprintedJson(repositoryRoot, 'registry') + const universities = readFingerprintedJson( + repositoryRoot, + 'universities', + ) + const sources = readFingerprintedJson(repositoryRoot, 'sources') + const programs = readFingerprintedJson(repositoryRoot, 'programs') + const admissionCycles = readFingerprintedJson( + repositoryRoot, + 'admissionCycles', + ) + const scholarships = readFingerprintedJson( + repositoryRoot, + 'scholarships', + ) + const build = buildSourceManifestCohort({ + registry: validateDoubleFirstClassRegistry(registry.value) as DoubleFirstClassRegistry, + universities: universities.value, + sources: sources.value, + programs: programs.value, + admissionCycles: admissionCycles.value, + scholarships: scholarships.value, + checkedAt, + }) + return { + build, + inputFingerprints: [ + registry.fingerprint, + universities.fingerprint, + sources.fingerprint, + programs.fingerprint, + admissionCycles.fingerprint, + scholarships.fingerprint, + ], + } +} + function isInside(parent: string, child: string): boolean { const path = relative(parent, child) - return path === '' || (!path.startsWith('..') && !path.startsWith(`..\\`) && !path.startsWith('../')) + return path === '' + || (path !== '..' && !path.startsWith('..' + sep) && !isAbsolute(path)) } -export function writeSourceManifestCohort( - build: SourceManifestCohortBuild, +function resolveThroughExistingAncestor(path: string): string { + let ancestor = resolve(path) + const suffix: string[] = [] + while (!existsSync(ancestor)) { + const parent = dirname(ancestor) + if (parent === ancestor) { + throw new Error('Unable to resolve artifact output ancestor for ' + path) + } + suffix.unshift(relative(parent, ancestor)) + ancestor = parent + } + return resolve(realpathSync(ancestor), ...suffix) +} + +function assertSafeEmptyArtifactOutput( + outputDirectory: string, + repositoryRoot: string, +): string { + const output = resolve(outputDirectory) + const formalManifestDirectory = resolve(repositoryRoot, 'content/source-manifests') + const effectiveOutput = resolveThroughExistingAncestor(output) + const effectiveFormalManifestDirectory = realpathSync(formalManifestDirectory) + if ( + isInside(formalManifestDirectory, output) + || isInside(effectiveFormalManifestDirectory, effectiveOutput) + ) { + throw new Error('Candidate artifact output must not be inside content/source-manifests') + } + if (!existsSync(output)) return output + const stats = lstatSync(output) + if (stats.isSymbolicLink() || !stats.isDirectory()) { + throw new Error('Candidate artifact output must be a real directory') + } + if (readdirSync(output).length > 0) { + throw new Error('Candidate artifact output must be empty to prevent stale-file leakage') + } + return output +} + +function sortedInputFingerprints( + inputFingerprints: SourceManifestCohortInputFingerprint[], +): SourceManifestCohortInputFingerprint[] { + const byName = new Map(inputFingerprints.map((input) => [input.name, input])) + if ( + inputFingerprints.length !== SOURCE_MANIFEST_COHORT_INPUT_ORDER.length + || byName.size !== SOURCE_MANIFEST_COHORT_INPUT_ORDER.length + ) { + throw new Error('Candidate artifact requires one fingerprint for every locked input') + } + return SOURCE_MANIFEST_COHORT_INPUT_ORDER.map((name) => { + const input = byName.get(name) + if ( + !input + || input.repositoryPath !== CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name] + || !/^[a-f0-9]{64}$/u.test(input.sha256) + || !Number.isSafeInteger(input.byteLength) + || input.byteLength <= 0 + ) { + throw new Error('Invalid locked input fingerprint: ' + name) + } + return input + }) +} + +function assertSafeArtifactRelativePath(path: string): void { + if ( + path.length === 0 + || path.includes('\\') + || isAbsolute(path) + || posix.normalize(path) !== path + || path === '..' + || path.startsWith('../') + ) { + throw new Error('Unsafe artifact-relative path: ' + path) + } +} + +function writeJsonArtifact( + outputDirectory: string, + path: string, + value: unknown, +): SourceManifestCohortArtifactFile { + assertSafeArtifactRelativePath(path) + const body = serializeJson(value) + const destination = join(outputDirectory, ...path.split('/')) + mkdirSync(dirname(destination), { recursive: true }) + writeFileSync(destination, body, 'utf8') + return { + path, + mediaType: 'application/json', + sha256: sha256(body), + byteLength: Buffer.byteLength(body), + } +} + +function walkArtifactFiles(directory: string, prefix = ''): string[] { + const files: string[] = [] + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.isSymbolicLink()) { + throw new Error('Artifact bundle contains a symbolic link: ' + entry.name) + } + const relativePath = prefix ? posix.join(prefix, entry.name) : entry.name + const absolutePath = join(directory, entry.name) + if (entry.isDirectory()) { + files.push(...walkArtifactFiles(absolutePath, relativePath)) + } else if (entry.isFile()) { + files.push(relativePath) + } else { + throw new Error('Artifact bundle contains an unsupported file: ' + relativePath) + } + } + return files.sort() +} + +function readArtifactManifest(outputDirectory: string): SourceManifestCohortArtifactManifest { + const raw = JSON.parse( + readFileSync(join(outputDirectory, 'artifact-manifest.v1.json'), 'utf8'), + ) as Partial + if ( + raw.format !== 'studyinchina.source-manifest-v2-candidate-bundle' + || raw.formatVersion !== 1 + || raw.disposition !== 'candidate_only' + || typeof raw.cohortId !== 'string' + || typeof raw.checkedAt !== 'string' + || !raw.summary + || !Array.isArray(raw.inputs) + || !Array.isArray(raw.files) + ) { + throw new Error('Invalid source-manifest candidate artifact manifest') + } + sortedInputFingerprints(raw.inputs) + return raw as SourceManifestCohortArtifactManifest +} + +export function verifySourceManifestCohortArtifact( outputDirectory: string, -): { outputDirectory: string; manifestDirectory: string; gapReportPath: string } { +): SourceManifestCohortArtifactVerification { const output = resolve(outputDirectory) - const formalManifestDirectory = resolve('content/source-manifests') - if (isInside(formalManifestDirectory, output)) { - throw new Error('Candidate output must not be inside content/source-manifests') + if (!existsSync(output) || lstatSync(output).isSymbolicLink()) { + throw new Error('Candidate artifact directory is missing or is a symbolic link') + } + const artifactManifest = readArtifactManifest(output) + const checksumBody = readFileSync(join(output, 'SHA256SUMS'), 'utf8') + const checksumEntries = checksumBody.split(/\r?\n/u).filter(Boolean).map((line) => { + const match = /^([a-f0-9]{64}) (.+)$/u.exec(line) + if (!match) throw new Error('Invalid SHA256SUMS line: ' + line) + const path = match[2]! + assertSafeArtifactRelativePath(path) + return { sha256: match[1]!, path } + }) + const checksumByPath = new Map( + checksumEntries.map((entry) => [entry.path, entry.sha256]), + ) + if (checksumByPath.size !== checksumEntries.length) { + throw new Error('SHA256SUMS contains duplicate paths') + } + + const describedPaths = artifactManifest.files.map((file) => file.path) + const expectedChecksummedPaths = [ + 'artifact-manifest.v1.json', + ...describedPaths, + ].sort() + if ( + JSON.stringify([...checksumByPath.keys()].sort()) + !== JSON.stringify(expectedChecksummedPaths) + ) { + throw new Error('SHA256SUMS paths do not match the artifact manifest') + } + const expectedDiskPaths = [...expectedChecksummedPaths, 'SHA256SUMS'].sort() + if (JSON.stringify(walkArtifactFiles(output)) !== JSON.stringify(expectedDiskPaths)) { + throw new Error('Candidate artifact contains missing or unexpected files') + } + + for (const entry of checksumEntries) { + const bytes = readFileSync(join(output, ...entry.path.split('/'))) + if (sha256(bytes) !== entry.sha256) { + throw new Error('Artifact checksum mismatch: ' + entry.path) + } } + for (const file of artifactManifest.files) { + assertSafeArtifactRelativePath(file.path) + const bytes = readFileSync(join(output, ...file.path.split('/'))) + if ( + file.mediaType !== 'application/json' + || file.sha256 !== sha256(bytes) + || file.byteLength !== bytes.byteLength + || checksumByPath.get(file.path) !== file.sha256 + ) { + throw new Error('Artifact manifest metadata mismatch: ' + file.path) + } + } + + const gapFile = artifactManifest.files.find((file) => file.path === 'gap-report.v1.json') + if (!gapFile) throw new Error('Candidate artifact is missing gap-report.v1.json') + const gapReport = JSON.parse( + readFileSync(join(output, gapFile.path), 'utf8'), + ) as Partial + if ( + gapReport.format !== 'studyinchina.source-manifest-v2-gap-report' + || gapReport.cohortId !== artifactManifest.cohortId + || gapReport.checkedAt !== artifactManifest.checkedAt + || JSON.stringify(gapReport.summary) !== JSON.stringify(artifactManifest.summary) + ) { + throw new Error('Gap report does not match the artifact manifest') + } + + const candidateFiles = artifactManifest.files.filter( + (file) => file.path.startsWith('manifests/'), + ) + if (candidateFiles.length !== artifactManifest.summary.candidateManifests) { + throw new Error('Candidate manifest count does not match the artifact summary') + } + const institutions = new Set() + let exactOfficialHttpsSources = 0 + for (const file of candidateFiles) { + if (!/^manifests\/\d{3}-[a-z0-9-]+\.v2\.candidate\.json$/u.test(file.path)) { + throw new Error('Unexpected candidate manifest filename: ' + file.path) + } + const candidate = sourceManifestV2Schema.parse(JSON.parse( + readFileSync(join(output, ...file.path.split('/')), 'utf8'), + )) + if ( + candidate.manifestStatus !== 'in_progress' + || candidate.catalogReconciliation.status !== 'in_progress' + || candidate.catalogReconciliation.entries.some((entry) => entry.status !== 'pending') + || candidate.sources.some( + (source) => source.enabled !== false || source.robots.mode !== 'blocked', + ) + ) { + throw new Error( + 'Candidate manifest is not safely disabled and pending: ' + file.path, + ) + } + if (institutions.has(candidate.institutionId)) { + throw new Error('Duplicate candidate institution: ' + candidate.institutionId) + } + institutions.add(candidate.institutionId) + exactOfficialHttpsSources += candidate.sources.length + } + if (exactOfficialHttpsSources !== artifactManifest.summary.exactOfficialHttpsSources) { + throw new Error('Official HTTPS source count does not match the artifact summary') + } + + return { + outputDirectory: output, + cohortId: artifactManifest.cohortId, + checkedAt: artifactManifest.checkedAt, + candidateManifests: candidateFiles.length, + exactOfficialHttpsSources, + verifiedFiles: checksumEntries.length, + } +} + +export function writeSourceManifestCohort( + build: SourceManifestCohortBuild, + outputDirectory: string, + inputFingerprints: SourceManifestCohortInputFingerprint[], + repositoryRoot = resolve('.'), +): SourceManifestCohortArtifactWrite { + const output = assertSafeEmptyArtifactOutput(outputDirectory, repositoryRoot) const manifestDirectory = join(output, 'manifests') - mkdirSync(manifestDirectory, { recursive: true }) - for (const candidate of build.candidates) { - writeFileSync( - join(manifestDirectory, candidate.fileName), - `${JSON.stringify(candidate.manifest, null, 2)}\n`, - 'utf8', - ) + const files: SourceManifestCohortArtifactFile[] = [] + const candidateNames = new Set() + for (const candidate of [...build.candidates].sort( + (left, right) => left.fileName.localeCompare(right.fileName), + )) { + if ( + candidateNames.has(candidate.fileName) + || !/^\d{3}-[a-z0-9-]+\.v2\.candidate\.json$/u.test(candidate.fileName) + ) { + throw new Error('Unsafe or duplicate candidate filename: ' + candidate.fileName) + } + candidateNames.add(candidate.fileName) + files.push(writeJsonArtifact( + output, + posix.join('manifests', candidate.fileName), + candidate.manifest, + )) + } + files.push(writeJsonArtifact(output, 'gap-report.v1.json', build.gapReport)) + files.sort((left, right) => left.path.localeCompare(right.path)) + + const artifactManifest: SourceManifestCohortArtifactManifest = { + format: 'studyinchina.source-manifest-v2-candidate-bundle', + formatVersion: 1, + cohortId: build.gapReport.cohortId, + checkedAt: build.gapReport.checkedAt, + disposition: 'candidate_only', + summary: build.summary, + policy: { + sourceOfTruth: + 'Read-only current catalog JSON and the locked Ministry of Education target registry.', + publication: + 'This bundle is review-only and must never be copied directly into content/source-manifests.', + network: + 'Generation performs no network requests and never invents or fuzzy-matches sources.', + }, + inputs: sortedInputFingerprints(inputFingerprints), + files, + } + const artifactManifestFile = writeJsonArtifact( + output, + 'artifact-manifest.v1.json', + artifactManifest, + ) + const checksummedFiles = [...files, artifactManifestFile].sort( + (left, right) => left.path.localeCompare(right.path), + ) + const checksumBody = checksummedFiles.map( + (file) => file.sha256 + ' ' + file.path, + ).join('\n') + const checksumPath = join(output, 'SHA256SUMS') + writeFileSync(checksumPath, checksumBody + '\n', 'utf8') + const verification = verifySourceManifestCohortArtifact(output) + return { + outputDirectory: output, + manifestDirectory, + gapReportPath: join(output, 'gap-report.v1.json'), + artifactManifestPath: join(output, 'artifact-manifest.v1.json'), + checksumPath, + verifiedFiles: verification.verifiedFiles, } - const gapReportPath = join(output, 'gap-report.v1.json') - writeFileSync(gapReportPath, `${JSON.stringify(build.gapReport, null, 2)}\n`, 'utf8') - return { outputDirectory: output, manifestDirectory, gapReportPath } } export function dryRunSummary(build: SourceManifestCohortBuild): string { return JSON.stringify({ mode: 'dry-run', ...build.summary }) } -function option(name: string): string | undefined { - const index = process.argv.indexOf(name) - return index >= 0 ? process.argv[index + 1] : undefined -} +const CLI_USAGE = [ + 'Usage:', + ' --checked-at --dry-run', + ' --checked-at --artifact-output ', + ' --verify-artifact ', +].join('\n') -function readJson(path: string): T { - return JSON.parse(readFileSync(resolve(path), 'utf8')) as T +export function parseSourceManifestCohortCli(argv: string[]): SourceManifestCohortCli { + let checkedAt: string | undefined + let artifactOutput: string | undefined + let verifyArtifact: string | undefined + let dryRun = false + const seen = new Set() + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]! + if (seen.has(argument)) { + throw new Error('Duplicate CLI option: ' + argument + '\n' + CLI_USAGE) + } + seen.add(argument) + if (argument === '--dry-run') { + dryRun = true + continue + } + if ( + argument !== '--checked-at' + && argument !== '--artifact-output' + && argument !== '--verify-artifact' + ) { + throw new Error('Unknown CLI option: ' + argument + '\n' + CLI_USAGE) + } + const value = argv[index + 1] + if (!value || value.startsWith('--')) { + throw new Error('Missing value for ' + argument + '\n' + CLI_USAGE) + } + index += 1 + if (argument === '--checked-at') checkedAt = value + if (argument === '--artifact-output') artifactOutput = value + if (argument === '--verify-artifact') verifyArtifact = value + } + if (verifyArtifact) { + if (checkedAt || artifactOutput || dryRun) { + throw new Error('--verify-artifact is an exclusive mode\n' + CLI_USAGE) + } + return { mode: 'verify-artifact', artifactOutput: verifyArtifact } + } + if (!checkedAt || dryRun === Boolean(artifactOutput)) { + throw new Error(CLI_USAGE) + } + return dryRun + ? { mode: 'dry-run', checkedAt } + : { mode: 'write-artifact', checkedAt, artifactOutput: artifactOutput! } } function runCli(): void { - const checkedAt = option('--checked-at') - if (!checkedAt) { - throw new Error('Usage requires --checked-at and either --dry-run or --output ') - } - const dryRun = process.argv.includes('--dry-run') - const output = option('--output') - if (!dryRun && !output) { - throw new Error('Refusing to write without an explicit --output directory; use --dry-run for a no-write summary') - } - const registry = validateDoubleFirstClassRegistry(readJson( - option('--registry') ?? 'content/source-manifests/double-first-class/targets.v1.json', - )) - const build = buildSourceManifestCohort({ - registry: registry as DoubleFirstClassRegistry, - universities: readJson(option('--universities') ?? 'content/data/universities.json'), - sources: readJson(option('--sources') ?? 'content/data/sources.json'), - programs: readJson(option('--programs') ?? 'content/data/programs.json'), - admissionCycles: readJson(option('--cycles') ?? 'content/data/admission-cycles.json'), - scholarships: readJson(option('--scholarships') ?? 'content/data/scholarships.json'), - checkedAt, - }) - if (dryRun) { - process.stdout.write(`${dryRunSummary(build)}\n`) + const cli = parseSourceManifestCohortCli(process.argv.slice(2)) + if (cli.mode === 'verify-artifact') { + process.stdout.write(JSON.stringify({ + mode: 'verify-artifact', + ...verifySourceManifestCohortArtifact(cli.artifactOutput), + }) + '\n') + return + } + const current = buildCurrentSourceManifestCohort(cli.checkedAt) + if (cli.mode === 'dry-run') { + process.stdout.write(dryRunSummary(current.build) + '\n') return } - const written = writeSourceManifestCohort(build, output!) - process.stdout.write(`${JSON.stringify({ mode: 'write', ...build.summary, ...written })}\n`) + const written = writeSourceManifestCohort( + current.build, + cli.artifactOutput, + current.inputFingerprints, + ) + process.stdout.write(JSON.stringify({ + mode: 'write-artifact', + ...current.build.summary, + ...written, + }) + '\n') } if (resolve(process.argv[1] ?? '') === fileURLToPath(import.meta.url)) { diff --git a/src/app/[locale]/cities/[slug]/page.tsx b/src/app/[locale]/cities/[slug]/page.tsx index aa39d81..59210be 100644 --- a/src/app/[locale]/cities/[slug]/page.tsx +++ b/src/app/[locale]/cities/[slug]/page.tsx @@ -1,13 +1,19 @@ import { notFound } from 'next/navigation' import { Badge, Card, PageHero, SectionHeading } from '@/components/ui' import { UniversityCard } from '@/components/features/RecordCards' -import { launchLocales } from '@/i18n/config' +import { indexedLocales } from '@/i18n/config' import { getMessages } from '@/i18n/messages' +import { selectCityPrebuildSlugs } from '@/lib/data/detail-prebuild' import { formatDate, localize } from '@/lib/data/format' import { regionLabels } from '@/lib/data/labels' import { getCityBySlug, getData } from '@/lib/data/load' import { pageMetadata, requireLocale } from '@/lib/site' -export function generateStaticParams() { const data = getData(); return launchLocales.flatMap((locale) => data.cities.map(({ slug }) => ({ locale, slug }))) } +export const dynamicParams = true + +export function generateStaticParams() { + const slugs = selectCityPrebuildSlugs(getData()) + return indexedLocales.flatMap((locale) => slugs.map((slug) => ({ locale, slug }))) +} export async function generateMetadata({ params }: { params: Promise<{ locale: string; slug: string }> }) { const { locale: raw, slug } = await params; const locale = requireLocale(raw) || 'en'; const city = getCityBySlug(slug); if (!city) return {}; return pageMetadata(locale, localize(city.name, locale), localize(city.overview, locale), `cities/${slug}`) } export default async function CityDetail({ params }: { params: Promise<{ locale: string; slug: string }> }) { const { locale: raw, slug } = await params; const locale = requireLocale(raw); if (!locale) notFound(); const city = getCityBySlug(slug); if (!city) notFound(); const messages = getMessages(locale); const data = getData(); const universities = data.universities.filter((item) => item.cityId === city.id); const fields = (id: string) => [...new Set(data.programs.filter((item) => item.universityId === id).map((item) => item.discipline))]; return <>{universities.length} {messages.nav.universities}{messages.common.lastVerified}: {formatDate(city.verifiedAt, locale, '—')}} />
{messages.cities.climate}

{messages.cities.climate}

{localize(city.climate, locale)}

{messages.cities.food}

{messages.cities.food}

    {city.foodHighlights.map((item, index) =>
  • {localize(item, locale)}
    • )}

{messages.cities.sights}

    {city.sights.map((item, index) =>
  • {localize(item, locale)}
    • )}
{universities.map((university) => )}
} diff --git a/src/app/[locale]/programs/[slug]/page.tsx b/src/app/[locale]/programs/[slug]/page.tsx index 79e5671..9c6cd67 100644 --- a/src/app/[locale]/programs/[slug]/page.tsx +++ b/src/app/[locale]/programs/[slug]/page.tsx @@ -4,9 +4,10 @@ import { ProgramCard } from '@/components/features/RecordCards' import { ScholarshipCard } from '@/components/features/ScholarshipCard' import { SourceTransparency } from '@/components/features/SourceTransparency' import { Badge, Card, PageHero, SectionHeading, VerificationBadge } from '@/components/ui' -import { launchLocales } from '@/i18n/config' +import { indexedLocales } from '@/i18n/config' import { getMessages } from '@/i18n/messages' import { getApplicationState, selectAdmissionCycle } from '@/lib/data/admission' +import { selectProgramPrebuildSlugs } from '@/lib/data/detail-prebuild' import { formatCny, formatDate, localize } from '@/lib/data/format' import { getTodayDate } from '@/lib/data/freshness' import { degreeLabels, disciplineLabels, languageLabel } from '@/lib/data/labels' @@ -14,9 +15,12 @@ import { getCatalogData, getCatalogProgramBySlug, getData } from '@/lib/data/loa import type { AdmissionCycle } from '@/lib/data/types' import { pageMetadata, requireLocale } from '@/lib/site' +export const dynamicParams = true + export function generateStaticParams() { const data = getData() - return launchLocales.flatMap((locale) => data.programs.map(({ slug }) => ({ locale, slug }))) + const slugs = selectProgramPrebuildSlugs(data, getTodayDate()) + return indexedLocales.flatMap((locale) => slugs.map((slug) => ({ locale, slug }))) } export async function generateMetadata({ params }: { params: Promise<{ locale: string; slug: string }> }) { diff --git a/src/app/[locale]/scholarships/[slug]/page.tsx b/src/app/[locale]/scholarships/[slug]/page.tsx index 6dd01de..454e55f 100644 --- a/src/app/[locale]/scholarships/[slug]/page.tsx +++ b/src/app/[locale]/scholarships/[slug]/page.tsx @@ -1,18 +1,21 @@ import { notFound } from 'next/navigation' import { SourceTransparency } from '@/components/features/SourceTransparency' import { Badge, Card, PageHero, VerificationBadge } from '@/components/ui' -import { launchLocales } from '@/i18n/config' +import { indexedLocales } from '@/i18n/config' import { getMessages } from '@/i18n/messages' +import { selectScholarshipPrebuildSlugs } from '@/lib/data/detail-prebuild' import { formatCny, formatDate, localize } from '@/lib/data/format' +import { getTodayDate } from '@/lib/data/freshness' import { getCatalogData, getCatalogScholarshipBySlug, getData } from '@/lib/data/load' import { coverageLabel, providerLabel } from '@/lib/data/scholarship' import { pageMetadata, requireLocale } from '@/lib/site' +export const dynamicParams = true + export function generateStaticParams() { const data = getData() - return launchLocales.flatMap((locale) => ( - data.scholarships.map(({ slug }) => ({ locale, slug })) - )) + const slugs = selectScholarshipPrebuildSlugs(data, getTodayDate()) + return indexedLocales.flatMap((locale) => slugs.map((slug) => ({ locale, slug }))) } export async function generateMetadata({ diff --git a/src/app/[locale]/universities/[slug]/page.tsx b/src/app/[locale]/universities/[slug]/page.tsx index ef8edd9..c068285 100644 --- a/src/app/[locale]/universities/[slug]/page.tsx +++ b/src/app/[locale]/universities/[slug]/page.tsx @@ -1,16 +1,22 @@ import { notFound } from 'next/navigation' import { Badge, Card, LinkButton, PageHero, SectionHeading, VerificationBadge } from '@/components/ui' import { ProgramCard } from '@/components/features/RecordCards' -import { launchLocales } from '@/i18n/config' +import { indexedLocales } from '@/i18n/config' import { getMessages } from '@/i18n/messages' import { selectAdmissionCycle } from '@/lib/data/admission' +import { selectUniversityPrebuildSlugs } from '@/lib/data/detail-prebuild' import { formatDate, localize } from '@/lib/data/format' import { getTodayDate } from '@/lib/data/freshness' import { disciplineLabels, regionLabels } from '@/lib/data/labels' import { getData, getUniversityBySlug } from '@/lib/data/load' import { pageMetadata, requireLocale } from '@/lib/site' -export function generateStaticParams() { const data = getData(); return launchLocales.flatMap((locale) => data.universities.map(({ slug }) => ({ locale, slug }))) } +export const dynamicParams = true + +export function generateStaticParams() { + const slugs = selectUniversityPrebuildSlugs(getData()) + return indexedLocales.flatMap((locale) => slugs.map((slug) => ({ locale, slug }))) +} export async function generateMetadata({ params }: { params: Promise<{ locale: string; slug: string }> }) { const { locale: raw, slug } = await params; const locale = requireLocale(raw) || 'en'; const university = getUniversityBySlug(slug) if (!university) return {}; return pageMetadata(locale, localize(university.name, locale), localize(university.summary, locale), `universities/${slug}`) diff --git a/src/lib/data/detail-prebuild.ts b/src/lib/data/detail-prebuild.ts new file mode 100644 index 0000000..2194a19 --- /dev/null +++ b/src/lib/data/detail-prebuild.ts @@ -0,0 +1,109 @@ +import { getApplicationState, selectAdmissionCycle } from './admission' +import type { DataBundle } from './types' + +export const DETAIL_PREBUILD_LIMITS = { + universities: 120, + programs: 120, + scholarships: 60, + cities: 30, +} as const + +type Slugged = { slug: string } + +function rankedSlugs( + items: T[], + score: (item: T) => number, + limit: number, +): string[] { + return [...items] + .sort((left, right) => score(right) - score(left) || left.slug.localeCompare(right.slug)) + .slice(0, Math.max(0, limit)) + .map((item) => item.slug) +} + +export function selectProgramPrebuildSlugs( + data: DataBundle, + today: string, + limit: number = DETAIL_PREBUILD_LIMITS.programs, +): string[] { + return rankedSlugs(data.programs, (program) => { + const cycle = selectAdmissionCycle(data.admissionCycles, program.id, today) + const applicationState = getApplicationState(cycle, today) + const cycleScore = { + open: 10, + rolling: 9, + upcoming: 8, + 'dates-published': 6, + 'not-announced': 2, + closed: 1, + 'previous-cycle': 0, + }[applicationState] + + return cycleScore + + (program.status === 'verified' ? 8 : 0) + + (program.details ? 6 : 0) + + (program.durationMonths !== null ? 3 : 0) + + (program.applyUrl ? 3 : 0) + + (program.teachingLanguages.length > 0 ? 2 : 0) + + (program.languageRequirements.length > 0 ? 2 : 0) + + (cycle?.tuitionCny != null ? 1 : 0) + }, limit) +} + +export function selectScholarshipPrebuildSlugs( + data: DataBundle, + today: string, + limit: number = DETAIL_PREBUILD_LIMITS.scholarships, +): string[] { + return rankedSlugs(data.scholarships, (scholarship) => ( + (scholarship.status === 'verified' ? 8 : 0) + + (scholarship.deadline && scholarship.deadline >= today ? 8 : 0) + + (scholarship.applicationUrl ? 4 : 0) + + (scholarship.summary ? 3 : 0) + + (scholarship.universityIds.length > 0 ? 2 : 0) + + (scholarship.coverage.tuition !== 'unknown' ? 1 : 0) + + (scholarship.coverage.accommodation !== 'unknown' ? 1 : 0) + ), limit) +} + +export function selectUniversityPrebuildSlugs( + data: DataBundle, + limit: number = DETAIL_PREBUILD_LIMITS.universities, +): string[] { + const programCounts = new Map() + const scholarshipCounts = new Map() + for (const program of data.programs) { + programCounts.set(program.universityId, (programCounts.get(program.universityId) ?? 0) + 1) + } + for (const scholarship of data.scholarships) { + for (const universityId of scholarship.universityIds) { + scholarshipCounts.set(universityId, (scholarshipCounts.get(universityId) ?? 0) + 1) + } + } + + return rankedSlugs(data.universities, (university) => ( + (university.status === 'verified' ? 8 : 0) + + (university.featured ? 8 : 0) + + Math.min(programCounts.get(university.id) ?? 0, 5) + + Math.min(scholarshipCounts.get(university.id) ?? 0, 3) + + (university.admissionsUrl ? 3 : 0) + + (university.summary ? 2 : 0) + ), limit) +} + +export function selectCityPrebuildSlugs( + data: DataBundle, + limit: number = DETAIL_PREBUILD_LIMITS.cities, +): string[] { + const universityCounts = new Map() + for (const university of data.universities) { + universityCounts.set(university.cityId, (universityCounts.get(university.cityId) ?? 0) + 1) + } + + return rankedSlugs(data.cities, (city) => ( + (city.status === 'verified' ? 6 : 0) + + (city.coordinates ? 5 : 0) + + Math.min(universityCounts.get(city.id) ?? 0, 8) + + (city.overview ? 2 : 0) + ), limit) +} diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index 936a8d2..0ef4d37 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -5,6 +5,7 @@ import { join } from 'node:path' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { buildLegacyRelease, readLegacyBundle } from '../../scripts/catalog/build-release' import worker from '../../workers/catalog-api/src/index' +import { chinaCalendarDate } from '../../workers/catalog-api/src/sql-data' import type { CatalogApiEnv, D1PreparedStatement, @@ -130,6 +131,10 @@ describe('Catalog D1 normalized v1 API', () => { expect(firstResponse.status).toBe(200) expect(first.meta.apiVersion).toBe('v1') expect(first.data).toHaveLength(1) + expect(first.meta.total).toBeGreaterThan(first.data.length) + expect(first.meta.facets?.universities?.length).toBeGreaterThan(1) + expect(first.meta.facets?.cities?.length).toBeGreaterThan(1) + expect(first.data[0]).toHaveProperty('currentCycle') expect(first.meta.nextCursor).toEqual(expect.any(String)) expect(first.data[0]).toMatchObject({ type: 'program', @@ -191,6 +196,68 @@ describe('Catalog D1 normalized v1 API', () => { expect(r2Reads).toBe(0) }) + it('filters and sorts scholarships with exact metadata and query-bound cursors', async () => { + const fundedResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/scholarships?funding=full-tuition&limit=1'), + environment, + ) + const funded = await fundedResponse.json() as ApiEnvelopeDto + expect(fundedResponse.status).toBe(200) + expect(funded.data).toHaveLength(1) + expect(funded.data[0]!.attributes.coverage.tuition).toBe('full') + expect(funded.meta.total).toBeGreaterThan(funded.data.length) + expect(funded.meta.facets?.universities?.length).toBeGreaterThan(0) + + const mismatchedCursor = await worker.fetch( + new Request( + `https://catalog.test/api/v1/scholarships?funding=partial-tuition&limit=1&cursor=${encodeURIComponent(funded.meta.nextCursor!)}`, + ), + environment, + ) + expect(mismatchedCursor.status).toBe(400) + + const stipendResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/scholarships?funding=stipend&sort=stipend-desc&limit=20'), + environment, + ) + const stipend = await stipendResponse.json() as ApiEnvelopeDto + const amounts = stipend.data.map((item) => item.attributes.coverage.stipendCnyPerMonth!) + expect(stipendResponse.status).toBe(200) + expect(amounts.length).toBeGreaterThan(1) + expect(amounts.every((amount) => amount > 0)).toBe(true) + expect(amounts).toEqual([...amounts].sort((left, right) => right - left)) + expect(stipend.data.every((item) => item.slug && /^[a-z0-9][a-z0-9-]*$/u.test(item.slug))).toBe(true) + const sortedDetailResponse = await worker.fetch( + new Request(`https://catalog.test/api/v1/scholarships/${stipend.data[0]!.slug}`), + environment, + ) + const sortedDetail = await sortedDetailResponse.json() as ApiEnvelopeDto + expect(sortedDetailResponse.status).toBe(200) + expect(sortedDetail.data.id).toBe(stipend.data[0]!.id) + + const futureResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/scholarships?deadline=future&limit=100'), + environment, + ) + const future = await futureResponse.json() as ApiEnvelopeDto + expect(futureResponse.status).toBe(200) + expect(future.data.length).toBeGreaterThan(0) + expect(future.data.every((item) => + item.attributes.deadline !== null && item.attributes.deadline >= chinaCalendarDate(), + )).toBe(true) + + const beforeDegree = queries.length + const degreeResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/scholarships?degree=master&limit=100'), + environment, + ) + expect(degreeResponse.status).toBe(200) + expect(queries.slice(beforeDegree).some(({ sql }) => + sql.includes('matched_program.degree_level = ?') + && sql.includes('scholarship_cycle_degree_levels'), + )).toBe(true) + }, 30_000) + it('serves normalized institution, program-cycle, and scholarship projections from D1', async () => { const institutionResponse = await worker.fetch( new Request('https://catalog.test/api/v1/institutions?limit=1'), diff --git a/tests/unit/data-health-platform-scorecard-workflow.test.ts b/tests/unit/data-health-platform-scorecard-workflow.test.ts new file mode 100644 index 0000000..388b07b --- /dev/null +++ b/tests/unit/data-health-platform-scorecard-workflow.test.ts @@ -0,0 +1,28 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' + +const workflow = readFileSync('.github/workflows/data-health.yml', 'utf8') + +describe('data-health platform scorecard workflow', () => { + it('generates and retains an advisory four-week scorecard', () => { + expect(workflow).toContain('npm ci --ignore-scripts') + expect(workflow).toContain('npm run quality:platform-scorecard --') + expect(workflow).toContain('--output "$RUNNER_TEMP/platform-data-quality.json"') + expect(workflow).toContain('${{ runner.temp }}/platform-data-quality.json') + + const scorecardStep = workflow.slice( + workflow.indexOf('- name: Build platform quality scorecard'), + workflow.indexOf('- name: Add report to workflow summary'), + ) + expect(scorecardStep).not.toContain('--strict') + }) + + it('fails only when scorecard generation fails, not while targets are incomplete', () => { + expect(workflow).toContain( + 'PLATFORM_SCORECARD_OUTCOME: ${{ steps.platform_scorecard.outcome }}', + ) + expect(workflow).toContain( + 'if [[ "$PLATFORM_SCORECARD_OUTCOME" == "failure" ]]; then', + ) + }) +}) diff --git a/tests/unit/detail-prebuild.test.ts b/tests/unit/detail-prebuild.test.ts new file mode 100644 index 0000000..02274bc --- /dev/null +++ b/tests/unit/detail-prebuild.test.ts @@ -0,0 +1,201 @@ +import { readFileSync } from 'node:fs' +import { describe, expect, it } from 'vitest' +import { + selectCityPrebuildSlugs, + selectProgramPrebuildSlugs, + selectScholarshipPrebuildSlugs, + selectUniversityPrebuildSlugs, +} from '../../src/lib/data/detail-prebuild' +import type { + AdmissionCycle, + City, + DataBundle, + Program, + Scholarship, + University, +} from '../../src/lib/data/types' + +const audit = { + sourceIds: ['source-official'], + verifiedAt: '2026-08-01', + reviewAfter: '2026-09-01', + status: 'verified' as const, +} + +function program(id: string, overrides: Partial = {}): Program { + return { + ...audit, + id, + slug: id, + universityId: 'university-a', + name: { en: id }, + degreeLevel: 'master', + discipline: 'engineering', + teachingLanguages: ['English'], + durationMonths: 24, + programUrl: `https://example.edu/${id}`, + applyUrl: 'https://apply.example.edu', + languageRequirements: [{ test: 'IELTS', minimum: '6.5' }], + ...overrides, + } +} + +function cycle(programId: string, overrides: Partial = {}): AdmissionCycle { + return { + ...audit, + id: `cycle-${programId}`, + programId, + academicYear: '2026-2027', + intake: 'autumn', + opensOn: '2026-07-01', + closesOn: '2026-10-01', + dateStatus: 'published', + tuitionCny: 30_000, + applicationFeeCny: 800, + ...overrides, + } +} + +function university(id: string, overrides: Partial = {}): University { + return { + ...audit, + id, + slug: id, + name: { en: id }, + cityId: 'city-a', + region: 'east', + officialUrl: `https://${id}.edu.cn`, + admissionsUrl: `https://${id}.edu.cn/admissions`, + summary: { en: id }, + featured: false, + ...overrides, + } +} + +function scholarship(id: string, overrides: Partial = {}): Scholarship { + return { + ...audit, + id, + slug: id, + name: { en: id }, + providerType: 'university', + universityIds: ['university-a'], + programIds: [], + coverage: { + tuition: 'full', + accommodation: 'full', + insurance: true, + stipendCnyPerMonth: 3_000, + }, + deadline: '2026-10-01', + applicationUrl: 'https://apply.example.edu/scholarship', + summary: { en: id }, + ...overrides, + } +} + +function city(id: string, overrides: Partial = {}): City { + return { + ...audit, + id, + slug: id, + name: { en: id }, + province: { en: 'Province' }, + region: 'east', + coordinates: { lat: 30, lng: 120 }, + overview: { en: id }, + climate: null, + foodHighlights: [], + sights: [], + ...overrides, + } +} + +function bundle(): DataBundle { + return { + sources: [], + cities: [city('city-a'), city('city-z', { coordinates: null, overview: null })], + universities: [ + university('university-a', { featured: true }), + university('university-z', { + cityId: 'city-z', + admissionsUrl: null, + summary: null, + }), + ], + programs: [ + program('program-open'), + program('program-thin', { + universityId: 'university-z', + status: 'stale', + teachingLanguages: [], + durationMonths: null, + applyUrl: null, + languageRequirements: [], + }), + ], + admissionCycles: [ + cycle('program-open'), + cycle('program-thin', { + opensOn: null, + closesOn: null, + dateStatus: 'not-announced', + tuitionCny: null, + }), + ], + scholarships: [ + scholarship('scholarship-current'), + scholarship('scholarship-thin', { + status: 'stale', + universityIds: [], + coverage: { + tuition: 'unknown', + accommodation: 'unknown', + insurance: 'unknown', + stipendCnyPerMonth: null, + }, + deadline: null, + applicationUrl: null, + summary: null, + }), + ], + } +} + +describe('detail-page prebuild selection', () => { + it('prioritizes complete, actionable records and obeys deterministic limits', () => { + const data = bundle() + expect(selectProgramPrebuildSlugs(data, '2026-08-06', 1)).toEqual(['program-open']) + expect(selectScholarshipPrebuildSlugs(data, '2026-08-06', 1)).toEqual([ + 'scholarship-current', + ]) + expect(selectUniversityPrebuildSlugs(data, 1)).toEqual(['university-a']) + expect(selectCityPrebuildSlugs(data, 1)).toEqual(['city-a']) + expect(selectProgramPrebuildSlugs(data, '2026-08-06', 0)).toEqual([]) + }) + + it('uses slug ordering as the stable tie-breaker', () => { + const data = bundle() + data.programs = [program('program-b'), program('program-a')] + data.admissionCycles = [] + expect(selectProgramPrebuildSlugs(data, '2026-08-06', 2)).toEqual([ + 'program-a', + 'program-b', + ]) + }) + + it('keeps beta locales out of build-time detail expansion', () => { + const detailPages = [ + 'src/app/[locale]/programs/[slug]/page.tsx', + 'src/app/[locale]/scholarships/[slug]/page.tsx', + 'src/app/[locale]/universities/[slug]/page.tsx', + 'src/app/[locale]/cities/[slug]/page.tsx', + ] + for (const path of detailPages) { + const source = readFileSync(path, 'utf8') + expect(source).toContain('indexedLocales') + expect(source).toContain('export const dynamicParams = true') + expect(source).not.toContain('launchLocales') + } + }) +}) diff --git a/tests/unit/source-manifest-cohort-builder.test.ts b/tests/unit/source-manifest-cohort-builder.test.ts index 494b717..d285626 100644 --- a/tests/unit/source-manifest-cohort-builder.test.ts +++ b/tests/unit/source-manifest-cohort-builder.test.ts @@ -1,11 +1,30 @@ -import { describe, expect, it } from 'vitest' +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join, resolve } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' import { SOURCE_CATEGORIES } from '../../workers/ingestion/src/manifest-schema' import { + buildCurrentSourceManifestCohort, buildSourceManifestCohort, + CURRENT_SOURCE_MANIFEST_COHORT_INPUTS, dryRunSummary, + parseSourceManifestCohortCli, + verifySourceManifestCohortArtifact, + writeSourceManifestCohort, type BuildSourceManifestCohortInput, + type SourceManifestCohortArtifactManifest, + type SourceManifestCohortInputFingerprint, + type SourceManifestCohortInputName, } from '../../scripts/ingestion/build-source-manifest-cohort' +const temporaryDirectories: string[] = [] + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + function fixture(): BuildSourceManifestCohortInput { return { checkedAt: '2026-08-06', @@ -116,6 +135,27 @@ function fixture(): BuildSourceManifestCohortInput { } } +function temporaryDirectory(): string { + const directory = mkdtempSync(join(tmpdir(), 'studyinchina-manifest-cohort-')) + temporaryDirectories.push(directory) + return directory +} + +function inputFingerprints(): SourceManifestCohortInputFingerprint[] { + return ( + Object.keys(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS) as SourceManifestCohortInputName[] + ).map((name, index) => ({ + name, + repositoryPath: CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name], + sha256: (index + 1).toString(16).padStart(64, '0'), + byteLength: index + 1, + })) +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, 'utf8')) as T +} + describe('SourceManifestV2 cohort candidate builder', () => { it('maps only exact official HTTPS relationships and leaves every candidate pending', () => { const build = buildSourceManifestCohort(fixture()) @@ -213,4 +253,127 @@ describe('SourceManifestV2 cohort candidate builder', () => { expect(() => buildSourceManifestCohort(input)).toThrow(/checkedAt/) }) + + it('loads the locked non-military cohort from the current catalog without network input', () => { + const current = buildCurrentSourceManifestCohort('2026-08-06', resolve('.')) + + expect(current.build.summary).toMatchObject({ + officialTargets: 147, + militaryExcluded: 3, + eligibleTargets: 144, + }) + expect( + current.build.summary.candidateManifests + + current.build.summary.targetsWithoutCandidate, + ).toBe(144) + expect(current.inputFingerprints.map((input) => input.repositoryPath)).toEqual( + Object.values(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS), + ) + expect(current.inputFingerprints.every( + (input) => /^[a-f0-9]{64}$/u.test(input.sha256) && input.byteLength > 0, + )).toBe(true) + }) + + it('writes and verifies a candidate-only bundle with an input ledger and checksums', () => { + const build = buildSourceManifestCohort(fixture()) + const output = temporaryDirectory() + const written = writeSourceManifestCohort( + build, + output, + inputFingerprints(), + resolve('.'), + ) + const artifact = readJson( + written.artifactManifestPath, + ) + const verification = verifySourceManifestCohortArtifact(output) + const repeatedOutput = temporaryDirectory() + const repeated = writeSourceManifestCohort( + build, + repeatedOutput, + inputFingerprints(), + resolve('.'), + ) + + expect(artifact.disposition).toBe('candidate_only') + expect(artifact.inputs.map((input) => input.name)).toEqual([ + 'registry', + 'universities', + 'sources', + 'programs', + 'admissionCycles', + 'scholarships', + ]) + expect(artifact.files.map((file) => file.path)).toEqual([ + 'gap-report.v1.json', + 'manifests/001-test-university.v2.candidate.json', + ]) + expect(readFileSync(written.checksumPath, 'utf8').trim().split('\n')).toHaveLength(3) + expect(verification).toMatchObject({ + cohortId: 'test-double-first-class', + checkedAt: '2026-08-06', + candidateManifests: 1, + exactOfficialHttpsSources: 3, + verifiedFiles: 3, + }) + expect(readFileSync(repeated.artifactManifestPath, 'utf8')).toBe( + readFileSync(written.artifactManifestPath, 'utf8'), + ) + expect(readFileSync(repeated.checksumPath, 'utf8')).toBe( + readFileSync(written.checksumPath, 'utf8'), + ) + }) + + it('fails closed for tampering, stale output, or formal-manifest output', () => { + const build = buildSourceManifestCohort(fixture()) + const tamperedOutput = temporaryDirectory() + const written = writeSourceManifestCohort( + build, + tamperedOutput, + inputFingerprints(), + resolve('.'), + ) + writeFileSync( + join(written.manifestDirectory, '001-test-university.v2.candidate.json'), + '{}\n', + 'utf8', + ) + expect(() => verifySourceManifestCohortArtifact(tamperedOutput)) + .toThrow(/checksum mismatch/) + + const staleOutput = temporaryDirectory() + writeFileSync(join(staleOutput, 'stale.json'), '{}\n', 'utf8') + expect(() => writeSourceManifestCohort( + build, + staleOutput, + inputFingerprints(), + resolve('.'), + )).toThrow(/must be empty/) + + expect(() => writeSourceManifestCohort( + build, + join(resolve('.'), 'content/source-manifests/candidate-artifact'), + inputFingerprints(), + resolve('.'), + )).toThrow(/must not be inside/) + }) + + it('accepts only one explicit CLI mode and rejects the former generic output flag', () => { + expect(parseSourceManifestCohortCli([ + '--checked-at', '2026-08-06', '--dry-run', + ])).toEqual({ mode: 'dry-run', checkedAt: '2026-08-06' }) + expect(parseSourceManifestCohortCli([ + '--checked-at', '2026-08-06', '--artifact-output', 'candidate-bundle', + ])).toEqual({ + mode: 'write-artifact', + checkedAt: '2026-08-06', + artifactOutput: 'candidate-bundle', + }) + expect(parseSourceManifestCohortCli([ + '--verify-artifact', 'candidate-bundle', + ])).toEqual({ mode: 'verify-artifact', artifactOutput: 'candidate-bundle' }) + expect(() => parseSourceManifestCohortCli([ + '--checked-at', '2026-08-06', '--output', 'content/source-manifests', + ])).toThrow(/Unknown CLI option/) + }) }) diff --git a/tests/unit/source-manifest-cohort-workflow.test.ts b/tests/unit/source-manifest-cohort-workflow.test.ts new file mode 100644 index 0000000..b501337 --- /dev/null +++ b/tests/unit/source-manifest-cohort-workflow.test.ts @@ -0,0 +1,30 @@ +import { readFileSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { describe, expect, it } from 'vitest' + +const workflowPath = join( + resolve('.'), + '.github', + 'workflows', + 'source-manifest-cohort-candidates.yml', +) + +describe('source-manifest candidate cohort workflow', () => { + it('is manually triggered, read-only, and uploads a runner-temp artifact', () => { + const workflow = readFileSync(workflowPath, 'utf8') + + expect(workflow).toContain('workflow_dispatch:') + expect(workflow).toContain('contents: read') + expect(workflow).not.toContain('contents: write') + expect(workflow).toContain('${{ runner.temp }}/source-manifest-cohort-candidates') + expect(workflow).toContain('npm run validate:double-first-class') + expect(workflow).not.toContain('npm run validate:data') + expect(workflow).toContain('--artifact-output "$ARTIFACT_DIRECTORY"') + expect(workflow).toContain( + 'pipeline:verify-source-manifest-candidates -- "$ARTIFACT_DIRECTORY"', + ) + expect(workflow).toContain('git diff --exit-code -- content/source-manifests') + expect(workflow).toContain('actions/upload-artifact@v6') + expect(workflow).not.toMatch(/\s--output(?:\s|$)/u) + }) +}) diff --git a/workers/catalog-api/src/index.ts b/workers/catalog-api/src/index.ts index 8ee83de..5b9c921 100644 --- a/workers/catalog-api/src/index.ts +++ b/workers/catalog-api/src/index.ts @@ -125,6 +125,19 @@ function stringParam( return value || undefined } +function enumParam( + params: URLSearchParams, + name: string, + values: ReadonlySet, +): Value | undefined { + const value = stringParam(params, name) + if (value === undefined) return undefined + if (!values.has(value as Value)) { + throw new InvalidRequestError(`${name} is invalid.`) + } + return value as Value +} + function integerParam(params: URLSearchParams, name: string, minimum: number, maximum: number) { const raw = stringParam(params, name) if (raw === undefined) return undefined @@ -226,6 +239,7 @@ async function publicCatalogResponse(request: Request, environment: CatalogApiEn tuitionMax: nonNegativeNumberParam(url.searchParams, 'tuitionMax'), applicationState: stringParam(url.searchParams, 'applicationState'), scholarship: stringParam(url.searchParams, 'scholarship'), + sort: stringParam(url.searchParams, 'sort'), cursor: stringParam(url.searchParams, 'cursor', 1_024), limit: integerParam(url.searchParams, 'limit', 1, 100), }), etag) @@ -248,6 +262,22 @@ async function publicCatalogResponse(request: Request, environment: CatalogApiEn provider: stringParam(url.searchParams, 'provider'), institution: stringParam(url.searchParams, 'institution'), program: stringParam(url.searchParams, 'program'), + degree: enumParam( + url.searchParams, + 'degree', + new Set(['bachelor', 'master', 'doctorate', 'language', 'foundation', 'other']), + ), + funding: enumParam( + url.searchParams, + 'funding', + new Set(['full-tuition', 'partial-tuition', 'stipend', 'accommodation', 'insurance']), + ), + deadline: enumParam( + url.searchParams, + 'deadline', + new Set(['future', 'next-30-days', 'next-90-days', 'announced', 'not-announced', 'closed']), + ), + sort: enumParam(url.searchParams, 'sort', new Set(['default', 'name', 'deadline', 'stipend-desc'])), cursor: stringParam(url.searchParams, 'cursor', 1_024), limit: integerParam(url.searchParams, 'limit', 1, 100), }), etag) diff --git a/workers/catalog-api/src/sql-api.ts b/workers/catalog-api/src/sql-api.ts index a518768..03a93f7 100644 --- a/workers/catalog-api/src/sql-api.ts +++ b/workers/catalog-api/src/sql-api.ts @@ -14,13 +14,14 @@ import { } from './sql-data' import type { ApiEnvelopeDto, + ApiFacetOptionDto, ApiMetaDto, ApplicationDto, InstitutionDto, InstitutionQuery, MoneyDto, ProgramCycleDto, - ProgramDto, + ProgramListDto, ProgramQuery, ProgramType, ReleaseInfoDto, @@ -69,6 +70,7 @@ type ProgramRow = SortableRow & { } type ScholarshipRow = SortableRow & { + slug: string | null provider_organization_id: string provider_slug: string | null provider_organization_type: string @@ -77,6 +79,17 @@ type ScholarshipRow = SortableRow & { official_url: string } +type CountRow = { + total: number +} + +type FacetRow = { + option_id: string + option_slug: string | null + locale: string | null + text_value: string | null +} + type ProgramCodeRow = { program_id: string code: string @@ -211,11 +224,32 @@ function routeApplication( } } +function cursorContext(query: Record) { + const entries = Object.entries(query) + .filter(([key, value]) => + key !== 'cursor' + && key !== 'limit' + && value !== undefined + && value !== '' + && value !== 'default', + ) + .sort(([left], [right]) => left.localeCompare(right)) + if (entries.length === 0) return 'default' + const input = JSON.stringify(entries) + let hash = 2_166_136_261 + for (let index = 0; index < input.length; index += 1) { + hash ^= input.charCodeAt(index) + hash = Math.imul(hash, 16_777_619) + } + return `q-${(hash >>> 0).toString(16).padStart(8, '0')}` +} + function pagination( rows: Row[], limit: number, resource: 'institutions' | 'programs' | 'scholarships', releaseId: string, + context = 'default', ) { const hasMore = rows.length > limit const items = hasMore ? rows.slice(0, limit) : rows @@ -223,11 +257,172 @@ function pagination( return { items, nextCursor: hasMore && last - ? encodeCursor(resource, releaseId, last.sort_slug, last.record_id) + ? encodeCursor(resource, releaseId, last.sort_slug, last.record_id, context) : null, } } +function facetOptions(rows: FacetRow[]): ApiFacetOptionDto[] { + const options = new Map() + for (const row of rows) { + const value = row.option_slug || row.option_id + const current = options.get(value) ?? { value, name: {} } + if (row.locale && row.text_value) current.name[row.locale] = row.text_value + options.set(value, current) + } + for (const option of options.values()) { + if (Object.keys(option.name).length === 0) option.name.en = option.value + } + return [...options.values()].sort((left, right) => left.value.localeCompare(right.value)) +} + +function scholarshipDeadlineSql() { + return `COALESCE( + ( + SELECT window.closes_on + FROM current_scholarship_cycles AS cycle + LEFT JOIN current_application_routes AS route + ON route.release_id = cycle.release_id + AND route.owner_record_id = cycle.scholarship_cycle_id + LEFT JOIN current_application_windows AS window + ON window.release_id = route.release_id + AND window.application_route_id = route.application_route_id + WHERE cycle.release_id = scholarship.release_id + AND cycle.scholarship_id = scholarship.scholarship_id + AND window.closes_on IS NOT NULL + ORDER BY + CASE window.application_state + WHEN 'open' THEN 0 + WHEN 'rolling' THEN 1 + WHEN 'upcoming' THEN 2 + WHEN 'not_announced' THEN 3 + ELSE 4 + END, + window.closes_on, + cycle.academic_year DESC, + cycle.scholarship_cycle_id + LIMIT 1 + ), + ( + SELECT CAST(json_extract(deadline.value_json, '$') AS TEXT) + FROM current_record_fields AS deadline + WHERE deadline.release_id = scholarship.release_id + AND deadline.record_id = scholarship.scholarship_id + AND deadline.field_path IN ('deadline', 'closes_on') + ORDER BY CASE deadline.field_path WHEN 'deadline' THEN 0 ELSE 1 END + LIMIT 1 + ) + )` +} + +function scholarshipStipendMinorSql() { + return `COALESCE( + ( + SELECT MAX(COALESCE(coverage.amount_max_minor, coverage.amount_min_minor)) + FROM current_scholarship_cycles AS cycle + JOIN current_scholarship_coverage AS coverage + ON coverage.release_id = cycle.release_id + AND coverage.scholarship_cycle_id = cycle.scholarship_cycle_id + WHERE cycle.release_id = scholarship.release_id + AND cycle.scholarship_id = scholarship.scholarship_id + AND coverage.coverage_type = 'stipend' + ), + ( + SELECT CAST(json_extract(stipend.value_json, '$') AS INTEGER) * 100 + FROM current_record_fields AS stipend + WHERE stipend.release_id = scholarship.release_id + AND stipend.record_id = scholarship.scholarship_id + AND stipend.field_path = 'coverage.stipendCnyPerMonth' + LIMIT 1 + ) + )` +} + +function scholarshipNameSql() { + return `COALESCE( + ( + SELECT localized.text_value + FROM current_localized_content AS localized + WHERE localized.release_id = scholarship.release_id + AND localized.record_id = scholarship.scholarship_id + AND localized.field_name = 'name' + ORDER BY CASE localized.locale WHEN 'en' THEN 0 WHEN 'zh' THEN 1 ELSE 2 END, + localized.locale + LIMIT 1 + ), + record.slug, + '' + )` +} + +function scholarshipSortSql(sort: ScholarshipQuery['sort']) { + const slug = `COALESCE(record.slug, '')` + if (sort === 'name') return `${scholarshipNameSql()} || ':' || ${slug}` + if (sort === 'deadline') { + const deadline = scholarshipDeadlineSql() + return `CASE + WHEN ${deadline} >= date('now', '+8 hours') THEN '0:' || ${deadline} + WHEN ${deadline} IS NOT NULL THEN '1:' || ${deadline} + ELSE '2:9999-12-31' + END || ':' || ${slug}` + } + if (sort === 'stipend-desc') { + const stipend = scholarshipStipendMinorSql() + return `CASE + WHEN ${stipend} IS NULL THEN '1:' + ELSE '0:' || printf('%015d', 999999999999999 - CAST(${stipend} AS INTEGER)) + END || ':' || ${slug}` + } + return slug +} + +function scholarshipFundingSql(funding: NonNullable) { + if (funding === 'stipend') return `${scholarshipStipendMinorSql()} > 0` + const legacyField = funding === 'full-tuition' || funding === 'partial-tuition' + ? 'coverage.tuition' + : funding === 'accommodation' + ? 'coverage.accommodation' + : 'coverage.insurance' + const legacyValues = funding === 'full-tuition' + ? `'full'` + : funding === 'partial-tuition' + ? `'partial'` + : funding === 'accommodation' + ? `'full', 'partial'` + : `1, 'true'` + const coverageType = funding === 'full-tuition' || funding === 'partial-tuition' + ? 'tuition' + : funding + const coverageModes = funding === 'full-tuition' + ? `'full', 'waiver'` + : funding === 'partial-tuition' + ? `'partial', 'fixed'` + : funding === 'accommodation' + ? `'full', 'partial', 'waiver'` + : `'full', 'partial', 'fixed', 'waiver'` + return `( + EXISTS ( + SELECT 1 + FROM current_scholarship_cycles AS cycle + JOIN current_scholarship_coverage AS coverage + ON coverage.release_id = cycle.release_id + AND coverage.scholarship_cycle_id = cycle.scholarship_cycle_id + WHERE cycle.release_id = scholarship.release_id + AND cycle.scholarship_id = scholarship.scholarship_id + AND coverage.coverage_type = '${coverageType}' + AND coverage.coverage_mode IN (${coverageModes}) + ) + OR EXISTS ( + SELECT 1 + FROM current_record_fields AS legacy_coverage + WHERE legacy_coverage.release_id = scholarship.release_id + AND legacy_coverage.record_id = scholarship.scholarship_id + AND legacy_coverage.field_path = '${legacyField}' + AND json_extract(legacy_coverage.value_json, '$') IN (${legacyValues}) + ) + )` +} + function identityMeta( decorations: RecordDecorations, row: RecordAuditRow, @@ -283,13 +478,25 @@ export class CatalogSqlApi { private envelope( data: T, - page?: { pageSize: number; nextCursor: string | null }, + page?: { + pageSize: number + nextCursor: string | null + total?: number + facets?: ApiMetaDto['facets'] + }, ): ApiEnvelopeDto { const meta: ApiMetaDto = { apiVersion: 'v1', release: this.release, notice: AUTOMATED_COLLECTION_NOTICE, - ...(page ? { pageSize: page.pageSize, nextCursor: page.nextCursor } : {}), + ...(page + ? { + pageSize: page.pageSize, + nextCursor: page.nextCursor, + ...(page.total === undefined ? {} : { total: page.total }), + ...(page.facets === undefined ? {} : { facets: page.facets }), + } + : {}), } return { data, meta } } @@ -653,8 +860,11 @@ export class CatalogSqlApi { ...cycleValues, ) } + const filteredConditions = [...conditions] + const filteredValues = [...values] + const context = cursorContext({ ...query }) if (query.cursor && exactSlug === undefined) { - const cursor = decodeCursor(query.cursor, 'programs', this.release.id) + const cursor = decodeCursor(query.cursor, 'programs', this.release.id, context) addCondition( conditions, values, @@ -667,7 +877,7 @@ export class CatalogSqlApi { } const limit = exactSlug === undefined ? pageLimit(query.limit) + 1 : 1 values.push(limit) - return queryAll(this.database, ` + const rows = await queryAll(this.database, ` SELECT record.record_id, COALESCE(record.slug, '') AS sort_slug, @@ -706,12 +916,13 @@ export class CatalogSqlApi { ORDER BY COALESCE(record.slug, ''), record.record_id LIMIT ? `, values) + return { rows, filteredConditions, filteredValues, context } } private async mapPrograms(rows: ProgramRow[]) { const programIds = rows.map((row) => row.record_id) const slots = programIds.length > 0 ? placeholders(programIds.length) : '' - const [decorations, disciplines, languages] = await Promise.all([ + const [decorations, disciplines, languages, currentCycles] = await Promise.all([ loadRecordDecorations( this.database, this.release.id, @@ -733,8 +944,9 @@ export class CatalogSqlApi { WHERE release_id = ? AND program_id IN (${slots}) ORDER BY program_id, role, language_code `, [this.release.id, ...programIds]), + this.currentProgramCycles(rows), ]) - return rows.map((row): ProgramDto => { + return rows.map((row): ProgramListDto => { const disciplineCodes = disciplines .filter((item) => item.program_id === row.record_id) .map((item) => item.code) @@ -786,24 +998,101 @@ export class CatalogSqlApi { applyUrl: decorations.meta(row, ['apply_url', 'applyUrl']), institution: identityMeta(decorations, row, 'institution_id'), }, + currentCycle: currentCycles.get(row.record_id) ?? null, } }) } + private async programListMetadata( + conditions: readonly string[], + values: readonly unknown[], + ) { + const where = conditions.join('\n AND ') + const joins = ` + FROM current_programs AS program + JOIN current_catalog_records AS record + ON record.release_id = program.release_id + AND record.record_id = program.program_id + JOIN current_institutions AS institution + ON institution.release_id = program.release_id + AND institution.institution_id = program.institution_id + JOIN current_catalog_records AS institution_record + ON institution_record.release_id = institution.release_id + AND institution_record.record_id = institution.institution_id + JOIN current_locations AS city + ON city.release_id = institution.release_id + AND city.location_id = institution.city_id + JOIN current_catalog_records AS city_record + ON city_record.release_id = city.release_id + AND city_record.record_id = city.location_id + ` + const [count, universities, cities] = await Promise.all([ + queryFirst(this.database, ` + SELECT COUNT(*) AS total + ${joins} + WHERE ${where} + `, [...values]), + queryAll(this.database, ` + SELECT DISTINCT + institution.institution_id AS option_id, + institution_record.slug AS option_slug, + localized.locale, + localized.text_value + ${joins} + LEFT JOIN current_localized_content AS localized + ON localized.release_id = institution.release_id + AND localized.record_id = institution.institution_id + AND localized.field_name = 'name' + WHERE ${where} + ORDER BY option_slug, option_id, localized.locale + `, [...values]), + queryAll(this.database, ` + SELECT DISTINCT + city.location_id AS option_id, + city_record.slug AS option_slug, + localized.locale, + localized.text_value + ${joins} + LEFT JOIN current_localized_content AS localized + ON localized.release_id = city.release_id + AND localized.record_id = city.location_id + AND localized.field_name = 'name' + WHERE ${where} + ORDER BY option_slug, option_id, localized.locale + `, [...values]), + ]) + return { + total: Number(count?.total ?? 0), + facets: { + universities: facetOptions(universities), + cities: facetOptions(cities), + }, + } + } + async listPrograms(query: ProgramQuery = {}) { const limit = pageLimit(query.limit) + const selected = await this.selectPrograms(query) const page = pagination( - await this.selectPrograms(query), + selected.rows, limit, 'programs', this.release.id, + selected.context, ) - const data = await this.mapPrograms(page.items) - return this.envelope(data, { pageSize: data.length, nextCursor: page.nextCursor }) + const [data, metadata] = await Promise.all([ + this.mapPrograms(page.items), + this.programListMetadata(selected.filteredConditions, selected.filteredValues), + ]) + return this.envelope(data, { + pageSize: data.length, + nextCursor: page.nextCursor, + ...metadata, + }) } async getProgram(slug: string) { - const row = (await this.selectPrograms({}, slug))[0] + const row = (await this.selectPrograms({}, slug)).rows[0] if (!row) return null return this.envelope((await this.mapPrograms([row]))[0]!) } @@ -870,13 +1159,93 @@ export class CatalogSqlApi { AND (matched_program.program_id = ? OR matched_record.slug = ?) )`, query.program, query.program) } + if (query.degree) { + const degreeValues: unknown[] = [] + const programDegree = ['bachelor', 'master', 'doctorate'].includes(query.degree) + ? (degreeValues.push(query.degree), 'matched_program.degree_level = ?') + : query.degree === 'language' + ? `matched_program.program_type = 'language'` + : query.degree === 'foundation' + ? `matched_program.program_type = 'foundation'` + : `matched_program.degree_level IS NULL + AND matched_program.program_type NOT IN ('language', 'foundation')` + const normalizedDegree = ['bachelor', 'master', 'doctorate'].includes(query.degree) + ? `OR EXISTS ( + SELECT 1 + FROM current_scholarship_cycles AS cycle + WHERE cycle.release_id = scholarship.release_id + AND cycle.scholarship_id = scholarship.scholarship_id + AND ( + ( + cycle.degree_scope = 'all' + AND NOT EXISTS ( + SELECT 1 FROM scholarship_cycle_degree_levels AS excluded + WHERE excluded.release_id = cycle.release_id + AND excluded.scholarship_cycle_id = cycle.scholarship_cycle_id + AND excluded.degree_level = ? + AND excluded.inclusion = 'exclude' + ) + ) + OR EXISTS ( + SELECT 1 FROM scholarship_cycle_degree_levels AS included + WHERE included.release_id = cycle.release_id + AND included.scholarship_cycle_id = cycle.scholarship_cycle_id + AND included.degree_level = ? + AND included.inclusion = 'include' + ) + ) + )` + : '' + if (normalizedDegree) degreeValues.push(query.degree, query.degree) + addCondition(conditions, values, `( + EXISTS ( + SELECT 1 + FROM current_record_fields AS scope + JOIN json_each(scope.value_json) AS scoped_program ON 1 = 1 + JOIN current_programs AS matched_program + ON matched_program.release_id = scope.release_id + AND matched_program.program_id = CAST(scoped_program.value AS TEXT) + WHERE scope.release_id = scholarship.release_id + AND scope.record_id = scholarship.scholarship_id + AND scope.field_path IN ('programIds', 'program_ids') + AND ${programDegree} + ) + ${normalizedDegree} + )`, ...degreeValues) + } + if (query.funding) addCondition(conditions, values, scholarshipFundingSql(query.funding)) + if (query.deadline) { + const deadline = scholarshipDeadlineSql() + if (query.deadline === 'not-announced') { + addCondition(conditions, values, `${deadline} IS NULL`) + } else if (query.deadline === 'announced') { + addCondition(conditions, values, `${deadline} IS NOT NULL`) + } else if (query.deadline === 'closed') { + addCondition(conditions, values, `${deadline} < ?`, this.today) + } else if (query.deadline === 'future') { + addCondition(conditions, values, `${deadline} >= ?`, this.today) + } else { + const days = query.deadline === 'next-30-days' ? 30 : 90 + addCondition( + conditions, + values, + `${deadline} >= ? AND ${deadline} <= date(?, '+${days} days')`, + this.today, + this.today, + ) + } + } + const sortSql = scholarshipSortSql(query.sort) + const filteredConditions = [...conditions] + const filteredValues = [...values] + const context = cursorContext({ ...query }) if (query.cursor && exactSlug === undefined) { - const cursor = decodeCursor(query.cursor, 'scholarships', this.release.id) + const cursor = decodeCursor(query.cursor, 'scholarships', this.release.id, context) addCondition( conditions, values, - `(COALESCE(record.slug, '') > ? - OR (COALESCE(record.slug, '') = ? AND record.record_id > ?))`, + `(${sortSql} > ? + OR (${sortSql} = ? AND record.record_id > ?))`, cursor.sortKey, cursor.sortKey, cursor.id, @@ -884,10 +1253,11 @@ export class CatalogSqlApi { } const limit = exactSlug === undefined ? pageLimit(query.limit) + 1 : 1 values.push(limit) - return queryAll(this.database, ` + const rows = await queryAll(this.database, ` SELECT record.record_id, - COALESCE(record.slug, '') AS sort_slug, + record.slug, + ${sortSql} AS sort_slug, record.verified_at AS record_verified_at, record.review_after AS record_review_after, scholarship.provider_organization_id, @@ -907,9 +1277,10 @@ export class CatalogSqlApi { ON provider_record.release_id = provider.release_id AND provider_record.record_id = provider.organization_id WHERE ${conditions.join('\n AND ')} - ORDER BY COALESCE(record.slug, ''), record.record_id + ORDER BY sort_slug, record.record_id LIMIT ? `, values) + return { rows, filteredConditions, filteredValues, context } } private async mapScholarships(rows: ScholarshipRow[]) { @@ -921,7 +1292,7 @@ export class CatalogSqlApi { return rows.map((row): ScholarshipDto => ({ type: 'scholarship', id: row.record_id, - slug: row.sort_slug || null, + slug: row.slug, attributes: { name: decorations.localized(row.record_id, 'name') ?? {}, summary: decorations.localized(row.record_id, 'summary'), @@ -971,20 +1342,86 @@ export class CatalogSqlApi { })) } + private async scholarshipListMetadata( + conditions: readonly string[], + values: readonly unknown[], + ) { + const where = conditions.join('\n AND ') + const joins = ` + FROM current_scholarships AS scholarship + JOIN current_catalog_records AS record + ON record.release_id = scholarship.release_id + AND record.record_id = scholarship.scholarship_id + JOIN current_organizations AS provider + ON provider.release_id = scholarship.release_id + AND provider.organization_id = scholarship.provider_organization_id + JOIN current_catalog_records AS provider_record + ON provider_record.release_id = provider.release_id + AND provider_record.record_id = provider.organization_id + ` + const [count, universities] = await Promise.all([ + queryFirst(this.database, ` + SELECT COUNT(*) AS total + ${joins} + WHERE ${where} + `, [...values]), + queryAll(this.database, ` + SELECT DISTINCT + matched_institution.institution_id AS option_id, + matched_record.slug AS option_slug, + localized.locale, + localized.text_value + ${joins} + JOIN current_record_fields AS scope + ON scope.release_id = scholarship.release_id + AND scope.record_id = scholarship.scholarship_id + AND scope.field_path IN ('universityIds', 'institution_ids') + JOIN json_each(scope.value_json) AS scoped_institution ON 1 = 1 + JOIN current_institutions AS matched_institution + ON matched_institution.release_id = scope.release_id + AND matched_institution.institution_id = CAST(scoped_institution.value AS TEXT) + JOIN current_catalog_records AS matched_record + ON matched_record.release_id = matched_institution.release_id + AND matched_record.record_id = matched_institution.institution_id + LEFT JOIN current_localized_content AS localized + ON localized.release_id = matched_institution.release_id + AND localized.record_id = matched_institution.institution_id + AND localized.field_name = 'name' + WHERE ${where} + ORDER BY option_slug, option_id, localized.locale + `, [...values]), + ]) + return { + total: Number(count?.total ?? 0), + facets: { + universities: facetOptions(universities), + }, + } + } + async listScholarships(query: ScholarshipQuery = {}) { const limit = pageLimit(query.limit) + const selected = await this.selectScholarships(query) const page = pagination( - await this.selectScholarships(query), + selected.rows, limit, 'scholarships', this.release.id, + selected.context, ) - const data = await this.mapScholarships(page.items) - return this.envelope(data, { pageSize: data.length, nextCursor: page.nextCursor }) + const [data, metadata] = await Promise.all([ + this.mapScholarships(page.items), + this.scholarshipListMetadata(selected.filteredConditions, selected.filteredValues), + ]) + return this.envelope(data, { + pageSize: data.length, + nextCursor: page.nextCursor, + ...metadata, + }) } async getScholarship(slug: string) { - const row = (await this.selectScholarships({}, slug))[0] + const row = (await this.selectScholarships({}, slug)).rows[0] if (!row) return null return this.envelope((await this.mapScholarships([row]))[0]!) } @@ -1030,6 +1467,172 @@ export class CatalogSqlApi { `, [this.release.id, ...ownerIds]) } + private async programCycleRows(programIds: string[]) { + if (programIds.length === 0) return [] + const slots = placeholders(programIds.length) + return queryAll(this.database, ` + SELECT + record.record_id, + record.slug, + record.verified_at AS record_verified_at, + record.review_after AS record_review_after, + cycle.program_cycle_id, + cycle.program_id, + cycle.academic_year, + cycle.intake_code, + cycle.sequence, + cycle.starts_on, + cycle.ends_on, + cycle.cycle_status + FROM current_program_cycles AS cycle + JOIN current_catalog_records AS record + ON record.release_id = cycle.release_id + AND record.record_id = cycle.program_cycle_id + WHERE cycle.release_id = ? AND cycle.program_id IN (${slots}) + ORDER BY cycle.program_id, cycle.academic_year DESC, + cycle.intake_code, cycle.sequence, cycle.program_cycle_id + `, [this.release.id, ...programIds]) + } + + private async mapProgramCycleRows( + rows: ProgramCycleRow[], + programSlugs: ReadonlyMap, + ) { + const ids = rows.map((row) => row.record_id) + const [decorations, routes, fees] = await Promise.all([ + loadRecordDecorations(this.database, this.release.id, ids), + this.routeWindows(ids), + this.fees(ids), + ]) + return rows.map((row): ProgramCycleDto => { + const route = routes.find((item) => item.owner_record_id === row.record_id) + const opensOn = route?.opens_on + ?? decorations.value(row.record_id, ['opens_on', 'opensOn']) + const closesOn = route?.closes_on + ?? decorations.value(row.record_id, ['closes_on', 'closesOn']) + const tuitionFee = moneyFromFee( + fees.find((item) => item.owner_record_id === row.record_id && item.fee_type === 'tuition'), + ) ?? legacyMoney( + decorations.value(row.record_id, ['tuitionCny', 'tuition_amount']), + decorations.value(row.record_id, ['tuitionPeriod', 'billing_period']), + ) + const applicationFee = moneyFromFee( + fees.find((item) => item.owner_record_id === row.record_id && item.fee_type === 'application'), + ) ?? legacyMoney( + decorations.value(row.record_id, ['applicationFeeCny', 'application_fee']), + 'one_time', + ) + return { + type: 'program_cycle', + id: row.record_id, + slug: row.slug, + attributes: { + academicYear: row.academic_year, + intake: row.intake_code, + sequence: Number(row.sequence), + cycleStatus: row.cycle_status, + startsOn: row.starts_on, + endsOn: row.ends_on, + application: routeApplication(route, { + opensOn, + closesOn, + applyUrl: decorations.value(row.record_id, ['apply_url', 'applyUrl']), + }, this.today), + tuition: tuitionFee, + applicationFee, + }, + relationships: { + program: { + id: row.program_id, + slug: programSlugs.get(row.program_id) ?? null, + }, + }, + sources: decorations.sources(row.record_id), + fieldMeta: { + academicYear: identityMeta(decorations, row, 'academic_year'), + intake: identityMeta(decorations, row, 'intake_code'), + sequence: identityMeta(decorations, row, 'sequence'), + cycleStatus: identityMeta(decorations, row, 'cycle_status'), + startsOn: decorations.meta(row, ['starts_on', 'startsOn']), + endsOn: decorations.meta(row, ['ends_on', 'endsOn']), + 'application.opensOn': decorations.meta(row, ['opens_on', 'opensOn']), + 'application.closesOn': decorations.meta(row, ['closes_on', 'closesOn']), + 'application.routeType': decorations.meta(row, ['route_type'], route !== undefined), + 'application.accessMode': decorations.meta( + row, + ['access_mode'], + route?.access_mode !== null && route?.access_mode !== undefined, + ), + 'application.applyUrl': decorations.meta(row, ['apply_url', 'applyUrl']), + 'application.rolling': decorations.meta( + row, + ['rolling'], + route?.rolling !== null && route?.rolling !== undefined, + ), + 'application.state': decorations.meta( + row, + ['application_state', 'opens_on', 'opensOn', 'closes_on', 'closesOn'], + route !== undefined, + ), + tuition: decorations.meta(row, ['tuitionCny', 'tuition_amount', 'amount_min_minor']), + applicationFee: decorations.meta(row, ['applicationFeeCny', 'application_fee']), + program: identityMeta(decorations, row, 'program_id'), + }, + } + }) + } + + private compareProgramCycles(left: ProgramCycleDto, right: ProgramCycleDto) { + const priority: Record = { + open: 0, + rolling: 1, + upcoming: 2, + 'dates-published': 3, + 'not-announced': 4, + closed: 5, + 'previous-cycle': 6, + } + const leftState = left.attributes.application.state + const rightState = right.attributes.application.state + const stateOrder = priority[leftState] - priority[rightState] + if (stateOrder !== 0) return stateOrder + if (leftState === 'open') { + const order = (left.attributes.application.closesOn ?? '9999-12-31') + .localeCompare(right.attributes.application.closesOn ?? '9999-12-31') + if (order !== 0) return order + } else if (leftState === 'upcoming') { + const order = (left.attributes.application.opensOn ?? '9999-12-31') + .localeCompare(right.attributes.application.opensOn ?? '9999-12-31') + if (order !== 0) return order + } else { + const leftDate = left.attributes.application.closesOn + ?? left.attributes.application.opensOn + ?? left.attributes.academicYear + const rightDate = right.attributes.application.closesOn + ?? right.attributes.application.opensOn + ?? right.attributes.academicYear + const order = rightDate.localeCompare(leftDate) + if (order !== 0) return order + } + return left.id.localeCompare(right.id) + } + + private async currentProgramCycles(rows: ProgramRow[]) { + const slugs = new Map(rows.map((row) => [row.record_id, row.sort_slug || null])) + const cycles = await this.mapProgramCycleRows( + await this.programCycleRows(rows.map((row) => row.record_id)), + slugs, + ) + const selected = new Map() + for (const cycle of cycles) { + const current = selected.get(cycle.relationships.program.id) + if (!current || this.compareProgramCycles(cycle, current) < 0) { + selected.set(cycle.relationships.program.id, cycle) + } + } + return selected + } + async getProgramCycles(slug: string) { const program = await queryFirst<{ program_id: string; slug: string | null }>(this.database, ` SELECT program.program_id, record.slug diff --git a/workers/catalog-api/src/sql-cursor.ts b/workers/catalog-api/src/sql-cursor.ts index 586c371..e89d75b 100644 --- a/workers/catalog-api/src/sql-cursor.ts +++ b/workers/catalog-api/src/sql-cursor.ts @@ -1,13 +1,16 @@ type CursorResource = 'institutions' | 'programs' | 'scholarships' type CursorPayload = { - v: 1 + v: 2 resource: CursorResource releaseId: string + context: string sortKey: string id: string } +type DecodedCursor = Pick + export class InvalidCursorError extends Error { constructor() { super('Invalid cursor.') @@ -35,31 +38,53 @@ export function encodeCursor( releaseId: string, sortKey: string, id: string, + context = 'default', ) { - return toBase64Url(JSON.stringify({ v: 1, resource, releaseId, sortKey, id } satisfies CursorPayload)) + return toBase64Url(JSON.stringify({ v: 2, resource, releaseId, context, sortKey, id } satisfies CursorPayload)) } export function decodeCursor( value: string, resource: CursorResource, releaseId: string, -): CursorPayload { + context = 'default', +): DecodedCursor { if (value.length > 1_024) throw new InvalidCursorError() try { - const parsed = JSON.parse(fromBase64Url(value)) as Partial + const parsed = JSON.parse(fromBase64Url(value)) as { + v?: unknown + resource?: unknown + releaseId?: unknown + context?: unknown + sortKey?: unknown + id?: unknown + } if ( - parsed.v !== 1 - || parsed.resource !== resource + parsed.resource !== resource || parsed.releaseId !== releaseId || typeof parsed.sortKey !== 'string' || typeof parsed.id !== 'string' - || parsed.sortKey.length > 160 + || parsed.sortKey.length > 320 || parsed.id.length === 0 || parsed.id.length > 200 + ) throw new InvalidCursorError() + const { sortKey, id } = parsed + + if (parsed.v === 1) { + if (context !== 'default') throw new InvalidCursorError() + return { sortKey, id } + } + + if ( + parsed.v !== 2 + || parsed.context !== context + || typeof parsed.context !== 'string' + || parsed.context.length === 0 + || parsed.context.length > 80 ) { throw new InvalidCursorError() } - return parsed as CursorPayload + return { sortKey, id } } catch (error) { if (error instanceof InvalidCursorError) throw error throw new InvalidCursorError() diff --git a/workers/catalog-api/src/sql-types.ts b/workers/catalog-api/src/sql-types.ts index 14d8fd7..6a9d302 100644 --- a/workers/catalog-api/src/sql-types.ts +++ b/workers/catalog-api/src/sql-types.ts @@ -48,6 +48,16 @@ export type ApiMetaDto = { notice: string pageSize?: number nextCursor?: string | null + total?: number + facets?: { + universities?: ApiFacetOptionDto[] + cities?: ApiFacetOptionDto[] + } +} + +export type ApiFacetOptionDto = { + value: string + name: LocalizedValue } export type ApiEnvelopeDto = { @@ -137,6 +147,10 @@ export type ProgramDto = RecordDto< } > +export type ProgramListDto = ProgramDto & { + currentCycle: ProgramCycleDto | null +} + export type ApplicationState = | 'open' | 'upcoming' @@ -273,6 +287,7 @@ export type ProgramQuery = ListOptions & { tuitionMax?: number applicationState?: string scholarship?: string + sort?: string } export type ScholarshipQuery = ListOptions & { @@ -280,4 +295,8 @@ export type ScholarshipQuery = ListOptions & { provider?: string institution?: string program?: string + degree?: string + funding?: string + deadline?: string + sort?: string } From 13b2844c3698d856162cf5b8bcfbebed3ac345aa Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 00:11:16 +0800 Subject: [PATCH 4/9] Fix cross-platform CI regressions --- .github/workflows/source-manifest-cohort-candidates.yml | 7 +++++-- tests/unit/catalog-sql-api.test.ts | 2 +- tests/unit/source-manifest-cohort-workflow.test.ts | 5 +++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/source-manifest-cohort-candidates.yml b/.github/workflows/source-manifest-cohort-candidates.yml index 66ec571..1ee05c7 100644 --- a/.github/workflows/source-manifest-cohort-candidates.yml +++ b/.github/workflows/source-manifest-cohort-candidates.yml @@ -20,7 +20,6 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 env: - ARTIFACT_DIRECTORY: ${{ runner.temp }}/source-manifest-cohort-candidates CHECKED_AT: ${{ inputs.checked_at }} steps: - name: Checkout repository @@ -40,6 +39,8 @@ jobs: - name: Build review-only candidate artifact shell: bash + env: + ARTIFACT_DIRECTORY: ${{ runner.temp }}/source-manifest-cohort-candidates run: | set -euo pipefail npm run pipeline:build-source-manifest-candidates -- \ @@ -48,6 +49,8 @@ jobs: - name: Verify checksums and publication safety shell: bash + env: + ARTIFACT_DIRECTORY: ${{ runner.temp }}/source-manifest-cohort-candidates run: | set -euo pipefail npm run pipeline:verify-source-manifest-candidates -- "$ARTIFACT_DIRECTORY" @@ -57,6 +60,6 @@ jobs: uses: actions/upload-artifact@v6 with: name: source-manifest-candidates-${{ github.run_id }} - path: ${{ env.ARTIFACT_DIRECTORY }} + path: ${{ runner.temp }}/source-manifest-cohort-candidates if-no-files-found: error retention-days: 14 diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index 0ef4d37..1f58a1b 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -194,7 +194,7 @@ describe('Catalog D1 normalized v1 API', () => { expect(filteredResponse.status).toBe(200) expect(filtered.data.map((item) => item.id)).toContain(program.id) expect(r2Reads).toBe(0) - }) + }, 30_000) it('filters and sorts scholarships with exact metadata and query-bound cursors', async () => { const fundedResponse = await worker.fetch( diff --git a/tests/unit/source-manifest-cohort-workflow.test.ts b/tests/unit/source-manifest-cohort-workflow.test.ts index b501337..9138190 100644 --- a/tests/unit/source-manifest-cohort-workflow.test.ts +++ b/tests/unit/source-manifest-cohort-workflow.test.ts @@ -12,11 +12,16 @@ const workflowPath = join( describe('source-manifest candidate cohort workflow', () => { it('is manually triggered, read-only, and uploads a runner-temp artifact', () => { const workflow = readFileSync(workflowPath, 'utf8') + const jobConfiguration = workflow.slice( + workflow.indexOf('jobs:'), + workflow.indexOf(' steps:'), + ) expect(workflow).toContain('workflow_dispatch:') expect(workflow).toContain('contents: read') expect(workflow).not.toContain('contents: write') expect(workflow).toContain('${{ runner.temp }}/source-manifest-cohort-candidates') + expect(jobConfiguration).not.toContain('runner.temp') expect(workflow).toContain('npm run validate:double-first-class') expect(workflow).not.toContain('npm run validate:data') expect(workflow).toContain('--artifact-output "$ARTIFACT_DIRECTORY"') From b9e59c542ac68934599bc5c8ecdd98a2418960bc Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 02:15:16 +0800 Subject: [PATCH 5/9] Secure manifest onboarding and refresh priority facts --- content/data/admission-cycles.json | 16 +- content/data/scholarships.json | 4 +- content/data/sources.json | 10 +- .../final-group-a-medical-refresh.v1.json | 71 +++ .../reconciliation/final-group-a.v1.json | 58 --- docs/source-manifest-cohort-candidates.md | 26 +- package.json | 1 + .../ingestion/build-source-manifest-cohort.ts | 304 ++++++++++++- .../promote-source-manifest-candidate.ts | 418 ++++++++++++++++++ .../double-first-class-coverage.json | 51 ++- src/lib/scholarship-catalog.ts | 15 + ...atalog-identity-program-visibility.test.ts | 2 +- tests/unit/catalog-release-builder.test.ts | 2 +- tests/unit/catalog-repository.test.ts | 8 +- tests/unit/catalog-sql-api.test.ts | 4 +- .../official-dependency-materializer.test.ts | 2 +- tests/unit/pipeline-bootstrap-import.test.ts | 2 +- tests/unit/scholarship-catalog.test.ts | 22 + ...ource-manifest-candidate-promotion.test.ts | 410 +++++++++++++++++ .../source-manifest-cohort-builder.test.ts | 124 +++++- 20 files changed, 1418 insertions(+), 132 deletions(-) create mode 100644 content/source-registry/reconciliation/final-group-a-medical-refresh.v1.json create mode 100644 scripts/ingestion/promote-source-manifest-candidate.ts create mode 100644 tests/unit/source-manifest-candidate-promotion.test.ts diff --git a/content/data/admission-cycles.json b/content/data/admission-cycles.json index 8fc348f..4e73518 100644 --- a/content/data/admission-cycles.json +++ b/content/data/admission-cycles.json @@ -3201,8 +3201,8 @@ "sourceIds": [ "src-schwarzman-application-2027" ], - "verifiedAt": "2026-07-29", - "reviewAfter": "2026-08-05", + "verifiedAt": "2026-08-07", + "reviewAfter": "2026-08-10", "status": "verified" }, { @@ -3222,8 +3222,8 @@ "sourceIds": [ "src-gdut-2026-international-admissions" ], - "verifiedAt": "2026-07-29", - "reviewAfter": "2026-08-05", + "verifiedAt": "2026-08-07", + "reviewAfter": "2026-08-14", "status": "verified" }, { @@ -7000,8 +7000,8 @@ "sourceIds": [ "src-gap-program-local-gxmu-b-clinical-medicine-cn" ], - "verifiedAt": "2026-07-30", - "reviewAfter": "2026-08-06", + "verifiedAt": "2026-08-07", + "reviewAfter": "2026-08-10", "status": "verified" }, { @@ -7052,8 +7052,8 @@ "sourceIds": [ "src-gap-program-local-gxmu-b-stomatology-cn" ], - "verifiedAt": "2026-07-30", - "reviewAfter": "2026-08-06", + "verifiedAt": "2026-08-07", + "reviewAfter": "2026-08-10", "status": "verified" }, { diff --git a/content/data/scholarships.json b/content/data/scholarships.json index a201828..e1003b9 100644 --- a/content/data/scholarships.json +++ b/content/data/scholarships.json @@ -2513,8 +2513,8 @@ "src-schwarzman-program-current", "src-schwarzman-application-2027" ], - "verifiedAt": "2026-07-29", - "reviewAfter": "2026-08-05", + "verifiedAt": "2026-08-07", + "reviewAfter": "2026-08-10", "status": "verified" }, { diff --git a/content/data/sources.json b/content/data/sources.json index 9364461..8de82fd 100644 --- a/content/data/sources.json +++ b/content/data/sources.json @@ -4677,7 +4677,7 @@ "kind": "program", "language": "en", "official": true, - "accessedAt": "2026-07-29" + "accessedAt": "2026-08-07" }, { "id": "src-schwarzman-application-2027", @@ -4687,7 +4687,7 @@ "kind": "admissions", "language": "en", "official": true, - "accessedAt": "2026-07-29" + "accessedAt": "2026-08-07" }, { "id": "src-uni-guangdong-university-of-technology", @@ -4707,7 +4707,7 @@ "kind": "program", "language": "en", "official": true, - "accessedAt": "2026-07-29" + "accessedAt": "2026-08-07" }, { "id": "src-zjut-2026-international-undergraduate", @@ -8567,7 +8567,7 @@ "kind": "program", "language": "en", "official": true, - "accessedAt": "2026-07-30" + "accessedAt": "2026-08-07" }, { "id": "src-gap-program-prog-gxmu-clinical-medicine-mbbs-2026", @@ -8597,7 +8597,7 @@ "kind": "program", "language": "en", "official": true, - "accessedAt": "2026-07-30" + "accessedAt": "2026-08-07" }, { "id": "src-gap-program-mew-scws-gxmzu-nondegree-chinese-language", diff --git a/content/source-registry/reconciliation/final-group-a-medical-refresh.v1.json b/content/source-registry/reconciliation/final-group-a-medical-refresh.v1.json new file mode 100644 index 0000000..34405e2 --- /dev/null +++ b/content/source-registry/reconciliation/final-group-a-medical-refresh.v1.json @@ -0,0 +1,71 @@ +{ + "format": "studyinchina.source-reconciliation", + "formatVersion": 1, + "groupId": "final-group-a-medical-refresh", + "checkedAt": "2026-08-07", + "requiredCategories": [ + "international_admissions_home", + "catalog_anchor", + "university_scholarship" + ], + "institutions": [ + { + "officialNameZh": "北京协和医学院", + "categories": [ + { + "category": "international_admissions_home", + "status": "officially_not_provided", + "officialUrl": null, + "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/", + "note": "重新核验北京协和医学院研究生招生官网、本科招生网、港澳台招生和国际合作栏目，未发现面向外国国籍学生的公开招生主页或个人申请入口；港澳台招生不等同于国际学生招生。", + "checkedAt": "2026-08-07" + }, + { + "category": "catalog_anchor", + "status": "officially_not_provided", + "officialUrl": null, + "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", + "note": "学校 2026 年硕士招生简章明确要求申请人为中华人民共和国公民；同年度博士和临床医学培养模式改革试点班简章也设置相同国籍条件，因此这些国内招生目录不能作为国际学生项目目录，官网未公开可核验的外国学生项目目录锚点。", + "checkedAt": "2026-08-07" + }, + { + "category": "university_scholarship", + "status": "officially_not_provided", + "officialUrl": null, + "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", + "note": "当前招生简章公布的是适用于中国公民招生路线的研究生奖助政策；原研究生奖助学金管理办法链接现已失效，在已核验的官方栏目中未发现面向外国学生的学校奖学金公开入口。", + "checkedAt": "2026-08-07" + } + ] + }, + { + "officialNameZh": "天津医科大学", + "categories": [ + { + "category": "international_admissions_home", + "status": "known", + "officialUrl": "https://www.imstmu.edu.cn/", + "evidenceUrl": "https://www.imstmu.edu.cn/", + "note": "天津医科大学国际医学院官方中英文网站现已通过 HTTPS 核验，网站页脚标注 International Medical School of Tianjin Medical University，并提供本科、研究生、短期项目和招生咨询入口。", + "checkedAt": "2026-08-07" + }, + { + "category": "catalog_anchor", + "status": "known", + "officialUrl": "https://www.imstmu.edu.cn/admission", + "evidenceUrl": "https://www.imstmu.edu.cn/archives/507", + "note": "国际医学院官方 Admission 索引汇总本科申请、2026 年研究生招生和短期项目；官方 Undergraduate Programs 页面列出专业、学制、授课语言与学位，可作为国际学生项目发现及逐项对账锚点。", + "checkedAt": "2026-08-07" + }, + { + "category": "university_scholarship", + "status": "officially_not_provided", + "officialUrl": null, + "evidenceUrl": "https://www.imstmu.edu.cn/archives/252", + "note": "已核验国际医学院招生索引、2026 年国际研究生招生简章及 2024 年费用与奖学金旧页；当前简章确认的是中国政府奖学金申请路线，未发现可作为当前事实发布的天津医科大学校级国际学生奖学金目录或当期指南。", + "checkedAt": "2026-08-07" + } + ] + } + ] +} diff --git a/content/source-registry/reconciliation/final-group-a.v1.json b/content/source-registry/reconciliation/final-group-a.v1.json index 25c5852..2bc0fea 100644 --- a/content/source-registry/reconciliation/final-group-a.v1.json +++ b/content/source-registry/reconciliation/final-group-a.v1.json @@ -9,35 +9,6 @@ "university_scholarship" ], "institutions": [ - { - "officialNameZh": "北京协和医学院", - "categories": [ - { - "category": "international_admissions_home", - "status": "officially_not_provided", - "officialUrl": null, - "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/", - "note": "核验北京协和医学院研究生招生官网及其硕士、博士、推免、港澳台招生栏目，未发现面向外国学生的公开招生主页或个人申请入口；港澳台招生不等同于国际学生招生。", - "checkedAt": "2026-07-26" - }, - { - "category": "catalog_anchor", - "status": "officially_not_provided", - "officialUrl": null, - "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", - "note": "已核验学校 2026 年硕士招生简章及招生目录入口；简章报考条件明确要求中华人民共和国公民，不能作为国际学生项目目录，官网未公开可核验的外国学生项目目录锚点。", - "checkedAt": "2026-07-26" - }, - { - "category": "university_scholarship", - "status": "officially_not_provided", - "officialUrl": null, - "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", - "note": "该官方简章包含国内全日制研究生奖助政策，但适用招生条件要求中华人民共和国公民；在已核验的官方招生栏目中未发现面向外国学生的学校奖学金公开入口。", - "checkedAt": "2026-07-26" - } - ] - }, { "officialNameZh": "外交学院", "categories": [ @@ -125,35 +96,6 @@ } ] }, - { - "officialNameZh": "天津医科大学", - "categories": [ - { - "category": "international_admissions_home", - "status": "source_unavailable", - "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "天津医科大学 2023—2024 年度官方信息公开报告确认国际医学院中英文网站存在，但报告仅登记 http://www.imstmu.edu.cn/；本次无法核验可用的官方 HTTPS 国际招生主页，因此不采用第三方 admissions.cn 页面替代。", - "checkedAt": "2026-07-26" - }, - { - "category": "catalog_anchor", - "status": "source_unavailable", - "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "官方信息公开报告将来华留学生信息指向仅提供 HTTP 的国际医学院站点；由于该来源无法通过官方 HTTPS 核验，无法确认其现行项目目录内容和更新状态。", - "checkedAt": "2026-07-26" - }, - { - "category": "university_scholarship", - "status": "source_unavailable", - "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "国际医学院来源当前无法通过官方 HTTPS 访问，故不能判断其是否公开国际学生奖学金；在来源恢复前保持 source_unavailable，不将国内学生奖助页面或第三方内容混入。", - "checkedAt": "2026-07-26" - } - ] - }, { "officialNameZh": "上海科技大学", "categories": [ diff --git a/docs/source-manifest-cohort-candidates.md b/docs/source-manifest-cohort-candidates.md index 2109eee..e4e53b8 100644 --- a/docs/source-manifest-cohort-candidates.md +++ b/docs/source-manifest-cohort-candidates.md @@ -3,18 +3,26 @@ ## Purpose This workflow turns the locked Ministry of Education Double First-Class target -registry and the current catalog JSON into a review queue for -`SourceManifestV2`. It does not discover new URLs, fetch websites, or publish -manifests. +registry, the current catalog JSON, and the validated official reconciliation +registry into a review queue for `SourceManifestV2`. It does not discover new +URLs, fetch websites, or publish manifests. The separation is intentional: -`official target registry + current catalog relationships -> candidate artifact -> evidence review -> formal manifest` +`official target registry + exact catalog relationships + validated reconciliation fallback -> candidate artifact -> evidence review -> formal manifest` The generator excludes the three military institutions, accepts only exact catalog relationships to sources already marked official and using HTTPS, and never fuzzy-matches a university or invents a missing source. +The reconciliation registry is used only when a school has no safe program or +scholarship catalog source. Matching uses the exact official Chinese name. A +verified category becomes parser-pending; `source_unavailable` and +`officially_not_provided` remain distinct. Each fallback is disabled, +robots-blocked, scoped as `limited_official_catalog`, and contains a synthetic +pending institution-level audit entry that is explicitly not a publishable +program. + ## Local use Inspect current coverage without writing: @@ -47,8 +55,9 @@ Every bundle contains: - `manifests/*.v2.candidate.json`: disabled, review-only candidates; - `gap-report.v1.json`: per-institution missing mappings, rejected sources, and uncovered source categories; -- `artifact-manifest.v1.json`: exact SHA-256 and byte length for all six - locked inputs and every generated JSON file; +- `artifact-manifest.v1.json`: exact SHA-256 and byte length for the six locked + catalog inputs, every `content/source-registry/reconciliation/*.v1.json` + input, and every generated JSON file; - `SHA256SUMS`: independent checksums for the artifact manifest, gap report, and every candidate. @@ -60,8 +69,9 @@ source remains disabled, robots-blocked, and pending review. `Build Source Manifest Candidate Cohort` is a manual, read-only workflow. The operator supplies an explicit evidence check date. It builds only under -`runner.temp`, validates the official target registry, and validates the -relevant current-catalog relationships while building each disabled candidate. +`runner.temp`, validates the official target and reconciliation registries, +and validates relevant current-catalog relationships while building each +disabled candidate. It then verifies the completed bundle, proves that `content/source-manifests` did not change, and uploads the result as a short-lived review artifact. diff --git a/package.json b/package.json index c02c24d..2e774f3 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "pipeline:build-source-cohorts": "tsx scripts/ingestion/build-official-source-cohort-import.ts", "pipeline:build-source-manifest-candidates": "tsx scripts/ingestion/build-source-manifest-cohort.ts", "pipeline:verify-source-manifest-candidates": "tsx scripts/ingestion/build-source-manifest-cohort.ts --verify-artifact", + "pipeline:promote-source-manifest-candidate": "tsx scripts/ingestion/promote-source-manifest-candidate.ts", "pipeline:build-sources": "tsx scripts/ingestion/build-source-import.ts", "benchmark:catalog": "tsx scripts/catalog/benchmark-catalog.ts", "benchmark:catalog:smoke": "tsx scripts/catalog/benchmark-catalog.ts --institutions 25 --programs 1000 --cycles 3000 --iterations 50 --warmup 10 --output .benchmark/catalog-performance-smoke.json", diff --git a/scripts/ingestion/build-source-manifest-cohort.ts b/scripts/ingestion/build-source-manifest-cohort.ts index 2a61b70..dc5b68b 100644 --- a/scripts/ingestion/build-source-manifest-cohort.ts +++ b/scripts/ingestion/build-source-manifest-cohort.ts @@ -24,6 +24,12 @@ import { validateDoubleFirstClassRegistry, type DoubleFirstClassRegistry, } from './double-first-class-registry' +import { + loadSourceReconciliations, + type ReconciledInstitution, + type ReconciledSourceCategory, +} from './source-reconciliation' + export const CURRENT_SOURCE_MANIFEST_COHORT_INPUTS = { registry: 'content/source-manifests/double-first-class/targets.v1.json', universities: 'content/data/universities.json', @@ -33,10 +39,17 @@ export const CURRENT_SOURCE_MANIFEST_COHORT_INPUTS = { scholarships: 'content/data/scholarships.json', } as const -export type SourceManifestCohortInputName = +export const SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY = + 'content/source-registry/reconciliation' + +type SourceManifestCohortCatalogInputName = keyof typeof CURRENT_SOURCE_MANIFEST_COHORT_INPUTS -const SOURCE_MANIFEST_COHORT_INPUT_ORDER: SourceManifestCohortInputName[] = [ +export type SourceManifestCohortInputName = + | SourceManifestCohortCatalogInputName + | `sourceReconciliation:${string}` + +const SOURCE_MANIFEST_COHORT_INPUT_ORDER: SourceManifestCohortCatalogInputName[] = [ 'registry', 'universities', 'sources', @@ -160,6 +173,7 @@ export type BuildSourceManifestCohortInput = { sources: CatalogSourceInput[] programs: CatalogProgramInput[] admissionCycles: CatalogAdmissionCycleInput[] + sourceReconciliations: ReconciledInstitution[] scholarships: CatalogScholarshipInput[] checkedAt: string } @@ -209,6 +223,8 @@ export type SourceManifestCohortGapReport = { officialTargets: number militaryExcluded: number eligibleTargets: number + catalogLinkedManifests: number + reconciliationFallbackManifests: number candidateManifests: number exactOfficialHttpsSources: number targetsWithoutCandidate: number @@ -381,6 +397,134 @@ function addReconciliationCandidate( candidates.set(mapKey, existing) } +function exactReconciliationSourceUrl( + category: ReconciledSourceCategory, +): { url: string; host: string } { + const sourceUrl = category.status === 'verified_official' + ? category.officialUrl + : category.evidenceUrl + if (!sourceUrl) { + throw new Error( + `${category.sourceCategory} verified reconciliation is missing officialUrl`, + ) + } + let parsed: URL + try { + parsed = new URL(sourceUrl) + } catch { + throw new Error(`${category.sourceCategory} reconciliation URL is invalid`) + } + if ( + parsed.protocol !== 'https:' + || parsed.username + || parsed.password + || parsed.port + || !parsed.hostname + ) { + throw new Error( + `${category.sourceCategory} reconciliation URL must be credential-free HTTPS`, + ) + } + return { url: sourceUrl, host: parsed.hostname.toLowerCase() } +} + +function reconciliationFallbackSourceId( + institutionId: string, + sourceCategory: ReconciledSourceCategory['sourceCategory'], +): string { + return manifestSourceId( + `source-reconciliation-${sourceCategory.replaceAll('_', '-')}`, + institutionId, + ) +} + +function buildReconciliationFallbackManifest( + institutionId: string, + officialNameZh: string, + checkedAt: string, + reconciliation: ReconciledInstitution, +): SourceManifestV2 { + const sources = reconciliation.categories.map((category): SourceManifestV1 => { + const exact = exactReconciliationSourceUrl(category) + return { + version: 1, + id: reconciliationFallbackSourceId(institutionId, category.sourceCategory), + institutionId, + entityType: 'program', + sourceCategory: category.sourceCategory, + officialUrl: exact.url, + allowedHosts: [exact.host], + enabled: false, + schedule: { + intervalHours: category.sourceCategory.includes('scholarship') ? 168 : 720, + }, + fetch: {}, + robots: { mode: 'blocked' }, + extraction: { + mode: 'rules-only', + schemaVersion: 'source-manifest-v2-reconciliation-fallback-v1', + fields: [{ path: 'candidateEvidence', type: 'object' }], + }, + } + }) + const sourceByCategory = new Map( + sources.map((source) => [source.sourceCategory, source]), + ) + const reconciliationByCategory = new Map< + SourceCategory, + ReconciledSourceCategory + >( + reconciliation.categories.map((category) => [category.sourceCategory, category]), + ) + const coverage: SourceManifestV2['coverage'] = SOURCE_CATEGORIES.map((sourceCategory) => { + const category = reconciliationByCategory.get(sourceCategory) + const source = sourceByCategory.get(sourceCategory) + if (!category || !source) { + return { + sourceCategory, + status: 'discovery_pending' as const, + note: 'This category is outside the validated three-category reconciliation fallback and still requires official-source discovery.', + } + } + const status = category.status === 'verified_official' + ? 'parser_pending' as const + : category.status + return { + sourceCategory, + status, + sourceIds: [source.id], + note: `Validated official reconciliation recorded ${category.status} on ${category.checkedAt}. The exact audit source remains disabled. ${category.note}`, + } + }) + const auditSource = sourceByCategory.get('catalog_anchor') ?? sources[0] + if (!auditSource) { + throw new Error(`No reconciliation fallback source for ${officialNameZh}`) + } + + return sourceManifestV2Schema.parse({ + version: 2, + institutionId, + catalogStatus: 'existing', + manifestStatus: 'in_progress', + checkedAt, + officialHosts: [...new Set(sources.flatMap((source) => source.allowedHosts))].sort(), + sources, + coverage, + catalogReconciliation: { + scope: 'limited_official_catalog', + status: 'in_progress', + entries: [{ + sourceId: auditSource.id, + officialKey: `audit-only:${institutionId}:international-catalog`, + officialName: `AUDIT ONLY - ${officialNameZh} institution-level international catalog`, + entityType: 'program', + status: 'pending', + note: 'Synthetic institution-level catalog audit placeholder; this is not a publishable program and must never be materialized as one.', + }], + note: 'Fallback candidate generated only from the exact validated official reconciliation registry. It is limited, disabled, pending review, and does not assert that a publishable international program exists.', + }, + }) +} function fileNameForTarget(target: InstitutionTarget, institutionId: string): string { return `${String(target.ordinal).padStart(3, '0')}-${institutionId.replace(/^uni-/, '')}.v2.candidate.json` } @@ -404,6 +548,20 @@ export function buildSourceManifestCohort( typeof program.id === 'string' ? [[program.id, program] as const] : [] )), ) + const sourceReconciliationByName = new Map() + for (const reconciliation of input.sourceReconciliations) { + if (sourceReconciliationByName.has(reconciliation.institutionNameZh)) { + throw new Error( + `Duplicate reconciled institution ${reconciliation.institutionNameZh}`, + ) + } + sourceReconciliationByName.set( + reconciliation.institutionNameZh, + reconciliation, + ) + } + let catalogLinkedManifests = 0 + let reconciliationFallbackManifests = 0 const candidates: CandidateManifestFile[] = [] const gaps: CohortGap[] = [] const militaryExclusions: SourceManifestCohortGapReport['militaryExclusions'] = [] @@ -528,6 +686,43 @@ export function buildSourceManifestCohort( status: 'pending' as const, }] }) + if (reconciliationEntries.length === 0) { + const officialReconciliation = sourceReconciliationByName.get( + target.officialNameZh, + ) + if (officialReconciliation) { + const manifest = buildReconciliationFallbackManifest( + institutionId, + target.officialNameZh, + input.checkedAt, + officialReconciliation, + ) + const mappedCategories = manifest.sources.map( + (source) => source.sourceCategory, + ) + institutionCoverage.push({ + targetId: target.targetId, + ordinal: target.ordinal, + officialNameZh: target.officialNameZh, + institutionId, + mappedSourceCount: manifest.sources.length, + reconciliationEntryCount: manifest.catalogReconciliation.entries.length, + mappedCategories, + discoveryPendingCategories: SOURCE_CATEGORIES.filter( + (category) => !mappedCategories.includes(category), + ), + rejectedSources: rejectedSources.sort( + (left, right) => left.sourceId.localeCompare(right.sourceId), + ), + }) + candidates.push({ + fileName: fileNameForTarget(target, institutionId), + manifest, + }) + reconciliationFallbackManifests += 1 + continue + } + } const sourcesByCategory = new Map() for (const source of manifestSources) { const ids = sourcesByCategory.get(source.sourceCategory) ?? [] @@ -592,6 +787,7 @@ export function buildSourceManifestCohort( fileName: fileNameForTarget(target, institutionId), manifest, }) + catalogLinkedManifests += 1 } const summary = { @@ -599,6 +795,8 @@ export function buildSourceManifestCohort( militaryExcluded: militaryExclusions.length, eligibleTargets: input.registry.targets.length - militaryExclusions.length, candidateManifests: candidates.length, + catalogLinkedManifests, + reconciliationFallbackManifests, exactOfficialHttpsSources: candidates.reduce( (total, candidate) => total + candidate.manifest.sources.length, 0, @@ -611,7 +809,7 @@ export function buildSourceManifestCohort( cohortId: input.registry.cohort.id, checkedAt: input.checkedAt, policy: { - mapping: 'Only exact sourceIds already related by current university, program, admission-cycle, or scholarship records are eligible; publisher and institution names are never fuzzy-matched.', + mapping: 'Exact current catalog relationships are preferred. Only when no safe program or scholarship source exists may an exact-name record from the validated official reconciliation registry create a disabled audit-only fallback; fuzzy matching remains forbidden.', missingCoverage: 'Unmapped categories are discovery_pending with an explicit note.', officialAbsence: 'officially_not_provided is never inferred; it requires separate explicit official evidence.', }, @@ -633,7 +831,7 @@ function serializeJson(value: unknown): string { function readFingerprintedJson( repositoryRoot: string, - name: SourceManifestCohortInputName, + name: SourceManifestCohortCatalogInputName, ): { value: T; fingerprint: SourceManifestCohortInputFingerprint } { const repositoryPath = CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name] const bytes = readFileSync(resolve(repositoryRoot, repositoryPath)) @@ -647,6 +845,47 @@ function readFingerprintedJson( }, } } +function readSourceReconciliationInputs( + repositoryRoot: string, +): { + value: ReconciledInstitution[] + fingerprints: SourceManifestCohortInputFingerprint[] +} { + const directory = resolve( + repositoryRoot, + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, + ) + const entries = readdirSync(directory, { withFileTypes: true }) + .filter((entry) => entry.name.endsWith('.v1.json')) + .sort((left, right) => left.name.localeCompare(right.name, 'en')) + if (entries.length === 0) { + throw new Error('Validated official reconciliation registry is empty') + } + const unsupported = entries.find((entry) => !entry.isFile()) + if (unsupported) { + throw new Error( + `Reconciliation input must be a regular file: ${unsupported.name}`, + ) + } + const fingerprints = entries.map((entry): SourceManifestCohortInputFingerprint => { + const repositoryPath = posix.join( + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, + entry.name, + ) + const bytes = readFileSync(resolve(repositoryRoot, repositoryPath)) + return { + name: `sourceReconciliation:${entry.name}`, + repositoryPath, + sha256: sha256(bytes), + byteLength: bytes.byteLength, + } + }) + return { + value: loadSourceReconciliations(directory), + fingerprints, + } +} + export function buildCurrentSourceManifestCohort( checkedAt: string, @@ -670,6 +909,9 @@ export function buildCurrentSourceManifestCohort( repositoryRoot, 'scholarships', ) + const sourceReconciliations = readSourceReconciliationInputs( + repositoryRoot, + ) const build = buildSourceManifestCohort({ registry: validateDoubleFirstClassRegistry(registry.value) as DoubleFirstClassRegistry, universities: universities.value, @@ -677,6 +919,7 @@ export function buildCurrentSourceManifestCohort( programs: programs.value, admissionCycles: admissionCycles.value, scholarships: scholarships.value, + sourceReconciliations: sourceReconciliations.value, checkedAt, }) return { @@ -688,6 +931,7 @@ export function buildCurrentSourceManifestCohort( programs.fingerprint, admissionCycles.fingerprint, scholarships.fingerprint, + ...sourceReconciliations.fingerprints, ], } } @@ -741,25 +985,57 @@ function sortedInputFingerprints( inputFingerprints: SourceManifestCohortInputFingerprint[], ): SourceManifestCohortInputFingerprint[] { const byName = new Map(inputFingerprints.map((input) => [input.name, input])) - if ( - inputFingerprints.length !== SOURCE_MANIFEST_COHORT_INPUT_ORDER.length - || byName.size !== SOURCE_MANIFEST_COHORT_INPUT_ORDER.length - ) { - throw new Error('Candidate artifact requires one fingerprint for every locked input') + if (byName.size !== inputFingerprints.length) { + throw new Error('Candidate artifact input fingerprints contain duplicate names') } - return SOURCE_MANIFEST_COHORT_INPUT_ORDER.map((name) => { + const validFingerprint = (input: SourceManifestCohortInputFingerprint): boolean => ( + /^[a-f0-9]{64}$/u.test(input.sha256) + && Number.isSafeInteger(input.byteLength) + && input.byteLength > 0 + ) + const lockedInputs = SOURCE_MANIFEST_COHORT_INPUT_ORDER.map((name) => { const input = byName.get(name) if ( !input || input.repositoryPath !== CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name] - || !/^[a-f0-9]{64}$/u.test(input.sha256) - || !Number.isSafeInteger(input.byteLength) - || input.byteLength <= 0 + || !validFingerprint(input) ) { throw new Error('Invalid locked input fingerprint: ' + name) } return input }) + const lockedNames = new Set(SOURCE_MANIFEST_COHORT_INPUT_ORDER) + const unexpected = inputFingerprints.find((input) => ( + !lockedNames.has(input.name) + && !input.name.startsWith('sourceReconciliation:') + )) + if (unexpected) { + throw new Error('Unexpected candidate artifact input: ' + unexpected.name) + } + const reconciliationInputs = inputFingerprints + .filter((input) => input.name.startsWith('sourceReconciliation:')) + .sort((left, right) => left.name.localeCompare(right.name, 'en')) + if (reconciliationInputs.length === 0) { + throw new Error('Candidate artifact requires reconciliation input fingerprints') + } + const repositoryPaths = new Set(lockedInputs.map((input) => input.repositoryPath)) + for (const input of reconciliationInputs) { + const fileName = input.name.slice('sourceReconciliation:'.length) + const expectedPath = posix.join( + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, + fileName, + ) + if ( + !/^[a-z0-9][a-z0-9.-]*\.v1\.json$/u.test(fileName) + || input.repositoryPath !== expectedPath + || !validFingerprint(input) + || repositoryPaths.has(input.repositoryPath) + ) { + throw new Error('Invalid reconciliation input fingerprint: ' + input.name) + } + repositoryPaths.add(input.repositoryPath) + } + return [...lockedInputs, ...reconciliationInputs] } function assertSafeArtifactRelativePath(path: string): void { @@ -989,7 +1265,7 @@ export function writeSourceManifestCohort( summary: build.summary, policy: { sourceOfTruth: - 'Read-only current catalog JSON and the locked Ministry of Education target registry.', + 'Read-only current catalog JSON, the locked Ministry of Education target registry, and the validated official reconciliation registry.', publication: 'This bundle is review-only and must never be copied directly into content/source-manifests.', network: diff --git a/scripts/ingestion/promote-source-manifest-candidate.ts b/scripts/ingestion/promote-source-manifest-candidate.ts new file mode 100644 index 0000000..185993b --- /dev/null +++ b/scripts/ingestion/promote-source-manifest-candidate.ts @@ -0,0 +1,418 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdirSync, + readFileSync, + realpathSync, + writeFileSync, +} from "node:fs"; +import { + dirname, + isAbsolute, + join, + posix, + relative, + resolve, + sep, +} from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import { + isCatalogReconciliationComplete, + loadSourceManifestFiles, + sourceManifestV2Schema, + validateSourceManifests, + type SourceManifestV2, +} from "../source-manifest-registry"; +import { + verifySourceManifestCohortArtifact, + type SourceManifestCohortArtifactManifest, +} from "./build-source-manifest-cohort"; + +const candidatePathSchema = z + .string() + .regex(/^manifests\/\d{3}-[a-z0-9-]+\.v2\.candidate\.json$/u); +const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/u); +function isRealUtcCalendarDate(value: string): boolean { + const match = /^(\d{4})-(\d{2})-(\d{2})$/u.exec(value); + if (!match) return false; + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const date = new Date(Date.UTC(year, month - 1, day)); + return ( + date.getUTCFullYear() === year && + date.getUTCMonth() === month - 1 && + date.getUTCDate() === day + ); +} + +const checkedAtSchema = z + .string() + .regex(/^\d{4}-\d{2}-\d{2}$/u) + .refine(isRealUtcCalendarDate, { + message: "reviewedAt must be a real ISO calendar date", + }); + +export const sourceManifestPromotionReviewSchema = z + .object({ + format: z.literal("studyinchina.source-manifest-v2-promotion-review"), + formatVersion: z.literal(1), + decision: z.literal("approve_complete_manifest"), + reviewedAt: checkedAtSchema, + reviewer: z.string().trim().min(3).max(200), + rationale: z.string().trim().min(20).max(4_000), + artifact: z + .object({ + cohortId: z.string().min(1), + candidatePath: candidatePathSchema, + candidateSha256: sha256Schema, + }) + .strict(), + manifest: sourceManifestV2Schema, + }) + .strict(); + +export type SourceManifestPromotionReview = z.infer< + typeof sourceManifestPromotionReviewSchema +>; + +export type PromoteSourceManifestCandidateOptions = { + artifactDirectory: string; + reviewDecisionPath: string; + repositoryRoot?: string; + write?: boolean; +}; + +export type SourceManifestPromotionResult = { + mode: "dry-run" | "write"; + cohortId: string; + candidatePath: string; + candidateSha256: string; + reviewDecisionSha256: string; + institutionId: string; + destinationPath: string; + manifestSha256: string; + sources: number; + reconciliationEntries: number; + qualityGate: { + catalogReconciliationComplete: true; + pendingEntries: 0; + discoveryPendingCoverage: 0; + auditOnlyMarkers: 0; + }; +}; + +type PromotionCli = { + artifactDirectory: string; + reviewDecisionPath: string; + write: boolean; +}; + +const CLI_USAGE = [ + "Usage:", + " --artifact --review-decision [--write]", + "", + "Without --write the command performs a dry-run and does not create a formal manifest.", +].join("\n"); + +function sha256(value: string | Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function serializeJson(value: unknown): string { + return JSON.stringify(value, null, 2) + "\n"; +} + +function isInside(parent: string, child: string): boolean { + const path = relative(parent, child); + return ( + path === "" || + (path !== ".." && !path.startsWith(".." + sep) && !isAbsolute(path)) + ); +} + +function resolveThroughExistingAncestor(path: string): string { + let ancestor = resolve(path); + const suffix: string[] = []; + while (!existsSync(ancestor)) { + const parent = dirname(ancestor); + if (parent === ancestor) { + throw new Error("Unable to resolve destination ancestor for " + path); + } + suffix.unshift(relative(parent, ancestor)); + ancestor = parent; + } + return resolve(realpathSync(ancestor), ...suffix); +} + +function readRegularFile(path: string, label: string): Buffer { + if (!existsSync(path)) throw new Error(label + " does not exist"); + const stats = lstatSync(path); + if (stats.isSymbolicLink() || !stats.isFile()) { + throw new Error(label + " must be a regular, non-symbolic-link file"); + } + return readFileSync(path); +} + +function readArtifactManifest( + artifactDirectory: string, +): SourceManifestCohortArtifactManifest { + return JSON.parse( + readFileSync(join(artifactDirectory, "artifact-manifest.v1.json"), "utf8"), + ) as SourceManifestCohortArtifactManifest; +} + +function containsAuditOnlyMarker(value: unknown): boolean { + if (typeof value === "string") return /audit[\s_-]*only/iu.test(value); + if (Array.isArray(value)) return value.some(containsAuditOnlyMarker); + if (typeof value === "object" && value !== null) { + return Object.values(value).some(containsAuditOnlyMarker); + } + return false; +} + +function assertNoExistingInstitution( + manifestDirectory: string, + institutionId: string, +): void { + for (const existing of loadSourceManifestFiles(manifestDirectory)) { + if ( + typeof existing.value === "object" && + existing.value !== null && + "institutionId" in existing.value && + existing.value.institutionId === institutionId + ) { + throw new Error( + `Formal manifest already exists for institutionId ${institutionId}: ${existing.filePath}`, + ); + } + } +} + +function formalDestination( + repositoryRoot: string, + candidatePath: string, +): { directory: string; path: string } { + if (!candidatePathSchema.safeParse(candidatePath).success) { + throw new Error("Unsafe candidate path: " + candidatePath); + } + const formalRoot = resolve(repositoryRoot, "content/source-manifests"); + if (!existsSync(formalRoot) || lstatSync(formalRoot).isSymbolicLink()) { + throw new Error("Formal manifest root must be a real directory"); + } + const candidateName = posix.basename(candidatePath); + const formalName = candidateName.replace(".v2.candidate.json", ".v2.json"); + const directory = resolve(formalRoot, "double-first-class/institutions"); + const path = resolve(directory, formalName); + const effectiveRoot = realpathSync(formalRoot); + const effectiveDestination = resolveThroughExistingAncestor(path); + if ( + !isInside(formalRoot, path) || + !isInside(effectiveRoot, effectiveDestination) + ) { + throw new Error( + "Formal manifest destination escapes content/source-manifests", + ); + } + if (existsSync(path)) { + throw new Error("Refusing to overwrite existing formal manifest: " + path); + } + return { directory, path }; +} + +function assertReviewLineage( + review: SourceManifestPromotionReview, + candidate: SourceManifestV2, +): void { + if (review.manifest.institutionId !== candidate.institutionId) { + throw new Error( + "Reviewed manifest institutionId does not match the candidate", + ); + } + if (review.manifest.catalogStatus !== candidate.catalogStatus) { + throw new Error( + "Reviewed manifest catalogStatus does not match the candidate", + ); + } + if (review.manifest.checkedAt < candidate.checkedAt) { + throw new Error("Reviewed manifest checkedAt predates the candidate"); + } + if (review.reviewedAt < review.manifest.checkedAt) { + throw new Error("Review decision predates the reviewed manifest"); + } +} + +function assertCompleteReviewedManifest(manifest: SourceManifestV2): void { + if (!isCatalogReconciliationComplete(manifest)) { + throw new Error( + "Promotion requires a complete manifest and complete catalog reconciliation", + ); + } + if ( + manifest.catalogReconciliation.entries.some( + (entry) => entry.status === "pending", + ) + ) { + throw new Error("Promotion refuses pending catalog reconciliation entries"); + } + if ( + manifest.coverage.some( + (coverage) => coverage.status === "discovery_pending", + ) + ) { + throw new Error("Promotion refuses discovery_pending coverage"); + } +} + +export function promoteSourceManifestCandidate( + options: PromoteSourceManifestCandidateOptions, +): SourceManifestPromotionResult { + const repositoryRoot = resolve(options.repositoryRoot ?? "."); + const artifactDirectory = resolve(options.artifactDirectory); + const reviewDecisionPath = resolve(options.reviewDecisionPath); + + // This verifies the artifact manifest, SHA256SUMS, every described file, + // candidate schema, disabled source state, and exact file inventory first. + const artifactVerification = + verifySourceManifestCohortArtifact(artifactDirectory); + const artifactManifest = readArtifactManifest(artifactDirectory); + const reviewBytes = readRegularFile(reviewDecisionPath, "Review decision"); + const review = sourceManifestPromotionReviewSchema.parse( + JSON.parse(reviewBytes.toString("utf8")), + ); + + if (review.artifact.cohortId !== artifactVerification.cohortId) { + throw new Error( + "Review decision cohortId does not match the verified artifact", + ); + } + const describedCandidate = artifactManifest.files.find( + (file) => file.path === review.artifact.candidatePath, + ); + if (!describedCandidate) { + throw new Error( + "Review decision candidatePath is absent from the verified artifact", + ); + } + const candidateBytes = readRegularFile( + join(artifactDirectory, ...review.artifact.candidatePath.split("/")), + "Candidate manifest", + ); + const candidateSha256 = sha256(candidateBytes); + if ( + review.artifact.candidateSha256 !== candidateSha256 || + describedCandidate.sha256 !== candidateSha256 + ) { + throw new Error( + "Review decision candidateSha256 does not exactly match the artifact", + ); + } + const candidate = sourceManifestV2Schema.parse( + JSON.parse(candidateBytes.toString("utf8")), + ); + if (containsAuditOnlyMarker(candidate)) { + throw new Error( + "AUDIT ONLY and audit-only synthetic candidates cannot be promoted", + ); + } + if (containsAuditOnlyMarker(review.manifest)) { + throw new Error( + "AUDIT ONLY and audit-only markers are forbidden in formal manifests", + ); + } + + assertReviewLineage(review, candidate); + assertCompleteReviewedManifest(review.manifest); + + const destination = formalDestination( + repositoryRoot, + review.artifact.candidatePath, + ); + const formalRoot = resolve(repositoryRoot, "content/source-manifests"); + assertNoExistingInstitution(formalRoot, review.manifest.institutionId); + validateSourceManifests( + [ + ...loadSourceManifestFiles(formalRoot), + { filePath: destination.path, value: review.manifest }, + ], + resolve(repositoryRoot, "content/data/universities.json"), + ); + + const manifestBody = serializeJson(review.manifest); + if (options.write === true) { + mkdirSync(destination.directory, { recursive: true }); + writeFileSync(destination.path, manifestBody, { + encoding: "utf8", + flag: "wx", + }); + } + + return { + mode: options.write === true ? "write" : "dry-run", + cohortId: artifactVerification.cohortId, + candidatePath: review.artifact.candidatePath, + candidateSha256, + reviewDecisionSha256: sha256(reviewBytes), + institutionId: review.manifest.institutionId, + destinationPath: destination.path, + manifestSha256: sha256(manifestBody), + sources: review.manifest.sources.length, + reconciliationEntries: review.manifest.catalogReconciliation.entries.length, + qualityGate: { + catalogReconciliationComplete: true, + pendingEntries: 0, + discoveryPendingCoverage: 0, + auditOnlyMarkers: 0, + }, + }; +} + +export function parseSourceManifestPromotionCli(argv: string[]): PromotionCli { + let artifactDirectory: string | undefined; + let reviewDecisionPath: string | undefined; + let write = false; + const seen = new Set(); + + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]!; + if (seen.has(argument)) { + throw new Error("Duplicate CLI option: " + argument + "\n" + CLI_USAGE); + } + seen.add(argument); + if (argument === "--write") { + write = true; + continue; + } + if (argument !== "--artifact" && argument !== "--review-decision") { + throw new Error("Unknown CLI option: " + argument + "\n" + CLI_USAGE); + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error("Missing value for " + argument + "\n" + CLI_USAGE); + } + index += 1; + if (argument === "--artifact") artifactDirectory = value; + if (argument === "--review-decision") reviewDecisionPath = value; + } + + if (!artifactDirectory || !reviewDecisionPath) throw new Error(CLI_USAGE); + return { artifactDirectory, reviewDecisionPath, write }; +} + +function runCli(): void { + const cli = parseSourceManifestPromotionCli(process.argv.slice(2)); + process.stdout.write( + JSON.stringify(promoteSourceManifestCandidate(cli)) + "\n", + ); +} + +if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) { + try { + runCli(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/src/data/generated/double-first-class-coverage.json b/src/data/generated/double-first-class-coverage.json index da0c631..4f9a204 100644 --- a/src/data/generated/double-first-class-coverage.json +++ b/src/data/generated/double-first-class-coverage.json @@ -1,7 +1,7 @@ { "format": "studyinchina.public-double-first-class-coverage", "formatVersion": 2, - "generatedAt": "2026-07-26T00:00:00.000Z", + "generatedAt": "2026-08-07T00:00:00.000Z", "officialRegistry": { "titleZh": "第二轮“双一流”建设高校及建设学科名单", "pageUrl": "https://www.moe.gov.cn/srcsite/A22/s7065/202202/t20220211_598710.html", @@ -18,7 +18,7 @@ "sourceManifestComplete": 135, "reconciledLimited": 9, "collecting": 0, - "verifiedOfficialSources": 416 + "verifiedOfficialSources": 418 }, "institutions": [ { @@ -666,7 +666,7 @@ "region": null, "province": null, "status": "reconciled_limited", - "checkedAt": "2026-07-26", + "checkedAt": "2026-08-07", "sourceCount": 0, "reconciliationCount": 3, "sources": [], @@ -676,21 +676,21 @@ "status": "officially_not_provided", "officialUrl": null, "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/", - "note": "核验北京协和医学院研究生招生官网及其硕士、博士、推免、港澳台招生栏目，未发现面向外国学生的公开招生主页或个人申请入口；港澳台招生不等同于国际学生招生。" + "note": "重新核验北京协和医学院研究生招生官网、本科招生网、港澳台招生和国际合作栏目，未发现面向外国国籍学生的公开招生主页或个人申请入口；港澳台招生不等同于国际学生招生。" }, { "category": "catalog_anchor", "status": "officially_not_provided", "officialUrl": null, "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", - "note": "已核验学校 2026 年硕士招生简章及招生目录入口；简章报考条件明确要求中华人民共和国公民，不能作为国际学生项目目录，官网未公开可核验的外国学生项目目录锚点。" + "note": "学校 2026 年硕士招生简章明确要求申请人为中华人民共和国公民；同年度博士和临床医学培养模式改革试点班简章也设置相同国籍条件，因此这些国内招生目录不能作为国际学生项目目录，官网未公开可核验的外国学生项目目录锚点。" }, { "category": "university_scholarship", "status": "officially_not_provided", "officialUrl": null, "evidenceUrl": "https://graduate.pumc.edu.cn/zsw/info/1003/2117.htm", - "note": "该官方简章包含国内全日制研究生奖助政策，但适用招生条件要求中华人民共和国公民；在已核验的官方招生栏目中未发现面向外国学生的学校奖学金公开入口。" + "note": "当前招生简章公布的是适用于中国公民招生路线的研究生奖助政策；原研究生奖助学金管理办法链接现已失效，在已核验的官方栏目中未发现面向外国学生的学校奖学金公开入口。" } ] }, @@ -1690,31 +1690,42 @@ "region": null, "province": null, "status": "reconciled_limited", - "checkedAt": "2026-07-26", - "sourceCount": 0, + "checkedAt": "2026-08-07", + "sourceCount": 2, "reconciliationCount": 3, - "sources": [], + "sources": [ + { + "category": "international_admissions_home", + "officialUrl": "https://www.imstmu.edu.cn/", + "verificationMethod": "天津医科大学国际医学院官方中英文网站现已通过 HTTPS 核验，网站页脚标注 International Medical School of Tianjin Medical University，并提供本科、研究生、短期项目和招生咨询入口。" + }, + { + "category": "catalog_anchor", + "officialUrl": "https://www.imstmu.edu.cn/admission", + "verificationMethod": "国际医学院官方 Admission 索引汇总本科申请、2026 年研究生招生和短期项目；官方 Undergraduate Programs 页面列出专业、学制、授课语言与学位，可作为国际学生项目发现及逐项对账锚点。" + } + ], "categories": [ { "category": "international_admissions_home", - "status": "source_unavailable", - "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "天津医科大学 2023—2024 年度官方信息公开报告确认国际医学院中英文网站存在，但报告仅登记 http://www.imstmu.edu.cn/；本次无法核验可用的官方 HTTPS 国际招生主页，因此不采用第三方 admissions.cn 页面替代。" + "status": "verified_official", + "officialUrl": "https://www.imstmu.edu.cn/", + "evidenceUrl": "https://www.imstmu.edu.cn/", + "note": "天津医科大学国际医学院官方中英文网站现已通过 HTTPS 核验，网站页脚标注 International Medical School of Tianjin Medical University，并提供本科、研究生、短期项目和招生咨询入口。" }, { "category": "catalog_anchor", - "status": "source_unavailable", - "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "官方信息公开报告将来华留学生信息指向仅提供 HTTP 的国际医学院站点；由于该来源无法通过官方 HTTPS 核验，无法确认其现行项目目录内容和更新状态。" + "status": "verified_official", + "officialUrl": "https://www.imstmu.edu.cn/admission", + "evidenceUrl": "https://www.imstmu.edu.cn/archives/507", + "note": "国际医学院官方 Admission 索引汇总本科申请、2026 年研究生招生和短期项目；官方 Undergraduate Programs 页面列出专业、学制、授课语言与学位，可作为国际学生项目发现及逐项对账锚点。" }, { "category": "university_scholarship", - "status": "source_unavailable", + "status": "officially_not_provided", "officialUrl": null, - "evidenceUrl": "https://www.tmu.edu.cn/_upload/article/files/78/90/ca47f6744687a6a3533cdf276596/b972f55d-643a-4245-8207-e5a76a24ec16.pdf", - "note": "国际医学院来源当前无法通过官方 HTTPS 访问，故不能判断其是否公开国际学生奖学金；在来源恢复前保持 source_unavailable，不将国内学生奖助页面或第三方内容混入。" + "evidenceUrl": "https://www.imstmu.edu.cn/archives/252", + "note": "已核验国际医学院招生索引、2026 年国际研究生招生简章及 2024 年费用与奖学金旧页；当前简章确认的是中国政府奖学金申请路线，未发现可作为当前事实发布的天津医科大学校级国际学生奖学金目录或当期指南。" } ] }, diff --git a/src/lib/scholarship-catalog.ts b/src/lib/scholarship-catalog.ts index 554c669..89bd360 100644 --- a/src/lib/scholarship-catalog.ts +++ b/src/lib/scholarship-catalog.ts @@ -395,6 +395,19 @@ export function scholarshipCatalogHref( page = filters.page, ): string { const params = new URLSearchParams() + let targetCursor = filters.cursor + let targetHistory = [...filters.cursorHistory] + if (page === filters.page + 1 && filters.nextCursor) { + targetHistory.push(filters.cursor || '~') + targetCursor = filters.nextCursor + } else if (page === filters.page - 1) { + const previous = targetHistory.pop() + targetCursor = previous && previous !== '~' ? previous : '' + } else if (page !== filters.page) { + targetCursor = '' + targetHistory = [] + } + const values: Array<[string, string]> = [ ['q', filters.query], ['institution', filters.institution], @@ -404,6 +417,8 @@ export function scholarshipCatalogHref( ['sort', filters.sort === 'default' ? '' : filters.sort], ] for (const [key, value] of values) if (value) params.set(key, value) + if (targetCursor) params.set('cursor', targetCursor) + if (targetHistory.length > 0) params.set('cursorHistory', targetHistory.join(',')) if (page > 1) params.set('page', String(page)) const query = params.toString() return `/${locale}/scholarships${query ? `?${query}` : ''}` diff --git a/tests/unit/catalog-identity-program-visibility.test.ts b/tests/unit/catalog-identity-program-visibility.test.ts index f91002e..2ccdcd1 100644 --- a/tests/unit/catalog-identity-program-visibility.test.ts +++ b/tests/unit/catalog-identity-program-visibility.test.ts @@ -64,5 +64,5 @@ describe('Catalog identity-only program visibility', () => { `).get(program.release_id, program.program_id)!.count).toBeGreaterThan(0) database.close() - }) + }, 30_000) }) diff --git a/tests/unit/catalog-release-builder.test.ts b/tests/unit/catalog-release-builder.test.ts index e613b36..66addb4 100644 --- a/tests/unit/catalog-release-builder.test.ts +++ b/tests/unit/catalog-release-builder.test.ts @@ -134,5 +134,5 @@ describe('legacy JSON release builder', () => { expect(database.prepare('SELECT count(*) AS count FROM release_activation_requests').get()) .toEqual({ count: 1 }) database.close() - }) + }, 30_000) }) diff --git a/tests/unit/catalog-repository.test.ts b/tests/unit/catalog-repository.test.ts index d1d27a5..62f9e48 100644 --- a/tests/unit/catalog-repository.test.ts +++ b/tests/unit/catalog-repository.test.ts @@ -16,6 +16,7 @@ import { type CatalogFetch, type CatalogRepository, } from '@/lib/catalog' +import { getDataReleaseDate } from '@/lib/data/release' import { bundleSchema } from '@/lib/data/schema' import type { DataBundle } from '@/lib/data/types' @@ -65,10 +66,11 @@ describe('CatalogRepository', () => { it('derives release metadata and all six record counts for JSON compatibility', async () => { const repository = createJsonCatalogRepository(() => copyBundle()) + const expectedDataDate = getDataReleaseDate(allData) await expect(repository.getRelease()).resolves.toEqual({ - id: 'json:2026-08-05', - dataDate: '2026-08-05', - generatedAt: '2026-08-05T00:00:00.000Z', + id: `json:${expectedDataDate}`, + dataDate: expectedDataDate, + generatedAt: `${expectedDataDate}T00:00:00.000Z`, recordCounts: getCatalogRecordCounts(allData), }) }) diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index 1f58a1b..0459fec 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -110,7 +110,7 @@ describe('Catalog D1 normalized v1 API', () => { }, CATALOG_API_TOKEN: 'shadow-secret', } - }) + }, 30_000) afterAll(() => database.close()) @@ -161,7 +161,7 @@ describe('Catalog D1 normalized v1 API', () => { expect(second.data.map((item) => item.id)).not.toContain(first.data[0]!.id) expect(r2Reads).toBe(0) expect(queries.some(({ sql }) => sql.includes('FROM current_programs AS program'))).toBe(true) - }) + }, 30_000) it('uses FTS5 only with current_search_documents and supports the locked filters', async () => { const seedResponse = await worker.fetch( diff --git a/tests/unit/official-dependency-materializer.test.ts b/tests/unit/official-dependency-materializer.test.ts index 9b2987b..38b6ce9 100644 --- a/tests/unit/official-dependency-materializer.test.ts +++ b/tests/unit/official-dependency-materializer.test.ts @@ -368,7 +368,7 @@ describe('official dependency canonicalizer', () => { integrity_check: 'ok', }) database.close() - }) + }, 30_000) it('materializes all ten existing dependencies twice without creating entities or cycles', () => { const artifacts = buildOfficialDependencyMaterialization( diff --git a/tests/unit/pipeline-bootstrap-import.test.ts b/tests/unit/pipeline-bootstrap-import.test.ts index edb109a..bddc3cb 100644 --- a/tests/unit/pipeline-bootstrap-import.test.ts +++ b/tests/unit/pipeline-bootstrap-import.test.ts @@ -271,7 +271,7 @@ describe('Pipeline stable-entity bootstrap', () => { FROM source_documents WHERE canonical_url = ? `).get(removable!.officialUrl)).toEqual({ active: 0 }) database.close() - }) + }, 30_000) it('keeps the bootstrap importer parseable and cross-platform', () => { const scriptPath = join( diff --git a/tests/unit/scholarship-catalog.test.ts b/tests/unit/scholarship-catalog.test.ts index 8c66ebc..0bf5db9 100644 --- a/tests/unit/scholarship-catalog.test.ts +++ b/tests/unit/scholarship-catalog.test.ts @@ -95,6 +95,28 @@ describe('server-side scholarship catalogue', () => { expect(href).toContain('page=2') }) + it('uses cursor links for adjacent repository pages without replaying earlier pages', () => { + const filters = { + ...parseScholarshipCatalogFilters({ + q: 'government', + page: '2', + cursor: 'page-2-cursor', + cursorHistory: '~', + }), + nextCursor: 'page-3-cursor', + } + + const next = new URL(scholarshipCatalogHref('en', filters, 3), 'https://example.test') + expect(next.searchParams.get('page')).toBe('3') + expect(next.searchParams.get('cursor')).toBe('page-3-cursor') + expect(next.searchParams.get('cursorHistory')).toBe('~,page-2-cursor') + + const previous = new URL(scholarshipCatalogHref('en', filters, 1), 'https://example.test') + expect(previous.searchParams.get('page')).toBeNull() + expect(previous.searchParams.get('cursor')).toBeNull() + expect(previous.searchParams.get('cursorHistory')).toBeNull() + }) + it('rejects unsupported filter values instead of passing them to queries', () => { const filters = parseScholarshipCatalogFilters({ degree: 'invalid', diff --git a/tests/unit/source-manifest-candidate-promotion.test.ts b/tests/unit/source-manifest-candidate-promotion.test.ts new file mode 100644 index 0000000..5670437 --- /dev/null +++ b/tests/unit/source-manifest-candidate-promotion.test.ts @@ -0,0 +1,410 @@ +import { createHash } from "node:crypto"; +import { + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + buildSourceManifestCohort, + CURRENT_SOURCE_MANIFEST_COHORT_INPUTS, + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, + writeSourceManifestCohort, + type BuildSourceManifestCohortInput, + type SourceManifestCohortArtifactManifest, + type SourceManifestCohortInputFingerprint, +} from "../../scripts/ingestion/build-source-manifest-cohort"; +import { + parseSourceManifestPromotionCli, + promoteSourceManifestCandidate, + type SourceManifestPromotionReview, +} from "../../scripts/ingestion/promote-source-manifest-candidate"; +import type { SourceManifestV2 } from "../../scripts/source-manifest-registry"; + +const temporaryDirectories: string[] = []; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } +}); + +function sha256(value: Buffer): string { + return createHash("sha256").update(value).digest("hex"); +} + +function temporaryRepository(): string { + const repositoryRoot = mkdtempSync(join(tmpdir(), "manifest-promotion-")); + temporaryDirectories.push(repositoryRoot); + mkdirSync(join(repositoryRoot, "content/source-manifests"), { + recursive: true, + }); + mkdirSync(join(repositoryRoot, "content/data"), { recursive: true }); + writeFileSync( + join(repositoryRoot, "content/data/universities.json"), + JSON.stringify([{ id: "uni-test-university" }]), + "utf8", + ); + return repositoryRoot; +} + +function cohortFixture(): BuildSourceManifestCohortInput { + return { + checkedAt: "2026-08-06", + registry: { + cohort: { id: "promotion-test-cohort" }, + targets: [ + { + targetId: "target-001", + ordinal: 1, + officialNameZh: "测试大学", + catalogInstitutionId: "uni-test-university", + }, + ], + }, + universities: [ + { + id: "uni-test-university", + slug: "test-university", + name: { en: "Test University", zh: "测试大学" }, + sourceIds: [], + }, + ], + sources: [ + { + id: "src-test-program", + url: "https://international.test.edu.cn/programs/master", + title: "Official international programme", + kind: "program", + official: true, + }, + ], + programs: [ + { + id: "program-test-master", + universityId: "uni-test-university", + name: { en: "Verified Master Programme" }, + sourceIds: ["src-test-program"], + }, + ], + admissionCycles: [], + scholarships: [], + sourceReconciliations: [], + }; +} + +function inputFingerprints(): SourceManifestCohortInputFingerprint[] { + const locked = ( + Object.keys(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS) as Array< + keyof typeof CURRENT_SOURCE_MANIFEST_COHORT_INPUTS + > + ).map((name, index): SourceManifestCohortInputFingerprint => ({ + name, + repositoryPath: CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name], + sha256: (index + 1).toString(16).padStart(64, "0"), + byteLength: index + 1, + })); + return [ + ...locked, + { + name: "sourceReconciliation:test.v1.json", + repositoryPath: `${SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY}/test.v1.json`, + sha256: "7".padStart(64, "0"), + byteLength: 7, + }, + ]; +} + +function completeManifest(candidate: SourceManifestV2): SourceManifestV2 { + return { + ...structuredClone(candidate), + manifestStatus: "complete", + sources: candidate.sources.map((source) => ({ + ...source, + enabled: true, + robots: { mode: "enforce" }, + })), + coverage: candidate.coverage.map((coverage) => + coverage.sourceIds?.length + ? { + sourceCategory: coverage.sourceCategory, + status: "registered" as const, + sourceIds: [...coverage.sourceIds], + } + : { + sourceCategory: coverage.sourceCategory, + status: "officially_not_provided" as const, + note: "A named reviewer resolved this category from official evidence.", + }, + ), + catalogReconciliation: { + ...structuredClone(candidate.catalogReconciliation), + status: "complete", + entries: candidate.catalogReconciliation.entries.map((entry, index) => ({ + ...entry, + status: "published" as const, + recordId: `program-reviewed-${index + 1}`, + note: undefined, + })), + note: "Every official catalog entry was resolved during evidence review.", + }, + }; +} + +function setupPromotion(options: { auditOnly?: boolean } = {}): { + artifactDirectory: string; + candidatePath: string; + candidateFilePath: string; + destinationPath: string; + repositoryRoot: string; + reviewDecisionPath: string; + review: SourceManifestPromotionReview; +} { + const repositoryRoot = temporaryRepository(); + const build = buildSourceManifestCohort(cohortFixture()); + const candidate = build.candidates[0]!; + if (options.auditOnly) { + candidate.manifest.catalogReconciliation.entries[0] = { + ...candidate.manifest.catalogReconciliation.entries[0]!, + officialKey: "audit-only:uni-test-university:international-catalog", + officialName: "AUDIT ONLY - not a publishable programme", + }; + } + const artifactDirectory = join(repositoryRoot, "candidate-bundle"); + writeSourceManifestCohort( + build, + artifactDirectory, + inputFingerprints(), + repositoryRoot, + ); + const artifact = JSON.parse( + readFileSync(join(artifactDirectory, "artifact-manifest.v1.json"), "utf8"), + ) as SourceManifestCohortArtifactManifest; + const candidateFile = artifact.files.find((file) => + file.path.startsWith("manifests/"), + )!; + const candidateFilePath = join( + artifactDirectory, + ...candidateFile.path.split("/"), + ); + const reviewedManifest = completeManifest(candidate.manifest); + const review: SourceManifestPromotionReview = { + format: "studyinchina.source-manifest-v2-promotion-review", + formatVersion: 1, + decision: "approve_complete_manifest", + reviewedAt: "2026-08-06", + reviewer: "reviewer@example.test", + rationale: + "Official evidence was inspected and every catalog outcome was resolved.", + artifact: { + cohortId: artifact.cohortId, + candidatePath: candidateFile.path, + candidateSha256: candidateFile.sha256, + }, + manifest: reviewedManifest, + }; + const reviewDecisionPath = join(repositoryRoot, "review-decision.json"); + writeFileSync( + reviewDecisionPath, + JSON.stringify(review, null, 2) + "\n", + "utf8", + ); + const destinationPath = join( + repositoryRoot, + "content/source-manifests/double-first-class/institutions", + candidate.fileName.replace(".v2.candidate.json", ".v2.json"), + ); + return { + artifactDirectory, + candidatePath: candidateFile.path, + candidateFilePath, + destinationPath, + repositoryRoot, + reviewDecisionPath, + review, + }; +} + +describe("SourceManifestV2 candidate promotion gate", () => { + it("is dry-run by default and writes one complete formal manifest only with --write", () => { + const setup = setupPromotion(); + + const dryRun = promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + }); + + expect(dryRun).toMatchObject({ + mode: "dry-run", + institutionId: "uni-test-university", + candidatePath: setup.candidatePath, + qualityGate: { + catalogReconciliationComplete: true, + pendingEntries: 0, + discoveryPendingCoverage: 0, + auditOnlyMarkers: 0, + }, + }); + expect(existsSync(setup.destinationPath)).toBe(false); + + const written = promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + write: true, + }); + + expect(written.mode).toBe("write"); + expect(JSON.parse(readFileSync(setup.destinationPath, "utf8"))).toEqual( + setup.review.manifest, + ); + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + write: true, + }), + ).toThrow(/Refusing to overwrite/); + }); + + it("verifies the whole artifact before accepting the exact reviewed candidate SHA", () => { + const setup = setupPromotion(); + writeFileSync(setup.candidateFilePath, "{}\n", "utf8"); + + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + }), + ).toThrow(/checksum mismatch/); + + const exactShaSetup = setupPromotion(); + exactShaSetup.review.artifact.candidateSha256 = sha256( + Buffer.from("wrong"), + ); + writeFileSync( + exactShaSetup.reviewDecisionPath, + JSON.stringify(exactShaSetup.review, null, 2) + "\n", + "utf8", + ); + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: exactShaSetup.artifactDirectory, + reviewDecisionPath: exactShaSetup.reviewDecisionPath, + repositoryRoot: exactShaSetup.repositoryRoot, + }), + ).toThrow(/candidateSha256/); + }); + + it("rejects audit-only synthetic entries and incomplete reviewed manifests", () => { + const auditOnly = setupPromotion({ auditOnly: true }); + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: auditOnly.artifactDirectory, + reviewDecisionPath: auditOnly.reviewDecisionPath, + repositoryRoot: auditOnly.repositoryRoot, + }), + ).toThrow(/AUDIT ONLY/); + + const incomplete = setupPromotion(); + incomplete.review.manifest.manifestStatus = "in_progress"; + incomplete.review.manifest.catalogReconciliation.status = "in_progress"; + writeFileSync( + incomplete.reviewDecisionPath, + JSON.stringify(incomplete.review, null, 2) + "\n", + "utf8", + ); + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: incomplete.artifactDirectory, + reviewDecisionPath: incomplete.reviewDecisionPath, + repositoryRoot: incomplete.repositoryRoot, + }), + ).toThrow(/requires a complete manifest/); + }); + + it("rejects a second formal manifest for the same institution", () => { + const setup = setupPromotion(); + const existingDirectory = join( + setup.repositoryRoot, + "content/source-manifests/existing", + ); + mkdirSync(existingDirectory, { recursive: true }); + writeFileSync( + join(existingDirectory, "same-school.v2.json"), + JSON.stringify(setup.review.manifest, null, 2) + "\n", + "utf8", + ); + + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + }), + ).toThrow(/already exists for institutionId/); + }); + + it("has no output path option and requires an explicit review decision", () => { + expect( + parseSourceManifestPromotionCli([ + "--artifact", + "candidate-bundle", + "--review-decision", + "review.json", + ]), + ).toEqual({ + artifactDirectory: "candidate-bundle", + reviewDecisionPath: "review.json", + write: false, + }); + expect( + parseSourceManifestPromotionCli([ + "--artifact", + "candidate-bundle", + "--review-decision", + "review.json", + "--write", + ]).write, + ).toBe(true); + expect(() => + parseSourceManifestPromotionCli([ + "--artifact", + "candidate-bundle", + "--output", + "../outside", + "--review-decision", + "review.json", + ]), + ).toThrow(/Unknown CLI option/); + expect(() => + parseSourceManifestPromotionCli(["--artifact", "candidate-bundle"]), + ).toThrow(/Usage/); + }); + + it("rejects calendar-normalized review dates such as February 31", () => { + const setup = setupPromotion(); + setup.review.reviewedAt = "2026-02-31"; + writeFileSync( + setup.reviewDecisionPath, + JSON.stringify(setup.review, null, 2) + "\n", + "utf8", + ); + + expect(() => + promoteSourceManifestCandidate({ + artifactDirectory: setup.artifactDirectory, + reviewDecisionPath: setup.reviewDecisionPath, + repositoryRoot: setup.repositoryRoot, + }), + ).toThrow(/reviewedAt must be a real ISO calendar date/); + }); +}); diff --git a/tests/unit/source-manifest-cohort-builder.test.ts b/tests/unit/source-manifest-cohort-builder.test.ts index d285626..960f50b 100644 --- a/tests/unit/source-manifest-cohort-builder.test.ts +++ b/tests/unit/source-manifest-cohort-builder.test.ts @@ -1,4 +1,10 @@ -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { afterEach, describe, expect, it } from 'vitest' @@ -7,6 +13,7 @@ import { buildCurrentSourceManifestCohort, buildSourceManifestCohort, CURRENT_SOURCE_MANIFEST_COHORT_INPUTS, + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, dryRunSummary, parseSourceManifestCohortCli, verifySourceManifestCohortArtifact, @@ -14,7 +21,6 @@ import { type BuildSourceManifestCohortInput, type SourceManifestCohortArtifactManifest, type SourceManifestCohortInputFingerprint, - type SourceManifestCohortInputName, } from '../../scripts/ingestion/build-source-manifest-cohort' const temporaryDirectories: string[] = [] @@ -132,6 +138,7 @@ function fixture(): BuildSourceManifestCohortInput { universityIds: ['uni-test-university'], sourceIds: ['src-test-scholarship'], }], + sourceReconciliations: [], } } @@ -142,14 +149,22 @@ function temporaryDirectory(): string { } function inputFingerprints(): SourceManifestCohortInputFingerprint[] { - return ( - Object.keys(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS) as SourceManifestCohortInputName[] - ).map((name, index) => ({ + const locked = ( + Object.keys(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS) as Array< + keyof typeof CURRENT_SOURCE_MANIFEST_COHORT_INPUTS + > + ).map((name, index): SourceManifestCohortInputFingerprint => ({ name, repositoryPath: CURRENT_SOURCE_MANIFEST_COHORT_INPUTS[name], sha256: (index + 1).toString(16).padStart(64, '0'), byteLength: index + 1, })) + return [...locked, { + name: 'sourceReconciliation:test.v1.json', + repositoryPath: `${SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY}/test.v1.json`, + sha256: '7'.padStart(64, '0'), + byteLength: 7, + }] } function readJson(path: string): T { @@ -195,6 +210,81 @@ describe('SourceManifestV2 cohort candidate builder', () => { )).toBe(false) }) + it('uses exact official reconciliation only as an audit fallback', () => { + const input = fixture() + input.sourceReconciliations = [{ + institutionNameZh: '缺口大学', + checkedAt: '2026-08-05', + categories: [ + { + sourceCategory: 'international_admissions_home', + status: 'verified_official', + officialUrl: 'https://international.gap.edu.cn/admissions', + evidenceUrl: 'https://international.gap.edu.cn/admissions', + note: 'Official international admissions entry exists.', + checkedAt: '2026-08-05', + }, + { + sourceCategory: 'catalog_anchor', + status: 'source_unavailable', + officialUrl: null, + evidenceUrl: 'https://www.gap.edu.cn/evidence/catalog-audit.pdf', + note: 'The official evidence records an unavailable catalog source.', + checkedAt: '2026-08-05', + }, + { + sourceCategory: 'university_scholarship', + status: 'officially_not_provided', + officialUrl: null, + evidenceUrl: 'https://international.gap.edu.cn/admissions', + note: 'The audited official section does not provide a scholarship catalog.', + checkedAt: '2026-08-05', + }, + ], + }] + + const build = buildSourceManifestCohort(input) + const fallback = build.candidates.find( + (candidate) => candidate.manifest.institutionId === 'uni-gap-university', + )?.manifest + + expect(build.summary).toMatchObject({ + candidateManifests: 2, + catalogLinkedManifests: 1, + reconciliationFallbackManifests: 1, + exactOfficialHttpsSources: 6, + targetsWithoutCandidate: 1, + }) + expect(fallback).toBeDefined() + expect(fallback?.manifestStatus).toBe('in_progress') + expect(fallback?.catalogReconciliation.scope).toBe('limited_official_catalog') + expect(fallback?.sources.map((source) => source.officialUrl)).toEqual([ + 'https://international.gap.edu.cn/admissions', + 'https://www.gap.edu.cn/evidence/catalog-audit.pdf', + 'https://international.gap.edu.cn/admissions', + ]) + expect(fallback?.sources.every( + (source) => source.enabled === false && source.robots.mode === 'blocked', + )).toBe(true) + expect(fallback?.coverage.filter((entry) => ( + entry.sourceCategory === 'international_admissions_home' + || entry.sourceCategory === 'catalog_anchor' + || entry.sourceCategory === 'university_scholarship' + )).map((entry) => [entry.sourceCategory, entry.status])).toEqual([ + ['international_admissions_home', 'parser_pending'], + ['university_scholarship', 'officially_not_provided'], + ['catalog_anchor', 'source_unavailable'], + ]) + expect(fallback?.catalogReconciliation.entries).toEqual([ + expect.objectContaining({ + officialKey: 'audit-only:uni-gap-university:international-catalog', + entityType: 'program', + status: 'pending', + note: expect.stringMatching(/not a publishable program/), + }), + ]) + }) + it('reports military, mapping, source-quality, and no-safe-entity gaps explicitly', () => { const build = buildSourceManifestCohort(fixture()) @@ -203,6 +293,8 @@ describe('SourceManifestV2 cohort candidate builder', () => { militaryExcluded: 1, eligibleTargets: 3, candidateManifests: 1, + catalogLinkedManifests: 1, + reconciliationFallbackManifests: 0, exactOfficialHttpsSources: 3, targetsWithoutCandidate: 2, }) @@ -261,14 +353,29 @@ describe('SourceManifestV2 cohort candidate builder', () => { officialTargets: 147, militaryExcluded: 3, eligibleTargets: 144, + candidateManifests: 144, + catalogLinkedManifests: 140, + reconciliationFallbackManifests: 4, + exactOfficialHttpsSources: 980, + targetsWithoutCandidate: 0, }) expect( current.build.summary.candidateManifests + current.build.summary.targetsWithoutCandidate, ).toBe(144) - expect(current.inputFingerprints.map((input) => input.repositoryPath)).toEqual( - Object.values(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS), - ) + expect(current.inputFingerprints.slice(0, 6).map( + (input) => input.repositoryPath, + )).toEqual(Object.values(CURRENT_SOURCE_MANIFEST_COHORT_INPUTS)) + const reconciliationPaths = readdirSync(resolve( + SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY, + )).filter((fileName) => fileName.endsWith('.v1.json')) + .sort((left, right) => left.localeCompare(right, 'en')) + .map((fileName) => ( + `${SOURCE_MANIFEST_COHORT_RECONCILIATION_DIRECTORY}/${fileName}` + )) + expect(current.inputFingerprints.slice(6).map( + (input) => input.repositoryPath, + )).toEqual(reconciliationPaths) expect(current.inputFingerprints.every( (input) => /^[a-f0-9]{64}$/u.test(input.sha256) && input.byteLength > 0, )).toBe(true) @@ -303,6 +410,7 @@ describe('SourceManifestV2 cohort candidate builder', () => { 'programs', 'admissionCycles', 'scholarships', + 'sourceReconciliation:test.v1.json', ]) expect(artifact.files.map((file) => file.path)).toEqual([ 'gap-report.v1.json', From aed642dbe7069a8ea4cb6d1fdcbd1ffa8a63ab09 Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 05:45:12 +0800 Subject: [PATCH 6/9] Paginate the institution catalog end to end --- scripts/catalog/build-release.ts | 29 +- src/app/[locale]/universities/page.tsx | 63 +++- src/app/api/v1/institutions/route.ts | 173 +++++++++- .../features/UniversityExplorerV2.tsx | 159 +++++++++ src/lib/catalog-api/http.ts | 7 +- src/lib/catalog-api/service.ts | 134 +++++++- src/lib/catalog-api/types.ts | 10 + src/lib/catalog/d1-list.ts | 138 +++++++- src/lib/catalog/d1.ts | 24 +- src/lib/catalog/index.ts | 6 + src/lib/catalog/json.ts | 123 ++++++- src/lib/catalog/list-cursor.ts | 4 +- src/lib/catalog/shadow.ts | 48 ++- src/lib/catalog/types.ts | 41 +++ src/lib/university-catalog.ts | 201 +++++++++++ tests/e2e/smoke.spec.ts | 6 +- tests/unit/catalog-api-service.test.ts | 191 ++++++++++- .../catalog-institution-api-route.test.ts | 134 ++++++++ .../catalog-institution-repository.test.ts | 322 ++++++++++++++++++ tests/unit/catalog-release-builder.test.ts | 107 +++++- tests/unit/catalog-repository.test.ts | 1 + tests/unit/catalog-sql-api.test.ts | 269 +++++++++++++++ tests/unit/university-catalog.test.ts | 151 ++++++++ tests/unit/university-explorer-v2.test.tsx | 71 ++++ workers/catalog-api/src/index.ts | 5 + workers/catalog-api/src/sql-api.ts | 258 ++++++++++---- workers/catalog-api/src/sql-data.ts | 5 +- workers/catalog-api/src/sql-types.ts | 4 +- 28 files changed, 2555 insertions(+), 129 deletions(-) create mode 100644 src/components/features/UniversityExplorerV2.tsx create mode 100644 src/lib/university-catalog.ts create mode 100644 tests/unit/catalog-institution-api-route.test.ts create mode 100644 tests/unit/catalog-institution-repository.test.ts create mode 100644 tests/unit/university-catalog.test.ts create mode 100644 tests/unit/university-explorer-v2.test.tsx diff --git a/scripts/catalog/build-release.ts b/scripts/catalog/build-release.ts index 927b55a..571a6d1 100644 --- a/scripts/catalog/build-release.ts +++ b/scripts/catalog/build-release.ts @@ -2,11 +2,12 @@ import { createHash } from 'node:crypto' import { mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { join, resolve } from 'node:path' import { pathToFileURL } from 'node:url' +import { classifyProgramField, programFieldTaxonomy } from '../../src/lib/data/fields' import { bundleSchema } from '../../src/lib/data/schema' import { getDataReleaseDate } from '../../src/lib/data/release' import type { AuditMeta, DataBundle, LocalizedText, Program, Source } from '../../src/lib/data/types' -const LEGACY_PROJECTION_VERSION = 2 +const LEGACY_PROJECTION_VERSION = 3 type SqlValue = string | number | boolean | null @@ -41,7 +42,7 @@ const sourceKind: Record = { other: 'other', } -const disciplineNames: Record = { +const legacyDisciplineNames: Record = { engineering: { en: 'Engineering', zh: '工学' }, business: { en: 'Business', zh: '商科' }, medicine: { en: 'Medicine', zh: '医学' }, @@ -53,6 +54,20 @@ const disciplineNames: Record = { other: { en: 'Other', zh: '其他' }, } +const programFieldNamesZh = new Map( + programFieldTaxonomy('zh').map((field) => [field.key, field.label]), +) +const disciplineNames: Record = { + ...legacyDisciplineNames, + ...Object.fromEntries(programFieldTaxonomy('en').map((field) => [ + field.key, + { + en: field.label, + zh: programFieldNamesZh.get(field.key) ?? field.label, + }, + ])), +} + function sha256(value: string) { return createHash('sha256').update(value).digest('hex') } @@ -362,7 +377,11 @@ export function buildLegacyRelease(bundleInput: DataBundle): ReleaseArtifacts { addLocalized(statements, releaseId, university.id, 'name', university.name) addLocalized(statements, releaseId, university.id, 'summary', university.summary) addSources(statements, releaseId, university.id, university.sourceIds) - for (const [field, value] of Object.entries({ officialUrl: university.officialUrl, admissionsUrl: university.admissionsUrl })) { + for (const [field, value] of Object.entries({ + summary: university.summary, + officialUrl: university.officialUrl, + admissionsUrl: university.admissionsUrl, + })) { addField(statements, releaseId, university.id, university, field, value, dataDate, hasOfficialSource(bundle, university.sourceIds)) } addSearch(statements, releaseId, university.id, 'organization', university.name, university.summary, `${university.region} ${university.cityId}`) @@ -393,6 +412,7 @@ export function buildLegacyRelease(bundleInput: DataBundle): ReleaseArtifacts { for (const program of bundle.programs) { const projection = programProjection(program) + const discipline = classifyProgramField(program) statements.push(recordRow(releaseId, program, 'program', program)) statements.push(insert('programs', { release_id: releaseId, @@ -413,7 +433,7 @@ export function buildLegacyRelease(bundleInput: DataBundle): ReleaseArtifacts { statements.push(insert('program_disciplines', { release_id: releaseId, program_id: program.id, - discipline_code: program.discipline, + discipline_code: discipline, is_primary: true, })) for (const language of program.teachingLanguages) { @@ -437,6 +457,7 @@ export function buildLegacyRelease(bundleInput: DataBundle): ReleaseArtifacts { addSources(statements, releaseId, program.id, program.sourceIds) const official = hasOfficialSource(bundle, program.sourceIds) for (const [field, value] of Object.entries({ + discipline, teachingLanguages: program.teachingLanguages, durationMonths: program.durationMonths, durationMonthsMax: program.durationMonthsMax ?? null, diff --git a/src/app/[locale]/universities/page.tsx b/src/app/[locale]/universities/page.tsx index b34d2c5..7a6dcf1 100644 --- a/src/app/[locale]/universities/page.tsx +++ b/src/app/[locale]/universities/page.tsx @@ -1,20 +1,59 @@ import { notFound } from 'next/navigation' -import { UniversityExplorer } from '@/components/features/UniversityExplorer' +import { UniversityExplorerV2 } from '@/components/features/UniversityExplorerV2' import { PageHero } from '@/components/ui' import { getMessages } from '@/i18n/messages' -import { getCatalogData } from '@/lib/data/load' +import { getCatalogRepository } from '@/lib/catalog' +import { + parseUniversityCatalogFilters, + queryUniversityCatalogRepository, + type UniversityCatalogSearchParams, +} from '@/lib/university-catalog' import { pageMetadata, requireLocale } from '@/lib/site' -export async function generateMetadata({ params }: { params: Promise<{ locale: string }> }) { const locale = requireLocale((await params).locale) || 'en'; const m = getMessages(locale); return pageMetadata(locale, m.universities.title, m.universities.intro, 'universities') } -export default async function UniversitiesPage({ params }: { params: Promise<{ locale: string }> }) { - const locale = requireLocale((await params).locale); if (!locale) notFound(); const messages = getMessages(locale); const data = await getCatalogData() +export async function generateMetadata({ params }: { params: Promise<{ locale: string }> }) { + const locale = requireLocale((await params).locale) || 'en' + const messages = getMessages(locale) + return pageMetadata( + locale, + messages.universities.title, + messages.universities.intro, + 'universities', + ) +} + +export default async function UniversitiesPage({ + params, + searchParams, +}: { + params: Promise<{ locale: string }> + searchParams: Promise +}) { + const locale = requireLocale((await params).locale) + if (!locale) notFound() + const messages = getMessages(locale) + const filters = parseUniversityCatalogFilters(await searchParams) + const result = await queryUniversityCatalogRepository(getCatalogRepository(), filters) const coverageLabel = { - zh: '查看 147 所双一流高校数据表', - en: 'View all 147 Double First-Class universities', - ru: 'Все 147 университетов Double First-Class', - de: 'Alle 147 Double-First-Class-Hochschulen', - fr: 'Voir les 147 établissements Double First-Class', - es: 'Ver las 147 universidades Double First-Class', + zh: '查看非军校双一流高校数据表', + en: 'View non-military Double First-Class coverage', + ru: 'Охват невоенных вузов Double First-Class', + de: 'Abdeckung ziviler Double-First-Class-Hochschulen', + fr: 'Couverture des universités civiles Double First-Class', + es: 'Cobertura de universidades civiles Double First-Class', }[locale] - return <>{coverageLabel} →} meta={<>{messages.common.authoritativeNotice}} />
+ const totalLabel = `${result.total}${result.totalExact ? '' : '+'}` + + return <> + {coverageLabel} →} + meta={<>{messages.common.authoritativeNotice}} + /> +
+ +
+ } diff --git a/src/app/api/v1/institutions/route.ts b/src/app/api/v1/institutions/route.ts index e82db1a..3f69b73 100644 --- a/src/app/api/v1/institutions/route.ts +++ b/src/app/api/v1/institutions/route.ts @@ -1,20 +1,181 @@ -import { getCatalogApiService } from '@/lib/catalog-api/runtime' -import { handleCatalogRequest, integerParam, ok, stringParam } from '@/lib/catalog-api/http' +import { + CatalogRepositoryError, + getCatalogRepository, + type CatalogInstitutionListItem, + type CatalogInstitutionListPage, + type CatalogInstitutionListQuery, + type CatalogInstitutionListSort, +} from '@/lib/catalog' +import { InvalidQueryError, handleCatalogRequest, integerParam, ok, stringParam } from '@/lib/catalog-api/http' +import { + AUTOMATED_COLLECTION_NOTICE, + type ApiEnvelope, + type FactStatus, + type FieldMeta, + type InstitutionRecord, +} from '@/lib/catalog-api/types' +import { getTodayDate } from '@/lib/data/freshness' export const runtime = 'nodejs' export const dynamic = 'force-dynamic' +const INSTITUTION_SORTS: ReadonlySet = new Set([ + 'default', + 'name', + 'programs-desc', + 'scholarships-desc', +]) + +function institutionSortParam( + params: URLSearchParams, +): CatalogInstitutionListSort | undefined { + const value = stringParam(params, 'sort') + if (value === undefined) return undefined + if (!INSTITUTION_SORTS.has(value as CatalogInstitutionListSort)) { + throw new InvalidQueryError('sort is invalid.') + } + return value as CatalogInstitutionListSort +} + +function institutionSearchParam(params: URLSearchParams): string | undefined { + const value = stringParam(params, 'q') + if (value === undefined) return undefined + const terms = value.normalize('NFKC').match(/[\p{L}\p{N}]+/gu) ?? [] + if (terms.length === 0 || terms.length > 20) { + throw new InvalidQueryError('Invalid search query.') + } + return terms.join(' ') +} + +function factStatus( + item: CatalogInstitutionListItem, + value: unknown, + today: string, + staleSensitive = false, +): FactStatus { + const institution = item.institution + if ( + staleSensitive + && (institution.status === 'stale' || institution.reviewAfter < today) + ) return 'stale' + return value === null || value === undefined || value === '' + ? 'officially_not_announced' + : 'known' +} + +function fieldMeta( + item: CatalogInstitutionListItem, + value: unknown, + today: string, + staleSensitive = false, +): FieldMeta { + const institution = item.institution + return { + status: factStatus(item, value, today, staleSensitive), + officialUrl: institution.admissionsUrl ?? institution.officialUrl, + sourceTitle: 'Official university website', + checkedAt: institution.verifiedAt, + } +} + +function institutionRecord( + item: CatalogInstitutionListItem, + today: string, +): InstitutionRecord { + const institution = item.institution + const officialUrl = institution.admissionsUrl ?? institution.officialUrl + const city = item.city + ? { + id: item.city.id, + slug: item.city.slug, + name: item.city.name, + province: null, + region: item.city.region, + } + : null + + return { + ...institution, + city, + disciplines: item.disciplines, + programCount: item.programCount, + scholarshipCount: item.scholarshipCount, + officialSources: [{ + url: officialUrl, + title: 'Official university website', + checkedAt: institution.verifiedAt, + }], + fieldMeta: { + id: fieldMeta(item, institution.id, today), + slug: fieldMeta(item, institution.slug, today), + name: fieldMeta(item, institution.name, today), + cityId: fieldMeta(item, institution.cityId, today), + city: fieldMeta(item, city, today), + region: fieldMeta(item, institution.region, today), + officialUrl: fieldMeta(item, institution.officialUrl, today), + admissionsUrl: fieldMeta(item, institution.admissionsUrl, today), + summary: fieldMeta(item, institution.summary, today, true), + featured: fieldMeta(item, institution.featured, today), + disciplines: fieldMeta(item, item.disciplines, today), + programCount: fieldMeta(item, item.programCount, today), + scholarshipCount: fieldMeta(item, item.scholarshipCount, today), + }, + } +} + +function envelope( + page: CatalogInstitutionListPage, + today: string, +): ApiEnvelope { + if (!page.release) { + throw new CatalogRepositoryError( + 'CATALOG_RELEASE_UNAVAILABLE', + 'The current catalog release is unavailable.', + ) + } + return { + data: page.items.map((item) => institutionRecord(item, today)), + meta: { + release: page.release, + notice: AUTOMATED_COLLECTION_NOTICE, + pageSize: page.items.length, + nextCursor: page.nextCursor, + ...(page.total === null ? {} : { total: page.total }), + facets: page.facets, + }, + } +} + +async function repositoryPage(query: CatalogInstitutionListQuery) { + try { + return await getCatalogRepository().listInstitutions(query) + } catch (error) { + if (error instanceof CatalogRepositoryError) { + if (error.code === 'INVALID_SEARCH_QUERY') { + throw new InvalidQueryError('Invalid search query.') + } + if (error.code === 'INVALID_LIST_CURSOR') { + throw new InvalidQueryError('Invalid cursor.') + } + } + throw error + } +} + export function GET(request: Request) { return handleCatalogRequest(async () => { const params = new URL(request.url).searchParams - const service = await getCatalogApiService() - return ok(service.listInstitutions({ - q: stringParam(params, 'q'), + const today = getTodayDate() + const page = await repositoryPage({ + q: institutionSearchParam(params), city: stringParam(params, 'city'), region: stringParam(params, 'region'), discipline: stringParam(params, 'discipline'), + sort: institutionSortParam(params), cursor: stringParam(params, 'cursor', { maxLength: 1024 }), limit: integerParam(params, 'limit', { min: 1, max: 100 }), - })) + today, + }) + return ok(envelope(page, today)) }) } diff --git a/src/components/features/UniversityExplorerV2.tsx b/src/components/features/UniversityExplorerV2.tsx new file mode 100644 index 0000000..279b069 --- /dev/null +++ b/src/components/features/UniversityExplorerV2.tsx @@ -0,0 +1,159 @@ +import { Badge, Button, Card, LinkButton, VerificationBadge } from '@/components/ui' +import type { LaunchLocale } from '@/i18n/config' +import type { Messages } from '@/i18n/messages' +import { + normalizeProgramField, + programFieldLabel, + programFieldTaxonomy, +} from '@/lib/data/fields' +import { localize } from '@/lib/data/format' +import { regionLabels } from '@/lib/data/labels' +import { + universityCatalogHref, + type UniversityCatalogResult, +} from '@/lib/university-catalog' +import styles from './ProgramExplorerV2.module.css' + +type ExplorerLabels = { + apply: string + defaultOrder: string + nameOrder: string + next: string + pagination: string + previous: string + programsMost: string + scholarshipsMost: string + sortBy: string +} + +const labels: Record = { + en: { apply: 'Apply filters', defaultOrder: 'Default order', nameOrder: 'University A–Z', next: 'Next', pagination: 'University catalogue pages', previous: 'Previous', programsMost: 'Most programs', scholarshipsMost: 'Most scholarships', sortBy: 'Sort by' }, + zh: { apply: '应用筛选', defaultOrder: '默认顺序', nameOrder: '大学名称 A–Z', next: '下一页', pagination: '高校目录分页', previous: '上一页', programsMost: '项目数量最多', scholarshipsMost: '奖学金数量最多', sortBy: '排序方式' }, + ru: { apply: 'Применить фильтры', defaultOrder: 'По умолчанию', nameOrder: 'Вуз A–Я', next: 'Далее', pagination: 'Страницы каталога вузов', previous: 'Назад', programsMost: 'Больше всего программ', scholarshipsMost: 'Больше всего стипендий', sortBy: 'Сортировка' }, + de: { apply: 'Filter anwenden', defaultOrder: 'Standardreihenfolge', nameOrder: 'Hochschule A–Z', next: 'Weiter', pagination: 'Hochschulkatalogseiten', previous: 'Zurück', programsMost: 'Meiste Studiengänge', scholarshipsMost: 'Meiste Stipendien', sortBy: 'Sortieren nach' }, + fr: { apply: 'Appliquer les filtres', defaultOrder: 'Ordre par défaut', nameOrder: 'Établissement A–Z', next: 'Suivant', pagination: 'Pages du catalogue des universités', previous: 'Précédent', programsMost: 'Plus de programmes', scholarshipsMost: 'Plus de bourses', sortBy: 'Trier par' }, + es: { apply: 'Aplicar filtros', defaultOrder: 'Orden predeterminado', nameOrder: 'Universidad A–Z', next: 'Siguiente', pagination: 'Páginas del catálogo de universidades', previous: 'Anterior', programsMost: 'Más programas', scholarshipsMost: 'Más becas', sortBy: 'Ordenar por' }, +} + +function disciplineLabel(value: string, locale: LaunchLocale): string { + const normalized = normalizeProgramField(value) + return normalized ? programFieldLabel(normalized, locale) : value +} + +export function UniversityExplorerV2({ + result, + locale, + messages, +}: { + result: UniversityCatalogResult + locale: LaunchLocale + messages: Messages +}) { + const text = labels[locale] + const filters = result.filters + + return <> +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + {messages.common.clear} +
+
+ +

+ {result.total}{result.totalExact ? '' : '+'} {messages.universities.results} +

+ + {result.items.length ? ( +
+ {result.items.map(({ institution, city, disciplines, programCount, scholarshipCount }) => { + const region = institution.region ?? city?.region ?? null + return +
+ {region ? regionLabels(locale)[region] : messages.common.unknown} + +
+
+

{localize(institution.name, locale)}

+ {city ?

⌖ {localize(city.name, locale)}

: null} +
+

{localize(institution.summary, locale)}

+
+ {disciplines.slice(0, 3).map((discipline) => ( + {disciplineLabel(discipline, locale)} + ))} +
+
+
{messages.universities.programs}
{programCount}
+
{messages.universities.funding}
{scholarshipCount}
+
+
+ {messages.common.viewDetails} + {institution.admissionsUrl + ? {messages.common.applyOfficial} ↗ + : {messages.common.officialSource} ↗} +
+ + })} +
+ ) :
{messages.universities.noResults}
} + + {result.pageCount > 1 ? ( + + ) : null} + +} diff --git a/src/lib/catalog-api/http.ts b/src/lib/catalog-api/http.ts index d9618bc..bdb9aa5 100644 --- a/src/lib/catalog-api/http.ts +++ b/src/lib/catalog-api/http.ts @@ -1,6 +1,7 @@ import { NextResponse } from 'next/server' import { CatalogRepositoryError } from '@/lib/catalog' import { InvalidCursorError } from './cursor' +import { InvalidSearchQueryError } from './service' const responseHeaders = { 'Cache-Control': 'public, max-age=60, s-maxage=300, stale-while-revalidate=3600', @@ -68,7 +69,11 @@ export async function handleCatalogRequest(operation: () => Promise 20) throw new InvalidSearchQueryError() + return terms.map((term) => term.toLocaleLowerCase()) +} + +function matchesInstitutionName(name: LocalizedText, query?: string) { + const terms = institutionSearchTerms(query) + if (!terms) return true + const tokens = searchable(name) + .normalize('NFKC') + .toLocaleLowerCase() + .match(/[\p{L}\p{N}]+/gu) ?? [] + return terms.every((term) => tokens.some((token) => token.startsWith(term))) +} + function matchesIdentity(value: { id: string; slug: string }, expected?: string) { return !expected || value.id === expected || value.slug === expected } +function institutionCursorContext(query: InstitutionQuery) { + const entries = Object.entries(query) + .filter(([key, value]) => + key !== 'cursor' + && key !== 'limit' + && value !== undefined + && value !== '' + && value !== 'default', + ) + .sort(([left], [right]) => left.localeCompare(right)) + if (entries.length === 0) return 'default' + const input = JSON.stringify(entries) + let hash = 2_166_136_261 + for (let index = 0; index < input.length; index += 1) { + hash ^= input.charCodeAt(index) + hash = Math.imul(hash, 16_777_619) + } + return `q-${(hash >>> 0).toString(16).padStart(8, '0')}` +} + + +function institutionSortKey(record: InstitutionRecord, sort: InstitutionSort = 'default') { + if (sort === 'name') return record.slug + if (sort === 'programs-desc') { + return `${String(999_999_999_999_999 - record.programCount).padStart(15, '0')}:${record.slug}` + } + if (sort === 'scholarships-desc') { + return `${String(999_999_999_999_999 - record.scholarshipCount).padStart(15, '0')}:${record.slug}` + } + return record.slug +} + +function paginateInstitutions( + records: InstitutionRecord[], + query: InstitutionQuery, +) { + const limit = Math.min(Math.max(query.limit ?? 24, 1), 100) + const context = institutionCursorContext(query) + const key = (record: InstitutionRecord) => + `${context}:${institutionSortKey(record, query.sort)}` + const sorted = [...records].sort((left, right) => + institutionSortKey(left, query.sort).localeCompare(institutionSortKey(right, query.sort)) + || left.id.localeCompare(right.id), + ) + const cursor = query.cursor ? decodeCursor(query.cursor) : null + const start = cursor + ? sorted.findIndex((item) => key(item) === cursor.sortKey && item.id === cursor.id) + 1 + : 0 + if (cursor && start === 0) throw new InvalidCursorError() + + const items = sorted.slice(start, start + limit) + const hasMore = start + items.length < sorted.length + const last = items.at(-1) + return { + items, + nextCursor: hasMore && last ? encodeCursor(key(last), last.id) : null, + } +} function matchesProgramDiscipline(program: Program, expected?: string) { if (!expected) return true return isProgramField(expected) @@ -112,13 +199,28 @@ export class CatalogApiService { private readonly today = new Date().toISOString().slice(0, 10), ) {} - private envelope(data: T, page?: { pageSize: number; nextCursor: string | null }): ApiEnvelope { + private envelope( + data: T, + page?: { + pageSize: number + nextCursor: string | null + total?: number + facets?: ApiMeta['facets'] + }, + ): ApiEnvelope { return { data, meta: { release: this.release, notice: AUTOMATED_COLLECTION_NOTICE, - ...(page ? { pageSize: page.pageSize, nextCursor: page.nextCursor } : {}), + ...(page + ? { + pageSize: page.pageSize, + nextCursor: page.nextCursor, + ...(page.total === undefined ? {} : { total: page.total }), + ...(page.facets === undefined ? {} : { facets: page.facets }), + } + : {}), }, } } @@ -126,6 +228,9 @@ export class CatalogApiService { private institutionRecord(university: DataBundle['universities'][number]): InstitutionRecord { const city = this.bundle.cities.find((item) => item.id === university.cityId) ?? null const relatedPrograms = this.bundle.programs.filter((item) => item.universityId === university.id) + const disciplines = [...new Set(relatedPrograms + .filter((item) => hasCurrentFacts(item, this.today)) + .map((item) => classifyProgramField(item)))].sort() const relatedScholarships = this.bundle.scholarships.filter((item) => item.universityIds.includes(university.id), ) @@ -145,6 +250,7 @@ export class CatalogApiService { return { ...university, summary: knownValue(dynamicMeta, 'summary', university.summary), + disciplines, city: city ? { id: city.id, slug: city.slug, name: city.name, province: city.province, region: city.region } : null, programCount: relatedPrograms.length, scholarshipCount: relatedScholarships.length, @@ -287,14 +393,26 @@ export class CatalogApiService { const filtered = this.bundle.universities.filter((university) => { const city = this.bundle.cities.find((item) => item.id === university.cityId) const programs = this.bundle.programs.filter((item) => item.universityId === university.id) - const currentSummary = hasCurrentFacts(university, this.today) ? university.summary : null - return matchesQuery([university.name, currentSummary, city?.name, programs.map((item) => item.name)], query.q) + return matchesInstitutionName(university.name, query.q) && (!query.city || Boolean(city && matchesIdentity(city, query.city))) && (!query.region || university.region === query.region) - && (!query.discipline || programs.some((item) => matchesProgramDiscipline(item, query.discipline))) + && (!query.discipline || programs.some((item) => + hasCurrentFacts(item, this.today) + && matchesProgramDiscipline(item, query.discipline) + )) }).map((item) => this.institutionRecord(item)) - const page = paginateBySlug(filtered, query) - return this.envelope(page.items, { pageSize: page.items.length, nextCursor: page.nextCursor }) + const cityIds = new Set(filtered.map((item) => item.city?.id).filter(Boolean)) + const cities = this.bundle.cities + .filter((city) => cityIds.has(city.id)) + .sort((left, right) => left.slug.localeCompare(right.slug) || left.id.localeCompare(right.id)) + .map((city) => ({ value: city.slug || city.id, name: city.name })) + const page = paginateInstitutions(filtered, query) + return this.envelope(page.items, { + pageSize: page.items.length, + nextCursor: page.nextCursor, + total: filtered.length, + facets: { cities }, + }) } getInstitution(slug: string): ApiEnvelope | null { diff --git a/src/lib/catalog-api/types.ts b/src/lib/catalog-api/types.ts index 958f5fd..8c74289 100644 --- a/src/lib/catalog-api/types.ts +++ b/src/lib/catalog-api/types.ts @@ -52,6 +52,7 @@ export type ProgramType = | 'other' export type InstitutionRecord = Omit & { summary: University['summary'] | null + disciplines: string[] city: Pick | null programCount: number scholarshipCount: number @@ -144,10 +145,19 @@ export type ScholarshipCycleRecord = { fieldMeta: Record } +export type ApiFacetOption = { + value: string + name: City['name'] +} + export type ApiMeta = { release: ReleaseInfo pageSize?: number nextCursor?: string | null + total?: number + facets?: { + cities?: ApiFacetOption[] + } notice: string } diff --git a/src/lib/catalog/d1-list.ts b/src/lib/catalog/d1-list.ts index 90660bb..b816d37 100644 --- a/src/lib/catalog/d1-list.ts +++ b/src/lib/catalog/d1-list.ts @@ -15,6 +15,9 @@ import type { import { parseCatalogReleaseInfo } from './release' import { CatalogRepositoryError, + type CatalogInstitutionCity, + type CatalogInstitutionListItem, + type CatalogInstitutionListPage, type CatalogListOption, type CatalogProgramListItem, type CatalogProgramListPage, @@ -69,11 +72,14 @@ function dateOnly(value: unknown, fallback: string): string { function fieldAudit(value: UnknownRecord, today: string) { const fieldMeta = isObject(value.fieldMeta) ? value.fieldMeta : {} const name = isObject(fieldMeta.name) ? fieldMeta.name : {} + const verifiedAt = dateOnly(value.verifiedAt ?? name.verifiedAt ?? name.checkedAt, today) + const reviewAfter = dateOnly(value.reviewAfter ?? name.reviewAfter, today) + const stale = value.status === 'stale' || name.status === 'stale' || reviewAfter < today return { - sourceIds: sourceIds(value), - verifiedAt: dateOnly(value.verifiedAt ?? name.verifiedAt ?? name.checkedAt, today), - reviewAfter: dateOnly(value.reviewAfter ?? name.reviewAfter, today), - status: value.status === 'stale' ? 'stale' as const : 'verified' as const, + sourceIds: sourceIds(value).sort(), + verifiedAt, + reviewAfter, + status: stale ? 'stale' as const : 'verified' as const, } } @@ -382,6 +388,130 @@ function pageMeta(meta: UnknownRecord) { return { nextCursor, total, release } } +function regionValue(value: unknown): University['region'] { + return value === 'north' + || value === 'northeast' + || value === 'east' + || value === 'south' + || value === 'central' + || value === 'southwest' + || value === 'northwest' + ? value + : null +} + +function relationshipCount( + value: UnknownRecord, + relationshipName: 'programs' | 'scholarships', + fallbackName: 'programCount' | 'scholarshipCount', +): number { + const relationships = isObject(value.relationships) ? value.relationships : {} + const relationship = isObject(relationships[relationshipName]) + ? relationships[relationshipName] + : {} + const count = relationship.count ?? value[fallbackName] + if (!Number.isInteger(count) || (count as number) < 0) { + invalid(`Institution list item has an invalid ${fallbackName}.`) + } + return count as number +} + +function institutionDisciplines(value: UnknownRecord): string[] { + const attributes = isObject(value.attributes) ? value.attributes : {} + const candidate = Array.isArray(attributes.disciplineCodes) + ? attributes.disciplineCodes + : Array.isArray(value.disciplines) + ? value.disciplines + : [] + return [...new Set(candidate.filter((item): item is string => ( + typeof item === 'string' && item.length > 0 + )))].sort() +} + +function normalizeInstitutionCity(value: unknown): CatalogInstitutionCity | null { + if (value === null || value === undefined) return null + if (!isObject(value)) invalid('Institution location relationship is invalid.') + const id = stringValue(value.id) + const slug = stringValue(value.slug) ?? id + if (!id || !slug) invalid('Institution location identity is invalid.') + return { + id, + slug, + name: localized(value.name, slug), + region: regionValue(value.region ?? value.regionCode), + } +} + +function normalizeInstitution(value: unknown, today: string): CatalogInstitutionListItem { + if (!isObject(value)) invalid('Catalog API returned an invalid institution list item.') + const nestedInstitution = isObject(value.institution) ? value.institution : value + const parsedInstitution = universitySchema.safeParse(nestedInstitution) + const relationships = isObject(value.relationships) ? value.relationships : {} + const cityValue = value.city ?? relationships.location + + if (parsedInstitution.success) { + return { + institution: parsedInstitution.data, + city: normalizeInstitutionCity(cityValue), + programCount: relationshipCount(value, 'programs', 'programCount'), + scholarshipCount: relationshipCount(value, 'scholarships', 'scholarshipCount'), + disciplines: institutionDisciplines(value), + } + } + + const attributes = isObject(value.attributes) ? value.attributes : value + const city = normalizeInstitutionCity(cityValue) + const id = stringValue(value.id) + const slug = stringValue(value.slug) + const officialUrl = safeHttps(attributes.officialUrl) + if (!id || !slug || !/^[a-z0-9-]+$/u.test(slug) || !officialUrl || !city) { + invalid('Catalog API institution identity is incomplete.') + } + const summary = attributes.summary === null || attributes.summary === undefined + ? null + : localized(attributes.summary, slug) + const institution = universitySchema.parse({ + ...fieldAudit(value, today), + id, + slug, + name: localized(attributes.name, slug), + cityId: city.id, + region: regionValue(attributes.region ?? city.region), + officialUrl, + admissionsUrl: safeHttps(attributes.admissionsUrl), + summary, + featured: attributes.featured === true, + }) + return { + institution, + city, + programCount: relationshipCount(value, 'programs', 'programCount'), + scholarshipCount: relationshipCount(value, 'scholarships', 'scholarshipCount'), + disciplines: institutionDisciplines(value), + } +} + +export function parseD1InstitutionList( + payload: unknown, + today: string, +): CatalogInstitutionListPage { + const { rows, meta, facets } = envelopeParts(payload) + const items = rows.map((row) => normalizeInstitution(row, today)) + const cities = Array.isArray(facets.cities) + ? facets.cities.flatMap((item) => option(item) ?? []) + : items.flatMap((item) => ( + item.city ? [{ value: item.city.slug, name: item.city.name }] : [] + )) + return { + items, + ...pageMeta(meta), + facets: { + cities: [...new Map(cities.map((item) => [item.value, item])).values()], + }, + } +} + + export function parseD1ProgramList(payload: unknown, today: string): CatalogProgramListPage { const { rows, meta, facets } = envelopeParts(payload) const items = rows.map((row) => normalizeProgram(row, today)) diff --git a/src/lib/catalog/d1.ts b/src/lib/catalog/d1.ts index 6fb3d65..29d948e 100644 --- a/src/lib/catalog/d1.ts +++ b/src/lib/catalog/d1.ts @@ -1,13 +1,15 @@ import { bundleSchema } from '@/lib/data/schema' import type { DataBundle } from '@/lib/data/types' import { getTodayDate } from '@/lib/data/freshness' -import { parseD1ProgramList, parseD1ScholarshipList } from './d1-list' +import { parseD1InstitutionList, parseD1ProgramList, parseD1ScholarshipList } from './d1-list' import { deriveCatalogRelease, parseCatalogRelease } from './release' import { CATALOG_LIST_DEFAULT_LIMIT, CATALOG_LIST_MAX_LIMIT, CatalogRepositoryError, type CatalogFetch, + type CatalogInstitutionListPage, + type CatalogInstitutionListQuery, type CatalogProgramListPage, type CatalogProgramListQuery, type CatalogRelease, @@ -193,6 +195,24 @@ export class D1CatalogRepository implements CatalogRepository { return (await this.getSnapshot()).release } + async listInstitutions( + query: CatalogInstitutionListQuery = {}, + ): Promise { + const url = this.publicEndpoint('institutions') + addParam(url, 'q', query.q) + addParam(url, 'city', query.city) + addParam(url, 'region', query.region) + addParam(url, 'discipline', query.discipline) + addParam(url, 'sort', query.sort) + addParam(url, 'cursor', query.cursor) + addParam(url, 'limit', listLimit(query.limit)) + return parseD1InstitutionList( + await this.fetchListPayload(url), + query.today ?? getTodayDate(), + ) + } + + async listPrograms( query: CatalogProgramListQuery = {}, ): Promise { @@ -241,7 +261,7 @@ export class D1CatalogRepository implements CatalogRepository { ) } - private publicEndpoint(resource: 'programs' | 'scholarships'): URL { + private publicEndpoint(resource: 'institutions' | 'programs' | 'scholarships'): URL { const url = new URL(this.parsedApiUrl) url.search = '' url.hash = '' diff --git a/src/lib/catalog/index.ts b/src/lib/catalog/index.ts index 067a294..3843366 100644 --- a/src/lib/catalog/index.ts +++ b/src/lib/catalog/index.ts @@ -39,6 +39,12 @@ export { type CatalogBundleLoader, type CatalogCollection, type CatalogFetch, + type CatalogInstitutionCity, + type CatalogInstitutionListFacets, + type CatalogInstitutionListItem, + type CatalogInstitutionListPage, + type CatalogInstitutionListQuery, + type CatalogInstitutionListSort, type CatalogListOption, type CatalogListPage, type CatalogProgramListFacets, diff --git a/src/lib/catalog/json.ts b/src/lib/catalog/json.ts index 6b19e21..bbd038b 100644 --- a/src/lib/catalog/json.ts +++ b/src/lib/catalog/json.ts @@ -2,7 +2,9 @@ import { readFileSync } from 'node:fs' import { join } from 'node:path' import { bundleSchema } from '@/lib/data/schema' import type { DataBundle } from '@/lib/data/types' -import { getTodayDate } from '@/lib/data/freshness' +import { getTodayDate, isCurrentVerifiedRecord } from '@/lib/data/freshness' +import { classifyProgramField } from '@/lib/data/fields' +import { selectCatalogApiData } from '@/lib/catalog-api/projection' import { selectPublishedData } from '@/lib/data/publication' import { parseProgramCatalogFilters, @@ -22,6 +24,8 @@ import { CATALOG_LIST_MAX_LIMIT, CatalogRepositoryError, type CatalogBundleLoader, + type CatalogInstitutionListPage, + type CatalogInstitutionListQuery, type CatalogProgramListPage, type CatalogProgramListQuery, type CatalogRelease, @@ -59,9 +63,18 @@ function listLimit(value: number | undefined): number { return Math.min(value, CATALOG_LIST_MAX_LIMIT) } +function searchTokens(values: Array): string[] { + return values + .filter((value): value is string => typeof value === 'string') + .join(' ') + .normalize('NFKC') + .toLocaleLowerCase() + .match(/[\p{L}\p{N}]+/gu) ?? [] +} + function queryFingerprint( - resource: 'programs' | 'scholarships', - query: CatalogProgramListQuery | CatalogScholarshipListQuery, + resource: 'institutions' | 'programs' | 'scholarships', + query: CatalogInstitutionListQuery | CatalogProgramListQuery | CatalogScholarshipListQuery, ): string { const entries = Object.entries(query) .filter(([key, value]) => ( @@ -75,8 +88,8 @@ function queryFingerprint( } function requestedPage( - resource: 'programs' | 'scholarships', - query: CatalogProgramListQuery | CatalogScholarshipListQuery, + resource: 'institutions' | 'programs' | 'scholarships', + query: CatalogInstitutionListQuery | CatalogProgramListQuery | CatalogScholarshipListQuery, ): number { return query.cursor ? decodeJsonListCursor(query.cursor, resource, queryFingerprint(resource, query)) @@ -105,6 +118,106 @@ export class JsonCatalogRepository implements CatalogRepository { async getRelease(): Promise { return deriveCatalogRelease(await this.getBundle()) } + async listInstitutions( + query: CatalogInstitutionListQuery = {}, + ): Promise { + const today = query.today ?? getTodayDate() + const data = selectCatalogApiData(await this.getBundle(), today) + const limit = listLimit(query.limit) + const page = requestedPage('institutions', query) + const requestedQuery = query.q?.trim() ?? '' + const queryTerms = searchTokens([query.q]) + if (requestedQuery && (queryTerms.length === 0 || queryTerms.length > 20)) { + throw new CatalogRepositoryError( + 'INVALID_SEARCH_QUERY', + 'Institution search must contain between 1 and 20 letter or number terms.', + ) + } + const cityById = new Map(data.cities.map((city) => [city.id, city])) + const programsByUniversity = new Map() + const scholarshipsByUniversity = new Map() + + for (const program of data.programs) { + const related = programsByUniversity.get(program.universityId) ?? [] + related.push(program) + programsByUniversity.set(program.universityId, related) + } + for (const scholarship of data.scholarships) { + if (!isCurrentVerifiedRecord(scholarship, today)) continue + for (const universityId of new Set(scholarship.universityIds)) { + scholarshipsByUniversity.set( + universityId, + (scholarshipsByUniversity.get(universityId) ?? 0) + 1, + ) + } + } + + const items = data.universities.flatMap((institution) => { + const safeInstitution = { + ...institution, + sourceIds: [...institution.sourceIds].sort(), + summary: isCurrentVerifiedRecord(institution, today) ? institution.summary : null, + } + const city = cityById.get(institution.cityId) ?? null + const relatedPrograms = programsByUniversity.get(institution.id) ?? [] + const disciplines = [...new Set( + relatedPrograms + .filter((program) => isCurrentVerifiedRecord(program, today)) + .map(classifyProgramField), + )].sort() + const nameTokens = searchTokens(Object.values(safeInstitution.name)) + if ( + queryTerms.some((term) => !nameTokens.some((token) => token.startsWith(term))) + || (query.city && city?.id !== query.city && city?.slug !== query.city) + || (query.region && (institution.region ?? city?.region) !== query.region) + || (query.discipline && !disciplines.some((discipline) => discipline === query.discipline)) + ) return [] + + return [{ + institution: safeInstitution, + city: city ? { + id: city.id, + slug: city.slug, + name: city.name, + region: city.region, + } : null, + programCount: relatedPrograms.length, + scholarshipCount: scholarshipsByUniversity.get(institution.id) ?? 0, + disciplines, + }] + }) + items.sort((left, right) => { + const sortComparison = query.sort === 'programs-desc' + ? right.programCount - left.programCount + : query.sort === 'scholarships-desc' + ? right.scholarshipCount - left.scholarshipCount + : 0 + return sortComparison + || left.institution.slug.localeCompare(right.institution.slug) + || left.institution.id.localeCompare(right.institution.id) + }) + + const pageCount = Math.max(1, Math.ceil(items.length / limit)) + if (page > pageCount) { + throw new CatalogRepositoryError('INVALID_LIST_CURSOR', 'Catalog list cursor is outside the result set.') + } + const nextCursor = page < pageCount + ? encodeJsonListCursor('institutions', queryFingerprint('institutions', query), page + 1) + : null + const cities = [...new Map(items.flatMap((item) => ( + item.city ? [[item.city.slug, { value: item.city.slug, name: item.city.name }] as const] : [] + ))).values()].sort((left, right) => left.value.localeCompare(right.value)) + + return { + items: items.slice((page - 1) * limit, page * limit), + nextCursor, + total: items.length, + facets: { cities }, + release: deriveCatalogRelease(data), + } + } + + async listPrograms(query: CatalogProgramListQuery = {}): Promise { const today = query.today ?? getTodayDate() const data = selectPublishedData(await this.getBundle(), today) diff --git a/src/lib/catalog/list-cursor.ts b/src/lib/catalog/list-cursor.ts index 0ecf6c4..3a7897e 100644 --- a/src/lib/catalog/list-cursor.ts +++ b/src/lib/catalog/list-cursor.ts @@ -4,7 +4,7 @@ import { CatalogRepositoryError } from './types' type JsonListCursor = { v: 1 backend: 'json' - resource: 'programs' | 'scholarships' + resource: 'institutions' | 'programs' | 'scholarships' fingerprint: string page: number } @@ -12,7 +12,7 @@ type JsonListCursor = { type ShadowListCursor = { v: 1 backend: 'shadow' - resource: 'programs' | 'scholarships' + resource: 'institutions' | 'programs' | 'scholarships' primary: string | null shadow: string | null } diff --git a/src/lib/catalog/shadow.ts b/src/lib/catalog/shadow.ts index 8ab3004..903ec3b 100644 --- a/src/lib/catalog/shadow.ts +++ b/src/lib/catalog/shadow.ts @@ -3,6 +3,8 @@ import { CATALOG_COLLECTIONS, type CatalogBackendMode, type CatalogCollection, + type CatalogInstitutionListPage, + type CatalogInstitutionListQuery, type CatalogProgramListPage, type CatalogProgramListQuery, type CatalogRelease, @@ -18,6 +20,7 @@ import { export type CatalogShadowOperation = | 'getBundle' | 'getRelease' + | 'listInstitutions' | 'listPrograms' | 'listScholarships' export type CatalogShadowStatus = 'match' | 'different' | 'shadow-error' @@ -251,7 +254,7 @@ function compareRelease( } function comparableListPage( - page: CatalogProgramListPage | CatalogScholarshipListPage, + page: CatalogInstitutionListPage | CatalogProgramListPage | CatalogScholarshipListPage, ): unknown { const facets = Object.fromEntries( Object.entries(page.facets).map(([name, values]) => [ @@ -267,9 +270,9 @@ function comparableListPage( } function compareListPage( - scope: 'programs' | 'scholarships', - primary: CatalogProgramListPage | CatalogScholarshipListPage, - shadow: CatalogProgramListPage | CatalogScholarshipListPage, + scope: 'universities' | 'programs' | 'scholarships', + primary: CatalogInstitutionListPage | CatalogProgramListPage | CatalogScholarshipListPage, + shadow: CatalogInstitutionListPage | CatalogProgramListPage | CatalogScholarshipListPage, maxDifferences: number, ): DifferenceCollector { const collector = new DifferenceCollector(maxDifferences) @@ -286,7 +289,7 @@ function compareListPage( function cursorInputs( cursor: string | undefined, - resource: 'programs' | 'scholarships', + resource: 'institutions' | 'programs' | 'scholarships', ): { primary?: string; shadow?: string } { if (!cursor) return {} try { @@ -302,7 +305,7 @@ function cursorInputs( } function combinedCursor( - resource: 'programs' | 'scholarships', + resource: 'institutions' | 'programs' | 'scholarships', primary: string | null, shadow: string | null, ): string | null { @@ -373,6 +376,39 @@ export class ShadowCatalogRepository implements CatalogRepository { return primaryResult.value } + async listInstitutions( + query: CatalogInstitutionListQuery = {}, + ): Promise { + const cursors = cursorInputs(query.cursor, 'institutions') + const [primaryResult, shadowResult] = await Promise.allSettled([ + this.primary.listInstitutions({ ...query, cursor: cursors.primary }), + this.shadow.listInstitutions({ ...query, cursor: cursors.shadow }), + ]) + if (primaryResult.status === 'rejected') throw primaryResult.reason + + if (shadowResult.status === 'rejected') { + await this.recordShadowError('listInstitutions', shadowResult.reason) + return { + ...primaryResult.value, + nextCursor: combinedCursor('institutions', primaryResult.value.nextCursor, null), + } + } + + await this.recordComparison( + 'listInstitutions', + compareListPage('universities', primaryResult.value, shadowResult.value, this.maxDifferences), + ) + return { + ...primaryResult.value, + nextCursor: combinedCursor( + 'institutions', + primaryResult.value.nextCursor, + shadowResult.value.nextCursor, + ), + } + } + + async listPrograms( query: CatalogProgramListQuery = {}, ): Promise { diff --git a/src/lib/catalog/types.ts b/src/lib/catalog/types.ts index 686f5cc..1fe28fa 100644 --- a/src/lib/catalog/types.ts +++ b/src/lib/catalog/types.ts @@ -1,5 +1,6 @@ import type { AdmissionCycle, + City, DataBundle, LocalizedText, Program, @@ -38,6 +39,45 @@ export type CatalogListPage = { release: CatalogRelease | null } +export type CatalogInstitutionListSort = + | 'default' + | 'name' + | 'programs-desc' + | 'scholarships-desc' + +export type CatalogInstitutionListQuery = { + q?: string + city?: string + region?: string + discipline?: string + sort?: CatalogInstitutionListSort + cursor?: string + limit?: number + today?: string +} + +export type CatalogInstitutionCity = Pick< + City, + 'id' | 'slug' | 'name' | 'region' +> + +export type CatalogInstitutionListItem = { + institution: University + city: CatalogInstitutionCity | null + programCount: number + scholarshipCount: number + disciplines: string[] +} + +export type CatalogInstitutionListFacets = { + cities: CatalogListOption[] +} + +export type CatalogInstitutionListPage = CatalogListPage< + CatalogInstitutionListItem, + CatalogInstitutionListFacets +> + export type CatalogProgramListQuery = { q?: string institution?: string @@ -128,6 +168,7 @@ export interface CatalogRepository { readonly mode: CatalogBackendMode getBundle(): Promise getRelease(): Promise + listInstitutions(query?: CatalogInstitutionListQuery): Promise listPrograms(query?: CatalogProgramListQuery): Promise listScholarships(query?: CatalogScholarshipListQuery): Promise } diff --git a/src/lib/university-catalog.ts b/src/lib/university-catalog.ts new file mode 100644 index 0000000..e1acae4 --- /dev/null +++ b/src/lib/university-catalog.ts @@ -0,0 +1,201 @@ +import { normalizeProgramField } from '@/lib/data/fields' +import type { Region } from '@/lib/data/types' +import type { + CatalogInstitutionListItem, + CatalogInstitutionListQuery, + CatalogInstitutionListSort, + CatalogListOption, + CatalogRepository, +} from '@/lib/catalog/types' + +export const UNIVERSITY_CATALOG_PAGE_SIZE = 24 + +const REGIONS = new Set([ + 'north', + 'northeast', + 'east', + 'south', + 'central', + 'southwest', + 'northwest', +]) +const SORT_ORDERS = new Set([ + 'default', + 'name', + 'programs-desc', + 'scholarships-desc', +]) + +export type UniversityCatalogSearchParams = Record + +export type UniversityCatalogFilters = { + query: string + city: string + region: string + discipline: string + sort: CatalogInstitutionListSort + page: number + cursor: string + cursorHistory: string[] + nextCursor?: string +} + +export type UniversityCatalogResult = { + items: CatalogInstitutionListItem[] + filters: UniversityCatalogFilters + total: number + totalExact: boolean + page: number + pageCount: number + pageSize: number + cityOptions: CatalogListOption[] +} + +function first(value: string | string[] | undefined): string { + return typeof value === 'string' ? value : '' +} + +function bounded(value: string | string[] | undefined, maxLength = 160): string { + return first(value).trim().slice(0, maxLength) +} + +function searchQuery(value: string | string[] | undefined): string { + const normalized = bounded(value).normalize('NFKC') + if (!normalized) return '' + const terms = normalized.match(/[\p{L}\p{N}]+/gu) ?? [] + if (terms.length === 0 || terms.length > 20) return '' + return terms.join(' ') +} + +function cursorValue(value: string | string[] | undefined): string { + const cursor = first(value).trim() + return cursor.length <= 1_024 ? cursor : '' +} + +function cursorHistory(value: string | string[] | undefined): string[] { + const history = first(value).trim() + if (!history || history.length > 8_192) return [] + const entries = history.split(',').slice(-50) + return entries.every((entry) => entry === '~' || (entry.length > 0 && entry.length <= 1_024)) + ? entries + : [] +} + +function allowed(value: string, values: ReadonlySet): string { + return values.has(value) ? value : '' +} + +export function parseUniversityCatalogFilters( + params: UniversityCatalogSearchParams, +): UniversityCatalogFilters { + const requestedDiscipline = bounded(params.discipline) + const requestedPage = Number.parseInt(first(params.page), 10) + + return { + query: searchQuery(params.q), + city: bounded(params.city), + region: allowed(bounded(params.region), REGIONS), + discipline: requestedDiscipline + ? normalizeProgramField(requestedDiscipline) ?? '' + : '', + sort: (allowed(bounded(params.sort), SORT_ORDERS) || 'default') as CatalogInstitutionListSort, + page: Number.isSafeInteger(requestedPage) && requestedPage > 0 && requestedPage <= 10_000 + ? requestedPage + : 1, + cursor: cursorValue(params.cursor), + cursorHistory: cursorHistory(params.cursorHistory), + } +} + +function repositoryInstitutionQuery( + filters: UniversityCatalogFilters, + cursor: string | undefined, +): CatalogInstitutionListQuery { + return { + q: filters.query || undefined, + city: filters.city || undefined, + region: filters.region || undefined, + discipline: filters.discipline || undefined, + sort: filters.sort === 'default' ? undefined : filters.sort, + cursor, + limit: UNIVERSITY_CATALOG_PAGE_SIZE, + } +} + +/** + * Executes exactly one bounded Repository request. Cursor-based pagination is + * intentionally not replayed from page one: a page number without its cursor + * is normalized to page one, while normal next/previous links carry both the + * current cursor and the cursor stack in the URL. + */ +export async function queryUniversityCatalogRepository( + repository: CatalogRepository, + filters: UniversityCatalogFilters, +): Promise { + const cursor = filters.cursor || undefined + const page = cursor ? filters.page : 1 + const history = cursor ? [...filters.cursorHistory] : [] + const pageResult = await repository.listInstitutions( + repositoryInstitutionQuery(filters, cursor), + ) + const lowerBound = (page - 1) * UNIVERSITY_CATALOG_PAGE_SIZE + + pageResult.items.length + + (pageResult.nextCursor ? 1 : 0) + const total = pageResult.total ?? lowerBound + const pageCount = pageResult.total === null + ? page + (pageResult.nextCursor ? 1 : 0) + : Math.max(1, Math.ceil(pageResult.total / UNIVERSITY_CATALOG_PAGE_SIZE)) + + return { + items: pageResult.items, + filters: { + ...filters, + page, + cursor: cursor ?? '', + cursorHistory: history, + nextCursor: pageResult.nextCursor ?? undefined, + }, + total, + totalExact: pageResult.total !== null, + page, + pageCount, + pageSize: UNIVERSITY_CATALOG_PAGE_SIZE, + cityOptions: pageResult.facets.cities, + } +} + +export function universityCatalogHref( + locale: string, + filters: UniversityCatalogFilters, + page = filters.page, +): string { + const params = new URLSearchParams() + let targetCursor = filters.cursor + let targetHistory = [...filters.cursorHistory] + + if (page === filters.page + 1 && filters.nextCursor) { + targetHistory.push(filters.cursor || '~') + targetCursor = filters.nextCursor + } else if (page === filters.page - 1) { + const previous = targetHistory.pop() + targetCursor = previous && previous !== '~' ? previous : '' + } else if (page !== filters.page) { + targetCursor = '' + targetHistory = [] + } + + const values: Array<[string, string]> = [ + ['q', filters.query], + ['city', filters.city], + ['region', filters.region], + ['discipline', filters.discipline], + ['sort', filters.sort === 'default' ? '' : filters.sort], + ] + for (const [key, value] of values) if (value) params.set(key, value) + if (targetCursor) params.set('cursor', targetCursor) + if (targetHistory.length > 0) params.set('cursorHistory', targetHistory.join(',')) + if (page > 1) params.set('page', String(page)) + + const query = params.toString() + return `/${locale}/universities${query ? `?${query}` : ''}` +} diff --git a/tests/e2e/smoke.spec.ts b/tests/e2e/smoke.spec.ts index 2d5e48a..1e26589 100644 --- a/tests/e2e/smoke.spec.ts +++ b/tests/e2e/smoke.spec.ts @@ -65,7 +65,9 @@ test('the public program catalogue excludes draft templates', async ({ page }) = await expect(search).toBeVisible() await expect(page.getByTestId('program-publication-note')).toHaveCount(0) expect(await page.locator('.record-card').count()).toBeGreaterThan(0) - await search.fill('Tsinghua University Computer Science and Technology') + await search.fill('Tsinghua University Business Administration') + await search.press('Enter') + await page.waitForURL(/q=Tsinghua\+University\+Business\+Administration/) await expect(page.locator('.record-card')).toHaveCount(0) }) @@ -92,7 +94,7 @@ test('a multi-cycle program promotes the next upcoming intake', async ({ page }) }) test('a draft program detail is not publicly routable', async ({ page }) => { - const response = await page.goto('/en/programs/tsinghua-university-computer-science-bachelor', { waitUntil: 'domcontentloaded' }) + const response = await page.goto('/en/programs/tsinghua-university-business-administration-master', { waitUntil: 'domcontentloaded' }) expect(response?.status()).toBe(404) await expect(page.locator('main')).toBeVisible() diff --git a/tests/unit/catalog-api-service.test.ts b/tests/unit/catalog-api-service.test.ts index 3da15aa..cc6c28d 100644 --- a/tests/unit/catalog-api-service.test.ts +++ b/tests/unit/catalog-api-service.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it } from 'vitest' import type { DataBundle } from '@/lib/data/types' import { InvalidCursorError } from '@/lib/catalog-api/cursor' -import { CatalogApiService, releaseFromBundle } from '@/lib/catalog-api/service' +import { + CatalogApiService, + InvalidSearchQueryError, + releaseFromBundle, +} from '@/lib/catalog-api/service' const text = (en: string, zh = en, ru = en) => ({ en, zh, ru }) @@ -84,6 +88,189 @@ describe('CatalogApiService', () => { .toEqual(['program-mtcsol']) }) + it('returns exact institution metadata, public disciplines, sorts, and query-bound cursors', () => { + const bundle = fixture() + bundle.cities.push({ + ...bundle.cities[0], + id: 'city-2', + slug: 'shanghai', + name: text('Shanghai'), + province: text('Shanghai'), + region: 'east', + }) + bundle.universities.push( + { + ...bundle.universities[0], + id: 'uni-2', + slug: 'alpha-university', + name: text('Alpha University'), + cityId: 'city-2', + region: 'east', + featured: false, + }, + { + ...bundle.universities[0], + id: 'uni-3', + slug: 'zeta-university', + name: text('Zeta University'), + featured: false, + }, + ) + bundle.programs.push( + { + ...bundle.programs[0], + id: 'program-2', + slug: 'business-administration', + universityId: 'uni-2', + name: text('Business Administration'), + discipline: 'business', + }, + { + ...bundle.programs[0], + id: 'program-3', + slug: 'mechanical-engineering', + universityId: 'uni-2', + name: text('Mechanical Engineering'), + }, + { + ...bundle.programs[0], + id: 'program-4', + slug: 'data-science', + universityId: 'uni-3', + name: text('Data Science'), + }, + ) + bundle.scholarships.push( + { + ...bundle.scholarships[0], + id: 'scholarship-2', + slug: 'alpha-scholarship-a', + universityIds: ['uni-2'], + programIds: ['program-2'], + }, + { + ...bundle.scholarships[0], + id: 'scholarship-3', + slug: 'alpha-scholarship-b', + universityIds: ['uni-2'], + programIds: ['program-3'], + }, + ) + const service = new CatalogApiService( + bundle, + releaseFromBundle(bundle, '2026-07-20'), + '2026-07-20', + ) + + const defaults = service.listInstitutions() + expect(defaults.meta.total).toBe(3) + expect(defaults.meta.facets?.cities?.map((item) => item.value)).toEqual([ + 'beijing', + 'shanghai', + ]) + expect(defaults.data.find((item) => item.id === 'uni-1')?.disciplines) + .toEqual(['computing-data']) + + expect(service.listInstitutions({ sort: 'name' }).data.map((item) => item.id)) + .toEqual(['uni-2', 'uni-1', 'uni-3']) + expect(service.listInstitutions({ sort: 'programs-desc' }).data.map((item) => item.id)) + .toEqual(['uni-2', 'uni-1', 'uni-3']) + expect(service.listInstitutions({ sort: 'scholarships-desc' }).data.map((item) => item.id)) + .toEqual(['uni-2', 'uni-1', 'uni-3']) + + const query = { + q: 'university', + city: 'beijing', + region: 'north', + discipline: 'computing-data', + sort: 'name' as const, + limit: 1, + } + const first = service.listInstitutions(query) + expect(first.data).toHaveLength(1) + expect(first.meta.total).toBe(2) + expect(first.meta.facets?.cities?.map((item) => item.value)).toEqual(['beijing']) + expect(first.meta.nextCursor).toBeTruthy() + + const second = service.listInstitutions({ + ...query, + cursor: first.meta.nextCursor ?? undefined, + }) + expect(second.data).toHaveLength(1) + expect(second.data[0].id).not.toBe(first.data[0].id) + + for (const changed of [ + { ...query, q: 'example' }, + { ...query, city: 'shanghai' }, + { ...query, region: 'east' }, + { ...query, discipline: 'business-economics' }, + { ...query, sort: 'programs-desc' as const }, + ]) { + expect(() => service.listInstitutions({ + ...changed, + cursor: first.meta.nextCursor ?? undefined, + })).toThrow(InvalidCursorError) + } + }) + + it('searches institution names only with NFKC AND token-prefix semantics', () => { + const bundle = fixture() + bundle.universities[0] = { + ...bundle.universities[0]!, + summary: text('Stale Summary Canary'), + reviewAfter: '2026-07-19', + status: 'stale', + } + bundle.cities[0] = { + ...bundle.cities[0]!, + name: text('City Search Canary'), + } + bundle.programs[0] = { + ...bundle.programs[0]!, + name: text('Program Search Canary'), + } + const service = new CatalogApiService( + bundle, + releaseFromBundle(bundle, '2026-07-20'), + '2026-07-20', + ) + + expect(service.listInstitutions({ q: 'Ｕｎｉ Ｅｘａ' }).data.map((item) => item.id)) + .toEqual(['uni-1']) + for (const query of ['stale summary', 'city search', 'program search']) { + expect(service.listInstitutions({ q: query }).data).toEqual([]) + } + expect(() => service.listInstitutions({ q: Array.from({ length: 21 }, () => 'term').join(' ') })) + .toThrow(InvalidSearchQueryError) + expect(() => service.listInstitutions({ q: '---' })).toThrow(InvalidSearchQueryError) + }) + it('uses 24 as the institution page default and never returns more than 100 records', () => { + const bundle = fixture() + for (let index = 2; index <= 105; index += 1) { + bundle.universities.push({ + ...bundle.universities[0], + id: `uni-${index}`, + slug: `university-${String(index).padStart(3, '0')}`, + name: text(`University ${index}`), + featured: false, + }) + } + const service = new CatalogApiService( + bundle, + releaseFromBundle(bundle, '2026-07-20'), + '2026-07-20', + ) + + const defaultPage = service.listInstitutions() + expect(defaultPage.data).toHaveLength(24) + expect(defaultPage.meta.total).toBe(105) + expect(defaultPage.meta.nextCursor).toBeTruthy() + + const maximumPage = service.listInstitutions({ limit: 101 }) + expect(maximumPage.data).toHaveLength(100) + expect(maximumPage.meta.total).toBe(105) + expect(maximumPage.meta.nextCursor).toBeTruthy() + }) it('uses stable opaque cursor pagination and rejects unknown cursors', () => { const bundle = fixture() bundle.universities.push({ ...bundle.universities[0], id: 'uni-2', slug: 'second-university' }) @@ -122,6 +309,7 @@ describe('CatalogApiService', () => { expect(institution?.name.en).toBe('Example University') expect(institution?.summary).toBeNull() + expect(institution?.disciplines).toEqual([]) expect(program?.name.en).toBe('Computer Science') expect(program?.durationMonths).toBeNull() expect(program?.teachingLanguages).toBeNull() @@ -135,6 +323,7 @@ describe('CatalogApiService', () => { expect(scholarshipCycle?.deadline).toBeNull() expect(scholarshipCycle?.academicYear).toBeNull() + expect(service.listInstitutions({ discipline: 'computing-data' }).data).toHaveLength(0) expect(service.listPrograms({ language: 'english' }).data).toHaveLength(0) expect(service.listPrograms({ tuitionMax: 35_000 }).data).toHaveLength(0) expect(service.listScholarships({ institution: 'example-university' }).data).toHaveLength(0) diff --git a/tests/unit/catalog-institution-api-route.test.ts b/tests/unit/catalog-institution-api-route.test.ts new file mode 100644 index 0000000..8f3c907 --- /dev/null +++ b/tests/unit/catalog-institution-api-route.test.ts @@ -0,0 +1,134 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { CatalogRepositoryError } from '@/lib/catalog' + +const repository = vi.hoisted(() => ({ + getBundle: vi.fn(), + getRelease: vi.fn(), + listInstitutions: vi.fn(), + listPrograms: vi.fn(), + listScholarships: vi.fn(), +})) + +vi.mock('@/lib/catalog', async () => { + const actual = await vi.importActual('@/lib/catalog') + return { + ...actual, + getCatalogRepository: () => ({ mode: 'd1', ...repository }), + } +}) + +import { GET } from '@/app/api/v1/institutions/route' + +const release = { + id: 'release-example', + dataDate: '2026-08-06', + generatedAt: '2026-08-06T00:00:00.000Z', + recordCounts: { + sources: 1, + cities: 1, + universities: 1, + programs: 7, + admissionCycles: 1, + scholarships: 3, + }, +} + +const page = { + items: [{ + institution: { + id: 'institution-example', + slug: 'example-university', + name: { en: 'Example University', zh: 'Example University ZH' }, + cityId: 'city-example', + region: 'east', + officialUrl: 'https://example.edu.cn/', + admissionsUrl: null, + summary: { en: 'Official summary' }, + featured: true, + sourceIds: ['source-example'], + verifiedAt: '2026-08-06', + reviewAfter: '2026-09-06', + status: 'verified', + }, + city: { + id: 'city-example', + slug: 'example-city', + name: { en: 'Example City' }, + region: 'east', + }, + programCount: 7, + scholarshipCount: 3, + disciplines: ['engineering-technology'], + }], + nextCursor: 'next-cursor', + total: 1, + facets: { cities: [{ value: 'example-city', name: { en: 'Example City' } }] }, + release, +} + +describe('institution catalog API route', () => { + beforeEach(() => { + vi.clearAllMocks() + repository.listInstitutions.mockResolvedValue(page) + }) + + it('uses the bounded Repository list path without reading a compatibility bundle', async () => { + const response = await GET(new Request( + 'https://example.test/api/v1/institutions?q=Example&sort=programs-desc&limit=24', + )) + const body = await response.json() + + expect(response.status).toBe(200) + expect(response.headers.get('cache-control')).toContain('s-maxage=300') + expect(repository.listInstitutions).toHaveBeenCalledTimes(1) + expect(repository.listInstitutions).toHaveBeenCalledWith(expect.objectContaining({ + q: 'Example', + sort: 'programs-desc', + limit: 24, + })) + expect(repository.getBundle).not.toHaveBeenCalled() + expect(repository.getRelease).not.toHaveBeenCalled() + expect(body).toMatchObject({ + data: [{ + id: 'institution-example', + city: { id: 'city-example', province: null }, + programCount: 7, + scholarshipCount: 3, + disciplines: ['engineering-technology'], + fieldMeta: { + admissionsUrl: { status: 'officially_not_announced' }, + summary: { status: 'known' }, + }, + }], + meta: { + release: { id: 'release-example' }, + pageSize: 1, + nextCursor: 'next-cursor', + total: 1, + facets: { cities: [{ value: 'example-city' }] }, + }, + }) + }) + + it('rejects invalid search, sort, and Repository cursors as client errors', async () => { + for (const url of [ + 'https://example.test/api/v1/institutions?q=---', + 'https://example.test/api/v1/institutions?sort=random', + ]) { + const response = await GET(new Request(url)) + expect(response.status).toBe(400) + } + expect(repository.listInstitutions).not.toHaveBeenCalled() + + repository.listInstitutions.mockRejectedValueOnce( + new CatalogRepositoryError('INVALID_LIST_CURSOR', 'Catalog cursor is invalid.'), + ) + const cursorResponse = await GET(new Request( + 'https://example.test/api/v1/institutions?cursor=bad-cursor', + )) + expect(cursorResponse.status).toBe(400) + expect(await cursorResponse.json()).toEqual({ + error: { code: 'invalid_request', message: 'Invalid cursor.' }, + }) + }) +}) diff --git a/tests/unit/catalog-institution-repository.test.ts b/tests/unit/catalog-institution-repository.test.ts new file mode 100644 index 0000000..37cca31 --- /dev/null +++ b/tests/unit/catalog-institution-repository.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, it } from 'vitest' +import sources from '../../content/data/sources.json' +import cities from '../../content/data/cities.json' +import universities from '../../content/data/universities.json' +import programs from '../../content/data/programs.json' +import admissionCycles from '../../content/data/admission-cycles.json' +import scholarships from '../../content/data/scholarships.json' +import { + CatalogRepositoryError, + createD1CatalogRepository, + createJsonCatalogRepository, + createShadowCatalogRepository, + deriveCatalogRelease, + type CatalogFetch, + type CatalogInstitutionListItem, + type CatalogInstitutionListSort, + type CatalogRepository, +} from '@/lib/catalog' +import { selectCatalogApiData } from '@/lib/catalog-api/projection' +import { classifyProgramField } from '@/lib/data/fields' +import { isCurrentVerifiedRecord } from '@/lib/data/freshness' +import { bundleSchema } from '@/lib/data/schema' +import type { DataBundle } from '@/lib/data/types' + +const TODAY = '2026-07-20' +const allData = bundleSchema.parse({ + sources, + cities, + universities, + programs, + admissionCycles, + scholarships, +}) + +function freshBundle(): DataBundle { + return structuredClone(allData) +} + +function compareItems( + left: CatalogInstitutionListItem, + right: CatalogInstitutionListItem, + sort: CatalogInstitutionListSort, +): number { + const comparison = sort === 'programs-desc' + ? right.programCount - left.programCount + : sort === 'scholarships-desc' + ? right.scholarshipCount - left.scholarshipCount + : 0 + return comparison + || left.institution.slug.localeCompare(right.institution.slug) + || left.institution.id.localeCompare(right.institution.id) +} + +describe('CatalogRepository institution lists', () => { + it('filters JSON institutions with the shared field taxonomy and exposes exact facets and counts', async () => { + const bundle = freshBundle() + const repository = createJsonCatalogRepository(() => bundle) + const publicBundle = selectCatalogApiData(bundle, TODAY) + const targetProgram = publicBundle.programs.find( + (program) => isCurrentVerifiedRecord(program, TODAY) + && classifyProgramField(program) === 'engineering-technology', + )! + const target = publicBundle.universities.find( + (institution) => institution.id === targetProgram.universityId, + )! + const city = bundle.cities.find((item) => item.id === target.cityId)! + const queryName = (target.name.en ?? target.name.zh)!.split(/\s+/u).slice(0, 2).join(' ') + + const result = await repository.listInstitutions({ + q: queryName, + city: city.slug, + region: target.region ?? city.region ?? undefined, + discipline: 'engineering-technology', + limit: 100, + today: TODAY, + }) + const item = result.items.find(({ institution }) => institution.id === target.id) + + expect(item).toMatchObject({ + institution: { id: target.id, slug: target.slug }, + city: { id: city.id, slug: city.slug }, + programCount: publicBundle.programs.filter((program) => program.universityId === target.id).length, + scholarshipCount: publicBundle.scholarships.filter( + (scholarship) => isCurrentVerifiedRecord(scholarship, TODAY) + && scholarship.universityIds.includes(target.id), + ).length, + }) + expect(item?.disciplines).toContain('engineering-technology') + expect(result.total).toBe(result.items.length) + expect(result.facets.cities).toEqual([{ value: city.slug, name: city.name }]) + + for (const discipline of ['engineering-technology', 'chinese-language'] as const) { + const page = await repository.listInstitutions({ discipline, limit: 100, today: TODAY }) + expect(page.total).toBeGreaterThan(0) + expect(page.items.every((entry) => entry.disciplines.includes(discipline))).toBe(true) + } + }) + + it('sorts JSON institutions deterministically and binds cursors to their query', async () => { + const repository = createJsonCatalogRepository(() => freshBundle()) + const sorts = ['default', 'name', 'programs-desc', 'scholarships-desc'] as const + + for (const sort of sorts) { + const page = await repository.listInstitutions({ sort, limit: 100, today: TODAY }) + expect(page.items.length).toBeGreaterThan(1) + for (let index = 1; index < page.items.length; index += 1) { + expect(compareItems(page.items[index - 1]!, page.items[index]!, sort)).toBeLessThanOrEqual(0) + } + } + + const first = await repository.listInstitutions({ sort: 'default', limit: 1, today: TODAY }) + expect(first.total).toBe(selectCatalogApiData(freshBundle(), TODAY).universities.length) + expect(first.nextCursor).not.toBeNull() + const second = await repository.listInstitutions({ + sort: 'default', + cursor: first.nextCursor!, + limit: 1, + today: TODAY, + }) + expect(second.items[0]?.institution.id).not.toBe(first.items[0]?.institution.id) + await expect(repository.listInstitutions({ + sort: 'name', + cursor: first.nextCursor!, + limit: 1, + today: TODAY, + })).rejects.toMatchObject({ code: 'INVALID_LIST_CURSOR' }) + }) + + it('maps the Worker institution DTO, field freshness, filters, facets, and cursor metadata', async () => { + const requests: string[] = [] + const release = deriveCatalogRelease(freshBundle(), 'd1') + const payload = { + data: [{ + type: 'institution', + id: 'institution-example', + slug: 'example-university', + attributes: { + name: { en: 'Example University', zh: 'Example University ZH' }, + summary: { en: 'Official summary' }, + institutionType: 'university', + officialUrl: 'https://example.edu.cn/', + admissionsUrl: 'https://example.edu.cn/admissions', + featured: true, + disciplineCodes: ['engineering-technology', 'chinese-language'], + }, + relationships: { + location: { + id: 'city-example', + slug: 'example-city', + name: { en: 'Example City', zh: 'Example City ZH' }, + countryCode: 'CN', + regionCode: 'east', + }, + programs: { count: 7 }, + scholarships: { count: 3 }, + }, + sources: [{ + id: 'source-example', + url: 'https://example.edu.cn/', + title: 'Official university website', + publisher: 'Example University', + languageCode: 'en', + authorityLevel: 'primary_official', + checkedAt: '2026-08-07', + }], + fieldMeta: { + name: { + status: 'stale', + officialUrl: 'https://example.edu.cn/', + sourceTitle: 'Official university website', + checkedAt: '2026-08-07', + verifiedAt: '2026-07-01', + reviewAfter: '2026-08-06', + sourceIds: ['source-example'], + }, + }, + }], + meta: { + release, + nextCursor: 'worker-next', + total: 1, + facets: { + cities: [{ value: 'example-city', name: { en: 'Example City', zh: 'Example City ZH' } }], + }, + }, + } + const fetcher: CatalogFetch = async (input) => { + requests.push(String(input)) + return new Response(JSON.stringify(payload), { status: 200 }) + } + const repository = createD1CatalogRepository({ + apiUrl: 'https://catalog.example.test/internal/v1/catalog-bundle', + fetch: fetcher, + }) + + const result = await repository.listInstitutions({ + q: 'example', + city: 'example-city', + region: 'east', + discipline: 'engineering-technology', + sort: 'programs-desc', + cursor: 'worker-cursor', + limit: 30, + today: TODAY, + }) + const request = new URL(requests[0]!) + + expect(request.pathname).toBe('/api/v1/institutions') + expect(Object.fromEntries(request.searchParams)).toMatchObject({ + q: 'example', + city: 'example-city', + region: 'east', + discipline: 'engineering-technology', + sort: 'programs-desc', + cursor: 'worker-cursor', + limit: '30', + }) + expect(result).toMatchObject({ + total: 1, + nextCursor: 'worker-next', + facets: { cities: [{ value: 'example-city' }] }, + items: [{ + institution: { + id: 'institution-example', + slug: 'example-university', + cityId: 'city-example', + region: 'east', + status: 'stale', + verifiedAt: '2026-07-01', + reviewAfter: '2026-08-06', + }, + city: { id: 'city-example', slug: 'example-city', region: 'east' }, + programCount: 7, + scholarshipCount: 3, + disciplines: ['chinese-language', 'engineering-technology'], + }], + }) + }) + + it('compares Shadow institution pages, combines cursors, and fails open on shadow errors', async () => { + const primaryBundle = freshBundle() + const shadowBundle = freshBundle() + const firstShadow = [...shadowBundle.universities] + .sort((left, right) => left.slug.localeCompare(right.slug))[0]! + firstShadow.featured = !firstShadow.featured + const repository = createShadowCatalogRepository({ + primary: createJsonCatalogRepository(() => primaryBundle), + shadow: createJsonCatalogRepository(() => shadowBundle), + }) + + const first = await repository.listInstitutions({ limit: 1, today: TODAY }) + expect(first.nextCursor).not.toBeNull() + expect(repository.getLastReport()).toMatchObject({ + operation: 'listInstitutions', + status: 'different', + matches: false, + }) + await expect(repository.listInstitutions({ + cursor: first.nextCursor!, + limit: 1, + today: TODAY, + })).resolves.toMatchObject({ items: expect.any(Array) }) + + const primary = createJsonCatalogRepository(() => primaryBundle) + const failingShadow: CatalogRepository = { + mode: 'd1', + getBundle: async () => { throw new Error('shadow unavailable') }, + getRelease: async () => { throw new Error('shadow unavailable') }, + listInstitutions: async () => { throw new Error('shadow unavailable') }, + listPrograms: async () => { throw new Error('shadow unavailable') }, + listScholarships: async () => { throw new Error('shadow unavailable') }, + } + const failOpen = createShadowCatalogRepository({ primary, shadow: failingShadow }) + await expect(failOpen.listInstitutions({ limit: 1, today: TODAY })).resolves.toMatchObject({ + items: expect.any(Array), + nextCursor: expect.any(String), + }) + expect(failOpen.getLastReport()).toMatchObject({ + operation: 'listInstitutions', + status: 'shadow-error', + shadowError: { message: 'shadow unavailable' }, + }) + }) + + it('rejects malformed Worker institution relationship counts', async () => { + const repository = createD1CatalogRepository({ + apiUrl: 'https://catalog.example.test/internal/v1/catalog-bundle', + fetch: async () => new Response(JSON.stringify({ + data: [{ + type: 'institution', + id: 'institution-example', + slug: 'example-university', + attributes: { + name: { en: 'Example University' }, + summary: null, + officialUrl: 'https://example.edu.cn/', + admissionsUrl: null, + featured: false, + disciplineCodes: [], + }, + relationships: { + location: { + id: 'city-example', + slug: 'example-city', + name: { en: 'Example City' }, + regionCode: 'east', + }, + programs: { count: -1 }, + scholarships: { count: 0 }, + }, + sources: [], + fieldMeta: {}, + }], + meta: { total: 1, nextCursor: null, facets: { cities: [] } }, + }), { status: 200 }), + }) + + await expect(repository.listInstitutions({ today: TODAY })).rejects.toBeInstanceOf( + CatalogRepositoryError, + ) + }) +}) diff --git a/tests/unit/catalog-release-builder.test.ts b/tests/unit/catalog-release-builder.test.ts index 66addb4..9ad6deb 100644 --- a/tests/unit/catalog-release-builder.test.ts +++ b/tests/unit/catalog-release-builder.test.ts @@ -3,10 +3,11 @@ import { createHash } from 'node:crypto' import { readFileSync } from 'node:fs' import { join } from 'node:path' import { describe, expect, it } from 'vitest' +import { classifyProgramField, programFieldTaxonomy } from '../../src/lib/data/fields' import { buildLegacyRelease, readLegacyBundle } from '../../scripts/catalog/build-release' function applyMigrations(database: DatabaseSync) { - for (const file of ['0001_release_core.sql', '0002_programs_scholarships.sql', '0003_search_views.sql', '0004_atomic_release_cutover.sql', '0009_release_compatibility_artifacts.sql']) { + for (const file of ['0001_release_core.sql', '0002_programs_scholarships.sql', '0003_search_views.sql', '0004_atomic_release_cutover.sql', '0005_public_projection_hardening.sql', '0009_release_compatibility_artifacts.sql']) { database.exec(readFileSync(join(process.cwd(), 'infra', 'd1', 'catalog', 'migrations', file), 'utf8')) } } @@ -22,6 +23,7 @@ describe('legacy JSON release builder', () => { expect(artifacts.sql).not.toContain("SET release_status = 'retired'") expect(artifacts.sql).not.toContain('UPDATE release_pointer SET current_release_id') expect(artifacts.sql).toContain('INSERT OR IGNORE INTO release_activation_requests') + expect(artifacts.release.id).toMatch(/^legacy-v3-/u) expect(artifacts.r2Key).toContain(artifacts.release.id) expect(artifacts.contentSha256).toBe( createHash('sha256').update(Buffer.from(artifacts.envelope, 'utf8')).digest('hex'), @@ -31,10 +33,11 @@ describe('legacy JSON release builder', () => { applyMigrations(database) database.exec(artifacts.sql) database.exec(artifacts.sql) - const release = database.prepare('SELECT release_id, release_status, content_sha256 FROM current_release').get() as Record + const release = database.prepare('SELECT release_id, release_status, schema_version, content_sha256 FROM current_release').get() as Record expect(release).toEqual({ release_id: artifacts.release.id, release_status: 'active', + schema_version: 3, content_sha256: artifacts.contentSha256, }) const counts = database.prepare(` @@ -50,6 +53,106 @@ describe('legacy JSON release builder', () => { cycles: bundle.admissionCycles.length, scholarships: bundle.scholarships.length, }) + const programFieldCodes = programFieldTaxonomy('en').map((field) => field.key) + expect(programFieldCodes).toHaveLength(17) + const disciplineDimensions = new Set( + (database.prepare(` + SELECT code FROM disciplines WHERE release_id = ? ORDER BY code + `).all(artifacts.release.id) as Array<{ code: string }>).map((item) => item.code), + ) + for (const code of programFieldCodes) expect(disciplineDimensions.has(code)).toBe(true) + for (const legacyCode of [ + 'engineering', + 'business', + 'medicine', + 'chinese-education', + 'humanities', + 'law-ir', + 'science', + 'art-design', + 'other', + ]) expect(disciplineDimensions.has(legacyCode)).toBe(true) + + const projectedDisciplines = database.prepare(` + SELECT program_id, discipline_code + FROM program_disciplines + WHERE release_id = ? + ORDER BY program_id + `).all(artifacts.release.id) as Array<{ program_id: string; discipline_code: string }> + expect(projectedDisciplines).toHaveLength(bundle.programs.length) + expect(projectedDisciplines.every((item) => + programFieldCodes.includes(item.discipline_code as (typeof programFieldCodes)[number]) + )).toBe(true) + + const disciplineFacts = database.prepare(` + SELECT + COUNT(*) AS total, + SUM(field_status = 'known') AS known + FROM record_field_status + WHERE release_id = ? AND field_path = 'discipline' + `).get(artifacts.release.id) as { total: number; known: number } + expect(disciplineFacts.total).toBe(bundle.programs.length) + expect(disciplineFacts.known).toBeGreaterThan(0) + + const currentDiscipline = database.prepare(` + SELECT program_id, discipline_code + FROM current_program_disciplines + ORDER BY program_id + LIMIT 1 + `).get() as { program_id: string; discipline_code: string } + expect(currentDiscipline).toBeTruthy() + const sourceProgram = bundle.programs.find( + (program) => program.id === currentDiscipline.program_id, + ) + expect(sourceProgram).toBeTruthy() + expect(currentDiscipline.discipline_code).toBe(classifyProgramField(sourceProgram!)) + expect(database.prepare(` + SELECT field_status, value_json + FROM current_record_fields + WHERE record_id = ? AND field_path = 'discipline' + `).get(currentDiscipline.program_id)).toEqual({ + field_status: 'known', + value_json: JSON.stringify(currentDiscipline.discipline_code), + }) + + const summaryFacts = database.prepare(` + SELECT + COUNT(*) AS total, + SUM(field_status = 'known') AS known + FROM record_field_status + WHERE release_id = ? AND field_path = 'summary' + AND record_id IN ( + SELECT institution_id FROM institutions WHERE release_id = ? + ) + `).get(artifacts.release.id, artifacts.release.id) as { total: number; known: number } + expect(summaryFacts.total).toBe(bundle.universities.length) + expect(summaryFacts.known).toBeGreaterThan(0) + + const currentSummary = database.prepare(` + SELECT localized.record_id, localized.locale, localized.text_value + FROM current_localized_content AS localized + JOIN current_institutions AS institution + ON institution.release_id = localized.release_id + AND institution.institution_id = localized.record_id + WHERE localized.field_name = 'summary' + ORDER BY localized.record_id, localized.locale + LIMIT 1 + `).get() as { record_id: string; locale: string; text_value: string } + expect(currentSummary.text_value.length).toBeGreaterThan(0) + const summaryUniversity = bundle.universities.find( + (university) => university.id === currentSummary.record_id, + ) + expect(summaryUniversity).toBeTruthy() + expect(Object.values(summaryUniversity!.summary!)).toContain(currentSummary.text_value) + expect(database.prepare(` + SELECT field_status, value_json + FROM current_record_fields + WHERE record_id = ? AND field_path = 'summary' + `).get(currentSummary.record_id)).toEqual({ + field_status: 'known', + value_json: JSON.stringify(summaryUniversity!.summary), + }) + const otherProgram = database.prepare(` SELECT program_type, degree_level FROM programs diff --git a/tests/unit/catalog-repository.test.ts b/tests/unit/catalog-repository.test.ts index 62f9e48..a0fa4b8 100644 --- a/tests/unit/catalog-repository.test.ts +++ b/tests/unit/catalog-repository.test.ts @@ -231,6 +231,7 @@ describe('CatalogRepository', () => { mode: 'd1', getBundle: async () => { throw new Error('shadow unavailable') }, getRelease: async () => { throw new Error('shadow unavailable') }, + listInstitutions: async () => { throw new Error('shadow unavailable') }, listPrograms: async () => { throw new Error('shadow unavailable') }, listScholarships: async () => { throw new Error('shadow unavailable') }, } diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index 0459fec..5f40430 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -258,6 +258,275 @@ describe('Catalog D1 normalized v1 API', () => { )).toBe(true) }, 30_000) + it('lists institutions with exact metadata, disciplines, sorting, and query-bound cursors', async () => { + const defaultResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/institutions'), + environment, + ) + const defaults = await defaultResponse.json() as ApiEnvelopeDto + expect(defaultResponse.status).toBe(200) + expect(defaults.data).toHaveLength(24) + expect(defaults.meta.total).toBeGreaterThan(defaults.data.length) + expect(defaults.meta.facets?.cities?.length).toBeGreaterThan(1) + expect(defaults.meta.nextCursor).toEqual(expect.any(String)) + expect(defaults.data[0]!.attributes.disciplineCodes).toEqual(expect.any(Array)) + expect(defaults.data[0]!.fieldMeta.disciplineCodes.status).toBe('known') + + const secondResponse = await worker.fetch( + new Request( + `https://catalog.test/api/v1/institutions?sort=default&cursor=${encodeURIComponent(defaults.meta.nextCursor!)}`, + ), + environment, + ) + const second = await secondResponse.json() as ApiEnvelopeDto + expect(secondResponse.status).toBe(200) + expect(second.data.map((item) => item.id)).not.toContain(defaults.data[0]!.id) + + const mismatchedCursor = await worker.fetch( + new Request( + `https://catalog.test/api/v1/institutions?sort=programs-desc&cursor=${encodeURIComponent(defaults.meta.nextCursor!)}`, + ), + environment, + ) + expect(mismatchedCursor.status).toBe(400) + + const programSortResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/institutions?sort=programs-desc&limit=100'), + environment, + ) + const programSort = await programSortResponse.json() as ApiEnvelopeDto + expect(programSortResponse.status).toBe(200) + const programCounts = programSort.data.map((item) => item.relationships.programs.count) + expect(programCounts).toEqual([...programCounts].sort((left, right) => right - left)) + expect(programSort.data.every((item) => + item.slug !== null && /^[a-z0-9][a-z0-9-]*$/u.test(item.slug) + )).toBe(true) + + const scholarshipSortResponse = await worker.fetch( + new Request('https://catalog.test/api/v1/institutions?sort=scholarships-desc&limit=100'), + environment, + ) + const scholarshipSort = await scholarshipSortResponse.json() as ApiEnvelopeDto + const scholarshipCounts = scholarshipSort.data.map( + (item) => item.relationships.scholarships.count, + ) + expect(scholarshipSortResponse.status).toBe(200) + expect(scholarshipCounts).toEqual( + [...scholarshipCounts].sort((left, right) => right - left), + ) + + const candidate = database.prepare(` + SELECT + institution.institution_id, + city_record.slug AS city_slug, + city.region_code, + discipline.discipline_code + FROM current_institutions AS institution + JOIN current_locations AS city + ON city.release_id = institution.release_id + AND city.location_id = institution.city_id + JOIN current_catalog_records AS city_record + ON city_record.release_id = city.release_id + AND city_record.record_id = city.location_id + JOIN current_programs AS program + ON program.release_id = institution.release_id + AND program.institution_id = institution.institution_id + JOIN current_program_disciplines AS discipline + ON discipline.release_id = program.release_id + AND discipline.program_id = program.program_id + WHERE city_record.slug IS NOT NULL AND city.region_code IS NOT NULL + ORDER BY institution.institution_id, discipline.discipline_code + LIMIT 1 + `).get() as { + institution_id: string + city_slug: string + region_code: string + discipline_code: string + } + const filteredResponse = await worker.fetch( + new Request( + `https://catalog.test/api/v1/institutions?city=${candidate.city_slug}®ion=${candidate.region_code}&discipline=${candidate.discipline_code}&limit=100`, + ), + environment, + ) + const filtered = await filteredResponse.json() as ApiEnvelopeDto + expect(filteredResponse.status).toBe(200) + expect(filtered.data.map((item) => item.id)).toContain(candidate.institution_id) + expect(filtered.data.every((item) => + item.relationships.location.slug === candidate.city_slug + && item.relationships.location.regionCode === candidate.region_code + && item.attributes.disciplineCodes.includes(candidate.discipline_code) + )).toBe(true) + expect(filtered.meta.total).toBe(filtered.data.length) + expect(filtered.meta.facets?.cities?.map((item) => item.value)).toEqual([ + candidate.city_slug, + ]) + + const invalidSort = await worker.fetch( + new Request('https://catalog.test/api/v1/institutions?sort=featured'), + environment, + ) + expect(invalidSort.status).toBe(400) + expect(r2Reads).toBe(0) + }, 30_000) + + it('publishes current institution summaries and withholds all 14 missing admissions URLs', async () => { + const missingAdmissions = readLegacyBundle().universities.filter( + (university) => university.admissionsUrl === null, + ) + expect(missingAdmissions).toHaveLength(14) + + const currentSummary = database.prepare(` + SELECT record.slug, localized.locale, localized.text_value + FROM current_institutions AS institution + JOIN current_catalog_records AS record + ON record.release_id = institution.release_id + AND record.record_id = institution.institution_id + JOIN current_localized_content AS localized + ON localized.release_id = institution.release_id + AND localized.record_id = institution.institution_id + AND localized.field_name = 'summary' + WHERE record.slug IS NOT NULL + ORDER BY record.slug, localized.locale + LIMIT 1 + `).get() as { slug: string; locale: string; text_value: string } + const summaryResponse = await worker.fetch( + new Request(`https://catalog.test/api/v1/institutions/${currentSummary.slug}`), + environment, + ) + const summary = await summaryResponse.json() as ApiEnvelopeDto + expect(summaryResponse.status).toBe(200) + expect(summary.data.attributes.summary?.[currentSummary.locale]).toBe(currentSummary.text_value) + expect(summary.data.fieldMeta.summary.status).toBe('known') + + const allInstitutions: InstitutionDto[] = [] + let cursor: string | null = null + do { + const url = new URL('https://catalog.test/api/v1/institutions') + url.searchParams.set('limit', '100') + if (cursor) url.searchParams.set('cursor', cursor) + const response = await worker.fetch(new Request(url), environment) + const page = await response.json() as ApiEnvelopeDto + expect(response.status).toBe(200) + allInstitutions.push(...page.data) + cursor = page.meta.nextCursor ?? null + } while (cursor) + + const missingIds = new Set(missingAdmissions.map((university) => university.id)) + const publiclyEligibleMissing = missingAdmissions.filter( + (university) => university.status === 'verified' || university.status === 'stale', + ) + const publishedMissing = allInstitutions.filter( + (institution) => missingIds.has(institution.id), + ) + expect(publishedMissing).toHaveLength(publiclyEligibleMissing.length) + expect(publishedMissing.every((institution) => + institution.attributes.admissionsUrl === null + && institution.fieldMeta.admissionsUrl.status === 'officially_not_announced' + )).toBe(true) + + const hiddenMissing = missingAdmissions.filter( + (university) => !publiclyEligibleMissing.some((item) => item.id === university.id), + ) + for (const university of hiddenMissing) { + const hiddenResponse = await worker.fetch( + new Request(`https://catalog.test/api/v1/institutions/${university.slug}`), + environment, + ) + expect(hiddenResponse.status).toBe(404) + } + + const detailResponse = await worker.fetch( + new Request(`https://catalog.test/api/v1/institutions/${publishedMissing[0]!.slug!}`), + environment, + ) + const detail = await detailResponse.json() as ApiEnvelopeDto + expect(detailResponse.status).toBe(200) + expect(detail.data.attributes.officialUrl).toMatch(/^https:\/\//u) + expect(detail.data.attributes.admissionsUrl).toBeNull() + expect(detail.data.fieldMeta.admissionsUrl.status).toBe('officially_not_announced') + expect(r2Reads).toBe(0) + }, 30_000) + + it('restricts institution search to normalized name-title tokens', async () => { + database.exec('BEGIN') + try { + const candidate = database.prepare(` + SELECT + institution.institution_id, + organization_search.search_rowid AS organization_search_rowid, + organization_search.title, + program_search.search_rowid AS program_search_rowid + FROM current_institutions AS institution + JOIN current_search_documents AS organization_search + ON organization_search.release_id = institution.release_id + AND organization_search.record_id = institution.institution_id + AND organization_search.record_kind = 'organization' + AND organization_search.locale = 'en' + JOIN current_programs AS program + ON program.release_id = institution.release_id + AND program.institution_id = institution.institution_id + JOIN current_search_documents AS program_search + ON program_search.release_id = program.release_id + AND program_search.record_id = program.program_id + AND program_search.record_kind = 'program' + AND program_search.locale = 'en' + ORDER BY institution.institution_id, program.program_id + LIMIT 1 + `).get() as { + institution_id: string + organization_search_rowid: number + title: string + program_search_rowid: number + } + database.prepare(` + UPDATE search_documents + SET body = 'StaleSummaryLeakCanary', filter_text = 'CityNameLeakCanary' + WHERE search_rowid = ? + `).run(candidate.organization_search_rowid) + database.prepare(` + UPDATE record_field_status + SET field_status = 'stale', value_json = NULL, review_after = '2020-01-01' + WHERE record_id = ? AND field_path = 'summary' + `).run(candidate.institution_id) + database.prepare(` + UPDATE search_documents + SET title = 'ProgramNameLeakCanary' + WHERE search_rowid = ? + `).run(candidate.program_search_rowid) + + const search = async (query: string) => { + const response = await worker.fetch( + new Request(`https://catalog.test/api/v1/institutions?q=${encodeURIComponent(query)}`), + environment, + ) + return { + response, + envelope: await response.json() as ApiEnvelopeDto, + } + } + const nameResult = await search(candidate.title) + expect(nameResult.response.status).toBe(200) + expect(nameResult.envelope.data.map((institution) => institution.id)) + .toContain(candidate.institution_id) + + for (const query of [ + 'StaleSummaryLeakCanary', + 'CityNameLeakCanary', + 'ProgramNameLeakCanary', + ]) { + const result = await search(query) + expect(result.response.status).toBe(200) + expect(result.envelope.data).toEqual([]) + } + + const tooManyTerms = await search(Array.from({ length: 21 }, () => 'term').join(' ')) + expect(tooManyTerms.response.status).toBe(400) + expect(r2Reads).toBe(0) + } finally { + database.exec('ROLLBACK') + } + }, 30_000) it('serves normalized institution, program-cycle, and scholarship projections from D1', async () => { const institutionResponse = await worker.fetch( new Request('https://catalog.test/api/v1/institutions?limit=1'), diff --git a/tests/unit/university-catalog.test.ts b/tests/unit/university-catalog.test.ts new file mode 100644 index 0000000..37f8d49 --- /dev/null +++ b/tests/unit/university-catalog.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi } from 'vitest' +import type { CatalogRepository } from '@/lib/catalog/types' +import { + parseUniversityCatalogFilters, + queryUniversityCatalogRepository, + UNIVERSITY_CATALOG_PAGE_SIZE, + universityCatalogHref, +} from '@/lib/university-catalog' + +function repositoryWithPage(overrides: Record = {}) { + const listInstitutions = vi.fn().mockResolvedValue({ + items: [], + nextCursor: 'cursor-next', + total: 72, + facets: { cities: [] }, + release: null, + ...overrides, + }) + return { + listInstitutions, + repository: { listInstitutions } as unknown as CatalogRepository, + } +} + +describe('repository-backed university catalogue', () => { + it('normalizes safe URL filters into the shared taxonomy', () => { + const filters = parseUniversityCatalogFilters({ + q: 'technology', + city: 'beijing', + region: 'north', + discipline: 'engineering', + sort: 'programs-desc', + page: '2', + cursor: 'cursor-current', + cursorHistory: '~', + }) + + expect(filters).toMatchObject({ + query: 'technology', + city: 'beijing', + region: 'north', + discipline: 'engineering-technology', + sort: 'programs-desc', + page: 2, + cursor: 'cursor-current', + cursorHistory: ['~'], + }) + }) + + it('rejects unsupported region, discipline and sort values', () => { + const filters = parseUniversityCatalogFilters({ + region: 'somewhere', + discipline: 'drop-table', + sort: 'random', + }) + + expect(filters.region).toBe('') + expect(filters.discipline).toBe('') + expect(filters.sort).toBe('default') + }) + + it('normalizes FTS terms and clears queries the Worker would reject', () => { + expect(parseUniversityCatalogFilters({ q: ' C++ / data-science ' }).query) + .toBe('C data science') + expect(parseUniversityCatalogFilters({ q: '!!!' }).query).toBe('') + expect(parseUniversityCatalogFilters({ + q: Array.from({ length: 21 }, (_, index) => `term${index}`).join(' '), + }).query).toBe('') + }) + + it('uses one bounded repository request for a cursor URL', async () => { + const { repository, listInstitutions } = repositoryWithPage() + const filters = parseUniversityCatalogFilters({ + q: 'medicine', + city: 'guangzhou', + region: 'south', + discipline: 'medicine-health', + sort: 'scholarships-desc', + page: '3', + cursor: 'cursor-page-3', + cursorHistory: '~,cursor-page-2', + }) + + const result = await queryUniversityCatalogRepository(repository, filters) + + expect(listInstitutions).toHaveBeenCalledTimes(1) + expect(listInstitutions).toHaveBeenCalledWith({ + q: 'medicine', + city: 'guangzhou', + region: 'south', + discipline: 'medicine-health', + sort: 'scholarships-desc', + cursor: 'cursor-page-3', + limit: UNIVERSITY_CATALOG_PAGE_SIZE, + }) + expect(result.page).toBe(3) + expect(result.filters.cursorHistory).toEqual(['~', 'cursor-page-2']) + }) + + it('does not replay earlier pages when a bare page number lacks its cursor', async () => { + const { repository, listInstitutions } = repositoryWithPage() + const result = await queryUniversityCatalogRepository( + repository, + parseUniversityCatalogFilters({ page: '9' }), + ) + + expect(listInstitutions).toHaveBeenCalledTimes(1) + expect(listInstitutions).toHaveBeenCalledWith({ limit: UNIVERSITY_CATALOG_PAGE_SIZE }) + expect(result.page).toBe(1) + expect(result.filters.cursorHistory).toEqual([]) + }) + + it('preserves all filters and the cursor stack in next and previous links', () => { + const filters = { + ...parseUniversityCatalogFilters({ + q: 'normal', + city: 'beijing', + region: 'north', + discipline: 'education', + sort: 'name', + }), + page: 3, + cursor: 'cursor-page-3', + cursorHistory: ['~', 'cursor-page-2'], + nextCursor: 'cursor-page-4', + } + const next = new URL(universityCatalogHref('en', filters, 4), 'https://example.test') + const previous = new URL(universityCatalogHref('en', filters, 2), 'https://example.test') + + expect(Object.fromEntries(next.searchParams)).toEqual({ + q: 'normal', + city: 'beijing', + region: 'north', + discipline: 'education', + sort: 'name', + cursor: 'cursor-page-4', + cursorHistory: '~,cursor-page-2,cursor-page-3', + page: '4', + }) + expect(Object.fromEntries(previous.searchParams)).toEqual({ + q: 'normal', + city: 'beijing', + region: 'north', + discipline: 'education', + sort: 'name', + cursor: 'cursor-page-2', + cursorHistory: '~', + page: '2', + }) + }) +}) diff --git a/tests/unit/university-explorer-v2.test.tsx b/tests/unit/university-explorer-v2.test.tsx new file mode 100644 index 0000000..14d1834 --- /dev/null +++ b/tests/unit/university-explorer-v2.test.tsx @@ -0,0 +1,71 @@ +import { render, screen } from '@testing-library/react' +import { describe, expect, it } from 'vitest' +import { UniversityExplorerV2 } from '@/components/features/UniversityExplorerV2' +import { getMessages } from '@/i18n/messages' +import type { LaunchLocale } from '@/i18n/config' +import { + parseUniversityCatalogFilters, + type UniversityCatalogResult, +} from '@/lib/university-catalog' + +const locales: LaunchLocale[] = ['en', 'zh', 'ru', 'de', 'fr', 'es'] + +function result(overrides: Partial = {}): UniversityCatalogResult { + return { + items: [], + filters: parseUniversityCatalogFilters({}), + total: 0, + totalExact: true, + page: 1, + pageCount: 1, + pageSize: 24, + cityOptions: [{ value: 'beijing', name: { en: 'Beijing', zh: '北京' } }], + ...overrides, + } +} + +describe('UniversityExplorerV2', () => { + it.each(locales)('renders the server-driven filter form in %s', (locale) => { + const messages = getMessages(locale) + render() + + expect(screen.getByRole('search', { name: messages.universities.title })).toBeVisible() + expect(screen.getByLabelText(messages.universities.cityFilter)).toHaveAttribute('name', 'city') + expect(screen.getByLabelText(messages.universities.regionFilter)).toHaveAttribute('name', 'region') + expect(screen.getByLabelText(messages.universities.fieldFilter)).toHaveAttribute('name', 'discipline') + }) + + it('shows the localized empty state without receiving a full catalogue', () => { + const messages = getMessages('en') + render() + + expect(screen.getByText(messages.universities.noResults)).toBeVisible() + expect(screen.getByText(`0 ${messages.universities.results}`)).toBeVisible() + }) + + it('renders cursor-backed previous and next links', () => { + const messages = getMessages('en') + const filters = { + ...parseUniversityCatalogFilters({ q: 'science', city: 'beijing', sort: 'name' }), + page: 2, + cursor: 'cursor-page-2', + cursorHistory: ['~'], + nextCursor: 'cursor-page-3', + } + render() + + const previous = screen.getByRole('link', { name: 'Previous' }) + const next = screen.getByRole('link', { name: 'Next' }) + expect(previous).toHaveAttribute('rel', 'prev') + expect(previous.getAttribute('href')).not.toContain('page=') + expect(previous.getAttribute('href')).not.toContain('cursor=') + expect(next).toHaveAttribute('rel', 'next') + expect(next.getAttribute('href')).toContain('cursor=cursor-page-3') + expect(next.getAttribute('href')).toContain('cursorHistory=%7E%2Ccursor-page-2') + expect(next.getAttribute('href')).toContain('page=3') + }) +}) diff --git a/workers/catalog-api/src/index.ts b/workers/catalog-api/src/index.ts index 5b9c921..5e22390 100644 --- a/workers/catalog-api/src/index.ts +++ b/workers/catalog-api/src/index.ts @@ -214,6 +214,11 @@ async function publicCatalogResponse(request: Request, environment: CatalogApiEn city: stringParam(url.searchParams, 'city'), region: stringParam(url.searchParams, 'region'), discipline: stringParam(url.searchParams, 'discipline'), + sort: enumParam( + url.searchParams, + 'sort', + new Set(['default', 'name', 'programs-desc', 'scholarships-desc']), + ), cursor: stringParam(url.searchParams, 'cursor', 1_024), limit: integerParam(url.searchParams, 'limit', 1, 100), }), etag) diff --git a/workers/catalog-api/src/sql-api.ts b/workers/catalog-api/src/sql-api.ts index 03a93f7..0ae507b 100644 --- a/workers/catalog-api/src/sql-api.ts +++ b/workers/catalog-api/src/sql-api.ts @@ -41,8 +41,8 @@ type SortableRow = RecordAuditRow & { } type InstitutionRow = SortableRow & { + slug: string | null institution_type: string - admissions_url: string featured: number official_url: string city_id: string @@ -95,6 +95,11 @@ type ProgramCodeRow = { code: string } +type InstitutionCodeRow = { + institution_id: string + code: string +} + type ProgramCycleRow = RecordAuditRow & { slug: string | null program_cycle_id: string @@ -276,6 +281,43 @@ function facetOptions(rows: FacetRow[]): ApiFacetOptionDto[] { return [...options.values()].sort((left, right) => left.value.localeCompare(right.value)) } +function institutionProgramCountSql() { + return `( + SELECT COUNT(*) + FROM current_programs AS counted_program + WHERE counted_program.release_id = institution.release_id + AND counted_program.institution_id = institution.institution_id + )` +} + +function institutionScholarshipCountSql() { + return `( + SELECT COUNT(DISTINCT counted_scholarship.scholarship_id) + FROM current_scholarships AS counted_scholarship + JOIN current_record_fields AS scope + ON scope.release_id = counted_scholarship.release_id + AND scope.record_id = counted_scholarship.scholarship_id + AND scope.field_path IN ('universityIds', 'institution_ids') + JOIN json_each(scope.value_json) AS scoped_institution ON 1 = 1 + WHERE counted_scholarship.release_id = institution.release_id + AND CAST(scoped_institution.value AS TEXT) = institution.institution_id + )` +} + + +function institutionSortSql(sort: InstitutionQuery['sort']) { + const slug = `COALESCE(record.slug, '')` + if (sort === 'name') return slug + if (sort === 'programs-desc') { + return `printf('%015d', 999999999999999 - CAST(${institutionProgramCountSql()} AS INTEGER)) + || ':' || ${slug}` + } + if (sort === 'scholarships-desc') { + return `printf('%015d', 999999999999999 - CAST(${institutionScholarshipCountSql()} AS INTEGER)) + || ':' || ${slug}` + } + return slug +} function scholarshipDeadlineSql() { return `COALESCE( ( @@ -519,7 +561,7 @@ export class CatalogSqlApi { AND search_document.release_id = institution.release_id AND search_document.record_id = institution.institution_id AND search_document.record_kind = 'organization' - )`, fts5Query(query.q)) + )`, fts5Query(query.q, 'title')) } if (query.city) { addCondition( @@ -543,13 +585,17 @@ export class CatalogSqlApi { AND related_discipline.discipline_code = ? )`, query.discipline) } + const sortSql = institutionSortSql(query.sort) + const filteredConditions = [...conditions] + const filteredValues = [...values] + const context = cursorContext({ ...query }) if (query.cursor && exactSlug === undefined) { - const cursor = decodeCursor(query.cursor, 'institutions', this.release.id) + const cursor = decodeCursor(query.cursor, 'institutions', this.release.id, context) addCondition( conditions, values, - `(COALESCE(record.slug, '') > ? - OR (COALESCE(record.slug, '') = ? AND record.record_id > ?))`, + `(${sortSql} > ? + OR (${sortSql} = ? AND record.record_id > ?))`, cursor.sortKey, cursor.sortKey, cursor.id, @@ -557,37 +603,22 @@ export class CatalogSqlApi { } const limit = exactSlug === undefined ? pageLimit(query.limit) + 1 : 1 values.push(limit) - return queryAll(this.database, ` + const rows = await queryAll(this.database, ` SELECT record.record_id, - COALESCE(record.slug, '') AS sort_slug, + record.slug, + ${sortSql} AS sort_slug, record.verified_at AS record_verified_at, record.review_after AS record_review_after, institution.institution_type, - institution.admissions_url, institution.featured, organization.official_url, institution.city_id, city_record.slug AS city_slug, city.country_code, city.region_code, - ( - SELECT COUNT(*) - FROM current_programs AS counted_program - WHERE counted_program.release_id = institution.release_id - AND counted_program.institution_id = institution.institution_id - ) AS program_count, - ( - SELECT COUNT(DISTINCT counted_scholarship.scholarship_id) - FROM current_scholarships AS counted_scholarship - JOIN current_record_fields AS scope - ON scope.release_id = counted_scholarship.release_id - AND scope.record_id = counted_scholarship.scholarship_id - AND scope.field_path IN ('universityIds', 'institution_ids') - JOIN json_each(scope.value_json) AS scoped_institution ON 1 = 1 - WHERE counted_scholarship.release_id = institution.release_id - AND CAST(scoped_institution.value AS TEXT) = institution.institution_id - ) AS scholarship_count + ${institutionProgramCountSql()} AS program_count, + ${institutionScholarshipCountSql()} AS scholarship_count FROM current_institutions AS institution JOIN current_organizations AS organization ON organization.release_id = institution.release_id @@ -602,73 +633,158 @@ export class CatalogSqlApi { ON city_record.release_id = city.release_id AND city_record.record_id = city.location_id WHERE ${conditions.join('\n AND ')} - ORDER BY COALESCE(record.slug, ''), record.record_id + ORDER BY sort_slug, record.record_id LIMIT ? `, values) + return { rows, filteredConditions, filteredValues, context } } private async mapInstitutions(rows: InstitutionRow[]) { - const decorations = await loadRecordDecorations( - this.database, - this.release.id, - rows.flatMap((row) => [row.record_id, row.city_id]), - ) - return rows.map((row): InstitutionDto => ({ - type: 'institution', - id: row.record_id, - slug: row.sort_slug || null, - attributes: { - name: decorations.localized(row.record_id, 'name') ?? {}, - summary: decorations.localized(row.record_id, 'summary'), - institutionType: row.institution_type, - officialUrl: row.official_url, - admissionsUrl: row.admissions_url, - featured: row.featured === 1, - }, - relationships: { - location: { - id: row.city_id, - slug: row.city_slug, - name: decorations.localized(row.city_id, 'name') ?? {}, - countryCode: row.country_code, - regionCode: row.region_code, + const institutionIds = rows.map((row) => row.record_id) + const cityIds = rows.map((row) => row.city_id) + const slots = institutionIds.length > 0 ? placeholders(institutionIds.length) : '' + const [institutionDecorations, cityDecorations, disciplines] = await Promise.all([ + loadRecordDecorations(this.database, this.release.id, institutionIds), + loadRecordDecorations(this.database, this.release.id, cityIds), + institutionIds.length === 0 + ? Promise.resolve([] as InstitutionCodeRow[]) + : queryAll(this.database, ` + SELECT DISTINCT + program.institution_id, + discipline.discipline_code AS code + FROM current_programs AS program + JOIN current_program_disciplines AS discipline + ON discipline.release_id = program.release_id + AND discipline.program_id = program.program_id + WHERE program.release_id = ? + AND program.institution_id IN (${slots}) + ORDER BY program.institution_id, discipline.discipline_code + `, [this.release.id, ...institutionIds]), + ]) + return rows.map((row): InstitutionDto => { + const disciplineCodes = disciplines + .filter((item) => item.institution_id === row.record_id) + .map((item) => item.code) + return { + type: 'institution', + id: row.record_id, + slug: row.slug, + attributes: { + name: institutionDecorations.localized(row.record_id, 'name') ?? {}, + summary: institutionDecorations.localized(row.record_id, 'summary'), + institutionType: row.institution_type, + disciplineCodes, + officialUrl: row.official_url, + admissionsUrl: institutionDecorations.value( + row.record_id, + ['admissionsUrl', 'admissions_url'], + ), + featured: row.featured === 1, }, - programs: { count: Number(row.program_count) }, - scholarships: { count: Number(row.scholarship_count) }, - }, - sources: decorations.sources(row.record_id), - fieldMeta: { - name: identityMeta(decorations, row, 'name'), - summary: decorations.meta(row, ['summary', 'localized.summary']), - institutionType: identityMeta(decorations, row, 'institution_type'), - officialUrl: identityMeta(decorations, row, 'official_url'), - admissionsUrl: identityMeta(decorations, row, 'admissions_url'), - featured: identityMeta(decorations, row, 'featured'), - location: identityMeta(decorations, row, 'city_id'), - programCount: identityMeta(decorations, row, 'program_count'), - scholarshipCount: identityMeta(decorations, row, 'scholarship_count'), - }, - })) + relationships: { + location: { + id: row.city_id, + slug: row.city_slug, + name: cityDecorations.localized(row.city_id, 'name') ?? {}, + countryCode: row.country_code, + regionCode: row.region_code, + }, + programs: { count: Number(row.program_count) }, + scholarships: { count: Number(row.scholarship_count) }, + }, + sources: institutionDecorations.sources(row.record_id), + fieldMeta: { + name: identityMeta(institutionDecorations, row, 'name'), + summary: institutionDecorations.meta(row, ['summary', 'localized.summary']), + institutionType: identityMeta(institutionDecorations, row, 'institution_type'), + disciplineCodes: identityMeta(institutionDecorations, row, 'disciplineCodes'), + officialUrl: identityMeta(institutionDecorations, row, 'official_url'), + admissionsUrl: institutionDecorations.meta( + row, + ['admissionsUrl', 'admissions_url'], + ), + featured: identityMeta(institutionDecorations, row, 'featured'), + location: identityMeta(institutionDecorations, row, 'city_id'), + programCount: identityMeta(institutionDecorations, row, 'program_count'), + scholarshipCount: identityMeta(institutionDecorations, row, 'scholarship_count'), + }, + } + }) + } + + private async institutionListMetadata( + conditions: readonly string[], + values: readonly unknown[], + ) { + const where = conditions.join('\n AND ') + const joins = ` + FROM current_institutions AS institution + JOIN current_organizations AS organization + ON organization.release_id = institution.release_id + AND organization.organization_id = institution.institution_id + JOIN current_catalog_records AS record + ON record.release_id = institution.release_id + AND record.record_id = institution.institution_id + JOIN current_locations AS city + ON city.release_id = institution.release_id + AND city.location_id = institution.city_id + JOIN current_catalog_records AS city_record + ON city_record.release_id = city.release_id + AND city_record.record_id = city.location_id + ` + const [count, cities] = await Promise.all([ + queryFirst(this.database, ` + SELECT COUNT(*) AS total + ${joins} + WHERE ${where} + `, [...values]), + queryAll(this.database, ` + SELECT DISTINCT + city.location_id AS option_id, + city_record.slug AS option_slug, + localized.locale, + localized.text_value + ${joins} + LEFT JOIN current_localized_content AS localized + ON localized.release_id = city.release_id + AND localized.record_id = city.location_id + AND localized.field_name = 'name' + WHERE ${where} + ORDER BY option_slug, option_id, localized.locale + `, [...values]), + ]) + return { + total: Number(count?.total ?? 0), + facets: { cities: facetOptions(cities) }, + } } async listInstitutions(query: InstitutionQuery = {}) { const limit = pageLimit(query.limit) + const selected = await this.selectInstitutions(query) const page = pagination( - await this.selectInstitutions(query), + selected.rows, limit, 'institutions', this.release.id, + selected.context, ) - const data = await this.mapInstitutions(page.items) - return this.envelope(data, { pageSize: data.length, nextCursor: page.nextCursor }) + const [data, metadata] = await Promise.all([ + this.mapInstitutions(page.items), + this.institutionListMetadata(selected.filteredConditions, selected.filteredValues), + ]) + return this.envelope(data, { + pageSize: data.length, + nextCursor: page.nextCursor, + ...metadata, + }) } async getInstitution(slug: string) { - const row = (await this.selectInstitutions({}, slug))[0] + const row = (await this.selectInstitutions({}, slug)).rows[0] if (!row) return null return this.envelope((await this.mapInstitutions([row]))[0]!) } - private async selectPrograms(query: ProgramQuery, exactSlug?: string) { const conditions = ['record.release_id = ?'] const values: unknown[] = [this.release.id] diff --git a/workers/catalog-api/src/sql-data.ts b/workers/catalog-api/src/sql-data.ts index 8128421..88534cd 100644 --- a/workers/catalog-api/src/sql-data.ts +++ b/workers/catalog-api/src/sql-data.ts @@ -76,12 +76,13 @@ export function placeholders(count: number) { return Array.from({ length: count }, () => '?').join(', ') } -export function fts5Query(value: string) { +export function fts5Query(value: string, column?: 'title') { const terms = value.normalize('NFKC').match(/[\p{L}\p{N}]+/gu) ?? [] if (terms.length === 0 || terms.length > 20) { throw new InvalidSearchQueryError('Invalid search query.') } - return terms.map((term) => `"${term}"*`).join(' AND ') + const columnFilter = column ? `${column} : ` : '' + return terms.map((term) => `${columnFilter}"${term}"*`).join(' AND ') } export function normalizeLanguageFilter(value: string) { diff --git a/workers/catalog-api/src/sql-types.ts b/workers/catalog-api/src/sql-types.ts index 6a9d302..5ac346b 100644 --- a/workers/catalog-api/src/sql-types.ts +++ b/workers/catalog-api/src/sql-types.ts @@ -93,8 +93,9 @@ export type InstitutionDto = RecordDto< name: LocalizedValue summary: LocalizedValue | null institutionType: string + disciplineCodes: string[] officialUrl: string - admissionsUrl: string + admissionsUrl: string | null featured: boolean }, { @@ -271,6 +272,7 @@ export type InstitutionQuery = ListOptions & { city?: string region?: string discipline?: string + sort?: 'default' | 'name' | 'programs-desc' | 'scholarships-desc' } export type ProgramQuery = ListOptions & { From 7e2ad6d313242e765a771debd19358104eb92680 Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 08:44:16 +0800 Subject: [PATCH 7/9] Stabilize the D1 projection integration test --- tests/unit/catalog-sql-api.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/catalog-sql-api.test.ts b/tests/unit/catalog-sql-api.test.ts index 5f40430..715d433 100644 --- a/tests/unit/catalog-sql-api.test.ts +++ b/tests/unit/catalog-sql-api.test.ts @@ -579,7 +579,7 @@ describe('Catalog D1 normalized v1 API', () => { attributes: { legacyProjection: true }, }) expect(r2Reads).toBe(0) - }) + }, 30_000) it('lists and resolves an unannounced zero-cycle scholarship without fabricated values', async () => { database.exec('BEGIN') From 147f9d92712a3be75a330194fcf72370415c031f Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 09:10:27 +0800 Subject: [PATCH 8/9] Upgrade Vitest to fix CI worker RPC timeouts --- package-lock.json | 1111 +++++++++++++------------ package.json | 2 +- vitest.config.ts => vitest.config.mts | 0 3 files changed, 587 insertions(+), 526 deletions(-) rename vitest.config.ts => vitest.config.mts (100%) diff --git a/package-lock.json b/package-lock.json index 1e7b445..ad43a5e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,7 +33,7 @@ "tailwindcss": "^3.4.0", "tsx": "^4.20.0", "typescript": "^5.9.0", - "vitest": "^3.2.0", + "vitest": "^4.1.10", "wrangler": "^4.112.0" }, "engines": { @@ -333,7 +333,7 @@ }, "node_modules/@cloudflare/kv-asset-handler": { "version": "0.5.0", - "resolved": "https://registry.npmmirror.com/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", + "resolved": "https://registry.npmjs.org/@cloudflare/kv-asset-handler/-/kv-asset-handler-0.5.0.tgz", "integrity": "sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==", "dev": true, "license": "MIT OR Apache-2.0", @@ -343,7 +343,7 @@ }, "node_modules/@cloudflare/unenv-preset": { "version": "2.16.1", - "resolved": "https://registry.npmmirror.com/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", + "resolved": "https://registry.npmjs.org/@cloudflare/unenv-preset/-/unenv-preset-2.16.1.tgz", "integrity": "sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==", "dev": true, "license": "MIT OR Apache-2.0", @@ -2087,6 +2087,16 @@ "node": ">=12.4.0" } }, + "node_modules/@oxc-project/types": { + "version": "0.143.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.143.0.tgz", + "integrity": "sha512-u6JZdLBTLotrNC9Vd6vPssINdzcCzleKAH6EJKImQb7GtYvX5keN2dxkoK44stCc4tffE6QQRtZTXVSzsLUlWA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@playwright/test": { "version": "1.61.1", "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", @@ -2145,24 +2155,10 @@ "dev": true, "license": "MIT" }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.2.tgz", - "integrity": "sha512-6o7ZLZK+BeenkZCFNDXqpbjw9bD6nuWonvS/lwQJp7NoVVxm6p3qE7qQ5jGuBjiFsgvqjD8mZAU5oWxTmbOeOg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.62.2.tgz", - "integrity": "sha512-BaH7BllCACHoH1LguOU56UItGfUWjujlO65kS9LAodViaN4bwIKd7oeW/ZHJ/4ljr/7MIiENnNy3HJ0zXv8Zkw==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.3.tgz", + "integrity": "sha512-zrJtHDcaZJ1Fp7xf4hNl+7seH9Cn/N5TwLYkhgXREtBwAd/jaqW3uqeHxpDugJLVICWg4eW44kOQEGJ1r6jCGw==", "cpu": [ "arm64" ], @@ -2171,12 +2167,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.62.2.tgz", - "integrity": "sha512-v39RCCvj4He82I9sFmk+M1VZ0PLM9sfsLVikjfx2hYBNALhrrOR2D3JjQA6AhlaSOgcR+RzrKY7e1+bT6SUO/A==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.3.tgz", + "integrity": "sha512-ieIiibVCp0tX7TLu2cafoNPv8wJyYi01ekXpbf8q2j7F4rGAhhXb/eQh7ge9DRBY78GwmRQtvjZDux7EDbA8kA==", "cpu": [ "arm64" ], @@ -2185,12 +2184,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.62.2.tgz", - "integrity": "sha512-yl0y2vq3S3lHeuXhEdss6TWfKW8vkujImO12tn4ZkG/4oghr09LvdYm2RElVjokTQiUvDUGXLGsYeLqUMCKpGA==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.3.tgz", + "integrity": "sha512-Zh9tCon19eDXJoihx0rqKhMUlMYqzwj3aPsSuHmI4RWZh62dWUL+DJN4C5YQya5TcQBJU/Fe8+rY0jhXTQITqA==", "cpu": [ "x64" ], @@ -2199,26 +2201,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.62.2.tgz", - "integrity": "sha512-tT4pvt4qXD+vEoezupCWi+a1F0vvDiksiHc+PxRlYTOH1I6/X4id9jPxTP+Fg+545euaFT1jJVs4CEdHZAU1vw==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.62.2.tgz", - "integrity": "sha512-6nU5F2wCW+qvCBhTn1pdIU3bzsIoF7EUwsCDRxilWGprQR6yd508YnH9+OKFCwpfS8pjZqDUmnCAr7exax0XCg==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.3.tgz", + "integrity": "sha512-nGbJWewA1wrXXZiQhjAT5rhibGfns5ZNkDVqxsO6zJ3f3YvpoDNNmGMSbbhLuXKjNScaBJVOAboztAWVespQMg==", "cpu": [ "x64" ], @@ -2227,26 +2218,15 @@ "optional": true, "os": [ "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.62.2.tgz", - "integrity": "sha512-n1GJHPOvpIfhi3TmrCeh6S6URt9BFCt0KQE3qvexyGCTAKpR4Lg+eWvNZEqu7epxwus/8ElT3hacYEucm49SZg==", - "cpu": [ - "arm" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.62.2.tgz", - "integrity": "sha512-JqgflS8wEB+UXV/vS1RpRbifGBeN4D5lz8D8oOFbFZw4vedvdOgCFAjfBmIMdW3yL10XpQQ0Ambepw6MXrhOnA==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.3.tgz", + "integrity": "sha512-QNniJr5Kml0kDEB98jiDOJjXNroxIIi0IXIbdYzY26Xt1pVbeP62+KnoIZLwirOymX/0jDk/2gI/bNUv7A7OIw==", "cpu": [ "arm" ], @@ -2255,26 +2235,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.62.2.tgz", - "integrity": "sha512-wnFJkogWvN4jm/hQRF2UBaeUmk20j5+DmHvoyWii2b8HJDyvz1MF2OU/6ynXt2KR63rbZLWkFpoytpdc/yBuSA==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.62.2.tgz", - "integrity": "sha512-HVu2bp0zhvJ8xHEV9+UUs7S90VadmBSY3LcIMvozbPo4AuMGDWlz3ymHLHZPX4hR67TKTt8Qp5PJ5RBg/i+RMQ==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.3.tgz", + "integrity": "sha512-TkqEAcmmvH3I/q4114NB4RVt6241Dao48pF45uLcFGrwAaIn0iITgTAKP/dLjbN0R4buJjGb91+UHSoFmpgIWw==", "cpu": [ "arm64" ], @@ -2283,54 +2252,32 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.62.2.tgz", - "integrity": "sha512-mQqqAV8QaoSgr9I2fKDLY2BAVvmKjWoGiu/cSYQonsLvtqwEn1E4QYfnCOcp5zoEqNhsDYin1s6jx/VJmrxlZg==", - "cpu": [ - "loong64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.62.2.tgz", - "integrity": "sha512-IxKLoxCQ2IWi6bT2akyDUBGsOImDKB+sPp4EsTmwFQ/fMwpCKm8uLSSgP/Kx/QYUgKis6SEZ5/Nlhup0DIA0PQ==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.3.tgz", + "integrity": "sha512-NHqjnxpsndf4MPymxteFAWHHfkTL8HjWh1KB7z23ofZ6QO2euONuxDXjat69dKZRALnGypg8k8SsK8vZJoXv1Q==", "cpu": [ - "loong64" + "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.62.2.tgz", - "integrity": "sha512-Mk5ha2RQSgyFfmYYLkBpPnUk8D8FriBxesO1u9O75X0mHgXL1UQcH5Itl2lurWL2tj0RxV9b9tJgipac0hRY9A==", - "cpu": [ - "ppc64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.62.2.tgz", - "integrity": "sha512-CjvEnqJL/0/TQ3TXX3OPIJ/kmBellrWd4heXUmHeJlTnmwjKpSJzoehLaL6Xk0ZnMHBu9dZuFADNOrtjF4v+2w==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.3.tgz", + "integrity": "sha512-6tbrbwfz5GB9DQ4Jwo6hy9v+vR31xZlvzZ6n5Xut6Hhx5PvrA9q/HsK8KMaYQp063iqZGXwNvZtYNLD7EM/x0w==", "cpu": [ "ppc64" ], @@ -2339,40 +2286,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.62.2.tgz", - "integrity": "sha512-1SiZbzwdkaDURsew/tSOrooKiYy7EQGT6m8ufavAi9NEyQb/6VuIxFXAL1fqa4iZe3g4NbNk4P7J32z2tw5Mgg==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.62.2.tgz", - "integrity": "sha512-nQts12zJ3NQRoE6uYljOH89v7szzLDvG2JD/vsX+vGXU8w/At1GowTZ5/7qeFQ8m7L55rpR8Okugnuo5bgjy2Q==", - "cpu": [ - "riscv64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.62.2.tgz", - "integrity": "sha512-E9/ll019jhPIJgpzfZoIkBGhcz+kKNgVWYRY0zr9srBdPPFVpvOKW8VaJKUbeK+eZXyQF9ltME+Kk6affeaPgg==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.3.tgz", + "integrity": "sha512-oyuXxXmoZHjXC917IAPFAAv4wWAa0cM9afk8nx1+9/jNNOX1uPf8yDA6p7G0RypOfw/X0PQt5IfoquY1um+zSg==", "cpu": [ "s390x" ], @@ -2381,12 +2303,15 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.62.2.tgz", - "integrity": "sha512-5BqxR/pshjey51iliyzTD5Xi3EN0aLmQ2lZ3lvefVV9c82BvrLo2/6OT55iifpWBufs6kdwWbuOKS841DrmK9A==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.3.tgz", + "integrity": "sha512-TytMwF2KVGqP2tgd0I1OY0PAv78dZRAYcF5ssDzjM34SUXCED3uXvSd5+lHoC0bTD6eEdFz7LdQNCO1y0oVk9w==", "cpu": [ "x64" ], @@ -2395,12 +2320,15 @@ "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.62.2.tgz", - "integrity": "sha512-uNN83XxQrRAh/w0/pmAfibcwyb6YWt4gP+dpnQKPVJshAloQ785ii8CT8ZCIxkGg9opVsvAlGhFitSm6D1Jjpg==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.3.tgz", + "integrity": "sha512-/E9m3qstrJFVPoULV25mVQblSNExY2+kBsYe4sy0Tn0yOOgJ8wZbZt3KnRbF/XeU2Gl1STKUQnDNTqhIE5MD4A==", "cpu": [ "x64" ], @@ -2409,26 +2337,15 @@ "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.62.2.tgz", - "integrity": "sha512-srjEIxSH3LRnJN6THczDHWQplqEMFiAJrTab0msUryh9kwNpkICf3Ea6q6MN/2cZwRFUNx5w+h6Hpi4QuHS6Zg==", - "cpu": [ - "x64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.62.2.tgz", - "integrity": "sha512-8hOJnxgbyObnCm5AlRA3A931xX19xq80RjVTKgJOvEKWqJruP/Uf12IbAOaDjjEXYRewwHLfmF0YRIdK3OwKWA==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.3.tgz", + "integrity": "sha512-Kr0OcsoQI816i6HOl3vFHpd1K0eZyh76zgfj4c1nTyaTsd5r2Mj1lwM4R90y/qaCfmTn9eHy0SKwi98eitRxug==", "cpu": [ "arm64" ], @@ -2437,12 +2354,15 @@ "optional": true, "os": [ "openharmony" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.62.2.tgz", - "integrity": "sha512-mmF4AY1i0hG/bLWUctUq59gtmgaSIRa3cu/A3JFRp/sCNEme2bgDEiDS22P9FbnJB8NJNF4jPJiSP5RHQpUTDg==", + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.3.tgz", + "integrity": "sha512-hOtMwTqnME+/gJcH/PCZ0wn0zPUjiWOgkHpxbSJpfGKMezHltx1S7/k1SitzVa7Ww2cqrDDaFbZEhcJZO8o+Jw==", "cpu": [ "arm64" ], @@ -2451,26 +2371,15 @@ "optional": true, "os": [ "win32" - ] - }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.62.2.tgz", - "integrity": "sha512-DZgkknc6jhHrk46V25vbAM0zZkyP0nSDkJB8/dRkLTxv470dOmWDqGoEJl/9A0dFfS7yE3REOwNDxpHwSLSt0Q==", - "cpu": [ - "ia32" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.62.2.tgz", - "integrity": "sha512-T6xr6ucWSFto+VGajA8YH26LdpHRuP4YLHEKAtCWvJDOlnmWcDZVCI2Jmjr+IFHDlt2zRaTAKE4tfjTaWLgJBg==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.3.tgz", + "integrity": "sha512-ekcqMMkI2PlhYnfzQnB/cEdYUVVJViWvoUyLrbzgDoi3Snfc1mVBwdnc306ufA5ejy8JSPjT2RlW1nQSjW7efg==", "cpu": [ "x64" ], @@ -2479,21 +2388,17 @@ "optional": true, "os": [ "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.62.2.tgz", - "integrity": "sha512-BfzEnDJOt9T8M989/lA37EcJgat01wLRnoi5dQf3QzOH7jzpqTAzdDbVfRljVr5r+jzKqpbHeyOfAaXxAd0PAA==", - "cpu": [ - "x64" ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] + "license": "MIT" }, "node_modules/@rtsao/scc": { "version": "1.1.0", @@ -2522,6 +2427,13 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, "node_modules/@swc/helpers": { "version": "0.5.15", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.15.tgz", @@ -3399,39 +3311,40 @@ } }, "node_modules/@vitest/expect": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.7.tgz", - "integrity": "sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", + "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", "dev": true, "license": "MIT", "dependencies": { + "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "3.2.7", - "@vitest/utils": "3.2.7", - "chai": "^5.2.0", - "tinyrainbow": "^2.0.0" + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/mocker": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.7.tgz", - "integrity": "sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", + "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "3.2.7", + "@vitest/spy": "4.1.10", "estree-walker": "^3.0.3", - "magic-string": "^0.30.17" + "magic-string": "^0.30.21" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "msw": "^2.4.9", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "msw": { @@ -3443,42 +3356,42 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.7.tgz", - "integrity": "sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", + "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", "dev": true, "license": "MIT", "dependencies": { - "tinyrainbow": "^2.0.0" + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/runner": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.7.tgz", - "integrity": "sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", + "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.7", - "pathe": "^2.0.3", - "strip-literal": "^3.0.0" + "@vitest/utils": "4.1.10", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.7.tgz", - "integrity": "sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", + "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.7", - "magic-string": "^0.30.17", + "@vitest/pretty-format": "4.1.10", + "@vitest/utils": "4.1.10", + "magic-string": "^0.30.21", "pathe": "^2.0.3" }, "funding": { @@ -3486,28 +3399,25 @@ } }, "node_modules/@vitest/spy": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.7.tgz", - "integrity": "sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", + "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", "dev": true, "license": "MIT", - "dependencies": { - "tinyspy": "^4.0.3" - }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.7.tgz", - "integrity": "sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", + "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.7", - "loupe": "^3.1.4", - "tinyrainbow": "^2.0.0" + "@vitest/pretty-format": "4.1.10", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" }, "funding": { "url": "https://opencollective.com/vitest" @@ -3928,7 +3838,7 @@ }, "node_modules/blake3-wasm": { "version": "2.1.5", - "resolved": "https://registry.npmmirror.com/blake3-wasm/-/blake3-wasm-2.1.5.tgz", + "resolved": "https://registry.npmjs.org/blake3-wasm/-/blake3-wasm-2.1.5.tgz", "integrity": "sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==", "dev": true, "license": "MIT" @@ -3991,16 +3901,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/call-bind": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.9.tgz", @@ -4092,18 +3992,11 @@ "license": "CC-BY-4.0" }, "node_modules/chai": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", - "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "dependencies": { - "assertion-error": "^2.0.1", - "check-error": "^2.1.1", - "deep-eql": "^5.0.1", - "loupe": "^3.1.0", - "pathval": "^2.0.0" - }, "engines": { "node": ">=18" } @@ -4125,16 +4018,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/check-error": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", - "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -4393,16 +4276,6 @@ "dev": true, "license": "MIT" }, - "node_modules/deep-eql": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", - "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -4689,9 +4562,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", - "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.1.tgz", + "integrity": "sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==", "dev": true, "license": "MIT" }, @@ -6526,42 +6399,303 @@ "node": ">= 0.8.0" } }, - "node_modules/lilconfig": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", - "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/antonk52" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/locate-path": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", - "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", "dev": true, - "license": "MIT", + "license": "MPL-2.0", "dependencies": { - "p-locate": "^5.0.0" + "detect-libc": "^2.0.3" }, "engines": { - "node": ">=10" + "node": ">= 12.0.0" }, "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lilconfig": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.3.tgz", + "integrity": "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", @@ -6582,13 +6716,6 @@ "loose-envify": "cli.js" } }, - "node_modules/loupe": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", - "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", - "dev": true, - "license": "MIT" - }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -7549,6 +7676,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obug": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", + "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -7672,7 +7813,7 @@ }, "node_modules/path-to-regexp": { "version": "6.3.0", - "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-6.3.0.tgz", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.3.0.tgz", "integrity": "sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==", "dev": true, "license": "MIT" @@ -7684,16 +7825,6 @@ "dev": true, "license": "MIT" }, - "node_modules/pathval": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", - "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.16" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -8212,49 +8343,37 @@ "node": ">=0.10.0" } }, - "node_modules/rollup": { - "version": "4.62.2", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.62.2.tgz", - "integrity": "sha512-RFnrW4lhXA3s3eqHDZvN654g8OTjzRfqpIRJYczCGB6HzphckVAi/Qh4tbPUbRuDi7s1Llv8g/NspLkttY3gTA==", + "node_modules/rolldown": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.3.tgz", + "integrity": "sha512-rn9wpmxplLf7NLNyCk9FyWh3FM43DbY8jOzCdEPzH7uflhTftRbCEpqi6Ly2osgoU8OwObtmavMbWLaWy4LX7A==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.9" + "@oxc-project/types": "=0.143.0", + "@rolldown/pluginutils": "^1.0.0" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.62.2", - "@rollup/rollup-android-arm64": "4.62.2", - "@rollup/rollup-darwin-arm64": "4.62.2", - "@rollup/rollup-darwin-x64": "4.62.2", - "@rollup/rollup-freebsd-arm64": "4.62.2", - "@rollup/rollup-freebsd-x64": "4.62.2", - "@rollup/rollup-linux-arm-gnueabihf": "4.62.2", - "@rollup/rollup-linux-arm-musleabihf": "4.62.2", - "@rollup/rollup-linux-arm64-gnu": "4.62.2", - "@rollup/rollup-linux-arm64-musl": "4.62.2", - "@rollup/rollup-linux-loong64-gnu": "4.62.2", - "@rollup/rollup-linux-loong64-musl": "4.62.2", - "@rollup/rollup-linux-ppc64-gnu": "4.62.2", - "@rollup/rollup-linux-ppc64-musl": "4.62.2", - "@rollup/rollup-linux-riscv64-gnu": "4.62.2", - "@rollup/rollup-linux-riscv64-musl": "4.62.2", - "@rollup/rollup-linux-s390x-gnu": "4.62.2", - "@rollup/rollup-linux-x64-gnu": "4.62.2", - "@rollup/rollup-linux-x64-musl": "4.62.2", - "@rollup/rollup-openbsd-x64": "4.62.2", - "@rollup/rollup-openharmony-arm64": "4.62.2", - "@rollup/rollup-win32-arm64-msvc": "4.62.2", - "@rollup/rollup-win32-ia32-msvc": "4.62.2", - "@rollup/rollup-win32-x64-gnu": "4.62.2", - "@rollup/rollup-win32-x64-msvc": "4.62.2", - "fsevents": "~2.3.2" + "@rolldown/binding-android-arm64": "1.2.3", + "@rolldown/binding-darwin-arm64": "1.2.3", + "@rolldown/binding-darwin-x64": "1.2.3", + "@rolldown/binding-freebsd-x64": "1.2.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.3", + "@rolldown/binding-linux-arm64-gnu": "1.2.3", + "@rolldown/binding-linux-arm64-musl": "1.2.3", + "@rolldown/binding-linux-ppc64-gnu": "1.2.3", + "@rolldown/binding-linux-s390x-gnu": "1.2.3", + "@rolldown/binding-linux-x64-gnu": "1.2.3", + "@rolldown/binding-linux-x64-musl": "1.2.3", + "@rolldown/binding-openharmony-arm64": "1.2.3", + "@rolldown/binding-win32-arm64-msvc": "1.2.3", + "@rolldown/binding-win32-x64-msvc": "1.2.3" } }, "node_modules/rrweb-cssom": { @@ -8381,7 +8500,7 @@ }, "node_modules/server-only": { "version": "0.0.1", - "resolved": "https://registry.npmmirror.com/server-only/-/server-only-0.0.1.tgz", + "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz", "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==", "license": "MIT" }, @@ -8622,9 +8741,9 @@ "license": "MIT" }, "node_modules/std-env": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", - "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.2.0.tgz", + "integrity": "sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==", "dev": true, "license": "MIT" }, @@ -8792,26 +8911,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strip-literal": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/strip-literal/-/strip-literal-3.1.0.tgz", - "integrity": "sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==", - "dev": true, - "license": "MIT", - "dependencies": { - "js-tokens": "^9.0.1" - }, - "funding": { - "url": "https://github.com/sponsors/antfu" - } - }, - "node_modules/strip-literal/node_modules/js-tokens": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-9.0.1.tgz", - "integrity": "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==", - "dev": true, - "license": "MIT" - }, "node_modules/styled-jsx": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", @@ -9012,11 +9111,14 @@ "license": "MIT" }, "node_modules/tinyexec": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", - "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", + "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", "dev": true, - "license": "MIT" + "license": "MIT", + "engines": { + "node": ">=18" + } }, "node_modules/tinyglobby": { "version": "0.2.17", @@ -9066,30 +9168,10 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, - "node_modules/tinypool": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", - "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^18.0.0 || >=20.0.0" - } - }, "node_modules/tinyrainbow": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-2.0.0.tgz", - "integrity": "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/tinyspy": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-4.0.4.tgz", - "integrity": "sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==", + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.1.tgz", + "integrity": "sha512-yau8yJdTt989Mm0Bd/236QnzEiPf2xLLTqUZRUJOo/3CB078LSwzei343DgtJVmfJKJE3TMINY1u42SQsP6mXw==", "dev": true, "license": "MIT", "engines": { @@ -9408,7 +9490,7 @@ }, "node_modules/unenv": { "version": "2.0.0-rc.24", - "resolved": "https://registry.npmmirror.com/unenv/-/unenv-2.0.0-rc.24.tgz", + "resolved": "https://registry.npmjs.org/unenv/-/unenv-2.0.0-rc.24.tgz", "integrity": "sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==", "dev": true, "license": "MIT", @@ -9503,18 +9585,17 @@ "license": "MIT" }, "node_modules/vite": { - "version": "7.3.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.6.tgz", - "integrity": "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg==", + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.1.tgz", + "integrity": "sha512-EU/eS7BH3XROHh2YnBefjM6DBKA6ZeMZEYQbj7NLWg5wHYlhB8B/Mayd5XsgWq+NFYccDOTemRpdETWR6Ka/lw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.27.0 || ^0.28.0", - "fdir": "^6.5.0", - "picomatch": "^4.0.3", - "postcss": "^8.5.6", - "rollup": "^4.43.0", - "tinyglobby": "^0.2.15" + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.25", + "rolldown": "~1.2.1", + "tinyglobby": "^0.2.17" }, "bin": { "vite": "bin/vite.js" @@ -9530,9 +9611,10 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0", + "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", - "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", @@ -9545,13 +9627,16 @@ "@types/node": { "optional": true }, - "jiti": { + "@vitejs/devtools": { "optional": true }, - "less": { + "esbuild": { + "optional": true + }, + "jiti": { "optional": true }, - "lightningcss": { + "less": { "optional": true }, "sass": { @@ -9577,47 +9662,6 @@ } } }, - "node_modules/vite-node": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-3.2.4.tgz", - "integrity": "sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cac": "^6.7.14", - "debug": "^4.4.1", - "es-module-lexer": "^1.7.0", - "pathe": "^2.0.3", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0" - }, - "bin": { - "vite-node": "vite-node.mjs" - }, - "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/vite/node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, "node_modules/vite/node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -9647,65 +9691,79 @@ } }, "node_modules/vitest": { - "version": "3.2.7", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.7.tgz", - "integrity": "sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "^5.2.2", - "@vitest/expect": "3.2.7", - "@vitest/mocker": "3.2.7", - "@vitest/pretty-format": "^3.2.7", - "@vitest/runner": "3.2.7", - "@vitest/snapshot": "3.2.7", - "@vitest/spy": "3.2.7", - "@vitest/utils": "3.2.7", - "chai": "^5.2.0", - "debug": "^4.4.1", - "expect-type": "^1.2.1", - "magic-string": "^0.30.17", + "version": "4.1.10", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", + "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.10", + "@vitest/mocker": "4.1.10", + "@vitest/pretty-format": "4.1.10", + "@vitest/runner": "4.1.10", + "@vitest/snapshot": "4.1.10", + "@vitest/spy": "4.1.10", + "@vitest/utils": "4.1.10", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", "pathe": "^2.0.3", - "picomatch": "^4.0.2", - "std-env": "^3.9.0", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", "tinybench": "^2.9.0", - "tinyexec": "^0.3.2", - "tinyglobby": "^0.2.14", - "tinypool": "^1.1.1", - "tinyrainbow": "^2.0.0", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0-0", - "vite-node": "3.2.4", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", "why-is-node-running": "^2.3.0" }, "bin": { "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/debug": "^4.1.12", - "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", - "@vitest/browser": "3.2.7", - "@vitest/ui": "3.2.7", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.10", + "@vitest/browser-preview": "4.1.10", + "@vitest/browser-webdriverio": "4.1.10", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/ui": "4.1.10", "happy-dom": "*", - "jsdom": "*" + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { "optional": true }, - "@types/debug": { + "@opentelemetry/api": { "optional": true }, "@types/node": { "optional": true }, - "@vitest/browser": { + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { "optional": true }, "@vitest/ui": { @@ -9716,6 +9774,9 @@ }, "jsdom": { "optional": true + }, + "vite": { + "optional": false } } }, @@ -9984,7 +10045,7 @@ }, "node_modules/wrangler/node_modules/fsevents": { "version": "2.3.3", - "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.3.tgz", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "dev": true, "hasInstallScript": true, diff --git a/package.json b/package.json index 2e774f3..d385128 100644 --- a/package.json +++ b/package.json @@ -82,7 +82,7 @@ "tailwindcss": "^3.4.0", "tsx": "^4.20.0", "typescript": "^5.9.0", - "vitest": "^3.2.0", + "vitest": "^4.1.10", "wrangler": "^4.112.0" } } diff --git a/vitest.config.ts b/vitest.config.mts similarity index 100% rename from vitest.config.ts rename to vitest.config.mts From 60844ea1ceaba4e8bd2738b971035c2de833a22b Mon Sep 17 00:00:00 2001 From: Henrythefoodie <13022037121@163.com> Date: Fri, 7 Aug 2026 10:23:07 +0800 Subject: [PATCH 9/9] Expire overdue dynamic facts safely --- content/data/admission-cycles.json | 398 +++++++++--------- content/data/scholarships.json | 4 +- .../regional-depth-wave-2026-08-05.test.ts | 13 +- ...sparse-school-expansion-2026-08-04.test.ts | 19 +- .../unit/wave3-integration-2026-07-30.test.ts | 8 +- 5 files changed, 225 insertions(+), 217 deletions(-) diff --git a/content/data/admission-cycles.json b/content/data/admission-cycles.json index 4e73518..a5b15f8 100644 --- a/content/data/admission-cycles.json +++ b/content/data/admission-cycles.json @@ -3250,7 +3250,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-ahmu-m-anesthesiology-2026-2027-autumn", @@ -3276,7 +3276,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-ahmu-d-clinical-pharmacy-2026-2027-autumn", @@ -3302,7 +3302,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-csw-ahu-chinese-language-literature-bachelor-2026-2027-other-fee-reference", @@ -3463,7 +3463,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-bjfu-international-economics-trade-bachelor-2026-2027-autumn-fee-reference", @@ -3484,7 +3484,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-bjfu-landscape-architecture-bachelor-2026-2027-autumn-fee-reference", @@ -3505,7 +3505,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave7-bipt-biological-pharmacy-bachelor-2026-2027-autumn-fee-reference", @@ -3547,7 +3547,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave7-bipt-chinese-language-program-2026-2027-spring", @@ -4223,7 +4223,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-cad-acting-bachelor-2026-2027-autumn", @@ -4487,7 +4487,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-cuit-artificial-intelligence-master-2026-2027-autumn-fee-reference", @@ -4635,7 +4635,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-cdut-b-mechanical-engineering-2026-2027-autumn-fee-reference", @@ -4656,7 +4656,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-cdut-b-resource-exploration-engineering-2026-2027-autumn-fee-reference", @@ -4677,7 +4677,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-final-cdutcm-self-funded-international-doctoral-route-2026-2027-autumn-fee-reference", @@ -4777,7 +4777,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-cugb-computer-science-bachelor-2026-2027-autumn", @@ -4804,7 +4804,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-cugb-environmental-science-engineering-doctorate-english-2026-2027-autumn-fee-reference", @@ -4826,7 +4826,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-cugb-geology-bachelor-2026-2027-autumn", @@ -4853,7 +4853,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-cugb-geology-master-english-2026-2027-autumn", @@ -4880,7 +4880,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-ecs-cumt-electrical-engineering-master-2026-2027-autumn", @@ -5262,7 +5262,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-cqmu-b-rehabilitation-science-2026-2027-autumn", @@ -5288,7 +5288,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-cqmu-b-stomatology-2026-2027-autumn", @@ -5314,7 +5314,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-cqnu-language-chinese-2026-2027-other", @@ -5340,7 +5340,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-ctbu-d-big-data-statistics-intelligent-computing-2026-2027-autumn", @@ -5366,7 +5366,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-ctbu-m-business-administration-en-2026-2027-autumn", @@ -5392,7 +5392,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-ctbu-d-digital-economy-2026-2027-autumn", @@ -5418,7 +5418,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-ctbu-b-international-economics-and-trade-2026-2027-autumn", @@ -5444,7 +5444,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-cuz-chinese-literature-bachelor-2026-2027-autumn", @@ -5707,7 +5707,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dmu-b-mbbs-2026-2027-other", @@ -5733,7 +5733,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dmu-pre-medical-2026-2027-other", @@ -5759,7 +5759,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dmu-b-stomatology-2026-2027-other-fee-reference", @@ -5780,7 +5780,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dlufl-b-chinese-culture-2026-2027-other-fee-reference", @@ -5801,7 +5801,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dlufl-b-chinese-international-trade-2026-2027-other-fee-reference", @@ -5822,7 +5822,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dlufl-b-tcsol-2026-2027-other-fee-reference", @@ -5843,7 +5843,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-dut-foundation-2026-2027-other-fee-reference", @@ -5939,7 +5939,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dufe-m-international-trade-en-2026-2027-other-fee-reference", @@ -5961,7 +5961,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-dufe-m-tourism-management-en-2026-2027-other-fee-reference", @@ -5983,7 +5983,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-dgut-food-and-nutrition-master-2026-2027-autumn", @@ -6060,7 +6060,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-dgut-social-work-master-2026-2027-autumn", @@ -6110,7 +6110,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-chinese-degree-ecnu-business-chinese-bachelor-2026-2027-autumn", @@ -6721,7 +6721,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-csw-gdufe-international-chinese-education-master-2026-2027-autumn-fee-reference", @@ -6770,7 +6770,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-csw-gdufe-international-trade-master-2026-2027-autumn-fee-reference", @@ -7143,7 +7143,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-gxtcmu-chinese-internal-medicine-master-2026-2027-autumn", @@ -7169,7 +7169,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-gxtcmu-chinese-medicine-bachelor-2026-2027-autumn", @@ -7195,7 +7195,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-gxtcmu-history-literature-chinese-medicine-doctorate-2026-2027-autumn", @@ -7221,7 +7221,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-gzucm-b-tcm-cn-2026-2027-other-fee-reference", @@ -7243,7 +7243,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-glmu-administrative-management-master-2026-2027-autumn", @@ -7295,7 +7295,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-glmu-epidemiology-health-statistics-master-2026-2027-autumn", @@ -7389,7 +7389,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-glmu-stomatology-bachelor-2026-2027-autumn-fee-reference", @@ -7431,7 +7431,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-guet-chinese-language-program-2026-2027-other-fee-reference", @@ -7452,7 +7452,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-guet-cs-bilingual-master-2026-2027-other-fee-reference", @@ -7473,7 +7473,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-guet-cs-english-bachelor-2026-2027-other-fee-reference", @@ -7494,7 +7494,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-gmu-b-clinical-medicine-cn-2026-2027-other", @@ -7520,7 +7520,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-gzmu-b-stomatology-2026-2027-other", @@ -7546,7 +7546,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-gznu-chinese-language-one-year-fall-2026-2026-2027-autumn-fee-reference", @@ -7702,7 +7702,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-hmu-nursing-bachelor-chinese-2026-2027-other-fee-reference", @@ -8131,7 +8131,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-hmu-d-international-postgraduate-programs-2026-2027-autumn", @@ -8158,7 +8158,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-hmu-m-international-postgraduate-programs-2026-2027-autumn", @@ -8185,7 +8185,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-hebmu-m-academic-masters-2026-2027-autumn", @@ -8212,7 +8212,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-hebmu-d-doctoral-2026-2027-autumn", @@ -8239,7 +8239,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-hebmu-b-international-undergraduate-2026-2027-autumn", @@ -8266,7 +8266,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-hebut-business-administration-bachelor-2026-2027-autumn-fee-reference", @@ -8435,7 +8435,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-fastpack-hlju-international-economics-trade-bachelor-2026-2027-autumn-fee-reference", @@ -8457,7 +8457,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-csw-henu-finance-master-2026-2027-autumn", @@ -8591,7 +8591,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-haut-nondegree-chinese-language-2026-2027-autumn", @@ -8617,7 +8617,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-haut-m-computer-science-and-technology-2026-2027-autumn", @@ -8643,7 +8643,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-haut-b-food-science-and-engineering-2026-2027-autumn", @@ -8670,7 +8670,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave5-hqu-business-administration-english-bachelor-2026-2027-autumn-fee-reference", @@ -9072,7 +9072,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-just-b-computer-science-en-2026-2027-other", @@ -9098,7 +9098,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-just-energy-power-master-2026-2027-autumn-fee-reference", @@ -9372,7 +9372,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-jnmc-m-clinical-medicine-cn-2026-2027-other-fee-reference", @@ -9394,7 +9394,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-jnmc-m-pharmacy-2026-2027-autumn-fee-reference", @@ -9415,7 +9415,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-jnmc-m-public-health-2026-2027-autumn-fee-reference", @@ -9436,7 +9436,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-kmmu-chinese-language-2026-2027-other", @@ -9461,7 +9461,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-kmmu-mbbs-preparatory-2026-2027-other", @@ -9487,7 +9487,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-kmmu-medical-advanced-training-2026-2027-other", @@ -9512,7 +9512,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-kust-b-civil-engineering-en-2026-2027-other", @@ -9539,7 +9539,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-kust-b-international-economics-trade-en-2026-2027-other", @@ -9566,7 +9566,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-lnnu-information-management-systems-bachelor-2026-2027-autumn-fee-reference", @@ -9630,7 +9630,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-lnu-international-business-master-english-2026-2027-autumn-fee-reference", @@ -9652,7 +9652,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-mnnu-iclt-year-chinese-language-literature-2026-2027-autumn", @@ -10331,7 +10331,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-ntu-b-computer-science-en-2026-2027-other-fee-reference", @@ -10352,7 +10352,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-ecs-ntu-stomatology-bachelor-2026-2027-other-fee-reference", @@ -10440,7 +10440,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-ncepu-chinese-language-2026-2027-autumn-fee-reference", @@ -10550,7 +10550,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-ncut-b-electronic-information-engineering-en-2026-2027-other", @@ -10576,7 +10576,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-ncut-b-mechatronics-engineering-en-2026-2027-other", @@ -10602,7 +10602,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-ne-neau-animal-husbandry-doctorate-2026-2027-autumn", @@ -10815,7 +10815,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-nenu-environmental-engineering-doctorate-english-2026-2027-autumn-fee-reference", @@ -10837,7 +10837,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-nenu-finance-bachelor-2026-2027-autumn-fee-reference", @@ -10859,7 +10859,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-nenu-international-chinese-education-master-2026-2027-autumn-fee-reference", @@ -10881,7 +10881,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-nenu-2026-one-year-chinese-literature-2026-2027-autumn", @@ -10929,7 +10929,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-nwnu-chinese-philology-master-2026-2027-other-fee-reference", @@ -11065,7 +11065,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-cs-doctorate-english-2026-2027-autumn-fee-reference", @@ -11086,7 +11086,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-gvs-chinese-language-2026-2027-autumn-fee-reference", @@ -11107,7 +11107,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-gvs-credit-courses-2026-2027-autumn-fee-reference", @@ -11128,7 +11128,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-chinese-degree-pku-international-chinese-education-master-2026-2027-autumn-fee-reference", @@ -11171,7 +11171,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-llm-chinese-law-2026-2027-autumn-fee-reference", @@ -11192,7 +11192,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-chinese-language-summer-school-2026-2027-other", @@ -11218,7 +11218,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-yuke-pre-university-2026-2027-autumn-fee-reference", @@ -11239,7 +11239,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-lse-environmental-double-master-2026-2027-autumn-fee-reference", @@ -11260,7 +11260,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-research-scholar-2026-2027-other", @@ -11287,7 +11287,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-senior-visiting-student-2026-2027-autumn", @@ -11313,7 +11313,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-spring-four-week-chinese-2026-2027-spring-fee-reference", @@ -11334,7 +11334,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-undergraduate-international-degree-entry-2026-2027-autumn", @@ -11362,7 +11362,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-pku-depth-yenching-china-studies-master-2026-2027-autumn", @@ -11388,7 +11388,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-remaining-qdu-nondegree-chinese-language-fall-2025-2025-2026-other", @@ -11414,7 +11414,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-qdu-international-chinese-education-master-2026-2027-autumn-fee-reference", @@ -11435,7 +11435,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-qdu-textile-science-engineering-doctorate-english-2026-2027-autumn-fee-reference", @@ -11457,7 +11457,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-qdu-textile-science-engineering-master-english-2026-2027-autumn-fee-reference", @@ -11479,7 +11479,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-qust-b-artificial-intelligence-en-2026-2027-other-fee-reference", @@ -11502,7 +11502,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-ne-qust-chinese-language-literature-bachelor-2026-2027-other-fee-reference", @@ -11545,7 +11545,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-ne-qust-chinese-language-semester-2026-2027-autumn-fee-reference", @@ -11721,7 +11721,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-sdjzu-b-civil-engineering-en-2026-2027-other-fee-reference", @@ -11744,7 +11744,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-sdjzu-b-civil-engineering-en-2026-2027-autumn", @@ -11772,7 +11772,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-sdu-2026-master-icle-2026-2027-other-fee-reference", @@ -11814,7 +11814,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sdufe-international-economics-trade-english-bachelor-2026-2027-autumn", @@ -11840,7 +11840,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sdufe-management-science-engineering-english-master-2026-2027-autumn-fee-reference", @@ -11861,7 +11861,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sdufe-international-business-english-master-2026-2027-autumn-fee-reference", @@ -11882,7 +11882,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-lixin-business-administration-bachelor-2026-2027-other-fee-reference", @@ -12032,7 +12032,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-shmtu-b-materials-science-engineering-en-2026-2027-other-fee-reference", @@ -12054,7 +12054,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-shmtu-b-mechatronic-engineering-bilingual-2026-2027-other-fee-reference", @@ -12076,7 +12076,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-shmtu-b-transport-management-shipping-bilingual-2026-2027-other-fee-reference", @@ -12098,7 +12098,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-shnu-2026-master-icle-2026-2027-autumn", @@ -12639,7 +12639,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-medicine-sxtcm-acupuncture-tuina-master-2026-2027-autumn", @@ -12666,7 +12666,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-medicine-sxtcm-pharmacy-bachelor-2026-2027-autumn", @@ -12693,7 +12693,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-sxtcm-b-traditional-chinese-medicine-cn-2026-2027-other", @@ -12719,7 +12719,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-nss-synu-four-week-chinese-study-2026-2027-other", @@ -12847,7 +12847,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-medicine-syphu-pharmaceutics-master-2026-2027-other-fee-reference", @@ -12870,7 +12870,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-remaining-syphu-b-pharmacy-2026-2027-other-fee-reference", @@ -12892,7 +12892,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sztu-ai-english-bachelor-2026-2027-autumn-fee-reference", @@ -12914,7 +12914,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sztu-cs-chinese-bachelor-2026-2027-autumn-fee-reference", @@ -12936,7 +12936,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sztu-international-business-english-bachelor-2026-2027-autumn-fee-reference", @@ -12958,7 +12958,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-sztu-pharmacy-english-bachelor-2026-2027-autumn-fee-reference", @@ -12980,7 +12980,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-szu-chinese-language-literature-bachelor-2026-2026-2027-autumn-fee-reference", @@ -13065,7 +13065,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-sicau-chinese-language-program-2026-2027-other-fee-reference", @@ -13086,7 +13086,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-sicau-international-doctoral-programs-2026-2026-2027-autumn-fee-reference", @@ -13107,7 +13107,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-sicau-international-master-programs-2026-2026-2027-autumn-fee-reference", @@ -13128,7 +13128,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-sisu-chinese-language-year-2026-2027-other", @@ -13153,7 +13153,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-sisu-chinese-language-2026-2027-2026-2027-autumn", @@ -13252,7 +13252,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-sisu-b-tcsol-cn-2026-2027-other-fee-reference", @@ -13273,7 +13273,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-sicnu-chinese-refresher-course-2026-2027-other-fee-reference", @@ -13336,7 +13336,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-clw-sw-scu-chinese-language-bachelor-2026-2027-autumn", @@ -13600,7 +13600,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-swpu-b-mechanical-engineering-2026-2027-autumn-fee-reference", @@ -13621,7 +13621,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-swpu-m-petroleum-and-natural-gas-engineering-2026-2027-autumn-fee-reference", @@ -13642,7 +13642,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-clw-sw-swu-icl-bachelor-2026-2027-autumn", @@ -14020,7 +14020,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-tjfsu-b-international-business-ecommerce-cn-2026-2027-other-fee-reference", @@ -14041,7 +14041,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-tjfsu-b-tcsol-cn-2026-2027-other-fee-reference", @@ -14062,7 +14062,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-tjfsu-b-translation-en-2026-2027-other-fee-reference", @@ -14083,7 +14083,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-tjnu-computer-science-bachelor-2026-2027-other-fee-reference", @@ -14104,7 +14104,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-tjnu-2026-doctor-icle-2026-2027-autumn", @@ -14151,7 +14151,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-tjnu-long-term-chinese-language-2026-2027-other-fee-reference", @@ -14172,7 +14172,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-tjnu-2026-master-icle-2026-2027-autumn", @@ -14220,7 +14220,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-chinese-degree-tju-chinese-cross-border-ecommerce-bachelor-2026-2027-autumn", @@ -14268,7 +14268,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-chinese-degree-tju-international-chinese-education-master-2026-2027-autumn", @@ -14321,7 +14321,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-tju-b-pharmacy-en-2026-2027-autumn-fee-reference", @@ -14343,7 +14343,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-remaining-tjufe-b-international-economics-trade-china-commerce-2026-2027-other-fee-reference", @@ -14364,7 +14364,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-mew-ne-tjutcm-acupuncture-master-en-2026-2027-autumn-fee-reference", @@ -14536,7 +14536,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-ujn-chinese-language-culture-2026-2027-other-fee-reference", @@ -14579,7 +14579,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-ujn-m-computer-science-and-technology-2026-2027-other-fee-reference", @@ -14601,7 +14601,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-ustc-scientific-management-leaders-mba-2026-2027-autumn", @@ -14648,7 +14648,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-usst-chinese-language-preparatory-2026-2026-2027-autumn-fee-reference", @@ -14716,7 +14716,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-usst-m-software-engineering-en-2026-2027-other-fee-reference", @@ -14737,7 +14737,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-wzu-chinese-language-literature-bachelor-2026-2027-autumn", @@ -14765,7 +14765,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-wzu-computer-science-technology-bachelor-2026-2027-autumn", @@ -14793,7 +14793,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-wzu-international-economics-trade-bachelor-2026-2027-autumn", @@ -14821,7 +14821,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave8-2-wtu-design-doctorate-2026-2027-autumn", @@ -15074,7 +15074,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-xisu-b-international-economics-and-trade-2026-2027-autumn-fee-reference", @@ -15096,7 +15096,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-xisu-foundation-pre-university-2026-2027-autumn-fee-reference", @@ -15117,7 +15117,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-xisu-b-russian-2026-2027-autumn-fee-reference", @@ -15139,7 +15139,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-xaut-d-international-doctoral-2026-2027-other-fee-reference", @@ -15161,7 +15161,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-csw-xtu-artificial-intelligence-bachelor-2026-2027-autumn-fee-reference", @@ -15276,7 +15276,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-xjau-b-animal-medicine-2026-2027-autumn", @@ -15303,7 +15303,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-xjau-m-veterinary-medicine-2026-2027-autumn", @@ -15329,7 +15329,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-xjnu-chinese-language-nondegree-2026-2027-autumn", @@ -15525,7 +15525,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-xzmu-b-mbbs-2026-2027-other", @@ -15551,7 +15551,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-breadth-xzhmu-b-stomatology-2026-2027-autumn", @@ -15577,7 +15577,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-yangtzeu-b-clinical-medicine-cn-2026-2027-other-fee-reference", @@ -15598,7 +15598,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mew-scws-yangtze-computer-science-bachelor-2026-2027-autumn-fee-reference", @@ -15682,7 +15682,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-yzu-animal-science-master-2026-2027-autumn", @@ -15791,7 +15791,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-yzu-m-software-engineering-en-2026-2027-other", @@ -15817,7 +15817,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-yzu-tcsol-bachelor-2026-2027-autumn", @@ -15954,7 +15954,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-ynnu-b-business-administration-cn-2026-2027-other", @@ -15979,7 +15979,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-ynnu-b-chinese-education-overseas-2026-2027-other", @@ -16004,7 +16004,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-clw-sw-ynnu-chinese-language-year-2026-2027-autumn-fee-reference", @@ -16072,7 +16072,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-ynu-ecology-doctorate-2026-2027-autumn-fee-reference", @@ -16094,7 +16094,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-ynu-finance-bachelor-2026-2027-autumn-fee-reference", @@ -16116,7 +16116,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-ynu-international-chinese-education-master-2026-2027-autumn-fee-reference", @@ -16138,7 +16138,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave3-depth-ynu-software-engineering-bachelor-english-2026-2027-autumn-fee-reference", @@ -16160,7 +16160,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-strong-ynufe-language-chinese-2026-2027-other", @@ -16186,7 +16186,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave7-ynufe-international-business-bachelor-2026-2027-autumn-fee-reference", @@ -16317,7 +16317,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-zjgsu-b-accounting-en-2026-2027-other-fee-reference", @@ -16339,7 +16339,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-prog-zjgsu-chinese-language-culture-2026-2026-2027-autumn", @@ -16387,7 +16387,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-zjgsu-b-international-business-en-2026-2027-other-fee-reference", @@ -16409,7 +16409,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-zjgsu-b-international-law-en-2026-2027-other-fee-reference", @@ -16431,7 +16431,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-zjnu-chinese-language-culture-2026-2027-other-fee-reference", @@ -16452,7 +16452,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-wave4-depth-zjnu-computer-science-master-english-2026-2027-other-fee-reference", @@ -16474,7 +16474,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-chinese-degree-zjnu-international-chinese-education-bachelor-2026-2027-autumn", @@ -16574,7 +16574,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-local-zstu-short-chinese-language-culture-2026-2027-other", @@ -16599,7 +16599,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-remaining-zufe-b-finance-2025-2026-other", @@ -16625,7 +16625,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-zust-chinese-robotics-2026-2027-autumn", @@ -16673,7 +16673,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-zust-intelligent-manufacturing-master-2026-2027-autumn", @@ -16773,7 +16773,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "cycle-gap-mve-jzh-zust-iclt-master-2026-2027-autumn", diff --git a/content/data/scholarships.json b/content/data/scholarships.json index e1003b9..e48ec0f 100644 --- a/content/data/scholarships.json +++ b/content/data/scholarships.json @@ -8463,7 +8463,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "sch-gap-pku-depth-law-international-student-scholarship", @@ -10872,7 +10872,7 @@ ], "verifiedAt": "2026-07-30", "reviewAfter": "2026-08-06", - "status": "verified" + "status": "stale" }, { "id": "sch-gap-local-tjfsu-tianjin-government-scholarship", diff --git a/tests/unit/regional-depth-wave-2026-08-05.test.ts b/tests/unit/regional-depth-wave-2026-08-05.test.ts index f30957e..76f3374 100644 --- a/tests/unit/regional-depth-wave-2026-08-05.test.ts +++ b/tests/unit/regional-depth-wave-2026-08-05.test.ts @@ -144,17 +144,20 @@ describe('regional depth wave on 2026-08-05', () => { } }) - it('preserves public catalog floors and leaves no university without a program', () => { - expect(published.universities.length).toBeGreaterThanOrEqual(257) - expect(published.programs.length).toBeGreaterThanOrEqual(1_152) - expect(published.scholarships.length).toBeGreaterThanOrEqual(334) + it('preserves catalog identity floors and leaves no university without a program identity', () => { + expect(data.universities.length).toBeGreaterThanOrEqual(263) + expect(data.programs.length).toBeGreaterThanOrEqual(1_173) + expect(data.scholarships.length).toBeGreaterThanOrEqual(358) const programCounts = new Map() - for (const program of published.programs) { + for (const program of data.programs) { programCounts.set(program.universityId, (programCounts.get(program.universityId) ?? 0) + 1) } expect(published.universities.every( (university) => (programCounts.get(university.id) ?? 0) >= 1, )).toBe(true) + expect(published.scholarships.every( + (scholarship) => scholarship.status === 'verified' && scholarship.reviewAfter >= TODAY, + )).toBe(true) }) }) diff --git a/tests/unit/sparse-school-expansion-2026-08-04.test.ts b/tests/unit/sparse-school-expansion-2026-08-04.test.ts index 6c76c16..8c0746d 100644 --- a/tests/unit/sparse-school-expansion-2026-08-04.test.ts +++ b/tests/unit/sparse-school-expansion-2026-08-04.test.ts @@ -393,16 +393,16 @@ describe('sparse-school and regional university expansion on 2026-08-04', () => .toContain(university?.id) } }) - it('publishes every new university with at least one program and preserves catalog floors', () => { - expect(published.universities.length).toBeGreaterThanOrEqual(256) - expect(published.programs.length).toBeGreaterThanOrEqual(1_146) - expect(published.scholarships.length).toBeGreaterThanOrEqual(333) + it('preserves catalog identity floors and imports every new university with a program identity', () => { + expect(data.universities.length).toBeGreaterThanOrEqual(263) + expect(data.programs.length).toBeGreaterThanOrEqual(1_173) + expect(data.scholarships.length).toBeGreaterThanOrEqual(358) - const publishedUniversityBySlug = new Map( - published.universities.map((university) => [university.slug, university]), + const universityBySlug = new Map( + data.universities.map((university) => [university.slug, university]), ) const programCounts = new Map() - for (const program of published.programs) { + for (const program of data.programs) { programCounts.set(program.universityId, (programCounts.get(program.universityId) ?? 0) + 1) } expect(published.universities.every( @@ -412,11 +412,14 @@ describe('sparse-school and regional university expansion on 2026-08-04', () => .toBeGreaterThanOrEqual(225) for (const candidateUniversity of candidateUniversities) { - const university = publishedUniversityBySlug.get(candidateUniversity.slug) + const university = universityBySlug.get(candidateUniversity.slug) expect(university, `${candidateUniversity.slug} was not imported`).toBeDefined() expect(programCounts.get(university?.id ?? '') ?? 0, candidateUniversity.slug) .toBeGreaterThanOrEqual(1) } + expect(published.scholarships.every( + (scholarship) => scholarship.status === 'verified' && scholarship.reviewAfter >= TODAY, + )).toBe(true) }) it('never leaves an overdue formal record marked as verified', () => { diff --git a/tests/unit/wave3-integration-2026-07-30.test.ts b/tests/unit/wave3-integration-2026-07-30.test.ts index 471bcb6..74450d5 100644 --- a/tests/unit/wave3-integration-2026-07-30.test.ts +++ b/tests/unit/wave3-integration-2026-07-30.test.ts @@ -98,17 +98,20 @@ describe('official coverage wave 3 on 2026-07-31', () => { it('publishes only safe current, grace-period, or date-free wave-3 cycles', () => { const programIds = new Set(wavePrograms.map((item) => item.id)) const published = selectPublishedData(data, TODAY) + const rawCycles = data.admissionCycles.filter((cycle) => + programIds.has(cycle.programId)) const cycles = published.admissionCycles.filter((cycle) => programIds.has(cycle.programId)) - expect(cycles.length).toBeGreaterThan(0) + expect(rawCycles.length).toBeGreaterThan(0) + expect(cycles.filter((cycle) => + cycle.status !== 'verified' || cycle.reviewAfter < TODAY)).toEqual([]) expect(cycles.filter((cycle) => cycle.closesOn !== null && !isWithinPostDeadlineGrace(cycle.closesOn, TODAY))).toEqual([]) const datedCycles = cycles.filter((cycle) => cycle.opensOn !== null || cycle.closesOn !== null) - expect(datedCycles.length).toBeGreaterThan(0) const datedClosedCycles = datedCycles.filter((cycle) => getApplicationState(cycle, TODAY) === 'closed') for (const cycle of datedClosedCycles) { @@ -121,7 +124,6 @@ describe('official coverage wave 3 on 2026-07-31', () => { const dateFreeCycles = cycles.filter((cycle) => cycle.opensOn === null && cycle.closesOn === null) - expect(dateFreeCycles.length).toBeGreaterThan(0) for (const cycle of dateFreeCycles) { if (cycle.id.includes('fee-reference')) { expect(cycle.dateStatus).toBe('not-announced')