diff --git a/.claude/bash-redirects.json b/.claude/bash-redirects.json index af6aaac..084a9a0 100644 --- a/.claude/bash-redirects.json +++ b/.claude/bash-redirects.json @@ -26,7 +26,9 @@ "Bash(grep *)", "Bash(find *)", "Bash(rg *)", - "Bash(awk *)" + "Bash(awk *)", + "Bash(head *)", + "Bash(tail *)" ], "reason": "Use the Read(), Write(), Edit() and Find() tools instead of invoking sed, grep, find, rg or awk directly" }, diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 62762c3..2eea4b9 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -23,6 +23,7 @@ "Bash(bun run build *)", "Bash(bun typecheck *)", "Bash(bun run typecheck *)", + "Bash(bun run typecheck:e2e *)", "Bash(bun lint *)", "Bash(bun run lint *)", "Bash(bun format *)", @@ -39,9 +40,7 @@ "Bash(git rm *)", "Bash(git show *)" ], - "deny": [ - "Write(supabase/migrations/*)" - ], + "deny": ["Write(supabase/migrations/*)"], "defaultMode": "acceptEdits" } } diff --git a/.github/workflows/schema.yml b/.github/workflows/schema.yml index a1b0775..903d14c 100644 --- a/.github/workflows/schema.yml +++ b/.github/workflows/schema.yml @@ -1,10 +1,10 @@ name: Schema +# Runs on every PR (not just schema-touching ones) so the check always reports a status - branch +# protection / auto-merge waits on it, and a path-filtered workflow that never starts blocks the +# merge forever. The expensive Supabase steps are gated below so an unrelated PR passes instantly. on: pull_request: - paths: - - 'supabase/schemas/**' - - 'supabase/migrations/**' workflow_dispatch: jobs: @@ -15,10 +15,33 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + - name: Detect schema changes + id: changes + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@v3 + with: + filters: | + schema: + - 'supabase/schemas/**' + - 'supabase/migrations/**' + + # Run the actual check when the schema/migrations changed, or always on a manual dispatch. + - name: Decide whether to run the schema check + id: decide + run: | + if [ "${{ github.event_name }}" != "pull_request" ] || [ "${{ steps.changes.outputs.schema }}" = "true" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "No schema or migration changes; passing without running the schema check." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + - name: Setup Bun + if: steps.decide.outputs.run == 'true' uses: oven-sh/setup-bun@v2 - name: Cache bun install cache + if: steps.decide.outputs.run == 'true' uses: actions/cache@v4 with: path: ~/.bun/install/cache @@ -26,20 +49,24 @@ jobs: restore-keys: bun-${{ runner.os }}- - name: Install dependencies + if: steps.decide.outputs.run == 'true' run: bun install --frozen-lockfile - name: Install Supabase CLI + if: steps.decide.outputs.run == 'true' uses: supabase/setup-cli@v1 with: version: latest - name: Start Supabase database + if: steps.decide.outputs.run == 'true' run: supabase db start # `supabase db diff` always prints progress lines and exits 0, so we can't gate on output being # empty. When the declarative schemas (supabase/schemas) and migrations match it prints # "No schema changes found"; otherwise it prints the reconciling migration. - name: Check declarative schema matches migrations + if: steps.decide.outputs.run == 'true' run: | diff_output=$(supabase db diff 2>&1) echo "$diff_output" @@ -49,12 +76,15 @@ jobs: fi - name: Reset database + if: steps.decide.outputs.run == 'true' run: supabase db reset - name: Regenerate Zapatos types + if: steps.decide.outputs.run == 'true' run: bun run schema - name: Check generated types are up to date + if: steps.decide.outputs.run == 'true' run: | if ! git diff --exit-code -- src/services/zapatos/schema.d.ts src/services/db/db.types.ts; then echo "::error::Generated DB types are out of date. Run 'bun schema' locally and commit the result." diff --git a/AGENTS.md b/AGENTS.md index 77eb632..cf78657 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,11 @@ The codebase uses Docker to run third-party services locally (Minio for a local - If you need to add dependencies, add them with `bun install XYZ` and ensure that the bun.lock file is updated and part of the commit or PR. +# Comments + +- Only add a comment if it either (a) explains something complex that is hard to read from the code itself, or (b) explains _why_ something is being done when it isn't clear why it's necessary. Otherwise leave it out - don't restate what the code already says. +- Don't commit comments that are only relevant to the session or task in which they were written (e.g. references to a code review, "fixes finding X", "RED test", etc.). + # Typescript - Use `== null` (double equals null) instead of either `=== null` or `=== undefined` (triple equals null / undefined), and the same for `!=`. This is to make `null` and `undefined` mean the same thing everywhere in our codebase to avoid any potential serialisation/deserialisation confusion or issues. @@ -65,7 +70,7 @@ The codebase uses Docker to run third-party services locally (Minio for a local - `bun format` will format the codebase with Prettier - `bun typecheck` will typecheck the codebase, and `bun lint` will lint it - `bun run test:unit` runs the pure unit tests (`*.unit.test.ts`) -- `bun run test:integration` runs the integration suite. It needs no live services: the database is an ephemeral in-memory PGlite (started by `tools/test.sh`, with the schema loaded from `supabase/config.toml`'s `schema_paths`), and Supabase, S3, and Axiom are faked (see `.env.test`). Do not run `bun test` (Bun's built-in runner); it's intercepted with a pointer to these scripts. +- `bun run test:integration` runs the integration suite. It needs no live services: the database is an ephemeral in-memory PGlite (started by `tools/test.sh`, with the real `supabase/migrations` applied by `tools/load_schema.ts` so it matches production), and Supabase, S3, and Axiom are faked (see `.env.test`). Do not run `bun test` (Bun's built-in runner); it's intercepted with a pointer to these scripts. # How you should work diff --git a/bun.lock b/bun.lock index 0d9f172..d702cb4 100644 --- a/bun.lock +++ b/bun.lock @@ -55,6 +55,8 @@ "@electric-sql/pglite": "^0.4.6", "@electric-sql/pglite-socket": "^0.1.6", "@playwright/test": "^1.60.0", + "@swc/core": "^1.15.40", + "@swc/jest": "^0.2.39", "@types/jest": "^29.5.5", "@types/pg": "^8.10.2", "@types/postcss-modules-values": "^4", @@ -503,6 +505,8 @@ "@jest/core": ["@jest/core@29.7.0", "", { "dependencies": { "@jest/console": "^29.7.0", "@jest/reporters": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "ansi-escapes": "^4.2.1", "chalk": "^4.0.0", "ci-info": "^3.2.0", "exit": "^0.1.2", "graceful-fs": "^4.2.9", "jest-changed-files": "^29.7.0", "jest-config": "^29.7.0", "jest-haste-map": "^29.7.0", "jest-message-util": "^29.7.0", "jest-regex-util": "^29.6.3", "jest-resolve": "^29.7.0", "jest-resolve-dependencies": "^29.7.0", "jest-runner": "^29.7.0", "jest-runtime": "^29.7.0", "jest-snapshot": "^29.7.0", "jest-util": "^29.7.0", "jest-validate": "^29.7.0", "jest-watcher": "^29.7.0", "micromatch": "^4.0.4", "pretty-format": "^29.7.0", "slash": "^3.0.0", "strip-ansi": "^6.0.0" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"] }, "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg=="], + "@jest/create-cache-key-function": ["@jest/create-cache-key-function@30.4.1", "", { "dependencies": { "@jest/types": "30.4.1" } }, "sha512-R+xGEtzA95NIsvpXJSROG4t01956dDOt17KpamguY4XOnGvdHNFFXE7Er0C1OAsRjOwiIxpKqOvGlznIGZIQlQ=="], + "@jest/environment": ["@jest/environment@29.7.0", "", { "dependencies": { "@jest/fake-timers": "^29.7.0", "@jest/types": "^29.6.3", "@types/node": "*", "jest-mock": "^29.7.0" } }, "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw=="], "@jest/expect": ["@jest/expect@29.7.0", "", { "dependencies": { "expect": "^29.7.0", "jest-snapshot": "^29.7.0" } }, "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ=="], @@ -513,6 +517,8 @@ "@jest/globals": ["@jest/globals@29.7.0", "", { "dependencies": { "@jest/environment": "^29.7.0", "@jest/expect": "^29.7.0", "@jest/types": "^29.6.3", "jest-mock": "^29.7.0" } }, "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ=="], + "@jest/pattern": ["@jest/pattern@30.4.0", "", { "dependencies": { "@types/node": "*", "jest-regex-util": "30.4.0" } }, "sha512-RAWn3+f9u8BsHijKJ71uHcFp6vmyEt6VvoWXkl6hKF3qVIuWNmudVjg12DlBPGup/frIl5UcUlH5HfEuvHpEXg=="], + "@jest/reporters": ["@jest/reporters@29.7.0", "", { "dependencies": { "@bcoe/v8-coverage": "^0.2.3", "@jest/console": "^29.7.0", "@jest/test-result": "^29.7.0", "@jest/transform": "^29.7.0", "@jest/types": "^29.6.3", "@jridgewell/trace-mapping": "^0.3.18", "@types/node": "*", "chalk": "^4.0.0", "collect-v8-coverage": "^1.0.0", "exit": "^0.1.2", "glob": "^7.1.3", "graceful-fs": "^4.2.9", "istanbul-lib-coverage": "^3.0.0", "istanbul-lib-instrument": "^6.0.0", "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^4.0.0", "istanbul-reports": "^3.1.3", "jest-message-util": "^29.7.0", "jest-util": "^29.7.0", "jest-worker": "^29.7.0", "slash": "^3.0.0", "string-length": "^4.0.1", "strip-ansi": "^6.0.0", "v8-to-istanbul": "^9.0.1" }, "peerDependencies": { "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" }, "optionalPeers": ["node-notifier"] }, "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg=="], "@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="], @@ -873,8 +879,40 @@ "@supabase/supabase-js": ["@supabase/supabase-js@2.89.0", "", { "dependencies": { "@supabase/auth-js": "2.89.0", "@supabase/functions-js": "2.89.0", "@supabase/postgrest-js": "2.89.0", "@supabase/realtime-js": "2.89.0", "@supabase/storage-js": "2.89.0" } }, "sha512-KlaRwSfFA0fD73PYVMHj5/iXFtQGCcX7PSx0FdQwYEEw9b2wqM7GxadY+5YwcmuEhalmjFB/YvqaoNVF+sWUlg=="], + "@swc/core": ["@swc/core@1.15.40", "", { "dependencies": { "@swc/counter": "^0.1.3", "@swc/types": "^0.1.26" }, "optionalDependencies": { "@swc/core-darwin-arm64": "1.15.40", "@swc/core-darwin-x64": "1.15.40", "@swc/core-linux-arm-gnueabihf": "1.15.40", "@swc/core-linux-arm64-gnu": "1.15.40", "@swc/core-linux-arm64-musl": "1.15.40", "@swc/core-linux-ppc64-gnu": "1.15.40", "@swc/core-linux-s390x-gnu": "1.15.40", "@swc/core-linux-x64-gnu": "1.15.40", "@swc/core-linux-x64-musl": "1.15.40", "@swc/core-win32-arm64-msvc": "1.15.40", "@swc/core-win32-ia32-msvc": "1.15.40", "@swc/core-win32-x64-msvc": "1.15.40" }, "peerDependencies": { "@swc/helpers": ">=0.5.17" }, "optionalPeers": ["@swc/helpers"] }, "sha512-2kwzJikRvgtNAG7MwVZY2vEzZjTxKIq5jXOihuSV/8U+Hej8Va22t65aKnJZs3P+NwojZvR8Mf8kyM7O+V8sQg=="], + + "@swc/core-darwin-arm64": ["@swc/core-darwin-arm64@1.15.40", "", { "os": "darwin", "cpu": "arm64" }, "sha512-PaYyclfmQ++77D8ityYvmmVzHv9aG8ROwt2GfG6/ccloy4Hgf80qtOnzb9VYvPsUT7Ty1uhuDRhv3XYpf62qhQ=="], + + "@swc/core-darwin-x64": ["@swc/core-darwin-x64@1.15.40", "", { "os": "darwin", "cpu": "x64" }, "sha512-HbbPzvfLBUXjIB1Ezks+//lNUjmLjfyd63XSwprJgrZaXYdm70kohXPJUWdqKZozolFxbPaO+xtBaiUp6BoueA=="], + + "@swc/core-linux-arm-gnueabihf": ["@swc/core-linux-arm-gnueabihf@1.15.40", "", { "os": "linux", "cpu": "arm" }, "sha512-SlRZsCjOCPR2LvFs0Ri/Xrx/5o5TCt8vl4gW6mX1hEZOG0a625RxzRHpHdAQNGykmAN/7IeaFAJG+QnNmxlHcA=="], + + "@swc/core-linux-arm64-gnu": ["@swc/core-linux-arm64-gnu@1.15.40", "", { "os": "linux", "cpu": "arm64" }, "sha512-Q8byxJt2fh8CR3EUX6snBpy47AoBVm+In/+Z3rjDHMjC38ZvR9/gtUUNCT0tfrn4EdVsO8/QPi59nxrxvqxvBQ=="], + + "@swc/core-linux-arm64-musl": ["@swc/core-linux-arm64-musl@1.15.40", "", { "os": "linux", "cpu": "arm64" }, "sha512-4z0MgHU+7M0pZDqBN1El7mFXDI1SBwinfcUkAyA4v8QrhOIUOZltySt2aStQLZGrdXVXM4Y4ylfiTC04ED+MoQ=="], + + "@swc/core-linux-ppc64-gnu": ["@swc/core-linux-ppc64-gnu@1.15.40", "", { "os": "linux", "cpu": "ppc64" }, "sha512-fLI4iUgeSZu0eRWUXwe6YzPFx9gHbFiPkl8Rp3mJfP8OpNR3nTQCGPvHdDh9xniW7mVvgMY4ni7A4VzqI1KrpA=="], + + "@swc/core-linux-s390x-gnu": ["@swc/core-linux-s390x-gnu@1.15.40", "", { "os": "linux", "cpu": "s390x" }, "sha512-YqeKMAb7d4nQSGMJQ454IlaCENpzcDqhvBE9+CPfdnYpnUXxd+BSrB6Xk0YjW8UyoEhUj4p6quATCxbsp6J3jg=="], + + "@swc/core-linux-x64-gnu": ["@swc/core-linux-x64-gnu@1.15.40", "", { "os": "linux", "cpu": "x64" }, "sha512-7HOuS1iGcme/j/TuL1TfmmLGiMQrjv/GmjyZeydl00FKPtpGXEldwqfI56xgd1YzrzoB2svWjxbGGyQ0TEASxg=="], + + "@swc/core-linux-x64-musl": ["@swc/core-linux-x64-musl@1.15.40", "", { "os": "linux", "cpu": "x64" }, "sha512-h4kZYHc7dpc9P9u4brRJaS8Pl7tPVHAeiLSzw7T5RfIJgAoSdaCMKzI/2Uay9gFhaw8uyCDl0L5q37r0EpAfIA=="], + + "@swc/core-win32-arm64-msvc": ["@swc/core-win32-arm64-msvc@1.15.40", "", { "os": "win32", "cpu": "arm64" }, "sha512-+mQgKZXSj6mV38Zh05QaxSjUDmGP/R2JWlXZTDLSPkDzHU6p3GxN9eeSf5dfyDVU86946fmCvSzyl/ucImx8+A=="], + + "@swc/core-win32-ia32-msvc": ["@swc/core-win32-ia32-msvc@1.15.40", "", { "os": "win32", "cpu": "ia32" }, "sha512-yvwdPLGd25mcj/mNatjNQ0lZujtQD6psH3v9PNmMb+fSzjbNG8KIDxjFWrcV+fsFVLOkyOmdJsFmX7NAFjVyPw=="], + + "@swc/core-win32-x64-msvc": ["@swc/core-win32-x64-msvc@1.15.40", "", { "os": "win32", "cpu": "x64" }, "sha512-OXtKsLU1bVtInzzDEAY2sYiF/rl4tvAnLLLpuMp3HzAOQZ5A+i69AKDhA1YLQTaMAqO3vzyYNVAYVRMPtSYD4w=="], + + "@swc/counter": ["@swc/counter@0.1.3", "", {}, "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="], + "@swc/helpers": ["@swc/helpers@0.5.15", "", { "dependencies": { "tslib": "^2.8.0" } }, "sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g=="], + "@swc/jest": ["@swc/jest@0.2.39", "", { "dependencies": { "@jest/create-cache-key-function": "^30.0.0", "@swc/counter": "^0.1.3", "jsonc-parser": "^3.2.0" }, "peerDependencies": { "@swc/core": "*" } }, "sha512-eyokjOwYd0Q8RnMHri+8/FS1HIrIUKK/sRrFp8c1dThUOfNeCWbLmBP1P5VsKdvmkd25JaH+OKYwEYiAYg9YAA=="], + + "@swc/types": ["@swc/types@0.1.26", "", { "dependencies": { "@swc/counter": "^0.1.3" } }, "sha512-lyMwd7WGgG79RS7EERZV3T8wMdmPq3xwyg+1nmAM64kIhx5yl+juO2PYIHb7vTiPgPCj8LYjsNV2T5wiQHUEaw=="], + "@tsconfig/node10": ["@tsconfig/node10@1.0.12", "", {}, "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ=="], "@tsconfig/node12": ["@tsconfig/node12@1.0.11", "", {}, "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag=="], @@ -1745,6 +1783,8 @@ "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "jsonc-parser": ["jsonc-parser@3.3.1", "", {}, "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ=="], + "jsonfile": ["jsonfile@6.2.0", "", { "dependencies": { "universalify": "^2.0.0" }, "optionalDependencies": { "graceful-fs": "^4.1.6" } }, "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg=="], "jsx-ast-utils": ["jsx-ast-utils@3.3.5", "", { "dependencies": { "array-includes": "^3.1.6", "array.prototype.flat": "^1.3.1", "object.assign": "^4.1.4", "object.values": "^1.1.6" } }, "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ=="], @@ -2505,6 +2545,10 @@ "@istanbuljs/load-nyc-config/js-yaml": ["js-yaml@3.14.2", "", { "dependencies": { "argparse": "^1.0.7", "esprima": "^4.0.0" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg=="], + "@jest/create-cache-key-function/@jest/types": ["@jest/types@30.4.1", "", { "dependencies": { "@jest/pattern": "30.4.0", "@jest/schemas": "30.4.1", "@types/istanbul-lib-coverage": "^2.0.6", "@types/istanbul-reports": "^3.0.4", "@types/node": "*", "@types/yargs": "^17.0.33", "chalk": "^4.1.2" } }, "sha512-f1x/vJXIfjOlEmejYpbkbgw1gOqpPECwMvMEtBqe47j7H2Hg8h8w3o3ikhSXq3MI15kg+oQ0exWO0uCtTNJLoQ=="], + + "@jest/pattern/jest-regex-util": ["jest-regex-util@30.4.0", "", {}, "sha512-mWlvLviKIgIQ8VCuM1xRdD0TWp3zlzionlmDBjuXVBs+VkmXq6FgW9T4Emr7oGz/Rk6feDCGyiugolcQEyp3mg=="], + "@jest/reporters/@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.31", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw=="], "@jest/reporters/glob": ["glob@7.2.3", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q=="], @@ -2671,6 +2715,8 @@ "@istanbuljs/load-nyc-config/js-yaml/argparse": ["argparse@1.0.10", "", { "dependencies": { "sprintf-js": "~1.0.2" } }, "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg=="], + "@jest/create-cache-key-function/@jest/types/@jest/schemas": ["@jest/schemas@30.4.1", "", { "dependencies": { "@sinclair/typebox": "^0.34.0" } }, "sha512-i6b4qw5qnP8c5FEeBJg/uZQ4ddrkN6Ca8qISJh0pr7a5hfn3h3v5x60BEbOC7OYAGZNMs1LfFLwnW2CuK8F57Q=="], + "@sentry/node/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], @@ -2701,6 +2747,8 @@ "@istanbuljs/load-nyc-config/find-up/locate-path/p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], + "@jest/create-cache-key-function/@jest/types/@jest/schemas/@sinclair/typebox": ["@sinclair/typebox@0.34.49", "", {}, "sha512-brySQQs7Jtn0joV8Xh9ZV/hZb9Ozb0pmazDIASBkYKCjXrXU3mpcFahmK/z4YDhGkQvP9mWJbVyahdtU5wQA+A=="], + "pkg-dir/find-up/locate-path/p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], "@istanbuljs/load-nyc-config/find-up/locate-path/p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], diff --git a/e2e/helpers/seed.ts b/e2e/helpers/seed.ts new file mode 100644 index 0000000..f876669 --- /dev/null +++ b/e2e/helpers/seed.ts @@ -0,0 +1,62 @@ +import { Page } from '@playwright/test'; +import { buildMapZip } from '../../src/services/maps/tests/map_generator'; + +// Seeds maps through the real submit flow over HTTP (presigned PUT to S3 + server-side validation + +// publish), rather than writing rows directly: it's slower than a raw insert but goes through the +// exact code path production does, so it can't drift from the schema the way hand-written SQL can. +// Uses `page.request` so it shares the (authenticated) browser context's cookies; the caller must +// have logged the page in first. + +export type SeededMap = { id: string; title: string }; + +/** + * Uploads `count` valid maps, all under one (caller-supplied, run-unique) artist so a search for it + * isolates exactly this run's maps. Returns the published map ids. + */ +export async function seedPublicMaps( + page: Page, + opts: { artist: string; count: number } +): Promise { + const { artist, count } = opts; + const maps: SeededMap[] = []; + for (let i = 0; i < count; i++) { + const n = String(i).padStart(3, '0'); + const title = `Infinite Scroll Map ${n}`; + const zip = buildMapZip({ folder: `IScroll${n}`, title, artist }); + + // 1. Reserve a map id + presigned S3 upload URL. + const submit = await page.request.post('/api/maps/submit', { data: { title: `${title}.zip` } }); + const submitBody = await submit.json(); + if (!submitBody.success) { + throw new Error(`submit failed for "${title}": ${submitBody.errorMessage}`); + } + const { id, url } = submitBody as { id: string; url: string }; + + // 2. Upload the archive to the presigned URL (real S3 / Minio). + const put = await page.request.put(url, { + data: zip, + headers: { 'Content-Type': 'application/zip' }, + }); + if (!put.ok()) { + throw new Error(`S3 upload failed for "${title}": ${put.status()}`); + } + + // 3. Validate + publish. + const complete = await page.request.post('/api/maps/submit/complete', { + data: { id, isReupload: false }, + }); + const completeBody = await complete.json(); + if (!completeBody.success) { + throw new Error(`complete failed for "${title}": ${completeBody.errorMessage}`); + } + maps.push({ id, title }); + } + return maps; +} + +/** Deletes the seeded maps via the real delete endpoint (cascades difficulties/favorites + S3). */ +export async function deleteSeededMaps(page: Page, ids: string[]): Promise { + for (const id of ids) { + await page.request.post(`/api/maps/${id}/delete`); + } +} diff --git a/e2e/infinite_scroll.e2e.ts b/e2e/infinite_scroll.e2e.ts new file mode 100644 index 0000000..f28752a --- /dev/null +++ b/e2e/infinite_scroll.e2e.ts @@ -0,0 +1,111 @@ +import { expect, Locator, Page, test } from '@playwright/test'; +import { makeUser, signUpAndConfirm } from './helpers/auth'; +import { deleteSeededMaps, seedPublicMaps } from './helpers/seed'; + +// SEARCH_LIMIT in src/app/map_list_presenter.ts. +const PAGE_SIZE = 20; +const TOTAL = 25; + +// Process-unique artist so re-runs against the persisted Supabase DB never collide; one seeded set +// is shared by both tests. +const unique = `${Date.now().toString(36)}${Math.floor(Math.random() * 1e6).toString(36)}`; +const artist = `infscroll${unique}`; + +// One row per map (each row links to /map/{id}); excludes the header and loading skeleton. +const mapRows = (page: Page) => page.locator('tr', { has: page.locator('a[href^="/map/"]') }); + +const nextPageResponse = (page: Page) => + page.waitForResponse( + (r) => + r.url().includes('/api/maps') && + r.request().method() === 'GET' && + new URL(r.url()).searchParams.get('offset') === String(PAGE_SIZE), + { timeout: 15_000 } + ); + +// Opens the list scoped (by the run-unique artist) to exactly this run's maps and asserts the +// first page rendered with "Load more" on offer. +async function openFirstPage(page: Page): Promise<{ rows: Locator; loadMore: Locator }> { + const rows = mapRows(page); + await page.goto(`/?q=${artist}`); + await page.waitForResponse( + (r) => r.url().includes('/api/maps') && r.request().method() === 'GET' + ); + await expect(rows).toHaveCount(PAGE_SIZE); + const loadMore = page.getByRole('button', { name: 'Load more' }); + await expect(loadMore).toBeVisible(); + return { rows, loadMore }; +} + +// Asserts the second page got appended: every map present exactly once, nothing more to load. +async function expectAllLoaded(rows: Locator, loadMore: Locator): Promise { + await expect(rows).toHaveCount(TOTAL); + await expect(loadMore).toHaveCount(0); + const ids = await rows.evaluateAll((trs) => + trs.map((tr) => tr.querySelector('a[href^="/map/"]')?.getAttribute('href')) + ); + expect(new Set(ids).size).toBe(TOTAL); +} + +// Parks the mouse over the (visible) centre of the map list table so wheel events scroll the list's +// container. Clamped into the viewport since a full page of rows overflows it. +async function moveMouseToTableCentre(page: Page): Promise { + const table = page.locator('table', { has: page.locator('a[href^="/map/"]') }); + const box = await table.boundingBox(); + const viewport = page.viewportSize(); + if (box == null || viewport == null) { + throw new Error('could not locate the map list table'); + } + // Centre of the table's visible region (the table is taller than the viewport). + const visibleTop = Math.max(box.y, 0); + const visibleBottom = Math.min(box.y + box.height, viewport.height); + const cx = Math.min(Math.max(box.x + box.width / 2, 1), viewport.width - 1); + const cy = (visibleTop + visibleBottom) / 2; + await page.mouse.move(cx, cy); +} + +// Scrolls the wheel over the table until the next page has loaded (the row count grows past the +// first page) - i.e. the infinite-scroll hook reached the bottom and appended the next page. The +// wheel scrolls the layout skeleton container (the list's scroll parent). We key off the row count +// rather than the button's loading/disabled state, which is too transient to observe reliably when +// the fetch is fast. +async function wheelUntilNextPageLoads(page: Page, rows: Locator): Promise { + await moveMouseToTableCentre(page); + for (let i = 0; i < 40; i++) { + await page.mouse.wheel(0, 400); + if ((await rows.count()) > PAGE_SIZE) { + return; + } + await page.waitForTimeout(150); // let the hook's 150ms scroll debounce + the fetch run + } + throw new Error('scrolling to the bottom did not load the next page'); +} + +test.describe('home page infinite scroll', () => { + // A dedicated, authenticated page seeds the maps once over the real API; both tests share them. + let seedPage: Page; + let seededIds: string[]; + + test.beforeAll(async ({ browser }) => { + seedPage = await browser.newPage(); + await signUpAndConfirm(seedPage, makeUser('isscroll')); + seededIds = (await seedPublicMaps(seedPage, { artist, count: TOTAL })).map((m) => m.id); + }); + + test.afterAll(async () => { + await deleteSeededMaps(seedPage, seededIds); + await seedPage.close(); + }); + + test('appends the next page when "Load more" is clicked', async ({ page }) => { + const { rows, loadMore } = await openFirstPage(page); + await Promise.all([nextPageResponse(page), loadMore.click()]); + await expectAllLoaded(rows, loadMore); + }); + + test('appends the next page when the mouse wheel reaches the bottom', async ({ page }) => { + const { rows, loadMore } = await openFirstPage(page); + await wheelUntilNextPageLoads(page, rows); + await expectAllLoaded(rows, loadMore); + }); +}); diff --git a/jest.config.integration.ts b/jest.config.integration.ts index 0344d74..717614d 100644 --- a/jest.config.integration.ts +++ b/jest.config.integration.ts @@ -3,7 +3,9 @@ import nextJest from 'next/jest'; const createJestConfig = nextJest({ dir: './' }); const config = { - testMatch: ['**/tests/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[jt]s?(x)'], + // Only files with a `.test`/`.spec` suffix are suites; plain modules under `tests/` (e.g. + // map_generator.ts) are helpers, not tests. + testMatch: ['**/tests/**/*.+(spec|test).[jt]s?(x)', '**/?(*.)+(spec|test).[jt]s?(x)'], // Unit tests run in their own config (jest.config.unit.ts) without DB/server setup. testPathIgnorePatterns: ['/node_modules/', '\\.unit\\.test\\.[jt]sx?$'], setupFilesAfterEnv: ['/src/services/jest_setup.ts'], diff --git a/jest.config.unit.ts b/jest.config.unit.ts index 9b76922..a5fec08 100644 --- a/jest.config.unit.ts +++ b/jest.config.unit.ts @@ -1,14 +1,29 @@ -import nextJest from 'next/jest'; - -const createJestConfig = nextJest({ dir: './' }); +import type { Config } from 'jest'; // Pure unit tests: run entirely within Jest with no server, Postgres, S3, or Supabase. // Tests opt in by using the `.unit.test.ts` suffix. Crucially, this config does NOT load // `jest_setup.ts`, which connects to and truncates a real database. -const config = { +// +// Transformed with @swc/jest (not next/jest's transformer) because the app uses standard-proposal +// decorators / `accessor` fields, which next/jest's SWC setup doesn't enable. Unit tests need no +// Next-specific transforms (CSS, fonts, etc.), so a plain SWC transform is enough. +const config: Config = { testMatch: ['**/*.unit.test.[jt]s?(x)'], setupFilesAfterEnv: ['/src/services/jest_setup.unit.ts'], modulePaths: ['/src'], + transform: { + '^.+\\.(t|j)sx?$': [ + '@swc/jest', + { + jsc: { + parser: { syntax: 'typescript', tsx: true, decorators: true }, + transform: { decoratorVersion: '2022-03' }, + target: 'es2022', + keepClassNames: true, + }, + }, + ], + }, }; -export default createJestConfig(config); +export default config; diff --git a/package.json b/package.json index 75b0dc5..14c467e 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "dev:prod": "dotenv -e .env.prod -- bun next dev", "build": "sudo docker compose -f docker/docker-compose.yml --env-file .env build", "start": "sudo docker compose -f docker/docker-compose.yml --env-file .env up --build", + "check": "bun run format && bun run lint && bun run typecheck && bun run typecheck:e2e && bun run test && bun run test:integration", "typecheck": "bun tsc", "typecheck:e2e": "bun tsc -p e2e/tsconfig.json", "lint": "eslint .", @@ -69,6 +70,8 @@ "@electric-sql/pglite": "^0.4.6", "@electric-sql/pglite-socket": "^0.1.6", "@playwright/test": "^1.60.0", + "@swc/core": "^1.15.40", + "@swc/jest": "^0.2.39", "@types/jest": "^29.5.5", "@types/pg": "^8.10.2", "@types/postcss-modules-values": "^4", diff --git a/src/app/api/api.ts b/src/app/api/api.ts index 838bb04..f11945f 100644 --- a/src/app/api/api.ts +++ b/src/app/api/api.ts @@ -2,6 +2,8 @@ import * as qs from 'qs'; import { ApiResponse } from 'schema/api'; import { encodeFilter } from 'schema/map_filter'; import { + CompleteUploadRequest, + CompleteUploadResponse, DeleteMapResponse, FindMapsResponse, GetMapResponse, @@ -34,6 +36,7 @@ export interface Api { getMap(id: string): Promise; deleteMap(id: string): Promise; submitMap(req: SubmitMapRequest): Promise; + completeMapUpload(req: CompleteUploadRequest): Promise; } export class HttpApi implements Api { @@ -88,6 +91,11 @@ export class HttpApi implements Api { const resp = await post(path(this.apiBase, 'maps', 'submit'), req); return SubmitMapResponse.parse(resp); } + + async completeMapUpload(req: CompleteUploadRequest): Promise { + const resp = await post(path(this.apiBase, 'maps', 'submit', 'complete'), req); + return CompleteUploadResponse.parse(resp); + } } function path(...parts: string[]) { diff --git a/src/app/api/fake_api.ts b/src/app/api/fake_api.ts index 788ac96..8bdf855 100644 --- a/src/app/api/fake_api.ts +++ b/src/app/api/fake_api.ts @@ -1,5 +1,7 @@ import { ApiResponse } from 'schema/api'; import { + CompleteUploadRequest, + CompleteUploadResponse, DeleteMapResponse, FindMapsResponse, GetMapResponse, @@ -75,6 +77,11 @@ export class FakeApi implements Api { await delay(); return { success: true, id: allStar.id, url: '' }; } + + async completeMapUpload(_req: CompleteUploadRequest): Promise { + await delay(); + return { success: true, map: allStar }; + } } const allStar: PDMap = { diff --git a/src/app/api/maps/submit/complete/actions.ts b/src/app/api/maps/submit/complete/actions.ts deleted file mode 100644 index 0435077..0000000 --- a/src/app/api/maps/submit/complete/actions.ts +++ /dev/null @@ -1,117 +0,0 @@ -'use server'; - -import * as Sentry from '@sentry/nextjs'; -import { MapValidity } from 'schema/maps'; -import { actionError } from 'services/helpers'; -import { submitErrorMap } from 'services/maps/maps_repo'; -import { getServerContext } from 'services/server_context'; -import { getUserSession } from 'services/session/session'; - -/** - * Handles the completion and validation of a new map upload. This will publish the map if it is - * valid. - * - * If it is a reupload, it will replace the existing map data. If it is a new map and it is invalid, - * it will rollback and delete the temporary Map record. - */ -export async function reportUploadComplete(id: string, isReupload: boolean) { - return Sentry.withServerActionInstrumentation('upload_complete', async () => { - const { mapsRepo, s3Handler } = await getServerContext(); - - const validityResult = await mapsRepo.setValidity( - id, - isReupload ? MapValidity.REUPLOADED : MapValidity.UPLOADED - ); - if (!validityResult.success) { - return actionError({ - errorBody: {}, - message: 'Could not update map upload status.', - resultError: validityResult, - shouldLog: true, - }); - } - - const dbMapResult = await mapsRepo.getMap(id); - const previousValidity = dbMapResult.success ? dbMapResult.value.validity : undefined; - - async function cleanupFailedUpload() { - // If this is a freshly created Map, delete the Map record - if (previousValidity === MapValidity.PENDING_UPLOAD) { - // Mark as invalid first in case any other parts of deletion fail - await mapsRepo.setValidity(id, MapValidity.INVALID); - await mapsRepo.deleteMap({ id }); - } else { - // Delete temp files - await s3Handler.deleteFiles(id, true); - // This is a reupload, so reset validity back to valid - await mapsRepo.setValidity(id, MapValidity.VALID); - } - } - - const session = await getUserSession(); - if (!session) { - await cleanupFailedUpload(); - return actionError({ - errorBody: {}, - message: 'You must be logged in to submit maps.', - }); - } - - // Technically this isn't required, but better safe than sorry - just enforce it for now for - // consistency - if (dbMapResult.success && dbMapResult.value.uploader !== session.id) { - await cleanupFailedUpload(); - if (previousValidity === MapValidity.PENDING_UPLOAD) { - return actionError({ - errorBody: {}, - message: - 'You must begin and complete the upload while logged into the same user session.', - }); - } else { - return actionError({ - errorBody: {}, - message: 'Only the original map uploader can reupload a map.', - }); - } - } - - // Fetch it and begin processing - const getMapResult = await s3Handler.getMapFile(id, true); - if (!getMapResult.success) { - await cleanupFailedUpload(); - return actionError({ - errorBody: {}, - message: 'The file could not be processed.', - resultError: getMapResult, - shouldLog: true, - }); - } - const mapFile = getMapResult.value; - if (mapFile.byteLength > 1024 * 1024 * 100) { - await cleanupFailedUpload(); - // 100MiB. We use MiB because that's what Windows displays in Explorer and therefore what users will expect. - return actionError({ - message: 'File is over the filesize limit (100MB)', - errorBody: {}, - }); - } - const processMapResult = await mapsRepo.validateUploadedMap({ - id, - mapFile, - uploader: session.id, - }); - if (!processMapResult.success) { - await cleanupFailedUpload(); - // TODO: report all errors back to the client and not just the first one - const [_statusCode, message] = submitErrorMap[processMapResult.errors[0].type]; - return actionError({ - message: processMapResult.errors[0].userMessage || message, - errorBody: {}, - resultError: processMapResult, - shouldLog: true, - }); - } - - return { success: true, value: processMapResult.value } as const; - }); -} diff --git a/src/app/api/maps/submit/complete/complete_upload.ts b/src/app/api/maps/submit/complete/complete_upload.ts new file mode 100644 index 0000000..b376d01 --- /dev/null +++ b/src/app/api/maps/submit/complete/complete_upload.ts @@ -0,0 +1,110 @@ +import { MapValidity } from 'schema/maps'; +import { actionError } from 'services/helpers'; +import { submitErrorMap } from 'services/maps/maps_repo'; +import { getServerContext } from 'services/server_context'; +import { getUserSession } from 'services/session/session'; + +/** + * Validates and publishes a map whose archive has already been uploaded to S3 (a fresh upload or a + * reupload), making it public when valid. A freshly-created map is rolled back on failure; a + * reupload reverts to its previous valid state. + * + * Invoked by the POST /api/maps/submit/complete route. Also called in-process by the integration + * tests: the in-memory fake S3 only round-trips within a single process, so they seed the archive + * and run this directly rather than over HTTP. + */ +export async function completeMapUpload(id: string, isReupload: boolean) { + const { mapsRepo, s3Handler } = await getServerContext(); + + const validityResult = await mapsRepo.setValidity( + id, + isReupload ? MapValidity.REUPLOADED : MapValidity.UPLOADED + ); + if (!validityResult.success) { + return actionError({ + errorBody: {}, + message: 'Could not update map upload status.', + resultError: validityResult, + shouldLog: true, + }); + } + + const dbMapResult = await mapsRepo.getMap(id); + + async function cleanupFailedUpload() { + // A fresh upload has no prior published state, so delete the placeholder record; a failed + // reupload keeps the existing map and reverts it to valid. (getMap can't tell them apart - it + // only returns public maps, and a fresh upload is still hidden here - so key off `isReupload`.) + if (!isReupload) { + // Mark as invalid first in case any other parts of deletion fail + await mapsRepo.setValidity(id, MapValidity.INVALID); + await mapsRepo.deleteMap({ id }); + } else { + await s3Handler.deleteFiles(id, true); + await mapsRepo.setValidity(id, MapValidity.VALID); + } + } + + const session = await getUserSession(); + if (!session) { + await cleanupFailedUpload(); + return actionError({ + errorBody: {}, + message: 'You must be logged in to submit maps.', + }); + } + + if (dbMapResult.success && dbMapResult.value.uploader !== session.id) { + await cleanupFailedUpload(); + if (!isReupload) { + return actionError({ + errorBody: {}, + message: 'You must begin and complete the upload while logged into the same user session.', + }); + } else { + return actionError({ + errorBody: {}, + message: 'Only the original map uploader can reupload a map.', + }); + } + } + + // Fetch it and begin processing + const getMapResult = await s3Handler.getMapFile(id, true); + if (!getMapResult.success) { + await cleanupFailedUpload(); + return actionError({ + errorBody: {}, + message: 'The file could not be processed.', + resultError: getMapResult, + shouldLog: true, + }); + } + const mapFile = getMapResult.value; + if (mapFile.byteLength > 1024 * 1024 * 100) { + await cleanupFailedUpload(); + // 100MiB. We use MiB because that's what Windows displays in Explorer and therefore what users will expect. + return actionError({ + message: 'File is over the filesize limit (100MB)', + errorBody: {}, + }); + } + const processMapResult = await mapsRepo.validateUploadedMap({ + id, + mapFile, + uploader: session.id, + }); + if (!processMapResult.success) { + await cleanupFailedUpload(); + // TODO: report all errors back to the client and not just the first one + const [_statusCode, message] = submitErrorMap[processMapResult.errors[0].type]; + return actionError({ + message: processMapResult.errors[0].userMessage || message, + errorBody: {}, + resultError: processMapResult, + shouldLog: true, + }); + } + + return { success: true, value: processMapResult.value } as const; +} diff --git a/src/app/api/maps/submit/complete/route.ts b/src/app/api/maps/submit/complete/route.ts new file mode 100644 index 0000000..0437d15 --- /dev/null +++ b/src/app/api/maps/submit/complete/route.ts @@ -0,0 +1,27 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { CompleteUploadRequest, CompleteUploadResponse } from 'schema/maps'; +import { completeMapUpload } from './complete_upload'; + +const send = (res: CompleteUploadResponse) => NextResponse.json(CompleteUploadResponse.parse(res)); + +/** + * Validates and publishes a map whose archive the client has already uploaded to the presigned S3 + * URL returned by /api/maps/submit. See {@link completeMapUpload}. + */ +export async function POST(req: NextRequest): Promise { + const reqResult = CompleteUploadRequest.safeParse(await req.json()); + if (!reqResult.success) { + return send({ + success: false, + statusCode: 400, + errorMessage: 'Invalid complete upload request.', + }); + } + const { id, isReupload } = reqResult.data; + const result = await completeMapUpload(id, isReupload); + if (!result.success) { + // completeMapUpload already logged the failure; just surface the message to the client. + return send({ success: false, statusCode: 400, errorMessage: result.errorMessage }); + } + return send({ success: true, map: result.value }); +} diff --git a/src/app/map_list_presenter.ts b/src/app/map_list_presenter.ts index 5cfe884..8eee87c 100644 --- a/src/app/map_list_presenter.ts +++ b/src/app/map_list_presenter.ts @@ -56,6 +56,10 @@ async function delay(ms: number = 5) { } export class MapListPresenter { + // Bumped on every new search so an in-flight load-more (or a superseded search) can tell its + // results are stale and drop them instead of clobbering the current list. + private searchSeq = 0; + constructor( private readonly api: Api, private readonly store: MapListStore @@ -142,6 +146,7 @@ export class MapListPresenter { } @action.bound async onSearch(trigger: 'sort' | 'search') { + const seq = ++this.searchSeq; if (trigger === 'search') { // Upon clearing search to go back to the original view, reset sort back to submission date if (this.store.query === '') { @@ -155,6 +160,7 @@ export class MapListPresenter { runInAction(() => { this.store.maps = undefined; this.store.totalCount = undefined; + this.store.loadingMore = false; }); const resp = await this.api.searchMaps({ query: this.store.query, @@ -163,17 +169,22 @@ export class MapListPresenter { filter: this.store.activeFilter, ...(sort || {}), }); + // A newer search started while this one was in flight; let it own the results. + if (seq !== this.searchSeq) { + return; + } if (resp.success) { runInAction(() => { this.store.maps = resp.maps; this.store.totalCount = resp.totalCount; - this.store.hasMore = resp.maps.length >= SEARCH_LIMIT; + this.store.hasMore = resp.maps.length < resp.totalCount; }); } this.store.lastSelectedMapIndex = undefined; } @action.bound async onLoadMore() { + const seq = this.searchSeq; const sort = this.getTableSortParams(); this.store.loadingMore = true; const resp = await this.api.searchMaps({ @@ -184,16 +195,15 @@ export class MapListPresenter { ...(sort || {}), }); runInAction(() => (this.store.loadingMore = false)); + // A search replaced the result set while this page was loading; drop the stale page. + if (seq !== this.searchSeq) { + return; + } if (resp.success) { runInAction(() => { - if (this.store.maps) { - this.store.maps.push(...resp.maps); - } else { - this.store.maps = resp.maps; - } - if (resp.maps.length < SEARCH_LIMIT) { - this.store.hasMore = false; - } + this.store.maps = [...(this.store.maps ?? []), ...resp.maps]; + this.store.totalCount = resp.totalCount; + this.store.hasMore = this.store.maps.length < resp.totalCount; }); } } diff --git a/src/app/tests/map_list_presenter.unit.test.ts b/src/app/tests/map_list_presenter.unit.test.ts new file mode 100644 index 0000000..f8eb120 --- /dev/null +++ b/src/app/tests/map_list_presenter.unit.test.ts @@ -0,0 +1,86 @@ +import { Api } from 'app/api/api'; +import { MapListPresenter, MapListStore } from 'app/map_list_presenter'; +import { FindMapsResponse, PDMap, SearchMapsRequest } from 'schema/maps'; +import type { Columns } from 'ui/base/table/table'; +import { TableSortStore } from 'ui/base/table/table_presenter'; + +// SEARCH_LIMIT in map_list_presenter.ts (kept in sync here; it isn't exported). +const SEARCH_LIMIT = 20; + +const makeMap = (id: string) => ({ id }) as unknown as PDMap; + +function createMapListStore() { + // sortColumn is cleared on search, and these tests never sort, so the columns are unused. + const sortStore = new TableSortStore([] as unknown as Columns, 0, 'desc'); + return new MapListStore('', sortStore); +} + +// Serves a fixed total number of maps, honouring the requested offset/limit so it models a real +// paged backend. +class FakeSearchApi { + constructor(private readonly total: number) {} + searchMaps(req: SearchMapsRequest): Promise { + const available = Math.max(0, this.total - req.offset); + const maps = Array.from({ length: Math.min(req.limit, available) }, (_, i) => + makeMap(`m${req.offset + i}`) + ); + return Promise.resolve({ success: true, maps, totalCount: this.total }); + } +} + +// Defers every searchMaps response so the test can resolve them out of order and reproduce races. +class FakeDeferredSearchApi { + readonly calls: Array<{ req: SearchMapsRequest; resolve: (maps: PDMap[]) => void }> = []; + searchMaps(req: SearchMapsRequest): Promise { + return new Promise((resolve) => { + this.calls.push({ + req, + resolve: (maps) => resolve({ success: true, maps, totalCount: maps.length }), + }); + }); + } +} + +describe('MapListPresenter pagination', () => { + it('does not report hasMore when the result count exactly fills one page', async () => { + const store = createMapListStore(); + const presenter = new MapListPresenter( + new FakeSearchApi(SEARCH_LIMIT) as unknown as Api, + store + ); + + presenter.onChangeQuery('anything'); + await presenter.onSearch('search'); + + expect(store.maps).toHaveLength(SEARCH_LIMIT); + expect(store.hasMore).toBe(false); + }); + + it('discards an in-flight load-more when a new search replaces the results', async () => { + const store = createMapListStore(); + const api = new FakeDeferredSearchApi(); + const presenter = new MapListPresenter(api as unknown as Api, store); + + // Initial search resolves with a short page. + presenter.onChangeQuery('first'); + const firstSearch = presenter.onSearch('search'); + api.calls[0].resolve([makeMap('a'), makeMap('b')]); + await firstSearch; + expect(store.maps).toEqual([makeMap('a'), makeMap('b')]); + + // Start loading more, but leave it in flight. + const loadMore = presenter.onLoadMore(); + + // A new search starts and finishes first, replacing the result set. + presenter.onChangeQuery('second'); + const secondSearch = presenter.onSearch('search'); + api.calls[2].resolve([makeMap('c')]); + await secondSearch; + expect(store.maps).toEqual([makeMap('c')]); + + // The stale load-more now resolves; its page belongs to the old query and must be discarded. + api.calls[1].resolve([makeMap('stale')]); + await loadMore; + expect(store.maps).toEqual([makeMap('c')]); + }); +}); diff --git a/src/schema/maps.ts b/src/schema/maps.ts index 8ae43b4..5b5cc32 100644 --- a/src/schema/maps.ts +++ b/src/schema/maps.ts @@ -126,3 +126,21 @@ export const SubmitMapResponse = z.discriminatedUnion('success', [ SubmitMapError, ]); export type SubmitMapResponse = z.infer; + +/* POST submitMap/complete */ +export const CompleteUploadRequest = z.object({ + id: z.string(), + isReupload: z.boolean(), +}); +export type CompleteUploadRequest = z.infer; + +export const CompleteUploadSuccess = ApiSuccess.extend({ + map: PDMap, +}); +export type CompleteUploadSuccess = z.infer; + +export const CompleteUploadResponse = z.discriminatedUnion('success', [ + CompleteUploadSuccess, + ApiError, +]); +export type CompleteUploadResponse = z.infer; diff --git a/src/services/helpers.ts b/src/services/helpers.ts index 74bd6f7..da8c1de 100644 --- a/src/services/helpers.ts +++ b/src/services/helpers.ts @@ -84,7 +84,19 @@ export function badRequest(message: string) { }); } +// Floor so a client can't page the list one map at a time; ceiling so one request can't ask for +// the whole table. The floor doubles as the default when `limit` is absent or unparseable. +const MIN_LIMIT = 20; +const MAX_LIMIT = 100; + export function getOffsetLimit(req: NextRequest) { const { offset, limit } = getQueryParams(req); - return { offset: Number(offset) || 0, limit: Number(limit) || 20 }; + const parsedOffset = Number(offset); + const parsedLimit = Number(limit); + return { + offset: Number.isFinite(parsedOffset) ? Math.max(0, Math.floor(parsedOffset)) : 0, + limit: Number.isFinite(parsedLimit) + ? Math.min(MAX_LIMIT, Math.max(MIN_LIMIT, Math.floor(parsedLimit))) + : MIN_LIMIT, + }; } diff --git a/src/services/maps/tests/map_generator.ts b/src/services/maps/tests/map_generator.ts new file mode 100644 index 0000000..0fd271e --- /dev/null +++ b/src/services/maps/tests/map_generator.ts @@ -0,0 +1,143 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +/** + * Programmatically builds valid Paradiddle map zips for tests, porting the Python generator that + * used to live in `e2e/fixtures/README.md`. A built zip carries controlled `recordingMetadata` + * (title/artist/creator/description/complexity) so tests can drive search, filtering and pagination + * from known values. The audio tracks and album art are reused from the committed fixtures under + * `files/` (`silence.ogg`, `album.jpg`), same as the Python script. + */ + +const FILES_DIR = path.resolve(__dirname, 'files'); +const albumArt = fs.readFileSync(path.join(FILES_DIR, 'album.jpg')); +const silence = fs.readFileSync(path.join(FILES_DIR, 'silence.ogg')); + +export type MapZipSpec = { + /** Top-level folder name. Must match the .rlrr filename prefix, which the validator enforces. */ + folder: string; + difficulty?: string; + title: string; + artist: string; + creator?: string; + description?: string; + complexity?: number; +}; + +export function buildMapZip(spec: MapZipSpec): Buffer { + const difficulty = spec.difficulty ?? 'Easy'; + const rlrr = { + version: 0.6, + recordingMetadata: { + title: spec.title, + description: spec.description ?? '', + coverImagePath: 'album.jpg', + artist: spec.artist, + creator: spec.creator ?? '', + length: 11.1814, + complexity: spec.complexity ?? 1, + }, + audioFileData: { songTracks: ['song.ogg'], drumTracks: ['drums.ogg'], calibrationOffset: 0.0 }, + instruments: [], + events: [], + bpmEvents: [{ bpm: 120.0, time: 0.0 }], + }; + + // The .rlrr must come first: the validator derives the map name from the first file entry. + return buildZip([ + { + name: `${spec.folder}/${spec.folder}_${difficulty}.rlrr`, + data: Buffer.from(JSON.stringify(rlrr, null, 2)), + }, + { name: `${spec.folder}/album.jpg`, data: albumArt }, + { name: `${spec.folder}/song.ogg`, data: silence }, + { name: `${spec.folder}/drums.ogg`, data: silence }, + ]); +} + +type ZipEntry = { name: string; data: Buffer }; + +// Minimal STORED (uncompressed) zip writer. STORED keeps this dependency-free, and `unzipper` (the +// reader the validator uses) handles it; compression buys nothing for tiny test fixtures. +function buildZip(entries: ZipEntry[]): Buffer { + const localParts: Buffer[] = []; + const centralParts: Buffer[] = []; + let offset = 0; + + for (const entry of entries) { + const nameBuf = Buffer.from(entry.name, 'utf8'); + const crc = crc32(entry.data); + const size = entry.data.length; + + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); // local file header signature + local.writeUInt16LE(20, 4); // version needed to extract + local.writeUInt16LE(0, 6); // flags + local.writeUInt16LE(0, 8); // compression method: stored + local.writeUInt16LE(0, 10); // mod time + local.writeUInt16LE(0, 12); // mod date + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(size, 18); // compressed size + local.writeUInt32LE(size, 22); // uncompressed size + local.writeUInt16LE(nameBuf.length, 26); + local.writeUInt16LE(0, 28); // extra field length + localParts.push(local, nameBuf, entry.data); + + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); // central directory header signature + central.writeUInt16LE(20, 4); // version made by + central.writeUInt16LE(20, 6); // version needed to extract + central.writeUInt16LE(0, 8); // flags + central.writeUInt16LE(0, 10); // compression method + central.writeUInt16LE(0, 12); // mod time + central.writeUInt16LE(0, 14); // mod date + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(size, 20); // compressed size + central.writeUInt32LE(size, 24); // uncompressed size + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt16LE(0, 30); // extra field length + central.writeUInt16LE(0, 32); // comment length + central.writeUInt16LE(0, 34); // disk number start + central.writeUInt16LE(0, 36); // internal attributes + central.writeUInt32LE(0, 38); // external attributes + central.writeUInt32LE(offset, 42); // relative offset of local header + centralParts.push(central, nameBuf); + + offset += local.length + nameBuf.length + entry.data.length; + } + + const localBuf = Buffer.concat(localParts); + const centralBuf = Buffer.concat(centralParts); + + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); // end of central directory signature + end.writeUInt16LE(0, 4); // disk number + end.writeUInt16LE(0, 6); // disk with central directory + end.writeUInt16LE(entries.length, 8); // central directory entries on this disk + end.writeUInt16LE(entries.length, 10); // total central directory entries + end.writeUInt32LE(centralBuf.length, 12); // size of central directory + end.writeUInt32LE(localBuf.length, 16); // offset of central directory + end.writeUInt16LE(0, 20); // comment length + + return Buffer.concat([localBuf, centralBuf, end]); +} + +const CRC_TABLE = (() => { + const table = new Uint32Array(256); + for (let n = 0; n < 256; n++) { + let c = n; + for (let k = 0; k < 8; k++) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c >>> 0; + } + return table; +})(); + +function crc32(buf: Buffer): number { + let crc = 0xffffffff; + for (let i = 0; i < buf.length; i++) { + crc = CRC_TABLE[(crc ^ buf[i]) & 0xff] ^ (crc >>> 8); + } + return (crc ^ 0xffffffff) >>> 0; +} diff --git a/src/services/maps/tests/maps.test.ts b/src/services/maps/tests/maps.test.ts index 22f6f59..7941b34 100644 --- a/src/services/maps/tests/maps.test.ts +++ b/src/services/maps/tests/maps.test.ts @@ -1,4 +1,4 @@ -import { reportUploadComplete } from 'app/api/maps/submit/complete/actions'; +import { completeMapUpload } from 'app/api/maps/submit/complete/complete_upload'; import { _unwrap } from 'base/result'; import * as fs from 'fs/promises'; import * as path from 'path'; @@ -30,7 +30,7 @@ describe('maps handler', () => { (await _unwrap(mapsRepo.createNewMap({ title: 'placeholder', uploader: uploaderId }))).id; (s3Handler as MemoryFakeS3Handler)._putMapFileForTesting(id, buffer); _setCurrentUserForTesting({ id: uploaderId, email: UPLOADER.email }); - const result = await reportUploadComplete(id, opts.isReupload ?? false); + const result = await completeMapUpload(id, opts.isReupload ?? false); return { result, id }; }; diff --git a/src/services/maps/tests/maps_pagination.test.ts b/src/services/maps/tests/maps_pagination.test.ts new file mode 100644 index 0000000..f73eb32 --- /dev/null +++ b/src/services/maps/tests/maps_pagination.test.ts @@ -0,0 +1,124 @@ +import { completeMapUpload } from 'app/api/maps/submit/complete/complete_upload'; +import { _unwrap } from 'base/result'; +import { MapVisibility } from 'schema/maps'; +import { IdDomain, generateId } from 'services/db/id_gen'; +import { MemoryFakeS3Handler } from 'services/maps/s3_handler_fake_memory'; +import { getServerContext } from 'services/server_context'; +import { _setCurrentUserForTesting } from 'services/session/supabase_fake'; +import { buildMapZip } from './map_generator'; + +// `maps.uploader` has no FK, so this id only has to match the session we install below. +const UPLOADER = { id: 'UPLOAD1', email: 'uploader@test.com' }; + +const TOTAL = 100; +const PAGE = 20; + +// Generates a map zip, seeds it into the fake S3 store, and drives the real completion flow that +// validates and publishes it (same path as maps.test.ts). Returns the published map id. +async function uploadGeneratedMap(index: number): Promise { + const n = String(index).padStart(3, '0'); + const { mapsRepo, s3Handler } = await getServerContext(); + const id = (await _unwrap(mapsRepo.createNewMap({ title: 'placeholder', uploader: UPLOADER.id }))) + .id; + // Zero-padded title/artist so lexical ordering matches numeric ordering, giving a unique total + // order to assert pagination against. + const zip = buildMapZip({ + folder: `PMap${n}`, + title: `Pagination Map ${n}`, + artist: `Artist ${n}`, + }); + (s3Handler as MemoryFakeS3Handler)._putMapFileForTesting(id, zip); + _setCurrentUserForTesting({ id: UPLOADER.id, email: UPLOADER.email }); + const result = await completeMapUpload(id, false); + if (!result.success) { + throw new Error(`Failed to publish generated map ${n}: ${result.errorMessage}`); + } + return result.value.id; +} + +async function page(offset: number, sortDirection: 'asc' | 'desc') { + const { mapsRepo } = await getServerContext(); + const result = await _unwrap( + mapsRepo.searchMaps({ query: '', offset, limit: PAGE, sort: 'title', sortDirection }) + ); + return result.maps; +} + +describe('maps repo pagination', () => { + it('pages through all generated maps with no gaps or duplicates', async () => { + // Start from a clean slate: drop the seed maps (see supabase/seed.sql) so the generated set is + // the only public data and counts are exact. + const { pool } = await getServerContext(); + await pool.query('TRUNCATE maps, difficulties, favorites CASCADE'); + + const expectedTitles: string[] = []; + for (let i = 0; i < TOTAL; i++) { + await uploadGeneratedMap(i); + expectedTitles.push(`Pagination Map ${String(i).padStart(3, '0')}`); + } + + // Walk every page sorted by title ascending; titles are unique, so the order is deterministic. + const collected: string[] = []; + for (let offset = 0; offset < TOTAL; offset += PAGE) { + const maps = await page(offset, 'asc'); + expect(maps).toHaveLength(PAGE); + collected.push(...maps.map((m) => m.title)); + } + // Exactly TOTAL maps, in order, with no duplicates or skipped rows. + expect(collected).toEqual(expectedTitles); + expect(new Set(collected).size).toBe(TOTAL); + + // The page past the end is empty. + expect(await page(TOTAL, 'asc')).toHaveLength(0); + + // Descending pagination yields the reverse order. + const descending: string[] = []; + for (let offset = 0; offset < TOTAL; offset += PAGE) { + const maps = await page(offset, 'desc'); + descending.push(...maps.map((m) => m.title)); + } + expect(descending).toEqual([...expectedTitles].reverse()); + }, 120000); + + // Direct-inserts public maps with a shared submission_date (which the upload flow can't set), + // using real generated ids. + it('orders maps with tied sort keys deterministically by id', async () => { + const { pool, mapsRepo } = await getServerContext(); + await pool.query('TRUNCATE maps, difficulties, favorites CASCADE'); + + // Every map shares one submission_date, so the default `submission_date DESC` order can't + // distinguish them; only a tiebreaker can. The ids are random, so insertion/scan order won't + // accidentally match id order - a stable total order has to come from the tiebreaker. + const TIED_DATE = '2022-01-01T00:00:00.000Z'; + const COUNT = 10; + const ids: string[] = []; + for (let i = 0; i < COUNT; i++) { + const id = await generateId( + IdDomain.MAPS, + async (candidate) => + ((await pool.query('SELECT 1 FROM maps WHERE id = $1', [candidate])).rowCount ?? 0) > 0 + ); + if (id == null) { + throw new Error('Could not generate a unique map id'); + } + ids.push(id); + await pool.query( + `INSERT INTO maps + (id, visibility, validity, submission_date, title, artist, uploader, download_count, complexity) + VALUES ($1, $2, 'valid', $3, $4, 'Artist', 'tester', 0, 1)`, + [id, MapVisibility.PUBLIC, TIED_DATE, `Title ${i}`] + ); + } + + const paged: string[] = []; + for (let offset = 0; offset < COUNT; offset += 4) { + const { maps } = await _unwrap(mapsRepo.searchMaps({ query: '', offset, limit: 4 })); + paged.push(...maps.map((m) => m.id)); + } + + // Every id is seen exactly once... + expect(new Set(paged).size).toBe(COUNT); + // ...and tied rows come back in the deterministic, id-ordered sequence the tiebreaker produces. + expect(paged).toEqual([...ids].sort()); + }); +}); diff --git a/src/services/maps/tests/maps_repo_filters.test.ts b/src/services/maps/tests/maps_repo_filters.test.ts index 62b74f5..b92b891 100644 --- a/src/services/maps/tests/maps_repo_filters.test.ts +++ b/src/services/maps/tests/maps_repo_filters.test.ts @@ -211,6 +211,32 @@ describe('maps repo search filters', () => { expect(result.value.totalCount).toBe(3); }); + it('treats an intra-word hyphen as a space so "spider-man" matches "Spider Man"', async () => { + // A hyphen between word characters isn't a search operator; searchMaps collapses it to a space + // before websearch_to_tsquery. Without that, "spider-man" compiles to a phrase with a + // 'spider-man' lexeme the spaced title doesn't have, and the map would be missed. + await insertMap({ id: '800', title: 'Spider Man', artist: 'Webhead' }); + await insertMap({ id: '801', title: 'Spider-Man', artist: 'Webhead' }); + const { mapsRepo } = await getServerContext(); + const result = await mapsRepo.searchMaps({ query: 'spider-man', offset: 0, limit: 50 }); + if (!result.success) { + throw new Error('searchMaps failed'); + } + expect(result.value.maps.map((m) => m.id).sort()).toEqual(['800', '801']); + }); + + it('keeps a whitespace-padded dash as a websearch negation', async () => { + // " - rose" excludes maps mentioning "rose"; only the dash *inside* a word is normalised away. + await insertMap({ id: '810', title: 'Kiss From a Rose', artist: 'Seal' }); + await insertMap({ id: '811', title: 'Kiss the Sky', artist: 'Seal' }); + const { mapsRepo } = await getServerContext(); + const result = await mapsRepo.searchMaps({ query: 'kiss - rose', offset: 0, limit: 50 }); + if (!result.success) { + throw new Error('searchMaps failed'); + } + expect(result.value.maps.map((m) => m.id)).toEqual(['811']); + }); + describe('LIKE-wildcard escaping', () => { it('treats % in a contains value literally', async () => { await insertMap({ id: '300', description: '100% complete' }); diff --git a/src/services/search/postgres.ts b/src/services/search/postgres.ts index 875aae2..e5c0979 100644 --- a/src/services/search/postgres.ts +++ b/src/services/search/postgres.ts @@ -26,7 +26,10 @@ export class PostgresIndex implements SearchIndex { async search(query: string, options?: SearchOptions): Promise { const pool = await getDbPool(); - // TODO: ensure correctness of pagination by using "WHERE >" instead of offset + limit + // Every ORDER BY below ends with `id` so tied sort keys (equal ranks, dates, etc.) get a stable + // total order, and offset pagination can't drop or duplicate rows across page boundaries. + // TODO: switch to keyset ("WHERE > last") pagination so a concurrent insert can't shift the + // offset window either. const offset = options?.offset ?? 0; const limit = options?.limit ?? 20; const filter = options?.filter; @@ -59,10 +62,10 @@ export class PostgresIndex implements SearchIndex { .select('maps', conditions, { columns: ['id'], lateral: sortLateral, - order: sortOrder ?? { - by: 'submission_date', - direction: 'DESC', - }, + order: [ + sortOrder ?? { by: 'submission_date', direction: 'DESC' }, + { by: 'id', direction: 'ASC' }, + ], limit, offset, }) @@ -74,14 +77,25 @@ export class PostgresIndex implements SearchIndex { if (query.trim() === '') { [results, totalCount] = await queryMostRecent(); } else { + // A hyphen between two alphanumeric characters (e.g. "spider-man") is part of a word, not a + // search operator, so collapse it to a space: the parts are then ANDed and a search matches + // "Spider-Man" and "Spider Man" alike. A whitespace-padded " - " is left intact as a + // websearch negation operator. + const normalizedQuery = query.replace(/(?<=[\p{L}\p{N}])-(?=[\p{L}\p{N}])/gu, ' '); const [{ tsquery }] = await db.sql< db.Parameter, [{ tsquery: string }] - >`select websearch_to_tsquery('english', ${db.param(query)})::text as tsquery`.run(pool); + >`select websearch_to_tsquery('english', ${db.param(normalizedQuery)})::text as tsquery`.run( + pool + ); if (tsquery.trim() === '') { [results, totalCount] = await queryMostRecent(); } else { - const tsqueryPartial = db.sql`(${db.param(tsquery)} || ':*')::tsquery`; + // Make the trailing lexeme a prefix match (search-as-you-type) - but only when the tsquery + // ends in a plain lexeme. A websearch result ending in ')' (a phrase or negation group) + // would become invalid tsquery syntax with a bare ':*' appended. + const prefixTsquery = tsquery.endsWith("'") ? `${tsquery}:*` : tsquery; + const tsqueryPartial = db.sql`${db.param(prefixTsquery)}::tsquery`; const lowerQuery = query.toLowerCase(); const exactMatch = db.sql`( @@ -92,9 +106,14 @@ export class PostgresIndex implements SearchIndex { const ftsMatch = db.sql`${'fts'} @@ ${tsqueryPartial}`; const rank = db.sql`ts_rank_cd(${'fts'}, ${tsqueryPartial})`; + // Filter on `ftsMatch` alone so the query can use the `fts` GIN index. `exactMatch` only + // boosts exact title/artist/author hits to the top of the FTS results in the ordering + // below, where it is evaluated over the matched rows rather than forcing a full scan. (A + // whitespace-padded "Artist - Title" name is a websearch negation and so isn't matched by + // FTS; we intentionally don't special-case it.) const conditions = db.conditions.and( { visibility: MapVisibility.PUBLIC }, - db.sql`(${exactMatch} OR ${ftsMatch})`, + ftsMatch, ...(filter ? [compileFilter(filter)] : []) ); [results, totalCount] = await Promise.all([ @@ -102,16 +121,15 @@ export class PostgresIndex implements SearchIndex { .select('maps', conditions, { columns: ['id'], lateral: sortLateral, - order: sortOrder ?? [ - { by: exactMatch, direction: 'DESC' }, - { by: rank, direction: 'DESC' }, - ], + order: sortOrder + ? [sortOrder, { by: 'id', direction: 'ASC' }] + : [ + { by: exactMatch, direction: 'DESC' }, + { by: rank, direction: 'DESC' }, + { by: 'id', direction: 'ASC' }, + ], limit, offset, - extras: { - rank, - exactMatch, - }, }) .run(pool), db.count('maps', conditions).run(pool), diff --git a/src/services/tests/helpers.test.ts b/src/services/tests/helpers.test.ts new file mode 100644 index 0000000..e940636 --- /dev/null +++ b/src/services/tests/helpers.test.ts @@ -0,0 +1,33 @@ +import { NextRequest } from 'next/server'; +import { getOffsetLimit } from 'services/helpers'; + +// getOffsetLimit reads `offset`/`limit` from the request query. It lives in a module that pulls in +// env-dependent logging at import, so this runs as an integration test (env is loaded) rather than +// a pure unit test. +const req = (qs: string) => new NextRequest(`http://localhost/api/maps${qs}`); + +describe('getOffsetLimit', () => { + it('defaults to offset 0 and limit 20 when the params are absent', () => { + expect(getOffsetLimit(req(''))).toEqual({ offset: 0, limit: 20 }); + }); + + it('parses valid offset and limit', () => { + expect(getOffsetLimit(req('?offset=40&limit=50'))).toEqual({ offset: 40, limit: 50 }); + }); + + // Invalid/out-of-range values are clamped rather than passed into the SQL LIMIT/OFFSET: a negative + // limit or offset would make Postgres throw, a tiny limit would let a client page one map at a + // time, and an unbounded limit would let one request pull the whole table. + it('clamps a small or negative limit up to the minimum page size', () => { + expect(getOffsetLimit(req('?limit=-5')).limit).toBe(20); + expect(getOffsetLimit(req('?limit=1')).limit).toBe(20); + }); + + it('clamps a negative offset up to 0', () => { + expect(getOffsetLimit(req('?offset=-5')).offset).toBeGreaterThanOrEqual(0); + }); + + it('caps an excessively large limit', () => { + expect(getOffsetLimit(req('?limit=100000')).limit).toBeLessThanOrEqual(100); + }); +}); diff --git a/src/services/zapatos/custom/index.d.ts b/src/services/zapatos/custom/index.d.ts index 88d8fc4..e098079 100644 --- a/src/services/zapatos/custom/index.d.ts +++ b/src/services/zapatos/custom/index.d.ts @@ -8,4 +8,4 @@ Released under the MIT licence: see LICENCE file */ // this empty declaration appears to fix relative imports in other custom type files -declare module 'zapatos/custom' { } +declare module 'zapatos/custom' {} diff --git a/src/ui/maps/submit/submit_map_presenter.ts b/src/ui/maps/submit/submit_map_presenter.ts index ff95c06..a2faac4 100644 --- a/src/ui/maps/submit/submit_map_presenter.ts +++ b/src/ui/maps/submit/submit_map_presenter.ts @@ -1,5 +1,4 @@ import { Api } from 'app/api/api'; -import { reportUploadComplete } from 'app/api/maps/submit/complete/actions'; import { action, computed, observable, runInAction } from 'mobx'; import { AppRouterInstance } from 'next/dist/shared/lib/app-router-context.shared-runtime'; import { getLog } from 'services/logging/client_logger'; @@ -84,7 +83,10 @@ export class ThrottledMapUploader { if (xhr.status !== 200) { throw new Error(`(${xhr.status}): ${xhr.responseText}`); } - const processMapResp = await reportUploadComplete(submitMapResp.id, !!reuploadMapId); + const processMapResp = await this.api.completeMapUpload({ + id: submitMapResp.id, + isReupload: !!reuploadMapId, + }); runInAction(() => { if (!processMapResp.success) { upload.state = 'error'; diff --git a/tools/load_schema.ts b/tools/load_schema.ts index ed9add3..640bd04 100644 --- a/tools/load_schema.ts +++ b/tools/load_schema.ts @@ -1,50 +1,38 @@ import * as fs from 'fs/promises'; import * as path from 'path'; import pg from 'pg'; -import { parse as parseToml } from 'smol-toml'; // Shared between the integration test global setup and the local dev bootstrap (tools/dev.sh): -// resolves the ordered schema files from `[db.migrations] schema_paths` in supabase/config.toml, -// the same list Supabase itself applies, then loads them into the target Postgres. PGlite can't -// run the auth-coupled / extension files Supabase real-deploys, so those are dropped. +// applies the real `supabase/migrations` in order to the target Postgres, so the test/dev DB matches +// exactly what Supabase deploys to production (rather than a hand-maintained declarative copy that +// can drift). +// +// PGlite isn't Supabase, so two migration concerns are handled here: the GRANTs target Supabase +// roles that don't exist (created in the prelude), and a couple of `create extension` statements +// reference extensions PGlite doesn't ship (stripped). The auth-coupled functions still load because +// the migration sets `check_function_bodies = off`, and RLS is a no-op under PGlite's superuser +// connection. const SUPABASE_DIR = path.resolve(process.cwd(), 'supabase'); -// functions.sql references the `auth` schema / `auth.uid()` / `crypt`; misc.sql creates extensions -// (hypopg, index_advisor) PGlite doesn't ship. The fake Supabase client reimplements what we need. -const SKIPPED_FILES = ['functions.sql', 'misc.sql']; +const MIGRATIONS_DIR = path.join(SUPABASE_DIR, 'migrations'); -async function resolveSchemaFiles(): Promise { - const config = parseToml(await fs.readFile(path.join(SUPABASE_DIR, 'config.toml'), 'utf8')) as { - db?: { migrations?: { schema_paths?: string[] } }; - }; - const patterns = config.db?.migrations?.schema_paths; - if (!patterns?.length) { - throw new Error('No [db.migrations] schema_paths found in supabase/config.toml'); - } +// Create the roles the migration GRANTs target. Idempotent so a `skipIfLoaded` re-run (dev) against +// an already-bootstrapped DB doesn't error on existing roles. +const MIGRATION_PRELUDE = ` +DO $$ BEGIN CREATE ROLE anon; EXCEPTION WHEN duplicate_object THEN NULL; END $$; +DO $$ BEGIN CREATE ROLE authenticated; EXCEPTION WHEN duplicate_object THEN NULL; END $$; +DO $$ BEGIN CREATE ROLE service_role; EXCEPTION WHEN duplicate_object THEN NULL; END $$; +`; - const ordered: string[] = []; - const seen = new Set(); - const add = (relPath: string) => { - if (!seen.has(relPath)) { - seen.add(relPath); - ordered.push(relPath); - } - }; - for (const raw of patterns) { - const pattern = raw.replace(/^\.\//, ''); - if (!pattern.includes('*')) { - add(pattern); - continue; - } - // Expand a glob (e.g. "schemas/*.sql") to its sorted matches, matching Supabase's - // dedupe-by-first-occurrence behaviour so already-listed files keep their explicit position. - const dir = path.dirname(pattern); - const matches = (await fs.readdir(path.join(SUPABASE_DIR, dir))) - .filter((f) => f.endsWith('.sql')) - .sort(); - for (const f of matches) add(path.join(dir, f)); - } - return ordered.filter((f) => !SKIPPED_FILES.includes(path.basename(f))); +// PGlite doesn't ship hypopg / index_advisor (advisory-only, unused at runtime), and `create +// extension` would error, so drop those statements. +function stripUnsupportedStatements(sql: string): string { + return sql.replace(/create extension[^;]*;/gi, ''); +} + +async function resolveMigrationFiles(): Promise { + // Supabase applies migrations in lexicographic (timestamp-prefixed) filename order. + return (await fs.readdir(MIGRATIONS_DIR)).filter((f) => f.endsWith('.sql')).sort(); } // PGlite's TCP socket can be accepting connections before the Postgres protocol is fully ready, @@ -76,8 +64,11 @@ export async function loadSchema( if (opts.skipIfLoaded && (await isSchemaLoaded(pool))) { return { loaded: false }; } - for (const file of await resolveSchemaFiles()) { - const sql = await fs.readFile(path.join(SUPABASE_DIR, file), 'utf8'); + await pool.query(MIGRATION_PRELUDE); + for (const file of await resolveMigrationFiles()) { + const sql = stripUnsupportedStatements( + await fs.readFile(path.join(MIGRATIONS_DIR, file), 'utf8') + ); await pool.query(sql); } if (opts.includeSeed) {