From a26352c859262b8c2a3a6e7b6b8443a9fa0fd308 Mon Sep 17 00:00:00 2001 From: peetzweg Date: Sun, 12 Jul 2026 21:31:14 +0200 Subject: [PATCH] Add indexes for hot query paths; bound the recent-lookups scan --- drizzle/0006_yummy_rattler.sql | 11 + drizzle/meta/0006_snapshot.json | 702 ++++++++++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + src/lib/commit-history.ts | 26 +- src/lib/db/schema.ts | 162 +++++--- 5 files changed, 857 insertions(+), 51 deletions(-) create mode 100644 drizzle/0006_yummy_rattler.sql create mode 100644 drizzle/meta/0006_snapshot.json diff --git a/drizzle/0006_yummy_rattler.sql b/drizzle/0006_yummy_rattler.sql new file mode 100644 index 0000000..543a058 --- /dev/null +++ b/drizzle/0006_yummy_rattler.sql @@ -0,0 +1,11 @@ +CREATE INDEX "entities_rank_commits_idx" ON "entities" USING btree ("total_commits" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_prs_idx" ON "entities" USING btree ("total_pull_requests" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_issues_idx" ON "entities" USING btree ("total_issues" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_reviews_idx" ON "entities" USING btree ("total_reviews" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_repos_idx" ON "entities" USING btree ("total_repos" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_private_idx" ON "entities" USING btree ("total_restricted" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_followers_idx" ON "entities" USING btree ("followers" DESC NULLS LAST,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_rank_total_idx" ON "entities" USING btree (("total_commits" + coalesce("total_issues", 0) + coalesce("total_pull_requests", 0) + coalesce("total_reviews", 0) + coalesce("total_repos", 0) + "total_restricted") desc,"id") WHERE "entities"."kind" = 'user' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "entities_org_rank_commits_idx" ON "entities" USING btree ("total_commits" DESC NULLS FIRST,"id") WHERE "entities"."kind" = 'org' and "entities"."suspended_at" is null and "entities"."built_at" is not null;--> statement-breakpoint +CREATE INDEX "lookups_searched_at_idx" ON "lookups" USING btree ("searched_at" DESC NULLS FIRST);--> statement-breakpoint +CREATE INDEX "org_members_org_commits_idx" ON "org_members" USING btree ("org_id","commits" DESC NULLS FIRST,"member_id") WHERE "org_members"."last_fetched" is not null; \ No newline at end of file diff --git a/drizzle/meta/0006_snapshot.json b/drizzle/meta/0006_snapshot.json new file mode 100644 index 0000000..74926be --- /dev/null +++ b/drizzle/meta/0006_snapshot.json @@ -0,0 +1,702 @@ +{ + "id": "d77af244-d306-4288-8d6d-76e1be4ac681", + "prevId": "36846044-13e2-4e4e-aa2a-3bf3cdc91e3c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.entities": { + "name": "entities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "login": { + "name": "login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "total_commits": { + "name": "total_commits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_restricted": { + "name": "total_restricted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "total_issues": { + "name": "total_issues", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_pull_requests": { + "name": "total_pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_reviews": { + "name": "total_reviews", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "total_repos": { + "name": "total_repos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "followers": { + "name": "followers", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "following": { + "name": "following", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "public_repos": { + "name": "public_repos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "company": { + "name": "company", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "location": { + "name": "location", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "twitter_username": { + "name": "twitter_username", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_verified": { + "name": "is_verified", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "github_node_id": { + "name": "github_node_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "member_count": { + "name": "member_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_fetched": { + "name": "last_fetched", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "built_at": { + "name": "built_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "suspended_reason": { + "name": "suspended_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "entities_rank_commits_idx": { + "name": "entities_rank_commits_idx", + "columns": [ + { + "expression": "total_commits", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_prs_idx": { + "name": "entities_rank_prs_idx", + "columns": [ + { + "expression": "total_pull_requests", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_issues_idx": { + "name": "entities_rank_issues_idx", + "columns": [ + { + "expression": "total_issues", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_reviews_idx": { + "name": "entities_rank_reviews_idx", + "columns": [ + { + "expression": "total_reviews", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_repos_idx": { + "name": "entities_rank_repos_idx", + "columns": [ + { + "expression": "total_repos", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_private_idx": { + "name": "entities_rank_private_idx", + "columns": [ + { + "expression": "total_restricted", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_followers_idx": { + "name": "entities_rank_followers_idx", + "columns": [ + { + "expression": "followers", + "isExpression": false, + "asc": false, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_rank_total_idx": { + "name": "entities_rank_total_idx", + "columns": [ + { + "expression": "(\"total_commits\" + coalesce(\"total_issues\", 0) + coalesce(\"total_pull_requests\", 0) + coalesce(\"total_reviews\", 0) + coalesce(\"total_repos\", 0) + \"total_restricted\") desc", + "asc": true, + "isExpression": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'user' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "entities_org_rank_commits_idx": { + "name": "entities_org_rank_commits_idx", + "columns": [ + { + "expression": "total_commits", + "isExpression": false, + "asc": false, + "nulls": "first" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"entities\".\"kind\" = 'org' and \"entities\".\"suspended_at\" is null and \"entities\".\"built_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.lookups": { + "name": "lookups", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "searched_at": { + "name": "searched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "lookups_searched_at_idx": { + "name": "lookups_searched_at_idx", + "columns": [ + { + "expression": "searched_at", + "isExpression": false, + "asc": false, + "nulls": "first" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "lookups_entity_id_entities_id_fk": { + "name": "lookups_entity_id_entities_id_fk", + "tableFrom": "lookups", + "tableTo": "entities", + "columnsFrom": [ + "entity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.monthly_commits": { + "name": "monthly_commits", + "schema": "", + "columns": { + "entity_id": { + "name": "entity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "month": { + "name": "month", + "type": "date", + "primaryKey": false, + "notNull": true + }, + "commits": { + "name": "commits", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "restricted": { + "name": "restricted", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "issues": { + "name": "issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "pull_requests": { + "name": "pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reviews": { + "name": "reviews", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "repos": { + "name": "repos", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": { + "monthly_commits_entity_id_entities_id_fk": { + "name": "monthly_commits_entity_id_entities_id_fk", + "tableFrom": "monthly_commits", + "tableTo": "entities", + "columnsFrom": [ + "entity_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "monthly_commits_entity_id_month_pk": { + "name": "monthly_commits_entity_id_month_pk", + "columns": [ + "entity_id", + "month" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_members": { + "name": "org_members", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'public_member'" + }, + "commits": { + "name": "commits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "pull_requests": { + "name": "pull_requests", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "reviews": { + "name": "reviews", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "issues": { + "name": "issues", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_fetched": { + "name": "last_fetched", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_members_member_idx": { + "name": "org_members_member_idx", + "columns": [ + { + "expression": "member_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_members_org_commits_idx": { + "name": "org_members_org_commits_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "commits", + "isExpression": false, + "asc": false, + "nulls": "first" + }, + { + "expression": "member_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"org_members\".\"last_fetched\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "org_members_org_id_entities_id_fk": { + "name": "org_members_org_id_entities_id_fk", + "tableFrom": "org_members", + "tableTo": "entities", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "org_members_member_id_entities_id_fk": { + "name": "org_members_member_id_entities_id_fk", + "tableFrom": "org_members", + "tableTo": "entities", + "columnsFrom": [ + "member_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "org_members_org_id_member_id_pk": { + "name": "org_members_org_id_member_id_pk", + "columns": [ + "org_id", + "member_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9f8e84d..dc1ddaf 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -43,6 +43,13 @@ "when": 1783493299385, "tag": "0005_clammy_bloodaxe", "breakpoints": true + }, + { + "idx": 6, + "version": "7", + "when": 1783884604657, + "tag": "0006_yummy_rattler", + "breakpoints": true } ] } \ No newline at end of file diff --git a/src/lib/commit-history.ts b/src/lib/commit-history.ts index 2dda4b4..92c219b 100644 --- a/src/lib/commit-history.ts +++ b/src/lib/commit-history.ts @@ -217,8 +217,26 @@ async function queryLeaderboard( return scoped.orderBy(order, tiebreak).limit(limit).offset(offset); } +/** + * How many raw lookup rows queryRecent inspects. The strip needs RECENT_LIMIT *distinct* entities, + * so this needs headroom for repeat searches of the same login (and the odd suspended/repo row) — + * but it must stay a constant: `lookups` is append-only and unbounded, and aggregating the whole + * table made every strip poll scan the full search history. + */ +const RECENT_SCAN_WINDOW = 400; + async function queryRecent(limit: number): Promise { if (!db) return []; + // Newest slice first (walks lookups_searched_at_idx, stops after the window), dedupe after. + const recent = db + .select({ + entityId: lookups.entityId, + searchedAt: lookups.searchedAt, + }) + .from(lookups) + .orderBy(desc(lookups.searchedAt)) + .limit(RECENT_SCAN_WINDOW) + .as("recent"); const rows = await db .select({ login: entities.login, @@ -226,10 +244,10 @@ async function queryRecent(limit: number): Promise { avatarUrl: entities.avatarUrl, kind: entities.kind, isVerified: entities.isVerified, - last: sql`max(${lookups.searchedAt})`, + last: sql`max(${recent.searchedAt})`, }) - .from(lookups) - .innerJoin(entities, eq(entities.id, lookups.entityId)) + .from(recent) + .innerJoin(entities, eq(entities.id, recent.entityId)) // Users and orgs both belong in the strip (each links to /$user, which resolves either); // repos are the only other kind and don't get a page here, so they're excluded. .where( @@ -239,7 +257,7 @@ async function queryRecent(limit: number): Promise { ), ) .groupBy(entities.id) - .orderBy(desc(sql`max(${lookups.searchedAt})`)) + .orderBy(desc(sql`max(${recent.searchedAt})`)) .limit(limit); return rows.map((r) => ({ login: r.login, diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index ed8e82b..abf3d5a 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -1,3 +1,4 @@ +import { sql } from "drizzle-orm"; import { boolean, date, @@ -14,44 +15,94 @@ import { * A tracked entity — currently a GitHub user; `kind` leaves room for orgs/repos later. * `id` is namespaced so the same login can't collide across kinds: e.g. "user:torvalds". */ -export const entities = pgTable("entities", { - id: text("id").primaryKey(), - kind: text("kind").notNull(), // 'user' | 'org' | 'repo' - login: text("login").notNull(), - name: text("name"), - avatarUrl: text("avatar_url"), - htmlUrl: text("html_url"), - createdAt: timestamp("created_at", { withTimezone: true }), // defines the window start - totalCommits: integer("total_commits").notNull().default(0), // public commits - totalRestricted: integer("total_restricted").notNull().default(0), // private contributions - // Additional public contribution-type lifetime totals. Nullable so null = "not yet backfilled - // with the new types" (drives the backfill script's default mode), distinct from a real 0 — - // same rationale as the profile-metadata columns below. - totalIssues: integer("total_issues"), // public issues opened - totalPullRequests: integer("total_pull_requests"), // public PRs opened - totalReviews: integer("total_reviews"), // public PR reviews - totalRepos: integer("total_repos"), // public repositories created - // Profile metadata — mutable, refreshed on the trailing-refresh path (see cache.ts). All - // nullable so an unknown value (older row, fetch failure) stays distinguishable from a real 0. - followers: integer("followers"), - following: integer("following"), - publicRepos: integer("public_repos"), - bio: text("bio"), - company: text("company"), - location: text("location"), - websiteUrl: text("website_url"), - twitterUsername: text("twitter_username"), - // Org-only metadata (null on user rows — same "unknown vs not applicable" convention). - isVerified: boolean("is_verified"), // org domain-verified badge - githubNodeId: text("github_node_id"), // GraphQL node id — keys contributionsCollection(organizationID:) without a profile round-trip - memberCount: integer("member_count"), // membersWithRole.totalCount (includes private members) — display only - lastFetched: timestamp("last_fetched", { withTimezone: true }), // staleness / trailing refresh; also "profile last updated" - builtAt: timestamp("built_at", { withTimezone: true }), // initial build completed; null = months still being fetched incrementally - // Moderation: null = active. When set, the entity is hidden from the leaderboard and - // "recently looked up" (still directly viewable, with an under-review notice) until cleared. - suspendedAt: timestamp("suspended_at", { withTimezone: true }), - suspendedReason: text("suspended_reason"), // internal note — never shown publicly -}); +export const entities = pgTable( + "entities", + { + id: text("id").primaryKey(), + kind: text("kind").notNull(), // 'user' | 'org' | 'repo' + login: text("login").notNull(), + name: text("name"), + avatarUrl: text("avatar_url"), + htmlUrl: text("html_url"), + createdAt: timestamp("created_at", { withTimezone: true }), // defines the window start + totalCommits: integer("total_commits").notNull().default(0), // public commits + totalRestricted: integer("total_restricted").notNull().default(0), // private contributions + // Additional public contribution-type lifetime totals. Nullable so null = "not yet backfilled + // with the new types" (drives the backfill script's default mode), distinct from a real 0 — + // same rationale as the profile-metadata columns below. + totalIssues: integer("total_issues"), // public issues opened + totalPullRequests: integer("total_pull_requests"), // public PRs opened + totalReviews: integer("total_reviews"), // public PR reviews + totalRepos: integer("total_repos"), // public repositories created + // Profile metadata — mutable, refreshed on the trailing-refresh path (see cache.ts). All + // nullable so an unknown value (older row, fetch failure) stays distinguishable from a real 0. + followers: integer("followers"), + following: integer("following"), + publicRepos: integer("public_repos"), + bio: text("bio"), + company: text("company"), + location: text("location"), + websiteUrl: text("website_url"), + twitterUsername: text("twitter_username"), + // Org-only metadata (null on user rows — same "unknown vs not applicable" convention). + isVerified: boolean("is_verified"), // org domain-verified badge + githubNodeId: text("github_node_id"), // GraphQL node id — keys contributionsCollection(organizationID:) without a profile round-trip + memberCount: integer("member_count"), // membersWithRole.totalCount (includes private members) — display only + lastFetched: timestamp("last_fetched", { withTimezone: true }), // staleness / trailing refresh; also "profile last updated" + builtAt: timestamp("built_at", { withTimezone: true }), // initial build completed; null = months still being fetched incrementally + // Moderation: null = active. When set, the entity is hidden from the leaderboard and + // "recently looked up" (still directly viewable, with an under-review notice) until cleared. + suspendedAt: timestamp("suspended_at", { withTimezone: true }), + suspendedReason: text("suspended_reason"), // internal note — never shown publicly + }, + (t) => { + // The user leaderboard (queryLeaderboard) and per-metric rank counts (metricRankFor) both + // scope to this population, then sort/range-scan a single metric column. One partial index + // per ranked metric turns each of those from a full-table scan into a short index scan. + // Column order (` DESC NULLS LAST, id ASC`) mirrors the board's ORDER BY exactly — + // including null placement, which Postgres pathkey matching is strict about. + const activeUsers = sql`${t.kind} = 'user' and ${t.suspendedAt} is null and ${t.builtAt} is not null`; + return [ + index("entities_rank_commits_idx") + .on(t.totalCommits.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_prs_idx") + .on(t.totalPullRequests.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_issues_idx") + .on(t.totalIssues.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_reviews_idx") + .on(t.totalReviews.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_repos_idx") + .on(t.totalRepos.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_private_idx") + .on(t.totalRestricted.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + index("entities_rank_followers_idx") + .on(t.followers.desc().nullsLast(), t.id.asc()) + .where(activeUsers), + // The `total` board/rank orders by this summed expression (plain DESC — it can never be + // null, every nullable term is coalesced). Must textually match commit-history.ts. + index("entities_rank_total_idx") + .on( + sql`(${t.totalCommits} + coalesce(${t.totalIssues}, 0) + coalesce(${t.totalPullRequests}, 0) + coalesce(${t.totalReviews}, 0) + coalesce(${t.totalRepos}, 0) + ${t.totalRestricted}) desc`, + t.id.asc(), + ) + .where(activeUsers), + // Org leaderboard (queryOrgLeaderboard) — same shape, org population. nullsFirst because + // that query orders by plain `desc(...)` (Postgres DESC default), and index usability + // depends on the null placement matching the ORDER BY even for non-null columns. + index("entities_org_rank_commits_idx") + .on(t.totalCommits.desc().nullsFirst(), t.id.asc()) + .where( + sql`${t.kind} = 'org' and ${t.suspendedAt} is null and ${t.builtAt} is not null`, + ), + ]; + }, +); /** Per-month commit counts. Past months are immutable; only the current month changes. */ export const monthlyCommits = pgTable( @@ -101,16 +152,33 @@ export const orgMembers = pgTable( primaryKey({ columns: [t.orgId, t.memberId] }), // Reverse lookup: "which orgs did this user contribute to" on personal profiles. index("org_members_member_idx").on(t.memberId), + // Within-org member leaderboard (queryOrgMembers): fetched members of one org ordered by + // commits. Matches that query's ORDER BY so big orgs skip the per-request sort; partial on + // the same `last_fetched is not null` predicate so pending rows never bloat it. + // nullsFirst: matches that query's plain `desc(...)` ordering (see entities' org index note). + index("org_members_org_commits_idx") + .on(t.orgId.asc(), t.commits.desc().nullsFirst(), t.memberId.asc()) + .where(sql`${t.lastFetched} is not null`), ], ); /** Every search — powers "recent lookups" and the all-time leaderboard. */ -export const lookups = pgTable("lookups", { - id: serial("id").primaryKey(), - entityId: text("entity_id") - .notNull() - .references(() => entities.id), - searchedAt: timestamp("searched_at", { withTimezone: true }) - .notNull() - .defaultNow(), -}); +export const lookups = pgTable( + "lookups", + { + id: serial("id").primaryKey(), + entityId: text("entity_id") + .notNull() + .references(() => entities.id), + searchedAt: timestamp("searched_at", { withTimezone: true }) + .notNull() + .defaultNow(), + }, + (t) => [ + // The "recently looked up" strip (queryRecent) reads the newest slice of this append-only + // table on every poll; this keeps that a bounded index scan no matter how large the full + // search history grows. + // nullsFirst: matches queryRecent's plain `desc(...)` ordering (see entities' org index note). + index("lookups_searched_at_idx").on(t.searchedAt.desc().nullsFirst()), + ], +);