fix: use more performant GitHub APIs to avoid rate limiting

README.md

@@ -25,7 +25,7 @@ Easiest path: run `gh auth login` (via [GitHub CLI](https://cli.github.com)) and
 
 Prefer a custom token? Create a [fine-grained token](https://github.com/settings/personal-access-tokens/new) instead and set these repository permissions to read:
 
-- **Contents:** commit history and the `dependabot.yml` config
+- **Contents:** the `dependabot.yml` config
 - **Pull requests:** the Dependabot PR backlog
 - **Administration:** branch-protection and ruleset coverage
 - **Dependabot alerts:** the CVE numbers
@@ -56,12 +56,11 @@ The report covers:
 
 Everything comes from `api.github.com` over a fixed 90-day window. For the org and its repos (archived repos and forks are skipped), it reads:
 
-- The repo list, visibility, and language breakdown
+- The repo list, visibility, and primary language metadata
 - Dependabot PRs in the window, including state, timing, reviews, and CI status
 - Open Dependabot security alerts (needs the `security_events` scope)
 - Each repo's `.github/dependabot.yml`
 - Branch-protection and ruleset settings on the default branch
-- Commit authors in the window, to count active humans
 
 All calls are read only. It writes nothing back to GitHub and pulls no file contents beyond the Dependabot config.
 

package.json

@@ -84,7 +84,6 @@
   "dependencies": {
     "@clack/prompts": "^1.4.0",
     "@js-temporal/polyfill": "^0.5.1",
-    "@octokit/graphql": "^9.0.3",
     "@octokit/plugin-retry": "^8.1.0",
     "@octokit/plugin-throttling": "^11.0.3",
     "@octokit/rest": "^22.0.1",
@@ -95,6 +94,7 @@
     "lucide-react": "^1.16.0",
     "neverthrow": "^8.2.0",
     "open": "^11.0.0",
+    "p-map": "^7.0.4",
     "pino": "^10.3.1",
     "pino-pretty": "^13.1.3",
     "posthog-js": "^1.370.0",

src/cli.test.ts

@@ -2,6 +2,17 @@ import { expect, test } from 'bun:test';
 import { main } from './cli.ts';
 import { createFakeContext } from './testHelpers/index.ts';
 
+function repoBatchResponse(nodes: unknown[] = []): Record<string, unknown> {
+  return { nodes };
+}
+
+const npmConfigBlob = {
+  __typename: 'Repository',
+  yml: { text: 'updates:\n  - package-ecosystem: "npm"\n' },
+  yaml: null,
+  defaultBranchRef: null,
+};
+
 test('prints usage and exits 0 when --help is passed', async () => {
   const { ctx, io } = createFakeContext();
   const result = await main(ctx, ['--help']);
@@ -56,6 +67,7 @@ test('writes a report when the GitHub calls succeed', async () => {
   githubClient.onPaginate('GET /orgs/{org}/repos', {}).resolves([
     {
       name: 'widgets',
+      node_id: 'R_kgDOwidgets',
       owner: { login: 'acme' },
       private: true,
       visibility: 'private',
@@ -65,17 +77,8 @@ test('writes a report when the GitHub calls succeed', async () => {
       pushed_at: '2026-04-01T00:00:00Z',
     },
   ]);
-  githubClient.onRequest('GET /repos/{owner}/{repo}/languages', {}).resolves({ TypeScript: 1000 });
-  githubClient.onRequest('GET /repos/{owner}/{repo}/contents/{path}', {}).resolves({
-    content: Buffer.from('updates:\n  - package-ecosystem: "npm"\n', 'utf8').toString('base64'),
-    encoding: 'base64',
-  });
-  githubClient.onPaginate('GET /repos/{owner}/{repo}/dependabot/alerts', {}).resolves([]);
-  githubClient
-    .onRequest('GET /repos/{owner}/{repo}/branches/{branch}/protection', {})
-    .fails({ kind: 'not-found', message: 'no protection' });
-  githubClient.onRequest('GET /repos/{owner}/{repo}/rules/branches/{branch}', {}).resolves([]);
-  githubClient.onPaginate('GET /repos/{owner}/{repo}/commits', {}).resolves([]);
+  githubClient.onGraphql('RepoMetadataBatch').resolves(repoBatchResponse([npmConfigBlob]));
+  githubClient.onPaginate('GET /orgs/{org}/dependabot/alerts', {}).resolves([]);
   githubClient.onGraphql('DependabotPrs').resolves({
     search: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] },
   });
@@ -127,6 +130,7 @@ test('excludes forked repos from the crawl', async () => {
   githubClient.onPaginate('GET /orgs/{org}/repos', {}).resolves([
     {
       name: 'widgets',
+      node_id: 'R_kgDOwidgets',
       owner: { login: 'acme' },
       private: true,
       visibility: 'private',
@@ -138,6 +142,7 @@ test('excludes forked repos from the crawl', async () => {
     },
     {
       name: 'upstream-fork',
+      node_id: 'R_kgDOfork',
       owner: { login: 'acme' },
       private: false,
       visibility: 'public',
@@ -148,17 +153,8 @@ test('excludes forked repos from the crawl', async () => {
       pushed_at: '2026-04-01T00:00:00Z',
     },
   ]);
-  githubClient.onRequest('GET /repos/{owner}/{repo}/languages', {}).resolves({ TypeScript: 1000 });
-  githubClient.onRequest('GET /repos/{owner}/{repo}/contents/{path}', {}).resolves({
-    content: Buffer.from('updates:\n  - package-ecosystem: "npm"\n', 'utf8').toString('base64'),
-    encoding: 'base64',
-  });
-  githubClient.onPaginate('GET /repos/{owner}/{repo}/dependabot/alerts', {}).resolves([]);
-  githubClient
-    .onRequest('GET /repos/{owner}/{repo}/branches/{branch}/protection', {})
-    .fails({ kind: 'not-found', message: 'no protection' });
-  githubClient.onRequest('GET /repos/{owner}/{repo}/rules/branches/{branch}', {}).resolves([]);
-  githubClient.onPaginate('GET /repos/{owner}/{repo}/commits', {}).resolves([]);
+  githubClient.onGraphql('RepoMetadataBatch').resolves(repoBatchResponse([npmConfigBlob]));
+  githubClient.onPaginate('GET /orgs/{org}/dependabot/alerts', {}).resolves([]);
   githubClient.onGraphql('DependabotPrs').resolves({
     search: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] },
   });
@@ -172,7 +168,76 @@ test('excludes forked repos from the crawl', async () => {
   });
 });
 
-test('captures run_failed when listOrgRepos fails', async () => {
+test('uses the per-repo CVE endpoint for user targets', async () => {
+  const { ctx, githubClient } = createFakeContext();
+
+  // User-target fallthrough: /orgs/{org}/repos 404s, /users/{username}/repos succeeds.
+  githubClient.onPaginate('GET /orgs/{org}/repos', {}).fails({ kind: 'not-found', message: 'no org' });
+  githubClient.onPaginate('GET /users/{username}/repos', {}).resolves([
+    {
+      name: 'solo',
+      node_id: 'R_kgDOsolo',
+      owner: { login: 'blimmer' },
+      private: false,
+      visibility: 'public',
+      archived: false,
+      fork: false,
+      default_branch: 'main',
+      language: 'TypeScript',
+      pushed_at: '2026-04-01T00:00:00Z',
+    },
+  ]);
+  githubClient.onGraphql('RepoMetadataBatch').resolves(repoBatchResponse([npmConfigBlob]));
+  // Per-repo CVE endpoint — the path Task 4 keeps for user targets.
+  githubClient.onPaginate('GET /repos/{owner}/{repo}/dependabot/alerts', {}).resolves([]);
+  githubClient.onGraphql('DependabotPrs').resolves({
+    search: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] },
+  });
+
+  const result = await main(ctx, ['blimmer']);
+  expect(result.kind).toBe('completed');
+
+  const paginateRoutes = githubClient.callsTo('paginate').flatMap((c) => (c.kind === 'paginate' ? [c.route] : []));
+  expect(paginateRoutes).toContain('GET /repos/{owner}/{repo}/dependabot/alerts');
+  expect(paginateRoutes).not.toContain('GET /orgs/{org}/dependabot/alerts');
+});
+
+test('falls back to the per-repo CVE endpoint when the org-level call fails', async () => {
+  const { ctx, githubClient } = createFakeContext();
+
+  githubClient.onPaginate('GET /orgs/{org}/repos', {}).resolves([
+    {
+      name: 'widgets',
+      node_id: 'R_kgDOwidgets',
+      owner: { login: 'acme' },
+      private: true,
+      visibility: 'private',
+      archived: false,
+      default_branch: 'main',
+      language: 'TypeScript',
+      pushed_at: '2026-04-01T00:00:00Z',
+    },
+  ]);
+  githubClient.onGraphql('RepoMetadataBatch').resolves(repoBatchResponse([npmConfigBlob]));
+  // Org-level endpoint refuses: token can see the org but not its alerts.
+  githubClient
+    .onPaginate('GET /orgs/{org}/dependabot/alerts', {})
+    .fails({ kind: 'forbidden', message: 'no access to org alerts' });
+  // Per-repo endpoint is the fallback so each repo still gets a real status.
+  githubClient.onPaginate('GET /repos/{owner}/{repo}/dependabot/alerts', {}).resolves([]);
+  githubClient.onGraphql('DependabotPrs').resolves({
+    search: { pageInfo: { hasNextPage: false, endCursor: null }, nodes: [] },
+  });
+
+  const result = await main(ctx, ['acme']);
+  expect(result.kind).toBe('completed');
+
+  const paginateRoutes = githubClient.callsTo('paginate').flatMap((c) => (c.kind === 'paginate' ? [c.route] : []));
+  expect(paginateRoutes).toContain('GET /orgs/{org}/dependabot/alerts');
+  expect(paginateRoutes).toContain('GET /repos/{owner}/{repo}/dependabot/alerts');
+});
+
+test('captures run_failed when listTargetRepos fails', async () => {
   const { ctx, githubClient, analytics } = createFakeContext();
   githubClient.onPaginate('GET /orgs/{org}/repos', {}).fails({ kind: 'forbidden', message: 'no access' });
 
@@ -182,7 +247,7 @@ test('captures run_failed when listOrgRepos fails', async () => {
   expect(failed?.properties).toMatchObject({ error_kind: 'forbidden' });
 });
 
-test('returns failed when listOrgRepos fails non-recoverably', async () => {
+test('returns failed when listTargetRepos fails non-recoverably', async () => {
   const { ctx, prompter, githubClient } = createFakeContext();
   githubClient.onPaginate('GET /orgs/{org}/repos', {}).fails({ kind: 'forbidden', message: 'no access' });
 

src/cli.ts

@@ -1,13 +1,11 @@
 import { join } from 'node:path';
 import { parseArgs } from 'node:util';
 import { Result, ResultAsync } from 'neverthrow';
-import { getBranchProtection } from './collectors/branchProtection.ts';
-import { listActiveCommitters } from './collectors/contributors.ts';
-import { getCveAlerts } from './collectors/cve.ts';
-import { getDependabotConfig } from './collectors/dependabotConfig.ts';
+import pMap from 'p-map';
+import { getCveAlerts, getOrgCveAlerts } from './collectors/cve.ts';
 import { listDependabotPrs } from './collectors/dependabotPrs.ts';
-import { getRepoLanguages, listOrgRepos } from './collectors/repos.ts';
-import { mapWithConcurrency } from './concurrency.ts';
+import { listRepoMetadataBatched } from './collectors/repoMetadata.ts';
+import { type TargetKind, listTargetRepos } from './collectors/repos.ts';
 import type { Context } from './context.ts';
 import { getErrorMessage } from './errors.ts';
 import { formatFsError } from './FileSystem.ts';
@@ -22,11 +20,9 @@ import type {
   BranchProtectionSlice,
   CollectedData,
   CollectorWarning,
-  ContributorSlice,
   CveSlice,
   DependabotConfigSlice,
   DependabotPr,
-  RepoLanguages,
   RepoMeta,
 } from './types.ts';
 
@@ -201,7 +197,7 @@ function renderReport(
     { target: opts.target, windowDays: opts.windowDays },
     `scanning ${opts.target} (${opts.windowDays}-day window)`,
   );
-  return listOrgRepos(ctx.githubClient, opts.target).andThen((repos) => {
+  return listTargetRepos(ctx.githubClient, opts.target).andThen(({ kind: targetKind, repos }) => {
     const filtered = filterRepos(repos, opts);
     ctx.logger.info(
       { total: repos.length, included: filtered.length },
@@ -211,7 +207,7 @@ function renderReport(
     const windowStart = now.subtract(Temporal.Duration.from({ hours: opts.windowDays * 24 }));
 
     return ResultAsync.fromSafePromise(
-      collectAll(ctx, filtered, opts.target, opts.windowDays, windowStart, now),
+      collectAll(ctx, filtered, opts.target, targetKind, opts.windowDays, windowStart, now),
     ).andThen((data) => {
       ctx.logger.info(
         { dependabotPrs: data.dependabotPrs.length, warnings: data.errors.length },
@@ -243,6 +239,7 @@ async function collectAll(
   ctx: Context,
   repos: RepoMeta[],
   target: string,
+  targetKind: TargetKind,
   windowDays: number,
   windowStart: Instant,
   now: Instant,
@@ -251,41 +248,32 @@ async function collectAll(
   const warnings: CollectorWarning[] = [];
   const windowStartIso = windowStart.toString();
 
-  const [languages, dependabotConfig, cve, branchProtection, contributors, dependabotPrs] = await Promise.all([
-    crawlPerRepo(repos, (r) => getRepoLanguages(client, { owner: r.owner, name: r.name }), warnings, 'languages').then(
-      (rows): RepoLanguages[] => rows.map((r) => ({ owner: r.ref.owner, name: r.ref.name, bytes: r.bytes })),
-    ),
-    crawlPerRepo<DependabotConfigSlice>(
-      repos,
-      (r) => getDependabotConfig(client, { owner: r.owner, name: r.name }),
-      warnings,
-      'dependabotConfig',
-    ),
-    crawlPerRepo<CveSlice>(repos, (r) => getCveAlerts(client, { owner: r.owner, name: r.name }), warnings, 'cve'),
-    crawlPerRepo<BranchProtectionSlice>(
-      repos,
-      (r) => getBranchProtection(client, { owner: r.owner, name: r.name }, r.defaultBranch),
-      warnings,
-      'branchProtection',
-    ),
-    crawlPerRepo<ContributorSlice>(
-      repos,
-      (r) => listActiveCommitters(client, { owner: r.owner, name: r.name }, windowStartIso),
-      warnings,
-      'contributors',
-    ),
+  const cvePromise: Promise<CveSlice[]> = (async () => {
+    const perRepoCrawl = (): Promise<CveSlice[]> =>
+      crawlPerRepo<CveSlice>(repos, (r) => getCveAlerts(client, { owner: r.owner, name: r.name }), warnings, 'cve');
+    if (targetKind === 'user') return perRepoCrawl();
+    // Try the org-level endpoint first (one call instead of N). If anything
+    // other than scope-missing comes back, fall back to per-repo so each
+    // repo gets a real status determination instead of an empty list.
+    const result = await getOrgCveAlerts(client, target, repos);
+    if (result.isOk()) return result.value;
+    warnings.push({ collector: 'cve', message: formatGithubError(result.error) });
+    return perRepoCrawl();
+  })();
+
+  const [metadata, cve, dependabotPrs] = await Promise.all([
+    runRepoMetadata(listRepoMetadataBatched(client, repos), warnings),
+    cvePromise,
     runResultAsync<DependabotPr[]>(listDependabotPrs(client, target, windowStartIso), [], warnings, 'dependabotPrs'),
   ]);
 
   return {
     ctx: { org: target, windowDays, windowStart, now },
     repos,
-    languages,
-    dependabotConfig,
+    dependabotConfig: metadata.dependabotConfig,
     dependabotPrs,
     cve,
-    branchProtection,
-    contributors,
+    branchProtection: metadata.branchProtection,
     errors: warnings,
   };
 }
@@ -296,10 +284,14 @@ async function crawlPerRepo<T>(
   warnings: CollectorWarning[],
   collector: string,
 ): Promise<T[]> {
-  const results = await mapWithConcurrency(repos, 8, async (repo) => {
-    const result = await fn(repo);
-    return { repo, result };
-  });
+  const results = await pMap(
+    repos,
+    async (repo) => {
+      const result = await fn(repo);
+      return { repo, result };
+    },
+    { concurrency: 8 },
+  );
   const ok: T[] = [];
   for (const { repo, result } of results) {
     if (result.isOk()) {
@@ -327,6 +319,23 @@ async function runResultAsync<T>(
   return fallback;
 }
 
+// listRepoMetadataBatched returns per-repo warnings inside the value; the outer
+// Err channel is reserved for systemic errors and is currently unused. This
+// helper forwards inner warnings up without double-counting and degrades to
+// empty slices if the outer Err ever fires.
+async function runRepoMetadata(
+  ra: ReturnType<typeof listRepoMetadataBatched>,
+  warnings: CollectorWarning[],
+): Promise<{ dependabotConfig: DependabotConfigSlice[]; branchProtection: BranchProtectionSlice[] }> {
+  const result = await ra;
+  if (result.isErr()) {
+    warnings.push({ collector: 'repoMetadata', message: formatGithubError(result.error) });
+    return { dependabotConfig: [], branchProtection: [] };
+  }
+  for (const w of result.value.warnings) warnings.push(w);
+  return { dependabotConfig: result.value.dependabotConfig, branchProtection: result.value.branchProtection };
+}
+
 export type ParseCliResult = { kind: 'ok'; value: CliOptions } | { kind: 'err'; message: string };
 
 const safeParseArgs = Result.fromThrowable(