diff --git a/src/cli.ts b/src/cli.ts index c127784..76b9ea1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,24 +3,35 @@ import { parseArgs } from 'node:util'; import { Result, ResultAsync } from 'neverthrow'; import { getBranchProtection } from './collectors/branchProtection.ts'; import { listActiveCommitters } from './collectors/contributors.ts'; -import { getCveAlerts } from './collectors/cve.ts'; +import { getCveAlerts, getOrgCveAlerts } from './collectors/cve.ts'; import { getDependabotConfig } from './collectors/dependabotConfig.ts'; import { listDependabotPrs } from './collectors/dependabotPrs.ts'; -import { getRepoLanguages, listOrgRepos } from './collectors/repos.ts'; +import { type TargetKind, getRepoLanguages, listTargetRepos as listRepos } from './collectors/repos.ts'; import { mapWithConcurrency } from './concurrency.ts'; import type { Context } from './context.ts'; import { getErrorMessage } from './errors.ts'; import { formatFsError } from './FileSystem.ts'; import { type GithubError, formatGithubError } from './github/errors.ts'; +import { getRateLimitStatus } from './github/rateLimit.ts'; import { promptForTarget } from './interactive/targetPrompt.ts'; import { formatPromptError } from './prompt/Prompter.ts'; import { aggregate } from './report/aggregate.ts'; import { type RenderError, renderHtml } from './report/html.ts'; import type { ReportAnalyticsConfig } from './report/reportAnalyticsConfig.ts'; +import { + COLLECTOR_KEYS, + DEFAULT_API_BUDGET, + type ScanPlan, + apiBudgetFromRateLimit, + buildScanPlan, + countSkipped, + isBudgetConstrained, +} from './scanPlan.ts'; import { type Instant, Temporal } from './time.ts'; import type { BranchProtectionSlice, CollectedData, + CollectorMeasurement, CollectorWarning, ContributorSlice, CveSlice, @@ -167,6 +178,9 @@ export async function main(ctx: Context, argv: readonly string[]): Promise { + return listRepos(ctx.githubClient, opts.target).andThen(({ targetKind, repos }) => { const filtered = filterRepos(repos, opts); ctx.logger.info( - { total: repos.length, included: filtered.length }, + { total: repos.length, included: filtered.length, targetKind }, `found ${repos.length} repos; ${filtered.length} included after filters`, ); const now = ctx.clock.now(); const windowStart = now.subtract(Temporal.Duration.from({ hours: opts.windowDays * 24 })); - return ResultAsync.fromSafePromise( - collectAll(ctx, filtered, opts.target, opts.windowDays, windowStart, now), - ).andThen((data) => { - ctx.logger.info( - { dependabotPrs: data.dependabotPrs.length, warnings: data.errors.length }, - `crawled ${data.dependabotPrs.length} Dependabot PRs; rendering report`, - ); - if (data.errors.length > 0) { + return getRateLimitStatus(ctx.githubClient) + .map(apiBudgetFromRateLimit) + .orElse((err) => { ctx.logger.warn( - { count: data.errors.length }, - `${data.errors.length} per-repo warnings were suppressed during crawl`, + { err: formatGithubError(err) }, + 'could not fetch GitHub rate limit; using conservative scan plan', ); - } - const bundle = aggregate(data); - return renderHtml(bundle, analytics).map( - (report): RenderedReport => ({ - report, - stats: { - reposTotal: repos.length, - reposIncluded: filtered.length, - dependabotPrs: data.dependabotPrs.length, - warnings: data.errors.length, - }, - }), - ); - }); + return ResultAsync.fromSafePromise(Promise.resolve(DEFAULT_API_BUDGET)); + }) + .andThen((apiBudget) => { + const plan = buildScanPlan({ targetKind, repoCount: filtered.length, apiBudget }); + ctx.logger.info( + { plan, budgetConstrained: isBudgetConstrained(plan) }, + 'using GitHub API budget-aware scan plan', + ); + return ResultAsync.fromSafePromise( + collectAll(ctx, filtered, opts.target, targetKind, opts.windowDays, windowStart, now, plan), + ).andThen((data) => { + ctx.logger.info( + { dependabotPrs: data.dependabotPrs.length, warnings: data.errors.length }, + `crawled ${data.dependabotPrs.length} Dependabot PRs; rendering report`, + ); + if (data.errors.length > 0) { + ctx.logger.warn( + { count: data.errors.length }, + `${data.errors.length} per-repo warnings were suppressed during crawl`, + ); + } + const bundle = aggregate(data); + return renderHtml(bundle, analytics).map( + (report): RenderedReport => ({ + report, + stats: { + reposTotal: repos.length, + reposIncluded: filtered.length, + dependabotPrs: data.dependabotPrs.length, + warnings: data.errors.length, + scanMode: isBudgetConstrained(plan) ? 'budgeted' : 'full', + skippedCollectors: countSkipped(plan), + restBudget: plan.restBudget, + }, + }), + ); + }); + }); }); } @@ -243,40 +279,40 @@ async function collectAll( ctx: Context, repos: RepoMeta[], target: string, + targetKind: TargetKind, windowDays: number, windowStart: Instant, now: Instant, + plan: ScanPlan, ): Promise { const client = ctx.githubClient; const warnings: CollectorWarning[] = []; + const measurements = measurementsFromPlan(plan); const windowStartIso = windowStart.toString(); + const concurrency = isBudgetConstrained(plan) ? 4 : 8; + + const dependabotPrsPromise = runResultAsyncWithStatus( + listDependabotPrs(client, target, windowStartIso), + [], + warnings, + measurements, + 'dependabotPrs', + ); - const [languages, dependabotConfig, cve, branchProtection, contributors, dependabotPrs] = await Promise.all([ - crawlPerRepo(repos, (r) => getRepoLanguages(client, { owner: r.owner, name: r.name }), warnings, 'languages').then( - (rows): RepoLanguages[] => rows.map((r) => ({ owner: r.ref.owner, name: r.ref.name, bytes: r.bytes })), - ), - crawlPerRepo( - repos, - (r) => getDependabotConfig(client, { owner: r.owner, name: r.name }), - warnings, - 'dependabotConfig', - ), - crawlPerRepo(repos, (r) => getCveAlerts(client, { owner: r.owner, name: r.name }), warnings, 'cve'), - crawlPerRepo( - repos, - (r) => getBranchProtection(client, { owner: r.owner, name: r.name }, r.defaultBranch), - warnings, - 'branchProtection', - ), - crawlPerRepo( - repos, - (r) => listActiveCommitters(client, { owner: r.owner, name: r.name }, windowStartIso), - warnings, - 'contributors', - ), - runResultAsync(listDependabotPrs(client, target, windowStartIso), [], warnings, 'dependabotPrs'), + const cvePromise = collectCveFromPlan(client, target, targetKind, repos, plan, warnings, measurements, concurrency); + const languagesPromise = collectLanguagesFromPlan(client, repos, plan, warnings, concurrency); + const configPromise = collectDependabotConfigFromPlan(client, repos, plan, warnings, measurements, concurrency); + + const [dependabotPrs, cve, languages, dependabotConfig] = await Promise.all([ + dependabotPrsPromise, + cvePromise, + languagesPromise, + configPromise, ]); + const branchProtection = await collectBranchProtectionFromPlan(client, repos, plan, warnings, concurrency); + const contributors = await collectContributorsFromPlan(client, repos, windowStartIso, plan, warnings, concurrency); + return { ctx: { org: target, windowDays, windowStart, now }, repos, @@ -286,17 +322,130 @@ async function collectAll( cve, branchProtection, contributors, + measurements, errors: warnings, }; } +async function collectCveFromPlan( + client: Context['githubClient'], + target: string, + targetKind: TargetKind, + repos: readonly RepoMeta[], + plan: ScanPlan, + warnings: CollectorWarning[], + measurements: CollectorMeasurement[], + concurrency: number, +): Promise { + if (!plan.include.has('cve')) { + addSkippedWarning(warnings, plan, 'cve'); + return []; + } + if (targetKind === 'org' && plan.modes.cve === 'org-endpoint') { + return runResultAsyncWithStatus(getOrgCveAlerts(client, target, repos), [], warnings, measurements, 'cve'); + } + return crawlPerRepo( + repos, + (r) => getCveAlerts(client, { owner: r.owner, name: r.name }), + warnings, + 'cve', + concurrency, + ); +} + +async function collectLanguagesFromPlan( + client: Context['githubClient'], + repos: readonly RepoMeta[], + plan: ScanPlan, + warnings: CollectorWarning[], + concurrency: number, +): Promise { + if (!plan.include.has('languages')) { + addSkippedWarning(warnings, plan, 'languages'); + return []; + } + if (plan.modes.languages === 'metadata') return []; + return crawlPerRepo( + repos, + (r) => getRepoLanguages(client, { owner: r.owner, name: r.name }), + warnings, + 'languages', + concurrency, + ).then((rows): RepoLanguages[] => rows.map((r) => ({ owner: r.ref.owner, name: r.ref.name, bytes: r.bytes }))); +} + +async function collectDependabotConfigFromPlan( + client: Context['githubClient'], + repos: readonly RepoMeta[], + plan: ScanPlan, + warnings: CollectorWarning[], + measurements: CollectorMeasurement[], + concurrency: number, +): Promise { + if (!plan.include.has('dependabotConfig')) { + addSkippedWarning(warnings, plan, 'dependabotConfig'); + return []; + } + const rows = await crawlPerRepo( + repos, + (r) => getDependabotConfig(client, { owner: r.owner, name: r.name }), + warnings, + 'dependabotConfig', + concurrency, + ); + if (rows.length < repos.length) markMeasurement(measurements, 'dependabotConfig', 'partial'); + return rows; +} + +async function collectBranchProtectionFromPlan( + client: Context['githubClient'], + repos: readonly RepoMeta[], + plan: ScanPlan, + warnings: CollectorWarning[], + concurrency: number, +): Promise { + if (!plan.include.has('branchProtection')) { + addSkippedWarning(warnings, plan, 'branchProtection'); + return []; + } + return crawlPerRepo( + repos, + (r) => getBranchProtection(client, { owner: r.owner, name: r.name }, r.defaultBranch), + warnings, + 'branchProtection', + concurrency, + ); +} + +async function collectContributorsFromPlan( + client: Context['githubClient'], + repos: readonly RepoMeta[], + windowStartIso: string, + plan: ScanPlan, + warnings: CollectorWarning[], + concurrency: number, +): Promise { + if (!plan.include.has('contributors')) { + addSkippedWarning(warnings, plan, 'contributors'); + return []; + } + return crawlPerRepo( + repos, + (r) => listActiveCommitters(client, { owner: r.owner, name: r.name }, windowStartIso), + warnings, + 'contributors', + concurrency, + ); +} + async function crawlPerRepo( repos: readonly RepoMeta[], fn: (repo: RepoMeta) => ResultAsync, warnings: CollectorWarning[], collector: string, + concurrency: number, ): Promise { - const results = await mapWithConcurrency(repos, 8, async (repo) => { + const results = await mapWithConcurrency(repos, concurrency, async (repo) => { const result = await fn(repo); return { repo, result }; }); @@ -315,18 +464,53 @@ async function crawlPerRepo( return ok; } -async function runResultAsync( +async function runResultAsyncWithStatus( ra: ResultAsync, fallback: T, warnings: CollectorWarning[], - collector: string, + measurements: CollectorMeasurement[], + collector: CollectorMeasurement['collector'], ): Promise { const result = await ra; if (result.isOk()) return result.value; warnings.push({ collector, message: formatGithubError(result.error) }); + markMeasurement(measurements, collector, 'failed', formatGithubError(result.error)); return fallback; } +function measurementsFromPlan(plan: ScanPlan): CollectorMeasurement[] { + return COLLECTOR_KEYS.map((collector) => ({ + collector, + status: plan.include.has(collector) ? 'measured' : 'skipped', + mode: plan.modes[collector], + reason: plan.skipped[collector]?.message, + estimatedRestCost: plan.estimatedRestCost[collector], + estimatedGraphqlCost: plan.estimatedGraphqlCost[collector], + })); +} + +function markMeasurement( + measurements: CollectorMeasurement[], + collector: CollectorMeasurement['collector'], + status: CollectorMeasurement['status'], + reason?: string, +): void { + const measurement = measurements.find((m) => m.collector === collector); + if (!measurement) return; + (measurement as { status: CollectorMeasurement['status']; reason?: string }).status = status; + if (reason) (measurement as { reason?: string }).reason = reason; +} + +function addSkippedWarning( + warnings: CollectorWarning[], + plan: ScanPlan, + collector: CollectorMeasurement['collector'], +): void { + const skipped = plan.skipped[collector]; + if (!skipped) return; + warnings.push({ collector, message: skipped.message }); +} + export type ParseCliResult = { kind: 'ok'; value: CliOptions } | { kind: 'err'; message: string }; const safeParseArgs = Result.fromThrowable( diff --git a/src/collectors/cve.ts b/src/collectors/cve.ts index aec3eb4..21df811 100644 --- a/src/collectors/cve.ts +++ b/src/collectors/cve.ts @@ -2,12 +2,12 @@ import { type ResultAsync, errAsync, okAsync } from 'neverthrow'; import { z } from 'zod'; import type { GithubError } from '../github/errors.ts'; import type { GithubClient } from '../github/GithubClient.ts'; -import type { CveAlert, CveSeverity, CveSlice, RepoRef } from '../types.ts'; +import type { CveAlert, CveSeverity, CveSlice, RepoMeta, RepoRef } from '../types.ts'; // `security_vulnerability` is null on alerts GitHub can't attribute to a // concrete vulnerability (e.g. auto-dismissed); those carry no severity or // package, so we skip them rather than crash. -const alertSchema = z.object({ +export const alertSchema = z.object({ number: z.number(), created_at: z.string(), security_advisory: z.object({ summary: z.string() }), @@ -19,6 +19,13 @@ const alertSchema = z.object({ .nullable(), }); +const orgAlertSchema = alertSchema.extend({ + repository: z.object({ + name: z.string(), + owner: z.object({ login: z.string() }), + }), +}); + export function getCveAlerts(client: GithubClient, ref: RepoRef): ResultAsync { return client .paginate( @@ -30,16 +37,7 @@ export function getCveAlerts(client: GithubClient, ref: RepoRef): ResultAsync { + return client + .paginate('GET /orgs/{org}/dependabot/alerts', { org, state: 'open', per_page: 100 }, orgAlertSchema) + .map((raw): CveSlice[] => { + const byRepo = new Map(); + const included = new Set(repos.map((repo) => repoKey(repo))); + for (const a of raw) { + const ref = { owner: a.repository.owner.login, name: a.repository.name }; + const key = repoKey(ref); + if (!included.has(key) || a.security_vulnerability === null) continue; + const alerts = byRepo.get(key) ?? []; + alerts.push(toCveAlert(ref, a)); + byRepo.set(key, alerts); + } + return repos.map((repo): CveSlice => { + if (repo.dependabotAlertsEnabled === false) + return { owner: repo.owner, name: repo.name, status: 'not-enabled' }; + return { owner: repo.owner, name: repo.name, status: 'ok', alerts: byRepo.get(repoKey(repo)) ?? [] }; + }); + }) + .orElse((err) => { + if (err.kind === 'scope-missing') { + return okAsync( + repos.map((repo) => ({ + owner: repo.owner, + name: repo.name, + status: 'scope-missing', + requiredScope: err.required, + })), + ); + } + return errAsync(err); + }); +} + +function toCveAlert(ref: RepoRef, raw: z.infer): CveAlert { + if (raw.security_vulnerability === null) { + return { + owner: ref.owner, + name: ref.name, + number: raw.number, + severity: 'low', + createdAt: raw.created_at, + packageName: '', + ecosystem: '', + summary: raw.security_advisory.summary, + }; + } + return { + owner: ref.owner, + name: ref.name, + number: raw.number, + severity: normalizeSeverity(raw.security_vulnerability.severity), + createdAt: raw.created_at, + packageName: raw.security_vulnerability.package.name, + ecosystem: raw.security_vulnerability.package.ecosystem, + summary: raw.security_advisory.summary, + }; +} + +function repoKey(ref: RepoRef): string { + return `${ref.owner}/${ref.name}`; +} + function normalizeSeverity(raw: string): CveSeverity { const v = raw.toLowerCase(); if (v === 'critical') return 'critical'; diff --git a/src/collectors/dependabotPrs.ts b/src/collectors/dependabotPrs.ts index 292db64..ed30cc4 100644 --- a/src/collectors/dependabotPrs.ts +++ b/src/collectors/dependabotPrs.ts @@ -1,4 +1,4 @@ -import { ResultAsync, okAsync } from 'neverthrow'; +import { Result, ResultAsync, err, errAsync, ok, okAsync } from 'neverthrow'; import type { GithubError } from '../github/errors.ts'; import type { GithubClient } from '../github/GithubClient.ts'; import { DependabotPrsDocument, type DependabotPrsQuery } from '../github/graphql/generated.ts'; @@ -6,7 +6,8 @@ import type { CheckSummary, DependabotPr, PrState } from '../types.ts'; // `search(type: ISSUE)` returns a union; nodes that aren't pull requests come // back as empty objects, so a real PR node is the arm carrying `number`. -type SearchNode = NonNullable[number]; +type SearchPayload = NonNullable; +type SearchNode = NonNullable[number]; export type PrNode = Extract; // GitHub's GraphQL Actor interface. `__typename` is the source of truth for @@ -33,16 +34,29 @@ function pageThrough( acc: DependabotPr[], ): ResultAsync { return client.graphql(DependabotPrsDocument, { searchQuery, cursor }).andThen((res) => { - for (const node of res.search.nodes ?? []) { + const searchResult = readSearch(res); + if (searchResult.isErr()) return errAsync(searchResult.error); + const search = searchResult.value; + for (const node of search.nodes ?? []) { if (isPrNode(node)) acc.push(toDependabotPr(node)); } - if (res.search.pageInfo.hasNextPage && res.search.pageInfo.endCursor) { - return pageThrough(client, searchQuery, res.search.pageInfo.endCursor, acc); + if (search.pageInfo.hasNextPage && search.pageInfo.endCursor) { + return pageThrough(client, searchQuery, search.pageInfo.endCursor, acc); } return okAsync(acc); }); } +function readSearch(res: DependabotPrsQuery | undefined): Result { + if (!res?.search?.pageInfo) { + return err({ + kind: 'malformed-response', + message: 'GitHub returned malformed data for Dependabot PR search; this run will continue without that section', + }); + } + return ok(res.search); +} + function isPrNode(node: SearchNode): node is PrNode { return node !== null && 'number' in node; } diff --git a/src/collectors/repos.ts b/src/collectors/repos.ts index 99a72cc..ef5ccf3 100644 --- a/src/collectors/repos.ts +++ b/src/collectors/repos.ts @@ -17,25 +17,37 @@ const repoSchema = z.object({ language: z.string().nullish(), pushed_at: z.string().nullish(), security_and_analysis: z - .object({ dependabot_security_updates: z.object({ status: z.enum(['enabled', 'disabled']) }).optional() }) + .object({ + dependabot_alerts: z.object({ status: z.enum(['enabled', 'disabled']) }).optional(), + dependabot_security_updates: z.object({ status: z.enum(['enabled', 'disabled']) }).optional(), + }) .nullish(), }); type RawRepo = z.infer; -export function listOrgRepos(client: GithubClient, org: string): ResultAsync { +export type TargetKind = 'org' | 'user'; + +export interface RepoListResult { + readonly targetKind: TargetKind; + readonly repos: RepoMeta[]; +} + +export function listTargetRepos(client: GithubClient, target: string): ResultAsync { return client - .paginate('GET /orgs/{org}/repos', { org, per_page: 100, type: 'all' }, repoSchema) + .paginate('GET /orgs/{org}/repos', { org: target, per_page: 100, type: 'all' }, repoSchema) + .map((repos): RepoListResult => ({ targetKind: 'org', repos: repos.map(toRepoMeta) })) .orElse((err) => { if (err.kind === 'not-found') { - return client.paginate( - 'GET /users/{username}/repos', - { username: org, per_page: 100, type: 'owner' }, - repoSchema, - ); + return client + .paginate('GET /users/{username}/repos', { username: target, per_page: 100, type: 'owner' }, repoSchema) + .map((repos): RepoListResult => ({ targetKind: 'user', repos: repos.map(toRepoMeta) })); } - return errAsync(err); - }) - .map((repos) => repos.map(toRepoMeta)); + return errAsync(err); + }); +} + +export function listOrgRepos(client: GithubClient, org: string): ResultAsync { + return listTargetRepos(client, org).map(({ repos }) => repos); } export function getRepoLanguages( @@ -50,6 +62,7 @@ export function getRepoLanguages( function toRepoMeta(raw: RawRepo): RepoMeta { const visibility: Visibility = raw.visibility === 'internal' ? 'internal' : raw.private ? 'private' : 'public'; const securityUpdates = raw.security_and_analysis?.dependabot_security_updates?.status; + const alertsStatus = raw.security_and_analysis?.dependabot_alerts?.status; return { owner: raw.owner.login, name: raw.name, @@ -60,5 +73,6 @@ function toRepoMeta(raw: RawRepo): RepoMeta { primaryLanguage: raw.language ?? null, pushedAt: raw.pushed_at ?? null, dependabotSecurityUpdates: securityUpdates === undefined ? null : securityUpdates === 'enabled', + dependabotAlertsEnabled: alertsStatus === undefined ? null : alertsStatus === 'enabled', }; } diff --git a/src/github/GithubClient.ts b/src/github/GithubClient.ts index d3b1fce..a40b30d 100644 --- a/src/github/GithubClient.ts +++ b/src/github/GithubClient.ts @@ -1,5 +1,4 @@ import type { TypedDocumentNode } from '@graphql-typed-document-node/core'; -import { graphql as graphqlBase } from '@octokit/graphql'; import type { PaginatingEndpoints } from '@octokit/plugin-paginate-rest'; import { retry } from '@octokit/plugin-retry'; import { throttling } from '@octokit/plugin-throttling'; @@ -43,7 +42,6 @@ export interface GithubClientImplOptions { export class GithubClientImpl implements GithubClient { private readonly rest: InstanceType; - private readonly graphqlClient: typeof graphqlBase; private readonly log: Logger; constructor(options: GithubClientImplOptions) { @@ -62,13 +60,9 @@ export class GithubClientImpl implements GithubClient { retry: { doNotRetry: [400, 401, 403, 404, 409, 422] }, throttle: { onRateLimit: (_retryAfter, _opts, _octokit, retryCount) => retryCount < 2, - onSecondaryRateLimit: () => true, + onSecondaryRateLimit: (_retryAfter, _opts, _octokit, retryCount) => retryCount < 2, }, }); - this.graphqlClient = graphqlBase.defaults({ - headers: { authorization: `token ${token}` }, - request: { log }, - }); } // Keep casts at the Octokit boundary; callers get route-derived types. @@ -99,6 +93,6 @@ export class GithubClientImpl implements GithubClient { document: TypedDocumentNode, variables: TVariables, ): ResultAsync { - return ResultAsync.fromPromise(this.graphqlClient(print(document), variables), toGithubError); + return ResultAsync.fromPromise(this.rest.graphql(print(document), variables), toGithubError); } } diff --git a/src/github/errors.ts b/src/github/errors.ts index e204507..7a03789 100644 --- a/src/github/errors.ts +++ b/src/github/errors.ts @@ -1,17 +1,21 @@ import { getErrorMessage } from '../errors.ts'; +import { Temporal } from '../time.ts'; export type GithubError = | { kind: 'network'; message: string; cause: Error } | { kind: 'not-found'; url?: string; message: string } | { kind: 'scope-missing'; required: string; message: string } | { kind: 'forbidden'; url?: string; message: string } + | { kind: 'rate-limited'; message: string; resetAt?: string; retryAfterSeconds?: number } + | { kind: 'malformed-response'; message: string } | { kind: 'http'; status: number; url?: string; message: string }; interface RequestErrorLike { status?: number; message?: string; request?: { url?: string }; - response?: { headers?: Record }; + response?: { headers?: Record; data?: unknown }; + errors?: Array<{ type?: string; message?: string; extensions?: { code?: string } }>; } export function toGithubError(err: unknown): GithubError { @@ -20,6 +24,9 @@ export function toGithubError(err: unknown): GithubError { const status = typeof e?.status === 'number' ? e.status : undefined; const message = getErrorMessage(err); + const rateLimit = detectRateLimit(e, message); + if (rateLimit) return rateLimit; + if (status === undefined) { return { kind: 'network', message, cause: err instanceof Error ? err : new Error(message) }; } @@ -41,6 +48,36 @@ export function toGithubError(err: unknown): GithubError { return { kind: 'http', status, url, message }; } +function detectRateLimit( + err: RequestErrorLike, + message: string, +): Extract | null { + const headers = err.response?.headers ?? {}; + const status = typeof err.status === 'number' ? err.status : undefined; + const remaining = headers['x-ratelimit-remaining']; + const retryAfter = parseNumber(headers['retry-after']); + const resetAt = resetHeaderToInstant(headers['x-ratelimit-reset']); + const graphQlRateLimited = err.errors?.some( + (e) => e.type === 'RATE_LIMITED' || e.extensions?.code === 'RATE_LIMITED' || /rate limit/i.test(e.message ?? ''), + ); + const secondary = /secondary rate limit/i.test(message) || retryAfter !== undefined; + if (graphQlRateLimited || ((status === 403 || status === 429) && (remaining === '0' || secondary))) { + return { kind: 'rate-limited', message, resetAt, retryAfterSeconds: retryAfter }; + } + return null; +} + +function parseNumber(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function resetHeaderToInstant(value: string | undefined): string | undefined { + const seconds = parseNumber(value); + return seconds === undefined ? undefined : Temporal.Instant.fromEpochMilliseconds(seconds * 1000).toString(); +} + function detectMissingScope(err: RequestErrorLike): string | null { const accepted = err.response?.headers?.['x-accepted-oauth-scopes']; const present = err.response?.headers?.['x-oauth-scopes']; @@ -74,6 +111,16 @@ export function formatGithubError(err: GithubError): string { return `${err.message}\n fix: gh auth refresh -s ${err.required}`; case 'forbidden': return `GitHub returned 403 for ${err.url ?? 'an API call'}: ${err.message}`; + case 'rate-limited': { + const retry = err.resetAt + ? `; try again after ${err.resetAt}` + : err.retryAfterSeconds + ? `; retry after ${err.retryAfterSeconds}s` + : ''; + return `GitHub rate limit exceeded${retry}`; + } + case 'malformed-response': + return err.message; case 'http': return `GitHub returned ${err.status} for ${err.url ?? 'an API call'}: ${err.message}`; } diff --git a/src/github/rateLimit.ts b/src/github/rateLimit.ts new file mode 100644 index 0000000..b5eb838 --- /dev/null +++ b/src/github/rateLimit.ts @@ -0,0 +1,43 @@ +import { Result, ResultAsync, errAsync, okAsync } from 'neverthrow'; +import { z } from 'zod'; +import { type Instant, Temporal } from '../time.ts'; +import type { GithubError } from './errors.ts'; +import type { GithubClient } from './GithubClient.ts'; + +export interface RateLimitBucket { + readonly limit: number; + readonly remaining: number; + readonly used: number; + readonly resetAt: Instant; +} + +export interface RateLimitStatus { + readonly rest: RateLimitBucket; + readonly graphql: RateLimitBucket; +} + +const bucketSchema = z.object({ limit: z.number(), remaining: z.number(), used: z.number(), reset: z.number() }); +const rateLimitSchema = z.object({ resources: z.object({ core: bucketSchema, graphql: bucketSchema }) }); + +export function getRateLimitStatus(client: GithubClient): ResultAsync { + return client.request('GET /rate_limit').andThen((raw) => { + const parsed = Result.fromThrowable( + () => rateLimitSchema.parse(raw), + (): GithubError => ({ + kind: 'malformed-response', + message: 'GitHub returned malformed data for rate-limit status', + }), + )(); + if (parsed.isErr()) return errAsync(parsed.error); + return okAsync({ rest: toBucket(parsed.value.resources.core), graphql: toBucket(parsed.value.resources.graphql) }); + }); +} + +function toBucket(raw: z.infer): RateLimitBucket { + return { + limit: raw.limit, + remaining: raw.remaining, + used: raw.used, + resetAt: Temporal.Instant.fromEpochMilliseconds(raw.reset * 1000), + }; +} diff --git a/src/report/aggregate.ts b/src/report/aggregate.ts index 600f299..f0b37de 100644 --- a/src/report/aggregate.ts +++ b/src/report/aggregate.ts @@ -2,6 +2,7 @@ import { classifyBumpType, isDevDependencyBump } from '../heuristics/bumpType.ts import { type Instant, Temporal, instantFromString } from '../time.ts'; import type { CollectedData, + CollectorMeasurement, CveAlert, CveSeverity, DependabotConfigSlice, @@ -12,6 +13,7 @@ import { ASSUMED_HOURLY_RATE_USD, ASSUMED_MIN_PER_PR, deriveCostEstimate, derive export interface ReportBundle { meta: ReportMeta; + measurements: ReportMeasurements; orgOverview: OrgOverview; dependabotCoverage: DependabotCoverage; prBacklog: PrBacklog; @@ -21,6 +23,11 @@ export interface ReportBundle { cve: CveExposure; } +export interface ReportMeasurements { + mode: 'full' | 'budgeted'; + collectors: Record; +} + export interface ReportMeta { org: string; windowDays: number; @@ -34,18 +41,20 @@ export interface OrgOverview { privateCount: number; internalCount: number; archivedExcluded: number; - topLanguages: Array<{ language: string; bytes: number; percentage: number }>; + topLanguages: Array<{ language: string; repoCount: number; bytes: number; percentage: number }>; + languageSource: 'bytes' | 'metadata' | 'skipped'; nodeTsRepoCount: number; nodeTsRepoPercentage: number; - activeHumanCommitters: number; - reposWithBranchProtection: number; + activeHumanCommitters: number | null; + reposWithBranchProtection: number | null; } export type CadenceLabel = 'daily' | 'weekly' | 'monthly' | 'unspecified'; export interface DependabotCoverage { - reposWithConfig: number; - reposWithConfigPercentage: number; + configStatus: 'measured' | 'skipped' | 'failed'; + reposWithConfig: number | null; + reposWithConfigPercentage: number | null; reposWithSecurityUpdates: number; reposWithSecurityUpdatesPercentage: number; ecosystemBreakdown: Array<{ ecosystem: string; repoCount: number }>; @@ -55,6 +64,7 @@ export interface DependabotCoverage { } export interface PrBacklog { + status: 'measured' | 'failed'; openCount: number; closedInWindowCount: number; mergedInWindowCount: number; @@ -70,6 +80,8 @@ export interface PrBacklog { } export interface StalledSignals { + status: 'measured' | 'skipped'; + reason?: string; reposAtPrCap: Array<{ repo: string; openPrs: number }>; reposWithConfigButNoRecentPrs: string[]; } @@ -94,7 +106,9 @@ export interface CostEstimate { } export interface CveExposure { - status: 'ok' | 'scope-missing'; + status: 'ok' | 'scope-missing' | 'not-measured'; + source?: 'org-endpoint' | 'repo-endpoint'; + disabledRepoStatus: 'measured' | 'metadata' | 'unknown'; requiredScope?: string; totalOpenAlerts: number; bySeverity: Record; @@ -116,6 +130,7 @@ export function aggregate(data: CollectedData): ReportBundle { totalReposScanned: data.repos.length, }; + const measurements = buildReportMeasurements(data); const orgOverview = buildOrgOverview(data); const dependabotCoverage = buildDependabotCoverage(data); const prBacklog = buildPrBacklog(data, now, windowStart); @@ -126,6 +141,7 @@ export function aggregate(data: CollectedData): ReportBundle { return { meta, + measurements, orgOverview, dependabotCoverage, prBacklog, @@ -143,21 +159,15 @@ function buildOrgOverview(data: CollectedData): OrgOverview { const privateCount = repos.filter((r) => r.visibility === 'private').length; const internalCount = repos.filter((r) => r.visibility === 'internal').length; - const aggregateBytes: LanguageBytes = {}; - for (const lang of data.languages) { - for (const [name, bytes] of Object.entries(lang.bytes)) { - aggregateBytes[name] = (aggregateBytes[name] ?? 0) + bytes; - } - } - const totalBytes = Object.values(aggregateBytes).reduce((a, b) => a + b, 0); - const topLanguages = Object.entries(aggregateBytes) - .map(([language, bytes]) => ({ - language, - bytes, - percentage: totalBytes > 0 ? round1((bytes / totalBytes) * 100) : 0, - })) - .sort((a, b) => b.bytes - a.bytes) - .slice(0, 10); + const languageMeasurement = findMeasurement(data, 'languages'); + const languageSource: OrgOverview['languageSource'] = + languageMeasurement?.status === 'skipped' + ? 'skipped' + : languageMeasurement?.mode === 'metadata' + ? 'metadata' + : 'bytes'; + const topLanguages = + languageSource === 'metadata' ? buildMetadataLanguages(repos) : buildByteLanguages(data.languages); const nodeTsRepoCount = repos.filter( (r) => r.primaryLanguage === 'TypeScript' || r.primaryLanguage === 'JavaScript', @@ -168,7 +178,10 @@ function buildOrgOverview(data: CollectedData): OrgOverview { for (const login of slice.activeHumanLogins) allCommitters.add(login); } - const reposWithBranchProtection = data.branchProtection.filter((b) => b.hasProtection).length; + const branchMeasurement = findMeasurement(data, 'branchProtection'); + const contributorMeasurement = findMeasurement(data, 'contributors'); + const reposWithBranchProtection = + branchMeasurement?.status === 'skipped' ? null : data.branchProtection.filter((b) => b.hasProtection).length; return { repoCount: repos.length, @@ -177,16 +190,24 @@ function buildOrgOverview(data: CollectedData): OrgOverview { internalCount, archivedExcluded, topLanguages, + languageSource, nodeTsRepoCount, nodeTsRepoPercentage: pct(nodeTsRepoCount, repos.length), - activeHumanCommitters: allCommitters.size, + activeHumanCommitters: contributorMeasurement?.status === 'skipped' ? null : allCommitters.size, reposWithBranchProtection, }; } function buildDependabotCoverage(data: CollectedData): DependabotCoverage { const liveRepos = data.repos.filter((r) => !r.archived); - const reposWithConfig = data.dependabotConfig.filter((c) => c.hasConfig).length; + const configMeasurement = findMeasurement(data, 'dependabotConfig'); + const configStatus: DependabotCoverage['configStatus'] = + configMeasurement?.status === 'skipped' + ? 'skipped' + : configMeasurement?.status === 'failed' + ? 'failed' + : 'measured'; + const reposWithConfig = configStatus === 'measured' ? data.dependabotConfig.filter((c) => c.hasConfig).length : null; const reposWithSecurity = liveRepos.filter((r) => r.dependabotSecurityUpdates === true).length; const ecoCounts = new Map(); @@ -215,8 +236,9 @@ function buildDependabotCoverage(data: CollectedData): DependabotCoverage { .filter((c) => c.entryCount > 0); return { + configStatus, reposWithConfig, - reposWithConfigPercentage: pct(reposWithConfig, liveRepos.length), + reposWithConfigPercentage: reposWithConfig === null ? null : pct(reposWithConfig, liveRepos.length), reposWithSecurityUpdates: reposWithSecurity, reposWithSecurityUpdatesPercentage: pct(reposWithSecurity, liveRepos.length), ecosystemBreakdown, @@ -227,6 +249,8 @@ function buildDependabotCoverage(data: CollectedData): DependabotCoverage { } function buildPrBacklog(data: CollectedData, now: Instant, windowStart: Instant): PrBacklog { + const status: PrBacklog['status'] = + findMeasurement(data, 'dependabotPrs')?.status === 'failed' ? 'failed' : 'measured'; const prs = data.dependabotPrs; const openPrs = prs.filter((p) => p.state === 'open'); const mergedInWindow = prs.filter((p) => p.merged && p.mergedAt && isAtOrAfter(p.mergedAt, windowStart)); @@ -294,6 +318,7 @@ function buildPrBacklog(data: CollectedData, now: Instant, windowStart: Instant) const timeToMergeP90Days = percentile(ttMergeDays, 90); return { + status, openCount: openPrs.length, closedInWindowCount: closedNotMergedInWindow.length, mergedInWindowCount: mergedInWindow.length, @@ -310,6 +335,10 @@ function buildPrBacklog(data: CollectedData, now: Instant, windowStart: Instant) } function buildStalledSignals(data: CollectedData, windowStart: Instant): StalledSignals { + const configMeasurement = findMeasurement(data, 'dependabotConfig'); + if (configMeasurement?.status === 'skipped' || configMeasurement?.status === 'failed') { + return { status: 'skipped', reason: configMeasurement.reason, reposAtPrCap: [], reposWithConfigButNoRecentPrs: [] }; + } const openByRepo = new Map(); for (const pr of data.dependabotPrs) { if (pr.state !== 'open') continue; @@ -340,6 +369,7 @@ function buildStalledSignals(data: CollectedData, windowStart: Instant): Stalled .sort(); return { + status: 'measured', reposAtPrCap, reposWithConfigButNoRecentPrs, }; @@ -401,10 +431,25 @@ export function isBotLogin(login: string): boolean { } function buildCveExposure(data: CollectedData, now: Instant): CveExposure { + const cveMeasurement = findMeasurement(data, 'cve'); + if (cveMeasurement?.status === 'skipped' || cveMeasurement?.status === 'failed') { + return { + status: 'not-measured', + disabledRepoStatus: 'unknown', + totalOpenAlerts: 0, + bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, + topReposBySeverity: [], + oldestCriticalDays: null, + oldestHighDays: null, + reposWithSecurityAlertsDisabled: [], + }; + } const scopeMissing = data.cve.find((s) => s.status === 'scope-missing'); if (scopeMissing && scopeMissing.status === 'scope-missing') { return { status: 'scope-missing', + source: cveMeasurement?.mode === 'org-endpoint' ? 'org-endpoint' : 'repo-endpoint', + disabledRepoStatus: 'unknown', requiredScope: scopeMissing.requiredScope, totalOpenAlerts: 0, bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, @@ -443,6 +488,8 @@ function buildCveExposure(data: CollectedData, now: Instant): CveExposure { return { status: 'ok', + source: cveMeasurement?.mode === 'org-endpoint' ? 'org-endpoint' : 'repo-endpoint', + disabledRepoStatus: cveMeasurement?.mode === 'org-endpoint' ? 'metadata' : 'measured', totalOpenAlerts: allAlerts.length, bySeverity, topReposBySeverity, @@ -452,6 +499,52 @@ function buildCveExposure(data: CollectedData, now: Instant): CveExposure { }; } +function buildReportMeasurements(data: CollectedData): ReportMeasurements { + const collectors = Object.fromEntries( + data.measurements.map((m) => [m.collector, m]), + ) as ReportMeasurements['collectors']; + const mode = data.measurements.some((m) => m.status === 'skipped' || m.mode === 'metadata') ? 'budgeted' : 'full'; + return { mode, collectors }; +} + +function findMeasurement( + data: CollectedData, + collector: CollectorMeasurement['collector'], +): CollectorMeasurement | undefined { + return data.measurements.find((m) => m.collector === collector); +} + +function buildByteLanguages(languages: CollectedData['languages']): OrgOverview['topLanguages'] { + const aggregateBytes: LanguageBytes = {}; + for (const lang of languages) { + for (const [name, bytes] of Object.entries(lang.bytes)) { + aggregateBytes[name] = (aggregateBytes[name] ?? 0) + bytes; + } + } + const totalBytes = Object.values(aggregateBytes).reduce((a, b) => a + b, 0); + return Object.entries(aggregateBytes) + .map(([language, bytes]) => ({ + language, + repoCount: 0, + bytes, + percentage: totalBytes > 0 ? round1((bytes / totalBytes) * 100) : 0, + })) + .sort((a, b) => b.bytes - a.bytes) + .slice(0, 10); +} + +function buildMetadataLanguages(repos: readonly { primaryLanguage: string | null }[]): OrgOverview['topLanguages'] { + const counts = new Map(); + for (const repo of repos) { + if (repo.primaryLanguage) counts.set(repo.primaryLanguage, (counts.get(repo.primaryLanguage) ?? 0) + 1); + } + const total = [...counts.values()].reduce((a, b) => a + b, 0); + return [...counts.entries()] + .map(([language, repoCount]) => ({ language, repoCount, bytes: 0, percentage: pct(repoCount, total) })) + .sort((a, b) => b.repoCount - a.repoCount) + .slice(0, 10); +} + function severityScore(rec: { critical: number; high: number; medium: number; low: number }): number { return rec.critical * 1000 + rec.high * 100 + rec.medium * 10 + rec.low; } diff --git a/src/report/testFactories.ts b/src/report/testFactories.ts index c4b4a25..f813cbc 100644 --- a/src/report/testFactories.ts +++ b/src/report/testFactories.ts @@ -8,6 +8,7 @@ import type { People, PrBacklog, ReportBundle, + ReportMeasurements, ReportMeta, StalledSignals, } from './aggregate.ts'; @@ -22,13 +23,26 @@ export const reportMeta = Factory.define(() => ({ totalReposScanned: 24, })); +export const reportMeasurements = Factory.define(() => ({ + mode: 'full', + collectors: { + languages: { collector: 'languages', status: 'measured', mode: 'exact' }, + dependabotConfig: { collector: 'dependabotConfig', status: 'measured', mode: 'exact' }, + cve: { collector: 'cve', status: 'measured', mode: 'exact' }, + branchProtection: { collector: 'branchProtection', status: 'measured', mode: 'exact' }, + contributors: { collector: 'contributors', status: 'measured', mode: 'exact' }, + dependabotPrs: { collector: 'dependabotPrs', status: 'measured', mode: 'exact' }, + }, +})); + export const orgOverview = Factory.define(() => ({ repoCount: 24, publicCount: 3, privateCount: 21, internalCount: 0, archivedExcluded: 1, - topLanguages: [{ language: 'TypeScript', bytes: 2_000_000, percentage: 65 }], + topLanguages: [{ language: 'TypeScript', repoCount: 0, bytes: 2_000_000, percentage: 65 }], + languageSource: 'bytes', nodeTsRepoCount: 18, nodeTsRepoPercentage: 75, activeHumanCommitters: 17, @@ -36,6 +50,7 @@ export const orgOverview = Factory.define(() => ({ })); export const dependabotCoverage = Factory.define(() => ({ + configStatus: 'measured', reposWithConfig: 20, reposWithConfigPercentage: 83.3, reposWithSecurityUpdates: 19, @@ -50,6 +65,7 @@ export const dependabotCoverage = Factory.define(() => ({ })); export const prBacklog = Factory.define(() => ({ + status: 'measured', openCount: 102, closedInWindowCount: 14, mergedInWindowCount: 273, @@ -74,6 +90,7 @@ export const prBacklog = Factory.define(() => ({ })); export const stalledSignals = Factory.define(() => ({ + status: 'measured', reposAtPrCap: [{ repo: 'acme/api', openPrs: 7 }], reposWithConfigButNoRecentPrs: ['acme/old-tool'], })); @@ -118,6 +135,8 @@ export const costEstimate = Factory.define(() => ({ export const cveExposureOk = Factory.define(() => ({ status: 'ok', + source: 'repo-endpoint', + disabledRepoStatus: 'measured', totalOpenAlerts: 7, bySeverity: { critical: 1, high: 3, medium: 2, low: 1 }, topReposBySeverity: [{ repo: 'acme/api', critical: 1, high: 2, medium: 1, low: 0 }], @@ -128,6 +147,8 @@ export const cveExposureOk = Factory.define(() => ({ export const cveExposureScopeMissing = Factory.define(() => ({ status: 'scope-missing', + source: 'repo-endpoint', + disabledRepoStatus: 'unknown', requiredScope: 'security_events', totalOpenAlerts: 0, bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, @@ -139,6 +160,7 @@ export const cveExposureScopeMissing = Factory.define(() => ({ export const reportBundle = Factory.define(() => ({ meta: reportMeta.build(), + measurements: reportMeasurements.build(), orgOverview: orgOverview.build(), dependabotCoverage: dependabotCoverage.build(), prBacklog: prBacklog.build(), diff --git a/src/report/web/App.browser.test.tsx b/src/report/web/App.browser.test.tsx index a9c2c0f..0dad2ef 100644 --- a/src/report/web/App.browser.test.tsx +++ b/src/report/web/App.browser.test.tsx @@ -161,6 +161,7 @@ describe('App report shell', () => { renderReport({ cve: { status: 'scope-missing', + disabledRepoStatus: 'unknown', requiredScope: 'security_events', totalOpenAlerts: 0, bySeverity: { critical: 0, high: 0, medium: 0, low: 0 }, diff --git a/src/report/web/App.stories.tsx b/src/report/web/App.stories.tsx index 277b1a8..2e50450 100644 --- a/src/report/web/App.stories.tsx +++ b/src/report/web/App.stories.tsx @@ -32,10 +32,10 @@ const sampleReport = toEmbeddedShape( reportBundle.build({ orgOverview: orgOverview.build({ topLanguages: [ - { language: 'TypeScript', bytes: 4_200_000, percentage: 58 }, - { language: 'Go', bytes: 1_600_000, percentage: 22 }, - { language: 'Python', bytes: 880_000, percentage: 12 }, - { language: 'Ruby', bytes: 560_000, percentage: 8 }, + { language: 'TypeScript', repoCount: 0, bytes: 4_200_000, percentage: 58 }, + { language: 'Go', repoCount: 0, bytes: 1_600_000, percentage: 22 }, + { language: 'Python', repoCount: 0, bytes: 880_000, percentage: 12 }, + { language: 'Ruby', repoCount: 0, bytes: 560_000, percentage: 8 }, ], }), cve: cveExposureOk.build({ diff --git a/src/report/web/App.tsx b/src/report/web/App.tsx index 7b3b55e..97c2c0a 100644 --- a/src/report/web/App.tsx +++ b/src/report/web/App.tsx @@ -30,6 +30,7 @@ export function App({ data }: { data: EmbeddedReportData }) {
+ {data.measurements.mode === 'budgeted' && } @@ -73,3 +74,12 @@ function ReportHeader({ org }: { org: string }) { ); } + +function MeasurementBanner() { + return ( +
+ Budgeted scan: GitHub API budget was limited, so some + lower-priority sections were not measured. Core Dependabot PR and CVE sections remain exact where shown. +
+ ); +} diff --git a/src/report/web/acts/CostStory.tsx b/src/report/web/acts/CostStory.tsx index 0acc250..3720565 100644 --- a/src/report/web/acts/CostStory.tsx +++ b/src/report/web/acts/CostStory.tsx @@ -26,6 +26,22 @@ export function CostStory() { const { humanMergeCount } = data.costEstimate; + if (data.prBacklog.status === 'failed') { + return ( +
+
+ {costStoryCopy.eyebrow} +
+

+ Dependabot PR toil was not measured +

+

+ PatchWave could not safely compute PR toil because GitHub returned incomplete PR data. +

+
+ ); + } + return (
diff --git a/src/report/web/acts/MethodologyAppendix.tsx b/src/report/web/acts/MethodologyAppendix.tsx index 2142d67..50fecca 100644 --- a/src/report/web/acts/MethodologyAppendix.tsx +++ b/src/report/web/acts/MethodologyAppendix.tsx @@ -6,6 +6,7 @@ import { useAssumptions } from '../hooks/useAssumptions.tsx'; import { type MethodologyTab, useAssumptionsDisclosure } from '../hooks/useAssumptionsDisclosure.tsx'; import { useRegisteredFootnotes } from '../hooks/useFootnotes.tsx'; import { Citation } from '../primitives/Citation.tsx'; +import { NotMeasured } from '../primitives/NotMeasured.tsx'; import { reposWithoutSecurityAlertsId } from './RiskStory.tsx'; export const methodologyAppendixTestIds = { @@ -158,8 +159,8 @@ export function MethodologyAppendix() { `${org.repoCount} active (${org.publicCount} public, ${org.privateCount} private, ${org.internalCount} internal)`, ], ['Archived excluded', org.archivedExcluded.toLocaleString()], - ['Active human committers', org.activeHumanCommitters.toLocaleString()], - ['Repos with branch protection', org.reposWithBranchProtection.toLocaleString()], + ['Active human committers', formatNullableNumber(org.activeHumanCommitters)], + ['Repos with branch protection', formatNullableNumber(org.reposWithBranchProtection)], ]} /> @@ -167,13 +168,24 @@ export function MethodologyAppendix() { - `${r.repo} (${r.openPrs} open)`)} - /> - + {stalled.status === 'skipped' ? ( +

+ + Stalled signals were not measured because Dependabot config was skipped. + +

+ ) : ( + <> + `${r.repo} (${r.openPrs} open)`)} + /> + + + )} {org.topLanguages.length > 0 && ( - +
    {org.topLanguages.map((l) => (
  • {l.language} - {fmtBytes(l.bytes)} ({l.percentage}%) + {org.languageSource === 'metadata' + ? `${l.repoCount.toLocaleString()} repos` + : fmtBytes(l.bytes)}{' '} + ({l.percentage}%)
    • ))} @@ -347,6 +375,10 @@ function DataPanel({ title, children }: { title: string; children: ReactNode }) ); } +function formatNullableNumber(value: number | null): string { + return value === null ? 'Not measured' : value.toLocaleString(); +} + function MetricList({ rows }: { rows: Array<[string, string]> }) { return (
    diff --git a/src/report/web/acts/OpenPrAgeStory.tsx b/src/report/web/acts/OpenPrAgeStory.tsx index df9f65d..66002d4 100644 --- a/src/report/web/acts/OpenPrAgeStory.tsx +++ b/src/report/web/acts/OpenPrAgeStory.tsx @@ -15,6 +15,21 @@ export const openPrAgeStoryCopy = { export function OpenPrAgeStory() { const { prBacklog: pr } = useEmbeddedData(); + if (pr.status === 'failed') { + return ( +
    +
    + {openPrAgeStoryCopy.eyebrow} +
    +

    + Dependabot PR backlog was not measured +

    +

    + GitHub returned incomplete PR search data, so PatchWave avoided showing a misleading zero backlog. +

    +
    + ); + } const hasOpenPrs = pr.openCount > 0; return ( diff --git a/src/report/web/acts/RiskStory.tsx b/src/report/web/acts/RiskStory.tsx index 15fcee1..76e9f9b 100644 --- a/src/report/web/acts/RiskStory.tsx +++ b/src/report/web/acts/RiskStory.tsx @@ -19,6 +19,7 @@ export const riskStoryTestIds = { export const riskStoryCopy = { eyebrow: 'CVE exposure', scopeMissingHeading: 'Not measured this run', + notMeasuredHeading: 'Not measured this run', noAlertsHeading: 'No open security alerts', } as const; @@ -29,6 +30,25 @@ export function RiskStory() { const { cve, orgOverview } = useEmbeddedData(); const { reveal } = useAssumptionsDisclosure(); + if (cve.status === 'not-measured') { + return ( +
    +
    + {riskStoryCopy.eyebrow} +
    +

    + {riskStoryCopy.notMeasuredHeading} +

    +

    + CVE exposure was not measured in this run to stay within the available GitHub API budget. +

    +
    + ); + } + if (cve.status === 'scope-missing') { return (
    @@ -70,7 +90,8 @@ export function RiskStory() {

    No open Dependabot security alerts across the repos in scope. That can mean you're caught up, or that security - alerts aren't enabled everywhere. Check the appendix data for repos with alerts disabled. + alerts aren't enabled everywhere. Check the appendix data for repos with alerts disabled when that metadata is + available.

    ); diff --git a/src/report/web/acts/Verdict.tsx b/src/report/web/acts/Verdict.tsx index 923f20a..4c9c509 100644 --- a/src/report/web/acts/Verdict.tsx +++ b/src/report/web/acts/Verdict.tsx @@ -19,9 +19,27 @@ export const verdictCopy = { export function Verdict() { const { derived } = useAssumptions(); - const { openCount } = useEmbeddedData().prBacklog; + const { prBacklog } = useEmbeddedData(); + const { openCount } = prBacklog; const analytics = useAnalytics(); + if (prBacklog.status === 'failed') { + return ( +
    +

    Dependabot PR toil

    +

    + Not measured +

    +

    + GitHub returned incomplete PR data, so PatchWave avoided showing a misleading zero-cost claim. +

    +
    + ); + } + return (

    {verdictCopy.costLeadIn}

    diff --git a/src/report/web/primitives/NotMeasured.tsx b/src/report/web/primitives/NotMeasured.tsx new file mode 100644 index 0000000..cbef43f --- /dev/null +++ b/src/report/web/primitives/NotMeasured.tsx @@ -0,0 +1,5 @@ +import type { ReactNode } from 'react'; + +export function NotMeasured({ children }: { children: ReactNode }) { + return {children}; +} diff --git a/src/scanPlan.ts b/src/scanPlan.ts new file mode 100644 index 0000000..aa170b9 --- /dev/null +++ b/src/scanPlan.ts @@ -0,0 +1,202 @@ +import type { TargetKind } from './collectors/repos.ts'; +import type { RateLimitStatus } from './github/rateLimit.ts'; + +export type CollectorKey = + | 'languages' + | 'dependabotConfig' + | 'cve' + | 'branchProtection' + | 'contributors' + | 'dependabotPrs'; + +export type CollectorMode = 'exact' | 'metadata' | 'org-endpoint'; + +export type SkipReason = 'budget' | 'unsupported-target' | 'rate-limit-unavailable'; + +export interface SkippedCollector { + readonly reason: SkipReason; + readonly message: string; +} + +export interface ScanPlan { + readonly restBudget: number; + readonly graphqlBudget: number; + readonly restReserve: number; + readonly include: ReadonlySet; + readonly modes: Partial>; + readonly skipped: Partial>; + readonly estimatedRestCost: Record; + readonly estimatedGraphqlCost: Record; +} + +export interface ApiBudget { + readonly restRemaining: number; + readonly graphqlRemaining: number; + readonly source: 'github' | 'default'; +} + +export interface BuildScanPlanInput { + readonly targetKind: TargetKind; + readonly repoCount: number; + readonly apiBudget: ApiBudget; +} + +export const COLLECTOR_KEYS: readonly CollectorKey[] = [ + 'languages', + 'dependabotConfig', + 'cve', + 'branchProtection', + 'contributors', + 'dependabotPrs', +]; + +const REST_RESERVE = 500; +const DEFAULT_REST_REMAINING = 5_000; +const DEFAULT_GRAPHQL_REMAINING = 5_000; +const ORG_CVE_ESTIMATED_REST_COST = 25; +const CONTRIBUTORS_MAX_REPOS_FOR_FULL_SCAN = 500; + +export const DEFAULT_API_BUDGET: ApiBudget = { + restRemaining: DEFAULT_REST_REMAINING, + graphqlRemaining: DEFAULT_GRAPHQL_REMAINING, + source: 'default', +}; + +export function apiBudgetFromRateLimit(rateLimit: RateLimitStatus): ApiBudget { + return { + restRemaining: rateLimit.rest.remaining, + graphqlRemaining: rateLimit.graphql.remaining, + source: 'github', + }; +} + +export function buildScanPlan(input: BuildScanPlanInput): ScanPlan { + const { targetKind, repoCount, apiBudget } = input; + const restBudget = Math.max(0, apiBudget.restRemaining - REST_RESERVE); + const graphqlBudget = apiBudget.graphqlRemaining; + let remainingRest = restBudget; + + const include = new Set(); + const modes: Partial> = {}; + const skipped: Partial> = {}; + const estimatedRestCost = buildRestEstimates(targetKind, repoCount); + const estimatedGraphqlCost = buildGraphqlEstimates(graphqlBudget); + + add(include, modes, 'dependabotPrs', 'exact'); + + if (targetKind === 'org' && remainingRest > 0) { + add(include, modes, 'cve', 'org-endpoint'); + remainingRest -= Math.min(remainingRest, estimatedRestCost.cve); + } else if (targetKind === 'user' && spend(remainingRest, estimatedRestCost.cve)) { + add(include, modes, 'cve', 'exact'); + remainingRest -= estimatedRestCost.cve; + } else { + skip( + skipped, + 'cve', + 'budget', + 'CVE exposure was not measured because the remaining REST budget could not cover it.', + ); + } + + if (repoCount <= 500 && spend(remainingRest, estimatedRestCost.languages)) { + add(include, modes, 'languages', 'exact'); + remainingRest -= estimatedRestCost.languages; + } else { + add(include, modes, 'languages', 'metadata'); + } + + if (spend(remainingRest, estimatedRestCost.dependabotConfig)) { + add(include, modes, 'dependabotConfig', 'exact'); + remainingRest -= estimatedRestCost.dependabotConfig; + } else { + skip( + skipped, + 'dependabotConfig', + 'budget', + 'Dependabot config parsing was skipped to stay within the GitHub API budget.', + ); + } + + if (spend(remainingRest, estimatedRestCost.branchProtection)) { + add(include, modes, 'branchProtection', 'exact'); + remainingRest -= estimatedRestCost.branchProtection; + } else { + skip(skipped, 'branchProtection', 'budget', 'Branch protection was skipped to stay within the GitHub API budget.'); + } + + if (repoCount <= CONTRIBUTORS_MAX_REPOS_FOR_FULL_SCAN && spend(remainingRest, estimatedRestCost.contributors)) { + add(include, modes, 'contributors', 'exact'); + } else { + skip( + skipped, + 'contributors', + 'budget', + 'Active committers were skipped because this collector can paginate heavily on large orgs.', + ); + } + + return { + restBudget, + graphqlBudget, + restReserve: REST_RESERVE, + include, + modes, + skipped, + estimatedRestCost, + estimatedGraphqlCost, + }; +} + +export function countSkipped(plan: ScanPlan): number { + return Object.keys(plan.skipped).length; +} + +export function isBudgetConstrained(plan: ScanPlan): boolean { + return countSkipped(plan) > 0 || Object.values(plan.modes).some((mode) => mode === 'metadata'); +} + +function buildRestEstimates(targetKind: TargetKind, repoCount: number): Record { + return { + dependabotPrs: 0, + cve: targetKind === 'org' ? ORG_CVE_ESTIMATED_REST_COST : repoCount, + languages: repoCount, + dependabotConfig: repoCount * 2, + branchProtection: repoCount * 2, + contributors: repoCount * 2, + }; +} + +function buildGraphqlEstimates(graphqlBudget: number): Record { + return { + dependabotPrs: Math.min(100, graphqlBudget), + cve: 0, + languages: 0, + dependabotConfig: 0, + branchProtection: 0, + contributors: 0, + }; +} + +function add( + include: Set, + modes: Partial>, + collector: CollectorKey, + mode: CollectorMode, +): void { + include.add(collector); + modes[collector] = mode; +} + +function skip( + skipped: Partial>, + collector: CollectorKey, + reason: SkipReason, + message: string, +): void { + skipped[collector] = { reason, message }; +} + +function spend(remaining: number, cost: number): boolean { + return cost <= remaining; +} diff --git a/src/testFactories.ts b/src/testFactories.ts index a614221..2ffb17b 100644 --- a/src/testFactories.ts +++ b/src/testFactories.ts @@ -32,6 +32,7 @@ export const repoMeta = Factory.define(() => ({ primaryLanguage: 'TypeScript', pushedAt: '2026-04-01T00:00:00Z', dependabotSecurityUpdates: true, + dependabotAlertsEnabled: true, })); export const checkSummary = Factory.define(() => ({ @@ -139,5 +140,13 @@ export const collectedData = Factory.define(() => ({ cve: [cveSliceOk.build()], branchProtection: [branchProtectionSlice.build()], contributors: [contributorSlice.build()], + measurements: [ + { collector: 'languages', status: 'measured', mode: 'exact' }, + { collector: 'dependabotConfig', status: 'measured', mode: 'exact' }, + { collector: 'cve', status: 'measured', mode: 'exact' }, + { collector: 'branchProtection', status: 'measured', mode: 'exact' }, + { collector: 'contributors', status: 'measured', mode: 'exact' }, + { collector: 'dependabotPrs', status: 'measured', mode: 'exact' }, + ], errors: [], })); diff --git a/src/testHelpers/FakeGithubClient.ts b/src/testHelpers/FakeGithubClient.ts index e674173..2ede483 100644 --- a/src/testHelpers/FakeGithubClient.ts +++ b/src/testHelpers/FakeGithubClient.ts @@ -40,8 +40,30 @@ export interface Stub { */ export class FakeGithubClient implements GithubClient { readonly calls: GithubCall[] = []; - private readonly paginateResponders: ParamResponder[] = []; - private readonly requestResponders: ParamResponder[] = []; + private readonly paginateResponders: ParamResponder[] = [ + { + route: 'GET /orgs/{org}/dependabot/alerts', + paramsMatcher: {}, + outcome: { kind: 'ok', value: [] }, + label: 'GET /orgs/{org}/dependabot/alerts {}', + }, + ]; + private readonly requestResponders: ParamResponder[] = [ + { + route: 'GET /rate_limit', + paramsMatcher: {}, + outcome: { + kind: 'ok', + value: { + resources: { + core: { limit: 5_000, remaining: 5_000, used: 0, reset: 1_779_456_000 }, + graphql: { limit: 5_000, remaining: 5_000, used: 0, reset: 1_779_456_000 }, + }, + }, + }, + label: 'GET /rate_limit {}', + }, + ]; private readonly graphqlResponders: GraphqlResponder[] = []; onPaginate(route: string, paramsMatcher: Record = {}): Stub { diff --git a/src/types.ts b/src/types.ts index 2f0297e..3d57ad5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,4 @@ +import type { CollectorKey } from './scanPlan.ts'; import type { Instant } from './time.ts'; export type Visibility = 'public' | 'private' | 'internal'; @@ -15,6 +16,7 @@ export interface RepoMeta extends RepoRef { primaryLanguage: string | null; pushedAt: string | null; dependabotSecurityUpdates: boolean | null; + dependabotAlertsEnabled: boolean | null; } export interface LanguageBytes { @@ -109,6 +111,17 @@ export interface CollectionContext { now: Instant; } +export type CollectorMeasurementStatus = 'measured' | 'partial' | 'skipped' | 'failed'; + +export interface CollectorMeasurement { + readonly collector: CollectorKey; + readonly status: CollectorMeasurementStatus; + readonly mode?: 'exact' | 'metadata' | 'org-endpoint'; + readonly reason?: string; + readonly estimatedRestCost?: number; + readonly estimatedGraphqlCost?: number; +} + export interface CollectedData { ctx: CollectionContext; repos: RepoMeta[]; @@ -118,6 +131,7 @@ export interface CollectedData { cve: CveSlice[]; branchProtection: BranchProtectionSlice[]; contributors: ContributorSlice[]; + measurements: CollectorMeasurement[]; errors: CollectorWarning[]; }