diff --git a/apps/bullmq/src/scheduled-jobs/__tests__/brain-collectors.test.ts b/apps/bullmq/src/scheduled-jobs/__tests__/brain-collectors.test.ts index 4046d21e9..b8bf1e35c 100644 --- a/apps/bullmq/src/scheduled-jobs/__tests__/brain-collectors.test.ts +++ b/apps/bullmq/src/scheduled-jobs/__tests__/brain-collectors.test.ts @@ -24,6 +24,7 @@ import { isSlackHumanProfile, runBrainCollectors, selectPersonIdentityBatch, + slackDirectoryPageUserIds, slackDirectoryProfileFromApi, type BrainCollector, type BrainSink, @@ -1056,6 +1057,7 @@ describe('person identity pages', () => { expect(page.slug).toMatch(/^people\/roomote-member-[a-f0-9]{16}$/); expect(page.title).toBe('Dan Riccio'); expect(page.content).toContain('type: person'); + expect(page.content).toContain('event_date: 2026-01-01'); expect(page.content).toContain('job_title: "VP of Engineering"'); expect(page.content).toContain('daniel-lxs'); expect(page.content).toContain('U08TMEM25CP'); @@ -1140,6 +1142,32 @@ describe('person identity pages', () => { ).toBe(false); }); + it('keeps a Slack profile anchored before later profile updates', () => { + const profile = slackDirectoryProfileFromApi({ + teamId: 'TROOMOTE', + teamName: 'Roomote', + firstKnownAt: new Date('2026-07-01T00:00:00Z'), + observedAt: new Date('2026-09-01T00:00:00Z'), + user: { + id: 'UADA', + updated: new Date('2026-08-15T00:00:00Z').getTime() / 1000, + }, + }); + + expect(profile?.firstKnownAt).toEqual(new Date('2026-07-01T00:00:00Z')); + }); + + it('scopes cached Slack profiles to the current API page', () => { + expect( + slackDirectoryPageUserIds([ + { id: ' U1 ' }, + { id: 'U2' }, + { id: 'U1' }, + {}, + ]), + ).toEqual(['U1', 'U2']); + }); + it('refreshes a newly connected Slack workspace immediately', () => { const now = new Date('2026-08-15T12:00:00Z'); @@ -1177,12 +1205,14 @@ describe('person identity pages', () => { isBot: false, isAppUser: false, profileUpdatedAt: new Date('2026-08-15T00:00:00Z'), + firstKnownAt: new Date('2026-07-01T00:00:00Z'), }; const page = buildSlackDirectoryPersonPage(profile); expect(page.slug).toMatch(/^people\/slack-member-[a-f0-9]{16}$/); expect(page.title).toBe('Ada'); expect(page.content).toContain('type: person'); + expect(page.content).toContain('event_date: 2026-07-01'); expect(page.content).toContain('job_title: "Mathematician"'); expect(page.content).toContain('Title: Mathematician'); expect(page.content).toContain('- Slack: ada (UADA)'); @@ -1202,11 +1232,17 @@ describe('person identity pages', () => { isBot: false, isAppUser: false, profileUpdatedAt: new Date('2026-08-15T00:00:00Z'), + firstKnownAt: new Date('2026-07-01T00:00:00Z'), + }, + { + slug: 'people/roomote-member-abc', + title: 'Dan Riccio', + effectiveDate: new Date('2026-08-01T00:00:00Z'), }, - { slug: 'people/roomote-member-abc', title: 'Dan Riccio' }, ); expect(page.content).toContain('type: person-alias'); + expect(page.content).toContain('event_date: 2026-07-01'); expect(page.content).toContain('canonical: "people/roomote-member-abc"'); expect(page.content).toContain('[Dan Riccio](people/roomote-member-abc)'); }); diff --git a/apps/bullmq/src/scheduled-jobs/__tests__/brain-outbox-drain.test.ts b/apps/bullmq/src/scheduled-jobs/__tests__/brain-outbox-drain.test.ts index 8d215e67e..f10c977a0 100644 --- a/apps/bullmq/src/scheduled-jobs/__tests__/brain-outbox-drain.test.ts +++ b/apps/bullmq/src/scheduled-jobs/__tests__/brain-outbox-drain.test.ts @@ -97,7 +97,7 @@ describe('PR fact resume cursor', () => { }); describe('pull request fact pages', () => { - it('uses the remote update date instead of the local sync date', () => { + it('uses the merge occurrence instead of later remote updates', () => { const page = buildPullRequestFactPage({ repositoryFullName: 'owner/repo', prNumber: 42, @@ -105,13 +105,41 @@ describe('pull request fact pages', () => { htmlUrl: 'https://example.test/owner/repo/pull/42', authorLogin: 'octocat', state: 'merged', + createdAtRemote: new Date('2026-08-01T09:00:00Z'), + closedAtRemote: new Date('2026-08-14T10:00:00Z'), mergedAtRemote: new Date('2026-08-14T10:00:00Z'), - updatedAtRemote: new Date('2026-08-13T11:00:00Z'), }); - expect(page.content).toContain('\ndate: 2026-08-13\n'); + expect(page.content).toContain('\nevent_date: 2026-08-14\n'); expect(page.content).toContain('\nmerged_at: 2026-08-14T10:00:00.000Z\n'); }); + + it('uses close then creation dates for unmerged pull requests', () => { + const base = { + repositoryFullName: 'owner/repo', + prNumber: 42, + title: 'Ship it', + htmlUrl: 'https://example.test/owner/repo/pull/42', + authorLogin: 'octocat', + createdAtRemote: new Date('2026-08-01T09:00:00Z'), + mergedAtRemote: null, + }; + + expect( + buildPullRequestFactPage({ + ...base, + state: 'closed', + closedAtRemote: new Date('2026-08-09T10:00:00Z'), + }).content, + ).toContain('\nevent_date: 2026-08-09\n'); + expect( + buildPullRequestFactPage({ + ...base, + state: 'open', + closedAtRemote: null, + }).content, + ).toContain('\nevent_date: 2026-08-01\n'); + }); }); describe('collector continuation orchestration', () => { @@ -168,8 +196,9 @@ describe('collector continuation orchestration', () => { htmlUrl: 'https://example.test/owner/repo/pull/42', authorLogin: 'octocat', state: 'merged', + createdAtRemote: new Date('2026-08-13T10:00:00Z'), + closedAtRemote: new Date('2026-08-14T10:00:00Z'), mergedAtRemote: new Date('2026-08-14T10:00:00Z'), - updatedAtRemote: new Date('2026-08-14T10:30:00Z'), updatedAt: new Date('2026-08-14T11:00:00Z'), }, ]); diff --git a/apps/bullmq/src/scheduled-jobs/brain-collectors.ts b/apps/bullmq/src/scheduled-jobs/brain-collectors.ts index 2460303ee..12c3d05f9 100644 --- a/apps/bullmq/src/scheduled-jobs/brain-collectors.ts +++ b/apps/bullmq/src/scheduled-jobs/brain-collectors.ts @@ -7,6 +7,7 @@ import { eq, getBrainSyncState, githubUserMappings, + inArray, isNull, listBrainCollectorItems, listBrainCollectorItemsBefore, @@ -1129,7 +1130,11 @@ export type PersonIdentityRecord = { providers: PersonIdentityProvider[]; }; -type PersonIdentityReference = { slug: string; title: string }; +type PersonIdentityReference = { + slug: string; + title: string; + effectiveDate?: Date; +}; const LEGACY_SETUP_BOOTSTRAP_USER_ID = 'setup-bootstrap-user'; @@ -1207,6 +1212,7 @@ export function buildPersonIdentityPage( 'type: person', `aliases: ${JSON.stringify(aliases)}`, `status: ${deleted ? 'deleted' : 'active'}`, + `event_date: ${formatUtcDay(record.createdAt)}`, ...(jobTitle ? [`job_title: ${JSON.stringify(jobTitle)}`] : []), 'provenance: roomote-person-identities', '---', @@ -1250,6 +1256,7 @@ export function buildPersonIdentityLookup( const reference = { slug: personIdentitySlug(record.userId), title: personIdentityDisplayName(record), + effectiveDate: record.createdAt, }; // Email is an internal linking hint for meeting attendees, not Brain page @@ -1389,7 +1396,8 @@ async function loadPersonIdentityRecords(): Promise { return [...byUserId.values()]; } -const SLACK_DIRECTORY_COLLECTOR_ID = 'slack-person-directory'; +const SLACK_DIRECTORY_COLLECTOR_ID = + 'slack-person-directory:occurrence-date-v2'; const SLACK_DIRECTORY_REFRESH_MS = 24 * 60 * 60 * 1000; const SLACK_DIRECTORY_PAGE_SIZE = 100; @@ -1419,6 +1427,7 @@ export type SlackDirectoryProfile = { isBot: boolean; isAppUser: boolean; profileUpdatedAt: Date | null; + firstKnownAt: Date; }; type RawSlackDirectoryUser = { @@ -1436,6 +1445,18 @@ type RawSlackDirectoryUser = { }; }; +export function slackDirectoryPageUserIds( + users: Array<{ id?: string }>, +): string[] { + return [ + ...new Set( + users + .map((user) => user.id?.trim()) + .filter((id): id is string => Boolean(id)), + ), + ]; +} + function slackDirectoryIdentityKey(teamId: string, userId: string): string { return `${teamId}/${userId}`; } @@ -1452,6 +1473,8 @@ export function slackDirectoryProfileFromApi(input: { teamId: string; teamName: string; user: RawSlackDirectoryUser; + firstKnownAt?: Date; + observedAt?: Date; }): SlackDirectoryProfile | null { const slackUserId = input.user.id?.trim(); if (!slackUserId) return null; @@ -1459,6 +1482,16 @@ export function slackDirectoryProfileFromApi(input: { const toOptional = (value: string | undefined) => value?.trim() || null; const updatedSeconds = input.user.updated; + const profileUpdatedAt = + typeof updatedSeconds === 'number' && Number.isFinite(updatedSeconds) + ? new Date(updatedSeconds * 1000) + : null; + const firstKnownAt = [input.firstKnownAt, profileUpdatedAt, input.observedAt] + .filter((date): date is Date => Boolean(date)) + .sort((a, b) => a.getTime() - b.getTime())[0]; + + if (!firstKnownAt) return null; + return { slackUserId, slackTeamId: input.teamId, @@ -1470,10 +1503,8 @@ export function slackDirectoryProfileFromApi(input: { isDeleted: input.user.deleted === true, isBot: input.user.is_bot === true, isAppUser: input.user.is_app_user === true, - profileUpdatedAt: - typeof updatedSeconds === 'number' && Number.isFinite(updatedSeconds) - ? new Date(updatedSeconds * 1000) - : null, + profileUpdatedAt, + firstKnownAt, }; } @@ -1509,6 +1540,14 @@ export function buildSlackDirectoryPersonPage( const name = slackDirectoryDisplayName(profile); const safeWorkspace = brainSafeIdentityValue(profile.slackTeamName) || 'Slack workspace'; + const effectiveDate = canonical?.effectiveDate + ? new Date( + Math.min( + canonical.effectiveDate.getTime(), + profile.firstKnownAt.getTime(), + ), + ) + : profile.firstKnownAt; if (canonical) { return { @@ -1518,6 +1557,7 @@ export function buildSlackDirectoryPersonPage( '---', 'type: person-alias', `canonical: ${JSON.stringify(canonical.slug)}`, + `event_date: ${formatUtcDay(effectiveDate)}`, 'provenance: slack-directory', '---', '', @@ -1547,6 +1587,7 @@ export function buildSlackDirectoryPersonPage( 'type: person', `aliases: ${JSON.stringify(aliases)}`, `status: ${profile.isDeleted ? 'deleted' : 'active'}`, + `event_date: ${formatUtcDay(effectiveDate)}`, ...(safeTitle ? [`job_title: ${JSON.stringify(safeTitle)}`] : []), 'provenance: slack-directory', `workspace: ${JSON.stringify(safeWorkspace)}`, @@ -1571,7 +1612,9 @@ async function loadSlackCanonicalIdentityLookup(): Promise< Map > { const mappings = await db.query.slackUserMappings.findMany({ - with: { user: { columns: { name: true, deletedAt: true } } }, + with: { + user: { columns: { name: true, deletedAt: true, createdAt: true } }, + }, }); const lookup = new Map(); @@ -1587,6 +1630,7 @@ async function loadSlackCanonicalIdentityLookup(): Promise< { slug: personIdentitySlug(mapping.userId), title: brainSafeIdentityValue(mapping.user.name) || 'Roomote member', + effectiveDate: mapping.user.createdAt, }, ); } @@ -1648,36 +1692,53 @@ async function readSlackDirectoryBatch(input: { now: Date; }): Promise { const client = createSlackWebClient(input.installation.botAccessToken); - const canonical = await loadSlackCanonicalIdentityLookup(); - const response = await client.users.list({ - limit: Math.min(input.limit, SLACK_DIRECTORY_PAGE_SIZE), - ...(input.slackCursor ? { cursor: input.slackCursor } : {}), - }); + const [canonical, response] = await Promise.all([ + loadSlackCanonicalIdentityLookup(), + client.users.list({ + limit: Math.min(input.limit, SLACK_DIRECTORY_PAGE_SIZE), + ...(input.slackCursor ? { cursor: input.slackCursor } : {}), + }), + ]); const listed = (response.members ?? []) as RawSlackDirectoryUser[]; + const pageUserIds = slackDirectoryPageUserIds(listed); // users.list supplies the directory and full profile shape. Refresh linked // Roomote identities with users.info so their canonical cards pick up the // freshest Slack name/title even if a paginated directory snapshot lags. - const refreshed = await Promise.all( - listed.map(async (user) => { - if ( - !user.id || - !canonical.has( - slackDirectoryIdentityKey(input.installation.teamId, user.id), - ) - ) { - return user; - } - try { - const info = await client.users.info({ user: user.id }); - return (info.user as RawSlackDirectoryUser | undefined) ?? user; - } catch (error) { - console.warn( - `${LOG_PREFIX} slack team ${input.installation.teamId}: users.info failed for a linked member; using users.list profile: ${error instanceof Error ? error.message : String(error)}`, - ); - return user; - } - }), + const [existingProfiles, refreshed] = await Promise.all([ + pageUserIds.length > 0 + ? db.query.slackDirectoryUsers.findMany({ + where: and( + eq(slackDirectoryUsers.slackTeamId, input.installation.teamId), + inArray(slackDirectoryUsers.slackUserId, pageUserIds), + ), + columns: { slackUserId: true, createdAt: true }, + }) + : Promise.resolve([]), + Promise.all( + listed.map(async (user) => { + if ( + !user.id || + !canonical.has( + slackDirectoryIdentityKey(input.installation.teamId, user.id), + ) + ) { + return user; + } + try { + const info = await client.users.info({ user: user.id }); + return (info.user as RawSlackDirectoryUser | undefined) ?? user; + } catch (error) { + console.warn( + `${LOG_PREFIX} slack team ${input.installation.teamId}: users.info failed for a linked member; using users.list profile: ${error instanceof Error ? error.message : String(error)}`, + ); + return user; + } + }), + ), + ]); + const existingCreatedAt = new Map( + existingProfiles.map((profile) => [profile.slackUserId, profile.createdAt]), ); const profiles = refreshed .map((user) => @@ -1685,6 +1746,8 @@ async function readSlackDirectoryBatch(input: { teamId: input.installation.teamId, teamName: input.installation.teamName, user, + firstKnownAt: user.id ? existingCreatedAt.get(user.id) : undefined, + observedAt: input.installation.createdAt, }), ) .filter((profile): profile is SlackDirectoryProfile => Boolean(profile)); @@ -1851,7 +1914,8 @@ const slackPersonDirectoryCollector: BrainCollector = { }, }; -const PERSON_IDENTITIES_STATE_ID = 'person-identities:members'; +const PERSON_IDENTITIES_STATE_ID = + 'person-identities:members:occurrence-date-v2'; const PERSON_IDENTITIES_RECONCILIATION_MS = 24 * 60 * 60 * 1000; const GRANOLA_MEETINGS_COLLECTOR_ID = 'granola-meetings'; @@ -3142,9 +3206,8 @@ const granolaMeetingsCollector: BrainCollector = { * limit can never masquerade as brain-side backpressure. */ const githubIssuesCollector: BrainCollector = { - // Versioned once so the historical backfill rewrites pages created before - // GitHub issue effective dates were explicit. - id: 'github-issues:effective-date-v2', + // Version when date semantics change so the deep backfill rewrites history. + id: 'github-issues:occurrence-date-v3', displayName: 'GitHub issues', async isEnabled() { return hasBrainGithubSources(); diff --git a/apps/bullmq/src/scheduled-jobs/brain-outbox-drain.ts b/apps/bullmq/src/scheduled-jobs/brain-outbox-drain.ts index d71d1793b..f84596853 100644 --- a/apps/bullmq/src/scheduled-jobs/brain-outbox-drain.ts +++ b/apps/bullmq/src/scheduled-jobs/brain-outbox-drain.ts @@ -26,15 +26,15 @@ import { runBrainCollectors } from './brain-collectors'; const LOG_PREFIX = '[brainOutboxDrain]'; /** Sync-state key for the one-time task-history backfill. */ -const TASK_MEMORY_COLLECTOR_ID = 'task-memory'; +const TASK_MEMORY_COLLECTOR_ID = 'task-memory:effective-date-v2'; const CLAIM_BATCH_SIZE = 10; // Backfill can enqueue a deployment's whole task history at once; drain up // to this many batches per tick so the backlog clears in minutes, not hours. const MAX_BATCHES_PER_TICK = 20; const MAX_ATTEMPTS = 5; -// Versioned once so existing pages that predate explicit effective dates are -// replayed and corrected instead of retaining their ingestion date forever. -const PR_FACTS_COLLECTOR_ID = 'pull-request-facts:effective-date-v2'; +// Versioned when date semantics change so existing pages are replayed and +// corrected instead of retaining stale effective dates forever. +const PR_FACTS_COLLECTOR_ID = 'pull-request-facts:occurrence-date-v3'; // PR analytics gives every repository in one sync the same timestamp but // writes repositories sequentially. Re-read a bounded window on each normal // collector tick so a row committed late with that shared timestamp cannot @@ -293,7 +293,9 @@ async function backfillTaskHistoryOnce(): Promise { return; } - const enqueued = await backfillBrainMemoryEvents(db); + const enqueued = await backfillBrainMemoryEvents(db, { + requeueCompleted: true, + }); await upsertBrainSyncState(db, TASK_MEMORY_COLLECTOR_ID, { backfillCompletedAt: new Date(), @@ -574,13 +576,16 @@ export function buildPullRequestFactPage(fact: { htmlUrl: string; authorLogin: string | null; state: string; + createdAtRemote: Date; + closedAtRemote: Date | null; mergedAtRemote: Date | null; - updatedAtRemote: Date; }): IngestPage { const merged = fact.mergedAtRemote?.toISOString(); + const occurredAt = + fact.mergedAtRemote ?? fact.closedAtRemote ?? fact.createdAtRemote; const content = [ '---', - `date: ${fact.updatedAtRemote.toISOString().slice(0, 10)}`, + `event_date: ${occurredAt.toISOString().slice(0, 10)}`, `repository: ${fact.repositoryFullName}`, `pr_number: ${fact.prNumber}`, `state: ${fact.state}`, @@ -635,8 +640,9 @@ async function syncPullRequestFacts( htmlUrl: pullRequestFacts.htmlUrl, authorLogin: pullRequestFacts.authorLogin, state: pullRequestFacts.state, + createdAtRemote: pullRequestFacts.createdAtRemote, + closedAtRemote: pullRequestFacts.closedAtRemote, mergedAtRemote: pullRequestFacts.mergedAtRemote, - updatedAtRemote: pullRequestFacts.updatedAtRemote, updatedAt: pullRequestFacts.updatedAt, }) .from(pullRequestFacts) diff --git a/packages/db/src/lib/__tests__/brain.test.ts b/packages/db/src/lib/__tests__/brain.test.ts index 64a0916b5..f5e52a704 100644 --- a/packages/db/src/lib/__tests__/brain.test.ts +++ b/packages/db/src/lib/__tests__/brain.test.ts @@ -247,6 +247,29 @@ describe('backfillBrainMemoryEvents', () => { expect(completedEvents).toHaveLength(1); expect(runningEvents).toHaveLength(0); }); + + it('requeues completed memories for a one-time metadata replay', async () => { + const completed = await makeCompletedRun(); + await saveBrainAgentSummary(db, completed.id, 'Keep this summary.'); + const [event] = await db + .select() + .from(brainMemoryEvents) + .where(eq(brainMemoryEvents.runId, completed.id)); + await markBrainMemoryEvent(db, event!.id, 'done'); + + await backfillBrainMemoryEvents(db, { requeueCompleted: true }); + + const [requeued] = await db + .select() + .from(brainMemoryEvents) + .where(eq(brainMemoryEvents.runId, completed.id)); + expect(requeued).toMatchObject({ + status: 'pending', + attempts: 0, + lastError: null, + agentSummary: 'Keep this summary.', + }); + }); }); describe('claimPendingBrainMemoryEvents', () => { diff --git a/packages/db/src/lib/brain.ts b/packages/db/src/lib/brain.ts index 646acc2b3..2cafeb4f8 100644 --- a/packages/db/src/lib/brain.ts +++ b/packages/db/src/lib/brain.ts @@ -215,7 +215,19 @@ export async function saveBrainAgentSummary( */ export async function backfillBrainMemoryEvents( database: DatabaseOrTransaction, + options: { requeueCompleted?: boolean } = {}, ): Promise { + const requeued = options.requeueCompleted + ? ((await database.execute( + sql`UPDATE ${brainMemoryEvents} AS event + SET status = 'pending', attempts = 0, last_error = NULL, updated_at = now() + FROM ${taskRuns} AS run + WHERE event.run_id = run.id + AND event.status = 'done' + AND run.status = 'completed' + RETURNING event.id`, + )) as unknown as Array<{ id: string }>) + : []; const rows = (await database.execute( sql`INSERT INTO ${brainMemoryEvents} (run_id) SELECT id FROM ${taskRuns} WHERE status = 'completed' @@ -223,7 +235,7 @@ export async function backfillBrainMemoryEvents( RETURNING id`, )) as unknown as Array<{ id: string }>; - return rows.length; + return requeued.length + rows.length; } /** diff --git a/packages/sdk/src/server/lib/__tests__/brain-github.test.ts b/packages/sdk/src/server/lib/__tests__/brain-github.test.ts index 80be9dadd..a49e46deb 100644 --- a/packages/sdk/src/server/lib/__tests__/brain-github.test.ts +++ b/packages/sdk/src/server/lib/__tests__/brain-github.test.ts @@ -83,7 +83,7 @@ describe('buildGithubIssuePage', () => { 'acme/widgets#42: Sandbox boots without the preview proxy', ); expect(page?.content).toContain('repository: acme/widgets'); - expect(page?.content).toContain('\ndate: 2026-08-03\n'); + expect(page?.content).toContain('\nevent_date: 2026-08-03\n'); expect(page?.content).toContain('state: closed'); expect(page?.content).toContain('labels: bug, previews'); expect(page?.content).toContain('author: ada'); @@ -107,14 +107,14 @@ describe('buildGithubIssuePage', () => { expect(page?.content).not.toContain('## Discussion'); }); - it('falls back to the creation date when no update date is available', () => { + it('uses the creation date for an open issue despite later updates', () => { const page = buildGithubIssuePage({ fullName: 'acme/widgets', - issue: { ...issue, updated_at: undefined, closed_at: undefined }, + issue: { ...issue, closed_at: undefined }, comments: [], }); - expect(page?.content).toContain('\ndate: 2026-08-01\n'); + expect(page?.content).toContain('\nevent_date: 2026-08-01\n'); }); it('caps long bodies and comment bodies', () => { @@ -165,8 +165,12 @@ describe('GitHub issue collector progress', () => { { fullName: 'acme/a', installationId: 1 }, { fullName: 'acme/b', installationId: 2 }, ]; - githubMocks.syncState.set('github-issues:acme/a', { watermark: start }); - githubMocks.syncState.set('github-issues:acme/b', { watermark: start }); + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { + watermark: start, + }); + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/b', { + watermark: start, + }); githubMocks.listForRepo.mockImplementation( async ({ repo, @@ -197,7 +201,7 @@ describe('GitHub issue collector progress', () => { const first = await collectBrainGithubIssues({ now, limit: 100 }); expect(first.pages).toHaveLength(100); expect(first.stateUpdates.map((update) => update.collectorId)).toEqual([ - 'github-issues:acme/a', + 'github-issues:occurrence-date-v2:acme/a', ]); for (const update of first.stateUpdates) { githubMocks.syncState.set(update.collectorId, { @@ -220,7 +224,9 @@ describe('GitHub issue collector progress', () => { const tiedAt = '2026-08-02T00:00:00Z'; const now = new Date('2026-08-15T00:00:00Z'); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { watermark: start }); + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { + watermark: start, + }); githubMocks.listForRepo.mockImplementation( async ({ page = 1 }: { page?: number }) => ({ data: @@ -234,7 +240,7 @@ describe('GitHub issue collector progress', () => { const first = await collectBrainGithubIssues({ now, limit: 100 }); expect(first.stateUpdates[0]).toMatchObject({ - collectorId: 'github-issues:acme/a', + collectorId: 'github-issues:occurrence-date-v2:acme/a', watermark: new Date(tiedAt), }); const firstCursor = JSON.parse(first.stateUpdates[0]!.cursor!) as { @@ -245,7 +251,7 @@ describe('GitHub issue collector progress', () => { expect(firstCursor.seen.map(([number]) => number)).toEqual( Array.from({ length: 100 }, (_, index) => index + 1), ); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: first.stateUpdates[0]!.watermark, backfillCursor: first.stateUpdates[0]!.cursor, }); @@ -262,7 +268,7 @@ describe('GitHub issue collector progress', () => { page: 2, }); expect(second.stateUpdates[0]).toMatchObject({ - collectorId: 'github-issues:acme/a', + collectorId: 'github-issues:occurrence-date-v2:acme/a', watermark: new Date(now.getTime() - 1000), }); expect(second.stateUpdates[0]?.cursor).not.toBeNull(); @@ -275,7 +281,9 @@ describe('GitHub issue collector progress', () => { new Date(start.getTime() + (index + 1) * 60_000).toISOString(), ); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { watermark: start }); + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { + watermark: start, + }); githubMocks.listForRepo .mockResolvedValueOnce({ data: Array.from({ length: 100 }, (_, index) => @@ -294,7 +302,7 @@ describe('GitHub issue collector progress', () => { }); const first = await collectBrainGithubIssues({ now, limit: 100 }); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: first.stateUpdates[0]!.watermark, backfillCursor: first.stateUpdates[0]!.cursor, }); @@ -321,7 +329,9 @@ describe('GitHub issue collector progress', () => { makeIssue(index + 2, tiedAt), ); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { watermark: start }); + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { + watermark: start, + }); githubMocks.listForRepo .mockResolvedValueOnce({ data: firstPage }) // The mutation occurs after this replay: issue 101 shifts into page 1, @@ -334,7 +344,7 @@ describe('GitHub issue collector progress', () => { .mockResolvedValueOnce({ data: [] }); const first = await collectBrainGithubIssues({ now, limit: 100 }); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: first.stateUpdates[0]!.watermark, backfillCursor: first.stateUpdates[0]!.cursor, }); @@ -342,7 +352,7 @@ describe('GitHub issue collector progress', () => { const missed = await collectBrainGithubIssues({ now, limit: 100 }); expect(missed.pages).toEqual([]); expect(missed.stateUpdates[0]?.cursor).not.toBeNull(); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: missed.stateUpdates[0]!.watermark, backfillCursor: missed.stateUpdates[0]!.cursor, }); @@ -359,7 +369,7 @@ describe('GitHub issue collector progress', () => { const original = makeIssue(7, boundary.toISOString()); const changed = { ...original, comments: 1 }; githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: boundary, backfillCursor: JSON.stringify({ boundary: boundary.toISOString(), @@ -389,7 +399,7 @@ describe('GitHub issue collector progress', () => { 'github/acme/a/issues/7', ]); expect(result.stateUpdates[0]).toMatchObject({ - collectorId: 'github-issues:acme/a', + collectorId: 'github-issues:occurrence-date-v2:acme/a', watermark: new Date(now.getTime() - 1000), }); expect(result.stateUpdates[0]?.cursor).not.toBeNull(); @@ -417,7 +427,7 @@ describe('GitHub issue collector progress', () => { [], ]); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: boundary, backfillCursor: JSON.stringify({ boundary: boundary.toISOString(), @@ -452,7 +462,7 @@ describe('GitHub issue collector progress', () => { comments: 1, })); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: boundary, backfillCursor: JSON.stringify({ boundary: boundary.toISOString(), @@ -507,7 +517,7 @@ describe('GitHub issue collector progress', () => { ); const newerIssue = makeIssue(32, nextBoundary.toISOString()); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: boundary, backfillCursor: JSON.stringify({ boundary: boundary.toISOString(), @@ -537,7 +547,7 @@ describe('GitHub issue collector progress', () => { boundary: boundary.toISOString(), commentProbeOffset: 30, }); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: first.stateUpdates[0]!.watermark, backfillCursor: first.stateUpdates[0]!.cursor, }); @@ -566,7 +576,7 @@ describe('GitHub issue collector progress', () => { [], ]); githubMocks.repositories = [{ fullName: 'acme/a', installationId: 1 }]; - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: boundary, backfillCursor: JSON.stringify({ boundary: boundary.toISOString(), @@ -595,7 +605,7 @@ describe('GitHub issue collector progress', () => { boundary: boundary.toISOString(), commentProbeOffset: 0, }); - githubMocks.syncState.set('github-issues:acme/a', { + githubMocks.syncState.set('github-issues:occurrence-date-v2:acme/a', { watermark: first.stateUpdates[0]!.watermark, backfillCursor: first.stateUpdates[0]!.cursor, }); diff --git a/packages/sdk/src/server/lib/brain-github.ts b/packages/sdk/src/server/lib/brain-github.ts index 54fffdcab..f1ad680ac 100644 --- a/packages/sdk/src/server/lib/brain-github.ts +++ b/packages/sdk/src/server/lib/brain-github.ts @@ -185,7 +185,7 @@ function labelNames(issue: GithubIssue): string[] { } function issueEffectiveDate(issue: GithubIssue): string | null { - for (const value of [issue.updated_at, issue.closed_at, issue.created_at]) { + for (const value of [issue.closed_at, issue.created_at]) { if (!value) { continue; } @@ -229,7 +229,7 @@ export function buildGithubIssuePage(input: { const content = [ '---', - ...(effectiveDate ? [`date: ${effectiveDate}`] : []), + ...(effectiveDate ? [`event_date: ${effectiveDate}`] : []), `repository: ${fullName}`, `issue_number: ${issue.number}`, `state: ${issue.state ?? 'unknown'}`, @@ -354,7 +354,7 @@ export async function collectBrainGithubIssues(input: { const commentBudget = { remaining: MAX_COMMENT_FETCHES_PER_PASS }; const repositoriesWithState = await Promise.all( repositoriesToScan.map(async (repository) => { - const stateId = `github-issues:${repository.fullName}`; + const stateId = `github-issues:occurrence-date-v2:${repository.fullName}`; return { ...repository,