Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions app/services/jobs/crawl.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ export const crawlJob = defineJob({
organizationId: z.string(),
refresh: z.boolean().default(false),
prNumbers: z.array(z.number()).optional(),
repoName: z.string().optional(),
}),
output: z.object({
fetchedRepos: z.number(),
Expand Down Expand Up @@ -53,14 +54,18 @@ export const crawlJob = defineJob({
githubAppLink: fullOrg.githubAppLink,
})

const repoCount = organization.repositories.length
const updatedPrNumbers = new Map<string, Set<number>>()

const FETCH_ALL_SENTINEL = '2000-01-01T00:00:00Z'

// Step 2: Fetch per repo
for (let i = 0; i < organization.repositories.length; i++) {
const repo = organization.repositories[i]
const targetRepos = input.repoName
? organization.repositories.filter((r) => r.repo === input.repoName)
: organization.repositories
const repoCount = targetRepos.length

for (let i = 0; i < targetRepos.length; i++) {
const repo = targetRepos[i]
const repoLabel = `${repo.owner}/${repo.repo}`

const store = createStore({
Expand Down
16 changes: 14 additions & 2 deletions batch/cli.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { cli, command } from 'cleye'
import consola from 'consola'
import 'dotenv/config'
import {
captureExceptionToSentry,
Expand All @@ -22,7 +23,12 @@ const crawl = command(
},
pr: {
type: [Number],
description: 'Specific PR numbers to refresh (e.g. --pr 123 --pr 456)',
description:
'Specific PR numbers to refresh (requires --repo). e.g. --repo falcon9 --pr 123',
},
repo: {
type: String,
description: 'Repository name to target (required with --pr)',
},
},
help: {
Expand All @@ -32,10 +38,16 @@ const crawl = command(
},
async (argv) => {
const { crawlCommand } = await import('./commands/crawl')
const prNumbers = argv.flags.pr?.length ? argv.flags.pr : undefined
if (prNumbers && !argv.flags.repo) {
consola.error('--repo is required when using --pr')
process.exit(1)
}
await crawlCommand({
organizationId: argv._.organizationId,
refresh: argv.flags.refresh,
prNumbers: argv.flags.pr?.length ? argv.flags.pr : undefined,
prNumbers,
repoName: argv.flags.repo,
})
},
)
Expand Down
16 changes: 9 additions & 7 deletions batch/commands/crawl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,30 @@ interface CrawlCommandProps {
organizationId?: string
refresh: boolean
prNumbers?: number[]
repoName?: string
}

export async function crawlCommand({
organizationId,
refresh,
prNumbers,
repoName,
}: CrawlCommandProps) {
const result = await requireOrganization(organizationId)
if (!result) return

const { orgId } = result

try {
const prLabel = prNumbers
? ` (PRs: ${prNumbers.join(', ')})`
: refresh
? ' (full refresh)'
: ''
consola.info(`Starting crawl for ${orgId}${prLabel}...`)
const labels: string[] = []
if (repoName) labels.push(`repo: ${repoName}`)
if (prNumbers) labels.push(`PRs: ${prNumbers.join(', ')}`)
if (refresh) labels.push('full refresh')
const label = labels.length > 0 ? ` (${labels.join(', ')})` : ''
consola.info(`Starting crawl for ${orgId}${label}...`)

const { output } = await durably.jobs.crawl.triggerAndWait(
{ organizationId: orgId, refresh, prNumbers },
{ organizationId: orgId, refresh, prNumbers, repoName },
{
concurrencyKey: `crawl:${orgId}`,
labels: { organizationId: orgId },
Expand Down
Loading