From a0bd454842bf56b5e8921eba459a252718682aab Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 18:46:17 +0100 Subject: [PATCH 01/10] feat: new prisma schema --- prisma/schema.prisma | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e73530c..2b3edc7 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8,19 +8,40 @@ datasource db { } model Language { - id String @id @unique + id String @id name String @unique - lastUpdatedAt DateTime @default(now()) extensions String[] highlightAlias String snippets Snippet[] + trackedFiles File[] +} + +model File { + id String @id @default(uuid()) + url String + languageId String + language Language @relation(fields: [languageId], references: [id]) + fileVersions FileVersion[] +} + +model FileVersion { + id String @id @default(uuid()) + commitSha String + createdAt DateTime @default(now()) + parsedAt DateTime? + fileId String + file File @relation(fields: [fileId], references: [id]) + snippets Snippet[] } model Snippet { - url String @id @unique - languageId String - isValid Boolean @default(true) - language Language @relation(fields: [languageId], references: [id]) + id String @id @default(uuid()) + hash String @unique + content String + languageId String + fileVersionId String + language Language @relation(fields: [languageId], references: [id]) + fileVersion FileVersion @relation(fields: [fileVersionId], references: [id]) @@index([languageId]) } From 926008f372fb46f38baeeba3bc448c8e711d0bd0 Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 18:51:27 +0100 Subject: [PATCH 02/10] feat: add new migration --- .../20250402143241_init/migration.sql | 31 ---------- .../migration.sql | 5 -- .../20251117175053_init/migration.sql | 61 +++++++++++++++++++ 3 files changed, 61 insertions(+), 36 deletions(-) delete mode 100644 prisma/migrations/20250402143241_init/migration.sql delete mode 100644 prisma/migrations/20250402153753_add_languageid_index/migration.sql create mode 100644 prisma/migrations/20251117175053_init/migration.sql diff --git a/prisma/migrations/20250402143241_init/migration.sql b/prisma/migrations/20250402143241_init/migration.sql deleted file mode 100644 index c48a269..0000000 --- a/prisma/migrations/20250402143241_init/migration.sql +++ /dev/null @@ -1,31 +0,0 @@ --- CreateTable -CREATE TABLE "Language" ( - "id" TEXT NOT NULL, - "name" TEXT NOT NULL, - "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "extensions" TEXT[], - "highlightAlias" TEXT NOT NULL, - - CONSTRAINT "Language_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Snippet" ( - "url" TEXT NOT NULL, - "languageId" TEXT NOT NULL, - "isValid" BOOLEAN NOT NULL, - - CONSTRAINT "Snippet_pkey" PRIMARY KEY ("url") -); - --- CreateIndex -CREATE UNIQUE INDEX "Language_id_key" ON "Language"("id"); - --- CreateIndex -CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); - --- CreateIndex -CREATE UNIQUE INDEX "Snippet_url_key" ON "Snippet"("url"); - --- AddForeignKey -ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/20250402153753_add_languageid_index/migration.sql b/prisma/migrations/20250402153753_add_languageid_index/migration.sql deleted file mode 100644 index e42d7ec..0000000 --- a/prisma/migrations/20250402153753_add_languageid_index/migration.sql +++ /dev/null @@ -1,5 +0,0 @@ --- AlterTable -ALTER TABLE "Snippet" ALTER COLUMN "isValid" SET DEFAULT true; - --- CreateIndex -CREATE INDEX "Snippet_languageId_idx" ON "Snippet"("languageId"); diff --git a/prisma/migrations/20251117175053_init/migration.sql b/prisma/migrations/20251117175053_init/migration.sql new file mode 100644 index 0000000..594f7b9 --- /dev/null +++ b/prisma/migrations/20251117175053_init/migration.sql @@ -0,0 +1,61 @@ +-- CreateTable +CREATE TABLE "Language" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "extensions" TEXT[], + "highlightAlias" TEXT NOT NULL, + + CONSTRAINT "Language_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "File" ( + "id" TEXT NOT NULL, + "url" TEXT NOT NULL, + "languageId" TEXT NOT NULL, + + CONSTRAINT "File_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "FileVersion" ( + "id" TEXT NOT NULL, + "commitSha" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "parsedAt" TIMESTAMP(3), + "fileId" TEXT NOT NULL, + + CONSTRAINT "FileVersion_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Snippet" ( + "id" TEXT NOT NULL, + "hash" TEXT NOT NULL, + "content" TEXT NOT NULL, + "languageId" TEXT NOT NULL, + "fileVersionId" TEXT NOT NULL, + + CONSTRAINT "Snippet_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); + +-- CreateIndex +CREATE UNIQUE INDEX "Snippet_hash_key" ON "Snippet"("hash"); + +-- CreateIndex +CREATE INDEX "Snippet_languageId_idx" ON "Snippet"("languageId"); + +-- AddForeignKey +ALTER TABLE "File" ADD CONSTRAINT "File_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FileVersion" ADD CONSTRAINT "FileVersion_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_fileVersionId_fkey" FOREIGN KEY ("fileVersionId") REFERENCES "FileVersion"("id") ON DELETE RESTRICT ON UPDATE CASCADE; From 3f5de5d388fe02565508460943a1f58ad1e7d82f Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 20:43:55 +0100 Subject: [PATCH 03/10] feat: add file seeding --- lib/snippet-helpers.ts | 121 +++++++++++++++ .../20251117175053_init/migration.sql | 61 -------- prisma/migrations/migration_lock.toml | 3 - prisma/schema.prisma | 12 +- prisma/seed.ts | 3 +- prisma/seed/files.ts | 62 ++++++++ prisma/seed/languages.ts | 1 + scripts/seed-missing-snippets.ts | 60 -------- scripts/seed-snippets.ts | 4 +- scripts/utils/snippet-helpers.ts | 140 ------------------ 10 files changed, 196 insertions(+), 271 deletions(-) create mode 100644 lib/snippet-helpers.ts delete mode 100644 prisma/migrations/20251117175053_init/migration.sql delete mode 100644 prisma/migrations/migration_lock.toml create mode 100644 prisma/seed/files.ts delete mode 100644 scripts/seed-missing-snippets.ts delete mode 100644 scripts/utils/snippet-helpers.ts diff --git a/lib/snippet-helpers.ts b/lib/snippet-helpers.ts new file mode 100644 index 0000000..fea37db --- /dev/null +++ b/lib/snippet-helpers.ts @@ -0,0 +1,121 @@ +import {ILanguage} from "@/shared/types/language"; + +const GITHUB_API_URL = "https://api.github.com"; +const MAX_ATTEMPTS = 3; +const PER_PAGE = 100; +const MIN_FILE_SIZE = 3000; + +interface FileMetadata { + repository: string; + path: string; + commit: string; + sha: string; +} + +interface GitHubSearchResponse { + items?: { + repository: {full_name: string}; + path: string; + url: string; + sha: string; + }[]; +} + +function getToken(): string { + const token = process.env.GITHUB_API_TOKEN; + + if (!token) { + throw new Error("Missing GITHUB_API_TOKEN environment variable"); + } + + return token; +} + +async function waitForGitHubRateLimitReset(token: string): Promise { + const res = await fetch(`${GITHUB_API_URL}/rate_limit`, { + headers: { + Authorization: `Bearer ${token}`, + }, + }); + + if (!res.ok) { + console.error(`Failed to fetch rate limit status: ${res.status} ${res.statusText}`); + return; + } + + const json = (await res.json()) as {resources: {code_search: {reset: number}}}; + const reset = json.resources.code_search.reset * 1000; + const now = Date.now(); + const waitTime = reset - now; + + if (waitTime > 0) { + console.warn(`Rate limited by GitHub. Waiting ${waitTime}ms until reset.`); + await new Promise((resolve) => setTimeout(resolve, waitTime + 3000)); + } +} + +export async function fetchFilesFromGithub(language: ILanguage, page: number): Promise { + const token = getToken(); + const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); + const params = new URLSearchParams({ + q: `language:${language.name} size:>${MIN_FILE_SIZE} ${extensionsFilter}`, + page: page.toString(), + per_page: PER_PAGE.toString(), + }); + + const request = () => + fetch(`${GITHUB_API_URL}/search/code?${params}`, { + headers: { + Authorization: `Bearer ${token}`, + }, + }); + + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { + const response = await request(); + + if (response.status === 403) { + console.warn(`Received 403 (rate limit) fetching ${language.name} files on page ${page}. Attempt ${attempt}/${MAX_ATTEMPTS}.`); + await waitForGitHubRateLimitReset(token); + continue; + } + + if (!response.ok) { + throw new Error(`GitHub search failed for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}) with status ${response.status}: ${response.statusText}`); + } + + const data = (await response.json()) as GitHubSearchResponse; + const items = data.items ?? []; + + if (items.length > 0) { + console.info(`Fetched ${items.length} files for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}).`); + return items.map((item) => { + return { + repository: item.repository.full_name, + path: item.path, + commit: item.url.split("?ref=")[1], + sha: item.sha, + }; + }); + } + + if (attempt < MAX_ATTEMPTS) { + console.warn(`No files returned for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}). Retrying...`); + } + } + + throw new Error(`No files returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); +} + +export async function getFilesForLanguage(language: ILanguage, {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { + const files: FileMetadata[] = []; + for (let page = 1; page <= pagesPerLanguage; page += 1) { + try { + const file = await fetchFilesFromGithub(language, page); + files.push(...file); + } catch (error) { + throw new Error(`Error fetching files for ${language.name} on page ${page}: ${error}`); + } + } + + return files; +} diff --git a/prisma/migrations/20251117175053_init/migration.sql b/prisma/migrations/20251117175053_init/migration.sql deleted file mode 100644 index 594f7b9..0000000 --- a/prisma/migrations/20251117175053_init/migration.sql +++ /dev/null @@ -1,61 +0,0 @@ --- CreateTable -CREATE TABLE "Language" ( - "id" TEXT NOT NULL, - "name" TEXT NOT NULL, - "extensions" TEXT[], - "highlightAlias" TEXT NOT NULL, - - CONSTRAINT "Language_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "File" ( - "id" TEXT NOT NULL, - "url" TEXT NOT NULL, - "languageId" TEXT NOT NULL, - - CONSTRAINT "File_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "FileVersion" ( - "id" TEXT NOT NULL, - "commitSha" TEXT NOT NULL, - "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "parsedAt" TIMESTAMP(3), - "fileId" TEXT NOT NULL, - - CONSTRAINT "FileVersion_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Snippet" ( - "id" TEXT NOT NULL, - "hash" TEXT NOT NULL, - "content" TEXT NOT NULL, - "languageId" TEXT NOT NULL, - "fileVersionId" TEXT NOT NULL, - - CONSTRAINT "Snippet_pkey" PRIMARY KEY ("id") -); - --- CreateIndex -CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); - --- CreateIndex -CREATE UNIQUE INDEX "Snippet_hash_key" ON "Snippet"("hash"); - --- CreateIndex -CREATE INDEX "Snippet_languageId_idx" ON "Snippet"("languageId"); - --- AddForeignKey -ALTER TABLE "File" ADD CONSTRAINT "File_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "FileVersion" ADD CONSTRAINT "FileVersion_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; - --- AddForeignKey -ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_fileVersionId_fkey" FOREIGN KEY ("fileVersionId") REFERENCES "FileVersion"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml deleted file mode 100644 index 648c57f..0000000 --- a/prisma/migrations/migration_lock.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Please do not edit this file manually -# It should be added in your version-control system (e.g., Git) -provider = "postgresql" \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 2b3edc7..20aad29 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -18,25 +18,31 @@ model Language { model File { id String @id @default(uuid()) - url String + repository String + path String languageId String language Language @relation(fields: [languageId], references: [id]) fileVersions FileVersion[] + + @@unique([repository, path]) } model FileVersion { id String @id @default(uuid()) - commitSha String + sha String + commit String createdAt DateTime @default(now()) parsedAt DateTime? fileId String file File @relation(fields: [fileId], references: [id]) snippets Snippet[] + + @@unique([fileId, sha]) } model Snippet { id String @id @default(uuid()) - hash String @unique + sha String @unique content String languageId String fileVersionId String diff --git a/prisma/seed.ts b/prisma/seed.ts index 94867bb..5668027 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -1,6 +1,7 @@ import dotenv from "dotenv"; import {PrismaClient} from "@prisma/client"; import {seedLanguages} from "./seed/languages"; +import {seedFiles} from "./seed/files"; dotenv.config(); @@ -8,7 +9,7 @@ const prisma = new PrismaClient(); async function main() { await seedLanguages(prisma); - console.log("Languages seeded successfully."); + await seedFiles(prisma); } main() diff --git a/prisma/seed/files.ts b/prisma/seed/files.ts new file mode 100644 index 0000000..de3a2c4 --- /dev/null +++ b/prisma/seed/files.ts @@ -0,0 +1,62 @@ +import {PrismaClient} from "@prisma/client"; +import {getFilesForLanguage} from "../../lib/snippet-helpers"; +import {SUPPORTED_LANGUAGES} from "./languages"; + +export async function seedFiles(prisma: PrismaClient) { + console.log("Seeding files for supported languages..."); + const languages = await prisma.language.findMany({ + select: { + id: true, + _count: {select: {trackedFiles: true}}, + }, + }); + + if (languages.length === 0) { + throw new Error("Language table is empty. Run the base seed first: ts-node prisma/seed.ts"); + } + + const missingLanguageIds = languages.filter((language) => language._count.trackedFiles === 0).map((language) => language.id); + + if (missingLanguageIds.length === 0) { + console.log("All languages already have snippets."); + return; + } + + console.log(`Seeding snippets for missing languages: ${missingLanguageIds.join(", ")}`); + + for (const languageId of missingLanguageIds) { + await seedFilesForLanguage(prisma, languageId); + } + + console.log("Finished seeding files for all missing languages."); +} + +async function seedFilesForLanguage(prisma: PrismaClient, languageId: string) { + const language = SUPPORTED_LANGUAGES.find((lang) => lang.id === languageId); + if (!language) { + throw new Error(`Language with ID ${languageId} is not supported.`); + } + + console.log(`Fetching files for language: ${language.name}`); + const files = await getFilesForLanguage(language); + + for (const file of files) { + const insertedFile = await prisma.file.create({ + data: { + repository: file.repository, + path: file.path, + languageId: language.id, + }, + }); + + await prisma.fileVersion.create({ + data: { + sha: file.sha, + commit: file.commit, + fileId: insertedFile.id, + }, + }); + } + + console.log(`Seeded ${files.length} files for language: ${language.name}`); +} diff --git a/prisma/seed/languages.ts b/prisma/seed/languages.ts index 1ead7e5..db0d518 100644 --- a/prisma/seed/languages.ts +++ b/prisma/seed/languages.ts @@ -77,4 +77,5 @@ export async function seedLanguages(prisma: PrismaClient) { console.log(`Language seeded: ${languageRecord.name}`); } + console.log("Languages seeded successfully."); } diff --git a/scripts/seed-missing-snippets.ts b/scripts/seed-missing-snippets.ts deleted file mode 100644 index 00c19dc..0000000 --- a/scripts/seed-missing-snippets.ts +++ /dev/null @@ -1,60 +0,0 @@ -import dotenv from "dotenv"; -import {PrismaClient} from "@prisma/client"; -import {parseNumberFlag} from "./utils/cli"; -import {seedLanguageSnippets} from "./utils/snippet-helpers"; - -dotenv.config(); - -const prisma = new PrismaClient(); -const rawArgs = process.argv.slice(2); - -interface CliOptions { - pagesPerLanguage?: number; -} - -function parseCliOptions(): CliOptions { - const pagesPerLanguage = parseNumberFlag(rawArgs, "--pages"); - return {pagesPerLanguage}; -} - -async function seedMissingLanguageSnippets(pagesPerLanguage?: number) { - const languages = await prisma.language.findMany({ - select: { - id: true, - _count: {select: {snippets: true}}, - }, - }); - - if (languages.length === 0) { - console.warn("Language table is empty. Run the base seed first: ts-node prisma/seed.ts"); - return; - } - - const missingLanguageIds = languages.filter((language) => language._count.snippets === 0).map((language) => language.id); - - if (missingLanguageIds.length === 0) { - console.log("All languages already have snippets."); - return; - } - - console.log(`Seeding snippets for missing languages: ${missingLanguageIds.join(", ")}`); - - for (const languageId of missingLanguageIds) { - await seedLanguageSnippets(prisma, languageId, pagesPerLanguage); - } -} - -async function main() { - const options = parseCliOptions(); - await seedMissingLanguageSnippets(options.pagesPerLanguage); -} - -main() - .then(async () => { - await prisma.$disconnect(); - }) - .catch(async (error) => { - console.error(error); - await prisma.$disconnect(); - process.exit(1); - }); diff --git a/scripts/seed-snippets.ts b/scripts/seed-snippets.ts index f14c866..ff01489 100644 --- a/scripts/seed-snippets.ts +++ b/scripts/seed-snippets.ts @@ -1,7 +1,7 @@ import dotenv from "dotenv"; import {PrismaClient} from "@prisma/client"; import {getFlagValue, parseNumberFlag} from "./utils/cli"; -import {seedLanguageSnippets as seedLanguageSnippetsHelper} from "./utils/snippet-helpers"; +import {getFilesForLanguage} from "../lib/snippet-helpers"; dotenv.config(); @@ -38,5 +38,3 @@ if (require.main === module) { } })(); } - -export {fetchLanguageSnippets, seedLanguageSnippets, seedSnippets} from "./utils/snippet-helpers"; diff --git a/scripts/utils/snippet-helpers.ts b/scripts/utils/snippet-helpers.ts deleted file mode 100644 index 3d3320e..0000000 --- a/scripts/utils/snippet-helpers.ts +++ /dev/null @@ -1,140 +0,0 @@ -import {PrismaClient} from "@prisma/client"; -import {SUPPORTED_LANGUAGES} from "../../prisma/seed/languages"; -import {ILanguage} from "@/shared/types/language"; - -const GITHUB_API_URL = "https://api.github.com"; -const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; -const MAX_ATTEMPTS = 3; - -interface SnippetData { - repository: {full_name: string}; - url: string; - path: string; -} - -interface GitHubSearchResponse { - items?: SnippetData[]; -} - -function getToken(): string { - const token = process.env.GITHUB_API_TOKEN; - - if (!token) { - throw new Error("Missing GITHUB_API_TOKEN environment variable"); - } - - return token; -} - -async function waitForGitHubRateLimitReset(token: string): Promise { - const res = await fetch(`${GITHUB_API_URL}/rate_limit`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - if (!res.ok) { - console.error(`Failed to fetch rate limit status: ${res.status} ${res.statusText}`); - return; - } - - const json = (await res.json()) as {resources: {code_search: {reset: number}}}; - const reset = json.resources.code_search.reset * 1000; - const now = Date.now(); - const waitTime = reset - now; - - if (waitTime > 0) { - console.warn(`Rate limited by GitHub. Waiting ${waitTime}ms until reset.`); - await new Promise((resolve) => setTimeout(resolve, waitTime + 3000)); - } -} - -function getSnippetRawLink(data: SnippetData): string { - const repository = data.repository.full_name; - const sha = data.url.split("?ref=")[1]; - const path = data.path; - return `${RAW_GITHUB_BASE_URL}/${repository}/${sha}/${path}`; -} - -export async function fetchLanguageSnippets(language: ILanguage, page: number): Promise { - const token = getToken(); - const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); - const params = new URLSearchParams({ - q: `language:${language.name} size:>3000 ${extensionsFilter}`, - page: page.toString(), - per_page: "100", - }); - - const request = () => - fetch(`${GITHUB_API_URL}/search/code?${params}`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { - const response = await request(); - - if (response.status === 403) { - console.warn(`Received 403 (rate limit) fetching ${language.name} snippets on page ${page}. Attempt ${attempt}/${MAX_ATTEMPTS}.`); - await waitForGitHubRateLimitReset(token); - continue; - } - - if (!response.ok) { - throw new Error(`GitHub search failed for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}) with status ${response.status}: ${response.statusText}`); - } - - const data = (await response.json()) as GitHubSearchResponse; - const items = data.items ?? []; - - if (items.length > 0) { - console.info(`Fetched ${items.length} snippets for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}).`); - return items.map((item) => getSnippetRawLink(item)); - } - - if (attempt < MAX_ATTEMPTS) { - console.warn(`No snippets returned for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}). Retrying...`); - } - } - - throw new Error(`No snippets returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); -} - -export async function seedSnippets(prisma: PrismaClient, languages: ILanguage[], {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { - for (const language of languages) { - for (let page = 1; page <= pagesPerLanguage; page += 1) { - try { - const snippets = await fetchLanguageSnippets(language, page); - - for (const snippet of snippets) { - await prisma.snippet.upsert({ - where: {url: snippet}, - update: {}, - create: { - url: snippet, - language: { - connect: { - id: language.id, - }, - }, - }, - }); - } - } catch (error) { - throw new Error(`Error fetching snippets for ${language.name} on page ${page}: ${error}`); - } - } - } -} - -export async function seedLanguageSnippets(prismaClient: PrismaClient, languageId: string, pagesPerLanguage?: number): Promise { - const language = SUPPORTED_LANGUAGES.find((lang) => lang.id === languageId); - - if (!language) { - throw new Error(`Unsupported language "${languageId}". Supported ids: ${SUPPORTED_LANGUAGES.map((lang) => lang.id).join(", ")}`); - } - - await seedSnippets(prismaClient, [language], {pagesPerLanguage}); - console.log(`Snippets seeded for ${language.name}.`); -} From 8d81f8af811265b7df1f9256188d3cb6f415f0f4 Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 23:41:51 +0100 Subject: [PATCH 04/10] feat: add functioning new architecture --- lib/snippets/config.ts | 27 ++++++ lib/snippets/index.ts | 26 ++++++ .../snippets/parsing/parser-factory.ts | 0 lib/snippets/parsing/parser.ts | 46 ++++++++++ .../snippets/processing/filter.ts | 16 ++-- .../snippets/processing/indentation.ts | 2 +- lib/snippets/processing/process.ts | 54 +++++++++++ .../snippets/processing/utils.ts | 9 ++ package-lock.json | 11 +++ package.json | 1 + prisma/schema.prisma | 11 +-- prisma/seed.ts | 2 + prisma/seed/languages.ts | 84 ++++++++--------- prisma/seed/snippets.ts | 89 +++++++++++++++++++ .../adapters/snippet-fetch.server.ts | 15 ---- .../repositories/snippet.repository.server.ts | 37 ++++---- .../logic/parsing/snippet-parser.server.ts | 46 +--------- .../snippets/logic/processing/filter.ts | 31 +++++++ .../processing/snippet-process.server.ts | 85 ------------------ .../services/get-random-snippets.server.ts | 36 +++----- src/features/snippets/types/snippet-source.ts | 4 - src/shared/db/prisma.ts | 21 +++-- tsconfig.json | 3 + 23 files changed, 402 insertions(+), 254 deletions(-) create mode 100644 lib/snippets/config.ts create mode 100644 lib/snippets/index.ts rename src/features/snippets/logic/parsing/parser-factory.server.ts => lib/snippets/parsing/parser-factory.ts (100%) create mode 100644 lib/snippets/parsing/parser.ts rename src/features/snippets/logic/processing/snippet-filters.server.ts => lib/snippets/processing/filter.ts (53%) rename src/features/snippets/logic/processing/indentation.server.ts => lib/snippets/processing/indentation.ts (99%) create mode 100644 lib/snippets/processing/process.ts rename src/features/snippets/logic/processing/text.server.ts => lib/snippets/processing/utils.ts (66%) create mode 100644 prisma/seed/snippets.ts delete mode 100644 src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts create mode 100644 src/features/snippets/logic/processing/filter.ts delete mode 100644 src/features/snippets/logic/processing/snippet-process.server.ts delete mode 100644 src/features/snippets/types/snippet-source.ts diff --git a/lib/snippets/config.ts b/lib/snippets/config.ts new file mode 100644 index 0000000..e9fb8e7 --- /dev/null +++ b/lib/snippets/config.ts @@ -0,0 +1,27 @@ +export const MAX_LINES = 11; +export const MIN_LINES = 3; + +export const MAX_LINE_LENGTH = 72; + +export const MAX_SNIPPET_LENGTH = 350; +export const MIN_SNIPPET_LENGTH = 75; + +export const VALID_NODES = [ + "function_declaration", + "function_definition", + "class_declaration", + "class_definition", + "function_item", + "method_declaration", + "method_definition", + "method_item", + "class_item", + "interface_declaration", + "interface_definition", + "interface_item", + "module", + "switch_statement", + "while_statement", + "for_statement", + "if_statement", +]; diff --git a/lib/snippets/index.ts b/lib/snippets/index.ts new file mode 100644 index 0000000..991eaca --- /dev/null +++ b/lib/snippets/index.ts @@ -0,0 +1,26 @@ +import {extractSnippets} from "./parsing/parser"; +import {filterSnippets} from "./processing/filter"; +import {formatCode} from "./processing/process"; +import {hashSnippet} from "./processing/utils"; + +interface ISnippetOutput { + sha: string; + content: string; + rawContent: string; +} + +export function getSnippetsFromFile(fileContent: string, languageId: string): ISnippetOutput[] { + const extractedSnippets = extractSnippets(fileContent, languageId); + + const formattedSnippets = extractedSnippets + .map((snippet) => ({ + sha: hashSnippet(snippet), + content: formatCode(snippet), + rawContent: snippet, + })) + .filter((s) => s.content !== null) as ISnippetOutput[]; + + const filteredSnippets = formattedSnippets.filter((snippet) => filterSnippets(snippet.content)); + + return filteredSnippets; +} diff --git a/src/features/snippets/logic/parsing/parser-factory.server.ts b/lib/snippets/parsing/parser-factory.ts similarity index 100% rename from src/features/snippets/logic/parsing/parser-factory.server.ts rename to lib/snippets/parsing/parser-factory.ts diff --git a/lib/snippets/parsing/parser.ts b/lib/snippets/parsing/parser.ts new file mode 100644 index 0000000..d967af5 --- /dev/null +++ b/lib/snippets/parsing/parser.ts @@ -0,0 +1,46 @@ +import {getInitialIndentation} from "@lib/snippets/processing/indentation"; +import {getTSParser} from "./parser-factory"; +import IParser from "tree-sitter"; +import {isValidNode} from "@lib/snippets/processing/filter"; + +export function findValidNodes(node: IParser.SyntaxNode): IParser.SyntaxNode[] { + let nodes: IParser.SyntaxNode[] = []; + if (isValidNode(node)) { + nodes.push(node); + } + + node.children.forEach((child) => { + if (child === null) return; + nodes = nodes.concat(findValidNodes(child)); + }); + + return nodes; +} + +export function convertSnippetToText(node: IParser.SyntaxNode, sourceCode: string): string { + const initialIndentation = getInitialIndentation(node.startIndex, sourceCode); + + return initialIndentation + getNodeText(node); +} + +export function extractSnippets(fileContent: string, languageId: string): string[] { + const parser = getTSParser(languageId); + + let parsedCode: IParser.Tree; + + try { + parsedCode = parser.parse(fileContent); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (error) { + return []; + } + + const validNodes = findValidNodes(parsedCode.rootNode); + const validSnippets = validNodes.map((node) => convertSnippetToText(node, fileContent)); + + return validSnippets; +} + +export function getNodeText(node: IParser.SyntaxNode): string { + return node.text; +} diff --git a/src/features/snippets/logic/processing/snippet-filters.server.ts b/lib/snippets/processing/filter.ts similarity index 53% rename from src/features/snippets/logic/processing/snippet-filters.server.ts rename to lib/snippets/processing/filter.ts index 976b797..8f81186 100644 --- a/src/features/snippets/logic/processing/snippet-filters.server.ts +++ b/lib/snippets/processing/filter.ts @@ -1,28 +1,28 @@ -import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH, VALID_NODES} from "@/features/snippets/config/snippets.server"; import IParser from "tree-sitter"; +import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH, VALID_NODES} from "../config"; -export function filterLinesNumber(snippet: string): boolean { +function filterLinesNumber(snippet: string): boolean { const numLines = snippet.split("\n").length; return numLines <= MAX_LINES && numLines >= MIN_LINES; } -export function filterLineLength(snippet: string): boolean { +function filterLineLength(snippet: string): boolean { const lines = snippet.split("\n"); return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH); } -export function filterSnippetLength(snippet: string): boolean { +function filterSnippetLength(snippet: string): boolean { return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH; } -export function filterSnippetSpecialCharacters(snippet: string): boolean { +function filterSnippetSpecialCharacters(snippet: string): boolean { const regex = /^[\x00-\x7F]+$/; const test = regex.test(snippet); return test; } -export function filterTabsInBetween(snippet: string): boolean { +function filterTabsInBetween(snippet: string): boolean { const regex = /(^|[^\\])(\\t)|(\t)/; return regex.test(snippet.trim()); } @@ -30,3 +30,7 @@ export function filterTabsInBetween(snippet: string): boolean { export function isValidNode(node: IParser.SyntaxNode) { return VALID_NODES.includes(node.type); } + +export function filterSnippets(content: string): boolean { + return filterSnippetLength(content) && filterLinesNumber(content) && filterLineLength(content) && filterSnippetSpecialCharacters(content) && filterTabsInBetween(content); +} diff --git a/src/features/snippets/logic/processing/indentation.server.ts b/lib/snippets/processing/indentation.ts similarity index 99% rename from src/features/snippets/logic/processing/indentation.server.ts rename to lib/snippets/processing/indentation.ts index daac2d8..3bf04d7 100644 --- a/src/features/snippets/logic/processing/indentation.server.ts +++ b/lib/snippets/processing/indentation.ts @@ -1,4 +1,4 @@ -import {countInitialWhitespaces, removeInitialCharacters} from "./text.server"; +import {countInitialWhitespaces, removeInitialCharacters} from "./utils"; type IIndentationStyle = {type: "none" | "mixed" | "tab"} | {type: "space"; value: number[]}; export function detectIndentationStyle(snippet: string): IIndentationStyle { diff --git a/lib/snippets/processing/process.ts b/lib/snippets/processing/process.ts new file mode 100644 index 0000000..18c52da --- /dev/null +++ b/lib/snippets/processing/process.ts @@ -0,0 +1,54 @@ +import {adjustIndentationOffset, applyIndentationToEmptyLines, detectIndentationStyle} from "./indentation"; +import {countInitialWhitespaces} from "./utils"; + +function cleanText(text: string): string { + // Remove whitespaces from empty lines + text = text.replace(/(?<=\n)[ \t]+(?=\n)/g, ""); + + return text; +} + +function hasProhibitedCharacters(input: string): boolean { + const regex = /((?:(?![ \t\n])\s)|\\)/; + return regex.test(input); +} + +export function formatCode(snippet: string): string | null { + if (hasProhibitedCharacters(snippet)) { + return null; + } + + // Clean raw snippet + snippet = cleanText(snippet); + + const indentationStyle = detectIndentationStyle(snippet); + + if (indentationStyle.type === "mixed") { + return null; + } + + if (indentationStyle.type === "none") { + return snippet; + } + + // Convert raw snippet indentation style with standardized '\t' indentation + let codeLines = snippet.split("\n").map((line) => { + const initialWhitespaces = countInitialWhitespaces(line); + + const trimmedLine = line.trim(); + + if (indentationStyle.type === "space") { + return "\t".repeat(indentationStyle.value.indexOf(initialWhitespaces)) + trimmedLine; + } + + return "\t".repeat(initialWhitespaces) + trimmedLine; + }); + + // Adjust indentation offset on every line + codeLines = adjustIndentationOffset(codeLines); + + // Applies indentation to empty lines (the minimum indentation of the surrounding lines) + codeLines = applyIndentationToEmptyLines(codeLines); + + return codeLines.join("\n"); +} diff --git a/src/features/snippets/logic/processing/text.server.ts b/lib/snippets/processing/utils.ts similarity index 66% rename from src/features/snippets/logic/processing/text.server.ts rename to lib/snippets/processing/utils.ts index 6004c5a..ecdee49 100644 --- a/src/features/snippets/logic/processing/text.server.ts +++ b/lib/snippets/processing/utils.ts @@ -1,3 +1,5 @@ +import {createHash} from "crypto"; + export function countInitialWhitespaces(text: string): number { let count = 0; for (let i = 0; i < text.length; i++) { @@ -21,3 +23,10 @@ export function removeInitialCharacters(lines: string[], quantity: number): stri return newLines; } + +export function hashSnippet(content: string): string { + // Remove all whitespaces + const newContent = content.replace(/\s+/g, "").trim(); + + return createHash("sha256").update(newContent).digest("hex"); +} diff --git a/package-lock.json b/package-lock.json index c6489cb..743d4c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "next": "16.0.1", "next-themes": "^0.4.6", "prisma": "^6.5.0", + "prisma-extension-random": "^0.2.2", "rate-limiter-flexible": "^7.0.0", "react": "19.2.0", "react-dom": "19.2.0", @@ -1667,6 +1668,7 @@ "integrity": "sha512-M6w1Ql/BeiGoZmhMdAZUXHu5sz5HubyVcKukbLs3l0ELcQb8hTUJxtGEChhv4SVJ0QJlwtLnwOLgIRQhpsm9dw==", "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=18.18" }, @@ -6917,6 +6919,15 @@ } } }, + "node_modules/prisma-extension-random": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/prisma-extension-random/-/prisma-extension-random-0.2.2.tgz", + "integrity": "sha512-tjY5w2pSIgwQh0tDT98KTTYJ0uadUwfbTH+6QbPCfLg+gesgeLZNO4oKv8O6/udx54QvEbyXgNC1LUcjtoypcg==", + "license": "MIT", + "peerDependencies": { + "@prisma/client": "*" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", diff --git a/package.json b/package.json index 69387aa..e0bd155 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "next": "16.0.1", "next-themes": "^0.4.6", "prisma": "^6.5.0", + "prisma-extension-random": "^0.2.2", "rate-limiter-flexible": "^7.0.0", "react": "19.2.0", "react-dom": "19.2.0", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 20aad29..d7eae6c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8,11 +8,10 @@ datasource db { } model Language { - id String @id - name String @unique + id String @id + name String @unique extensions String[] highlightAlias String - snippets Snippet[] trackedFiles File[] } @@ -25,6 +24,7 @@ model File { fileVersions FileVersion[] @@unique([repository, path]) + @@index([languageId]) } model FileVersion { @@ -44,10 +44,7 @@ model Snippet { id String @id @default(uuid()) sha String @unique content String - languageId String + rawContent String fileVersionId String - language Language @relation(fields: [languageId], references: [id]) fileVersion FileVersion @relation(fields: [fileVersionId], references: [id]) - - @@index([languageId]) } diff --git a/prisma/seed.ts b/prisma/seed.ts index 5668027..12cf7b5 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -2,6 +2,7 @@ import dotenv from "dotenv"; import {PrismaClient} from "@prisma/client"; import {seedLanguages} from "./seed/languages"; import {seedFiles} from "./seed/files"; +import {seedSnippets} from "./seed/snippets"; dotenv.config(); @@ -10,6 +11,7 @@ const prisma = new PrismaClient(); async function main() { await seedLanguages(prisma); await seedFiles(prisma); + await seedSnippets(prisma); } main() diff --git a/prisma/seed/languages.ts b/prisma/seed/languages.ts index db0d518..11f008f 100644 --- a/prisma/seed/languages.ts +++ b/prisma/seed/languages.ts @@ -14,48 +14,48 @@ export const SUPPORTED_LANGUAGES: ILanguage[] = [ extensions: ["ts", "tsx"], highlightAlias: "typescript", }, - { - id: "c", - name: "C", - extensions: ["c"], - highlightAlias: "cpp", - }, - { - id: "cpp", - name: "C++", - extensions: ["cpp"], - highlightAlias: "cpp", - }, - { - id: "cs", - name: "C#", - extensions: ["cs"], - highlightAlias: "csharp", - }, - { - id: "java", - name: "Java", - extensions: ["java"], - highlightAlias: "java", - }, - { - id: "py", - name: "Python", - extensions: ["py"], - highlightAlias: "python", - }, - { - id: "lua", - name: "Lua", - extensions: ["lua"], - highlightAlias: "lua", - }, - { - id: "r", - name: "R", - extensions: ["r", "R"], - highlightAlias: "r", - }, + // { + // id: "c", + // name: "C", + // extensions: ["c"], + // highlightAlias: "cpp", + // }, + // { + // id: "cpp", + // name: "C++", + // extensions: ["cpp"], + // highlightAlias: "cpp", + // }, + // { + // id: "cs", + // name: "C#", + // extensions: ["cs"], + // highlightAlias: "csharp", + // }, + // { + // id: "java", + // name: "Java", + // extensions: ["java"], + // highlightAlias: "java", + // }, + // { + // id: "py", + // name: "Python", + // extensions: ["py"], + // highlightAlias: "python", + // }, + // { + // id: "lua", + // name: "Lua", + // extensions: ["lua"], + // highlightAlias: "lua", + // }, + // { + // id: "r", + // name: "R", + // extensions: ["r", "R"], + // highlightAlias: "r", + // }, ]; export async function seedLanguages(prisma: PrismaClient) { diff --git a/prisma/seed/snippets.ts b/prisma/seed/snippets.ts new file mode 100644 index 0000000..cc95dba --- /dev/null +++ b/prisma/seed/snippets.ts @@ -0,0 +1,89 @@ +import {getSnippetsFromFile} from "@lib/snippets"; +import {Prisma, PrismaClient} from "@prisma/client"; +const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; +import pLimit from "p-limit"; + +const limit = pLimit(100); + +export async function seedSnippets(prisma: PrismaClient) { + console.log("Seeding snippets for unparsed files.."); + const unparsedFiles = await prisma.fileVersion.findMany({ + select: { + id: true, + file: { + select: { + repository: true, + path: true, + languageId: true, + }, + }, + commit: true, + }, + where: { + parsedAt: null, + }, + }); + + if (unparsedFiles.length === 0) { + throw new Error("All files are parsed. No snippets to seed."); + } + + await Promise.all( + unparsedFiles.map(async (file) => { + const {repository, path} = file.file; + const url = `${RAW_GITHUB_BASE_URL}/${repository}/${file.commit}/${path}`; + + const fileContent = await limit(async () => { + const res = await fetch(url).catch((error) => { + console.warn(`Could not fetch file content from ${url}:`, error); + return null; + }); + + if (!res) return null; + + if (!res.ok) { + if (res.status === 404 || res.status === 400) { + await prisma.fileVersion.delete({ + where: {id: file.id}, + }); + console.warn(`Deleted file version ${file.id} due to 404 (file missing).`); + } + return null; + } + + return res.text(); + }); + + if (!fileContent) return; + + const snippets = getSnippetsFromFile(fileContent, file.file.languageId); + + await Promise.all( + snippets.map(async (snippet) => { + console.log(`Inserting snippet with SHA ${snippet.sha} from file version ${file.id}`); + try { + await prisma.snippet.create({ + data: { + sha: snippet.sha, + content: snippet.content, + rawContent: snippet.rawContent, + fileVersionId: file.id, + }, + }); + } catch (error) { + // Ignore unique constraint violation (since multiple files can produce the same snippet). Upsert is not an option since we are seeding in parallel. + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") return; + throw error; + } + }) + ); + + await prisma.fileVersion.update({ + where: {id: file.id}, + data: {parsedAt: new Date()}, + }); + }) + ); + + console.log("Finished seeding snippets for all unparsed files."); +} diff --git a/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts b/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts deleted file mode 100644 index 9a82c07..0000000 --- a/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {SnippetSourceFile} from "@/features/snippets/types/snippet-source"; - -async function getFileContent(rawUrl: string): Promise { - const response = await fetch(rawUrl); - return response.text(); -} - -export async function getFilesFromUrls(snippetUrls: string[]): Promise { - const filePromises = snippetUrls.map(async (snippetUrl: string) => ({ - url: snippetUrl, - content: await getFileContent(snippetUrl), - })); - - return Promise.all(filePromises); -} diff --git a/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts b/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts index cd8b9e5..c5084ba 100644 --- a/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts +++ b/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts @@ -1,25 +1,20 @@ import {prisma} from "@/shared/db/prisma"; +import {Snippet} from "@prisma/client"; -export async function fetchRandomFiles(languageId: string, quantity: number, alreadyFetched: string[]): Promise { - const alreadyFetchedList = alreadyFetched.length > 0 ? `AND "url" NOT IN (${alreadyFetched.map((_, i) => `$${i + 2}`).join(", ")})` : ""; - - const query = ` - SELECT "url" FROM "Snippet" - WHERE "languageId" = $1 - AND "isValid" = true - ${alreadyFetchedList} - ORDER BY RANDOM() - LIMIT $${alreadyFetched.length + 2}; - `; - - const files = (await prisma.$queryRawUnsafe(query, languageId, ...alreadyFetched, quantity)) as {url: string}[]; - - return files.map((snippet) => snippet.url); -} - -export async function setSnippetAsNonValid(url: string) { - await prisma.snippet.update({ - where: {url}, - data: {isValid: false}, +export async function findRandomSnippets(languageId: string, quantity: number): Promise[]> { + const snippets = await prisma.snippet.findManyRandom(quantity, { + select: { + id: true, + content: true, + }, + where: { + fileVersion: { + file: { + languageId, + }, + }, + }, }); + + return snippets; } diff --git a/src/features/snippets/logic/parsing/snippet-parser.server.ts b/src/features/snippets/logic/parsing/snippet-parser.server.ts index 703a831..bfbadab 100644 --- a/src/features/snippets/logic/parsing/snippet-parser.server.ts +++ b/src/features/snippets/logic/parsing/snippet-parser.server.ts @@ -1,28 +1,6 @@ -import {getTSParser} from "./parser-factory.server"; -import {getInitialIndentation} from "@/features/snippets/logic/processing/indentation.server"; -import {isValidNode} from "@/features/snippets/logic/processing/snippet-filters.server"; +import {getTSParser} from "@lib/snippets/parsing/parser-factory"; import IParser from "tree-sitter"; -export function findValidNodes(node: IParser.SyntaxNode): IParser.SyntaxNode[] { - let nodes: IParser.SyntaxNode[] = []; - if (isValidNode(node)) { - nodes.push(node); - } - - node.children.forEach((child) => { - if (child === null) return; - nodes = nodes.concat(findValidNodes(child)); - }); - - return nodes; -} - -export function convertSnippetToText(node: IParser.SyntaxNode, sourceCode: string): string { - const initialIndentation = getInitialIndentation(node.startIndex, sourceCode); - - return initialIndentation + getNodeText(node); -} - export function extractAutoCompleteDisabledRanges(fileContent: string, languageId: string): {startIndex: number; endIndex: number}[] { const parser = getTSParser(languageId); @@ -71,25 +49,3 @@ export function extractAutoCompleteDisabledRanges(fileContent: string, languageI return ranges; } - -export function extractSnippets(fileContent: string, languageId: string): string[] { - const parser = getTSParser(languageId); - - let parsedCode: IParser.Tree; - - try { - parsedCode = parser.parse(fileContent); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (error) { - return []; - } - - const validNodes = findValidNodes(parsedCode.rootNode); - const validSnippets = validNodes.map((node) => convertSnippetToText(node, fileContent)); - - return validSnippets; -} - -export function getNodeText(node: IParser.SyntaxNode): string { - return node.text; -} diff --git a/src/features/snippets/logic/processing/filter.ts b/src/features/snippets/logic/processing/filter.ts new file mode 100644 index 0000000..aa4674d --- /dev/null +++ b/src/features/snippets/logic/processing/filter.ts @@ -0,0 +1,31 @@ +import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH} from "@/features/snippets/config/snippets.server"; + +function filterLinesNumber(snippet: string): boolean { + const numLines = snippet.split("\n").length; + return numLines <= MAX_LINES && numLines >= MIN_LINES; +} + +function filterLineLength(snippet: string): boolean { + const lines = snippet.split("\n"); + return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH); +} + +function filterSnippetLength(snippet: string): boolean { + return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH; +} + +function filterSnippetSpecialCharacters(snippet: string): boolean { + const regex = /^[\x00-\x7F]+$/; + const test = regex.test(snippet); + + return test; +} + +function filterTabsInBetween(snippet: string): boolean { + const regex = /(^|[^\\])(\\t)|(\t)/; + return regex.test(snippet.trim()); +} + +export function filterSnippets(content: string): boolean { + return filterSnippetLength(content) && filterLinesNumber(content) && filterLineLength(content) && filterSnippetSpecialCharacters(content) && filterTabsInBetween(content); +} diff --git a/src/features/snippets/logic/processing/snippet-process.server.ts b/src/features/snippets/logic/processing/snippet-process.server.ts deleted file mode 100644 index c0d0088..0000000 --- a/src/features/snippets/logic/processing/snippet-process.server.ts +++ /dev/null @@ -1,85 +0,0 @@ -import {adjustIndentationOffset, applyIndentationToEmptyLines, detectIndentationStyle} from "./indentation.server"; -import {extractAutoCompleteDisabledRanges, extractSnippets} from "@/features/snippets/logic/parsing/snippet-parser.server"; -import {countInitialWhitespaces} from "./text.server"; -import {filterLineLength, filterLinesNumber, filterSnippetLength, filterSnippetSpecialCharacters, filterTabsInBetween} from "./snippet-filters.server"; -import {ISnippet} from "@/shared/types/snippet.server"; - -function filterSnippets(snippets: string[]): string[] { - return snippets - .filter((snippet) => filterSnippetLength(snippet)) - .filter((snippet) => filterLinesNumber(snippet)) - .filter((snippet) => filterLineLength(snippet)) - .filter((snippet) => filterSnippetSpecialCharacters(snippet)) - .filter((snippet) => filterTabsInBetween(snippet)); -} - -function cleanText(text: string): string { - // Remove whitespaces from empty lines - text = text.replace(/(?<=\n)[ \t]+(?=\n)/g, ""); - - return text; -} - -function hasProhibitedCharacters(input: string): boolean { - const regex = /((?:(?![ \t\n])\s)|\\)/; - return regex.test(input); -} - -function formatCode(snippet: string): string | null { - if (hasProhibitedCharacters(snippet)) { - return null; - } - - // Clean raw snippet - snippet = cleanText(snippet); - - const indentationStyle = detectIndentationStyle(snippet); - - if (indentationStyle.type === "mixed") { - return null; - } - - if (indentationStyle.type === "none") { - return snippet; - } - - // Convert raw snippet indentation style with standardized '\t' indentation - let codeLines = snippet.split("\n").map((line) => { - const initialWhitespaces = countInitialWhitespaces(line); - - const trimmedLine = line.trim(); - - if (indentationStyle.type === "space") { - return "\t".repeat(indentationStyle.value.indexOf(initialWhitespaces)) + trimmedLine; - } - - return "\t".repeat(initialWhitespaces) + trimmedLine; - }); - - // Adjust indentation offset on every line - codeLines = adjustIndentationOffset(codeLines); - - // Applies indentation to empty lines (the minimum indentation of the surrounding lines) - codeLines = applyIndentationToEmptyLines(codeLines); - - return codeLines.join("\n"); -} - -export function processSnippets(fileContent: string, languageId: string): ISnippet[] { - const extractedSnippets = extractSnippets(fileContent, languageId); - - const formattedSnippets = extractedSnippets.map((snippet) => formatCode(snippet)).filter((snippet) => snippet !== null); - - const filteredSnippets = filterSnippets(formattedSnippets); - - const finalSnippets = filteredSnippets.map((snippet) => { - const disabledRanges = extractAutoCompleteDisabledRanges(snippet, languageId); - - return { - content: snippet, - disabledRanges, - }; - }); - - return finalSnippets; -} diff --git a/src/features/snippets/services/get-random-snippets.server.ts b/src/features/snippets/services/get-random-snippets.server.ts index ead613f..f0e0dbe 100644 --- a/src/features/snippets/services/get-random-snippets.server.ts +++ b/src/features/snippets/services/get-random-snippets.server.ts @@ -1,39 +1,29 @@ import {MIN_SNIPPETS_PER_LANGUAGE, MAX_SNIPPETS_FETCH_ATTEMPTS, RANDOM_FILES_FETCHED} from "@/features/snippets/config/snippets.server"; -import {fetchRandomFiles as getRandomFiles, setSnippetAsNonValid} from "@/features/snippets/infrastructure/repositories/snippet.repository.server"; -import {getFilesFromUrls} from "@/features/snippets/infrastructure/adapters/snippet-fetch.server"; -import {processSnippets as processFile} from "@/features/snippets/logic/processing/snippet-process.server"; -import type {SnippetSourceFile} from "@/features/snippets/types/snippet-source"; import {ISnippet} from "@/shared/types/snippet.server"; -import {isDev} from "@/core/config/env"; +import {filterSnippets} from "@/features/snippets/logic/processing/filter"; +import {findRandomSnippets} from "@/features/snippets/infrastructure/repositories/snippet.repository.server"; +import {extractAutoCompleteDisabledRanges} from "@/features/snippets/logic/parsing/snippet-parser.server"; export async function getRandomSnippets(languageId: string): Promise { const snippets: ISnippet[] = []; - const fetchedFilesUrls: string[] = []; let attempts = 0; while (snippets.length < MIN_SNIPPETS_PER_LANGUAGE && attempts < MAX_SNIPPETS_FETCH_ATTEMPTS) { - const fileUrls = await getRandomFiles(languageId, RANDOM_FILES_FETCHED, fetchedFilesUrls).catch((error) => { - console.error("Error fetching random files:", error); - return []; - }); - - fetchedFilesUrls.push(...fileUrls); + const randomSnippets = await findRandomSnippets(languageId, RANDOM_FILES_FETCHED); - const fetchedFiles: SnippetSourceFile[] = await getFilesFromUrls(fileUrls); + const filteredSnippets = randomSnippets.filter((s) => filterSnippets(s.content)); - const extractedSnippets = fetchedFiles - .map((file) => { - const snippetsFromFile = processFile(file.content, languageId); + const finalSnippets = filteredSnippets.map((snippet) => { + const disabledRanges = extractAutoCompleteDisabledRanges(snippet.content, languageId); - if (snippetsFromFile.length === 0 && !isDev) { - setSnippetAsNonValid(file.url); - } + return { + content: snippet.content, + disabledRanges, + }; + }); - return snippetsFromFile; - }) - .flat(); + snippets.push(...finalSnippets); - snippets.push(...extractedSnippets); attempts++; } diff --git a/src/features/snippets/types/snippet-source.ts b/src/features/snippets/types/snippet-source.ts deleted file mode 100644 index 7fd10f7..0000000 --- a/src/features/snippets/types/snippet-source.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface SnippetSourceFile { - url: string; - content: string; -} diff --git a/src/shared/db/prisma.ts b/src/shared/db/prisma.ts index 62e15ca..3f2eb73 100644 --- a/src/shared/db/prisma.ts +++ b/src/shared/db/prisma.ts @@ -1,11 +1,22 @@ import {PrismaClient} from "@prisma/client"; +import prismaRandom from "prisma-extension-random"; -const globalForPrisma = globalThis as unknown as {prisma: PrismaClient | undefined}; +const globalForPrisma = globalThis as unknown as { + prisma: ExtendedPrismaClient | undefined; +}; -export const prisma = - globalForPrisma.prisma ?? - new PrismaClient({ +const createPrisma = () => { + const base = new PrismaClient({ log: ["error", "warn"], }); + const extended = base.$extends(prismaRandom()); + return extended; +}; -if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma; +export type ExtendedPrismaClient = ReturnType; + +export const prisma: ExtendedPrismaClient = globalForPrisma.prisma ?? createPrisma(); + +if (process.env.NODE_ENV !== "production") { + globalForPrisma.prisma = prisma; +} diff --git a/tsconfig.json b/tsconfig.json index b575f7d..f946b5b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,6 +25,9 @@ "paths": { "@/*": [ "./src/*" + ], + "@lib/*": [ + "./lib/*" ] } }, From 030b49fce40c571a1fe3ab383e54339d4f723a3c Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 23:53:22 +0100 Subject: [PATCH 05/10] refactor: cleanup --- prisma/seed/languages.ts | 84 +++++++++---------- prisma/seed/snippets.ts | 2 +- scripts/seed-snippets.ts | 40 --------- .../snippets/config/snippets.server.ts | 6 +- .../snippets/logic/{processing => }/filter.ts | 0 .../services/get-random-snippets.server.ts | 12 +-- 6 files changed, 52 insertions(+), 92 deletions(-) delete mode 100644 scripts/seed-snippets.ts rename src/features/snippets/logic/{processing => }/filter.ts (100%) diff --git a/prisma/seed/languages.ts b/prisma/seed/languages.ts index 11f008f..db0d518 100644 --- a/prisma/seed/languages.ts +++ b/prisma/seed/languages.ts @@ -14,48 +14,48 @@ export const SUPPORTED_LANGUAGES: ILanguage[] = [ extensions: ["ts", "tsx"], highlightAlias: "typescript", }, - // { - // id: "c", - // name: "C", - // extensions: ["c"], - // highlightAlias: "cpp", - // }, - // { - // id: "cpp", - // name: "C++", - // extensions: ["cpp"], - // highlightAlias: "cpp", - // }, - // { - // id: "cs", - // name: "C#", - // extensions: ["cs"], - // highlightAlias: "csharp", - // }, - // { - // id: "java", - // name: "Java", - // extensions: ["java"], - // highlightAlias: "java", - // }, - // { - // id: "py", - // name: "Python", - // extensions: ["py"], - // highlightAlias: "python", - // }, - // { - // id: "lua", - // name: "Lua", - // extensions: ["lua"], - // highlightAlias: "lua", - // }, - // { - // id: "r", - // name: "R", - // extensions: ["r", "R"], - // highlightAlias: "r", - // }, + { + id: "c", + name: "C", + extensions: ["c"], + highlightAlias: "cpp", + }, + { + id: "cpp", + name: "C++", + extensions: ["cpp"], + highlightAlias: "cpp", + }, + { + id: "cs", + name: "C#", + extensions: ["cs"], + highlightAlias: "csharp", + }, + { + id: "java", + name: "Java", + extensions: ["java"], + highlightAlias: "java", + }, + { + id: "py", + name: "Python", + extensions: ["py"], + highlightAlias: "python", + }, + { + id: "lua", + name: "Lua", + extensions: ["lua"], + highlightAlias: "lua", + }, + { + id: "r", + name: "R", + extensions: ["r", "R"], + highlightAlias: "r", + }, ]; export async function seedLanguages(prisma: PrismaClient) { diff --git a/prisma/seed/snippets.ts b/prisma/seed/snippets.ts index cc95dba..826a24b 100644 --- a/prisma/seed/snippets.ts +++ b/prisma/seed/snippets.ts @@ -1,8 +1,8 @@ import {getSnippetsFromFile} from "@lib/snippets"; import {Prisma, PrismaClient} from "@prisma/client"; -const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; import pLimit from "p-limit"; +const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; const limit = pLimit(100); export async function seedSnippets(prisma: PrismaClient) { diff --git a/scripts/seed-snippets.ts b/scripts/seed-snippets.ts deleted file mode 100644 index ff01489..0000000 --- a/scripts/seed-snippets.ts +++ /dev/null @@ -1,40 +0,0 @@ -import dotenv from "dotenv"; -import {PrismaClient} from "@prisma/client"; -import {getFlagValue, parseNumberFlag} from "./utils/cli"; -import {getFilesForLanguage} from "../lib/snippet-helpers"; - -dotenv.config(); - -const rawArgs = process.argv.slice(2); - -interface CliOptions { - languageId: string; - pagesPerLanguage?: number; -} - -function parseCliOptions(): CliOptions { - const languageId = getFlagValue(rawArgs, "--language"); - - if (!languageId) { - throw new Error("Missing --language flag. Example: ts-node scripts/seed-snippets.ts --language js"); - } - - const pagesPerLanguage = parseNumberFlag(rawArgs, "--pages"); - - return {languageId, pagesPerLanguage}; -} - -if (require.main === module) { - (async () => { - const prisma = new PrismaClient(); - try { - const options = parseCliOptions(); - await seedLanguageSnippetsHelper(prisma, options.languageId, options.pagesPerLanguage); - await prisma.$disconnect(); - } catch (error) { - console.error(error); - await prisma.$disconnect(); - process.exit(1); - } - })(); -} diff --git a/src/features/snippets/config/snippets.server.ts b/src/features/snippets/config/snippets.server.ts index 4e54f85..ac0794b 100644 --- a/src/features/snippets/config/snippets.server.ts +++ b/src/features/snippets/config/snippets.server.ts @@ -26,6 +26,6 @@ export const VALID_NODES = [ "if_statement", ]; -export const MIN_SNIPPETS_PER_LANGUAGE = 10; -export const RANDOM_FILES_FETCHED = 30; -export const MAX_SNIPPETS_FETCH_ATTEMPTS = 20; +export const MIN_SNIPPETS_NUMBER = 10; +export const SNIPPETS_RETRIEVED_PER_QUERY = 30; +export const MAX_GET_SNIPPETS_ATTEMPTS = 10; diff --git a/src/features/snippets/logic/processing/filter.ts b/src/features/snippets/logic/filter.ts similarity index 100% rename from src/features/snippets/logic/processing/filter.ts rename to src/features/snippets/logic/filter.ts diff --git a/src/features/snippets/services/get-random-snippets.server.ts b/src/features/snippets/services/get-random-snippets.server.ts index f0e0dbe..f337d8d 100644 --- a/src/features/snippets/services/get-random-snippets.server.ts +++ b/src/features/snippets/services/get-random-snippets.server.ts @@ -1,15 +1,15 @@ -import {MIN_SNIPPETS_PER_LANGUAGE, MAX_SNIPPETS_FETCH_ATTEMPTS, RANDOM_FILES_FETCHED} from "@/features/snippets/config/snippets.server"; import {ISnippet} from "@/shared/types/snippet.server"; -import {filterSnippets} from "@/features/snippets/logic/processing/filter"; +import {filterSnippets} from "@/features/snippets/logic/filter"; import {findRandomSnippets} from "@/features/snippets/infrastructure/repositories/snippet.repository.server"; import {extractAutoCompleteDisabledRanges} from "@/features/snippets/logic/parsing/snippet-parser.server"; +import {MAX_GET_SNIPPETS_ATTEMPTS, MIN_SNIPPETS_NUMBER, SNIPPETS_RETRIEVED_PER_QUERY} from "@/features/snippets/config/snippets.server"; export async function getRandomSnippets(languageId: string): Promise { const snippets: ISnippet[] = []; let attempts = 0; - while (snippets.length < MIN_SNIPPETS_PER_LANGUAGE && attempts < MAX_SNIPPETS_FETCH_ATTEMPTS) { - const randomSnippets = await findRandomSnippets(languageId, RANDOM_FILES_FETCHED); + while (snippets.length < MIN_SNIPPETS_NUMBER && attempts < MAX_GET_SNIPPETS_ATTEMPTS) { + const randomSnippets = await findRandomSnippets(languageId, SNIPPETS_RETRIEVED_PER_QUERY); const filteredSnippets = randomSnippets.filter((s) => filterSnippets(s.content)); @@ -27,8 +27,8 @@ export async function getRandomSnippets(languageId: string): Promise attempts++; } - if (snippets.length < MIN_SNIPPETS_PER_LANGUAGE) { - throw new Error(`Failed to fetch enough snippets in ${MAX_SNIPPETS_FETCH_ATTEMPTS} attempts`); + if (snippets.length < MIN_SNIPPETS_NUMBER) { + throw new Error(`Failed to fetch enough snippets in ${MAX_GET_SNIPPETS_ATTEMPTS} attempts`); } return snippets.sort(() => Math.random() - 0.5); From 7185a0d609358c81806fe6aee38b6ba1beda1004 Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Mon, 17 Nov 2025 23:58:31 +0100 Subject: [PATCH 06/10] feat: add init migration --- .../20251117225653_init/migration.sql | 66 +++++++++++++++++++ prisma/migrations/migration_lock.toml | 3 + 2 files changed, 69 insertions(+) create mode 100644 prisma/migrations/20251117225653_init/migration.sql create mode 100644 prisma/migrations/migration_lock.toml diff --git a/prisma/migrations/20251117225653_init/migration.sql b/prisma/migrations/20251117225653_init/migration.sql new file mode 100644 index 0000000..6bd40a1 --- /dev/null +++ b/prisma/migrations/20251117225653_init/migration.sql @@ -0,0 +1,66 @@ +-- CreateTable +CREATE TABLE "Language" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "extensions" TEXT[], + "highlightAlias" TEXT NOT NULL, + + CONSTRAINT "Language_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "File" ( + "id" TEXT NOT NULL, + "repository" TEXT NOT NULL, + "path" TEXT NOT NULL, + "languageId" TEXT NOT NULL, + + CONSTRAINT "File_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "FileVersion" ( + "id" TEXT NOT NULL, + "sha" TEXT NOT NULL, + "commit" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "parsedAt" TIMESTAMP(3), + "fileId" TEXT NOT NULL, + + CONSTRAINT "FileVersion_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Snippet" ( + "id" TEXT NOT NULL, + "sha" TEXT NOT NULL, + "content" TEXT NOT NULL, + "rawContent" TEXT NOT NULL, + "fileVersionId" TEXT NOT NULL, + + CONSTRAINT "Snippet_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); + +-- CreateIndex +CREATE INDEX "File_languageId_idx" ON "File"("languageId"); + +-- CreateIndex +CREATE UNIQUE INDEX "File_repository_path_key" ON "File"("repository", "path"); + +-- CreateIndex +CREATE UNIQUE INDEX "FileVersion_fileId_sha_key" ON "FileVersion"("fileId", "sha"); + +-- CreateIndex +CREATE UNIQUE INDEX "Snippet_sha_key" ON "Snippet"("sha"); + +-- AddForeignKey +ALTER TABLE "File" ADD CONSTRAINT "File_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FileVersion" ADD CONSTRAINT "FileVersion_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_fileVersionId_fkey" FOREIGN KEY ("fileVersionId") REFERENCES "FileVersion"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..648c57f --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (e.g., Git) +provider = "postgresql" \ No newline at end of file From 67c87ab7904f78905316d563e036922e70971b01 Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Tue, 18 Nov 2025 00:50:49 +0100 Subject: [PATCH 07/10] refactor: add helper function for github api --- lib/github.ts | 146 +++++++++++++++++++++++++++++++++++++++++ lib/snippet-helpers.ts | 121 ---------------------------------- prisma/seed/files.ts | 76 ++++++++++++++++++++- scripts/utils/cli.ts | 31 --------- 4 files changed, 221 insertions(+), 153 deletions(-) create mode 100644 lib/github.ts delete mode 100644 lib/snippet-helpers.ts delete mode 100644 scripts/utils/cli.ts diff --git a/lib/github.ts b/lib/github.ts new file mode 100644 index 0000000..ed14c8c --- /dev/null +++ b/lib/github.ts @@ -0,0 +1,146 @@ +const GITHUB_API_URL = "https://api.github.com"; +const RETRY_DELAY_MS = 1000; +const RATE_LIMIT_PADDING_MS = 3000; + +type GitHubRequestParams = Record; + +export interface GitHubRequestOptions extends RequestInit { + params?: GitHubRequestParams; + maxAttempts?: number; + retry?: boolean; + shouldRetry?: (context: {response: Response; attempt: number; maxAttempts: number; url: string}) => Promise | boolean; +} + +export async function callGitHubApi(path: string, options: GitHubRequestOptions = {}): Promise { + const token = getToken(); + const {params, maxAttempts = 3, retry = true, shouldRetry, headers: customHeaders, ...fetchOptions} = options; + + const url = buildGitHubUrl(path, params); + const headers = new Headers({ + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + }); + + if (customHeaders) { + new Headers(customHeaders).forEach((value, key) => { + headers.set(key, value); + }); + } + + const attempts = retry ? maxAttempts : 1; + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...fetchOptions, + headers, + }); + + if (isRateLimitStatus(response.status)) { + await handleRateLimit(response); + continue; + } + + if (!response.ok) { + if (attempt >= attempts) { + throw new Error(`GitHub API request failed for ${url} (attempt ${attempt}/${attempts}) with status ${response.status}: ${response.statusText}`); + } + + await wait(RETRY_DELAY_MS); + continue; + } + + if (retry && shouldRetry && attempt < attempts) { + const shouldRetryResult = await shouldRetry({ + response: response.clone(), + attempt, + maxAttempts, + url, + }); + + if (shouldRetryResult) { + await wait(RETRY_DELAY_MS); + continue; + } + } + + return response; + } + + throw new Error(`GitHub API request failed after ${attempts} attempts for ${url}`); +} + +function getToken(): string { + const token = process.env.GITHUB_API_TOKEN; + + if (!token) { + throw new Error("Missing GITHUB_API_TOKEN environment variable"); + } + + return token; +} + +function buildGitHubUrl(path: string, params?: GitHubRequestParams): string { + const baseUrl = path.startsWith("http") ? new URL(path) : new URL(path.startsWith("/") ? path : `/${path}`, GITHUB_API_URL); + + if (params) { + Object.entries(params).forEach(([key, value]) => { + if (value === undefined) return; + baseUrl.searchParams.set(key, String(value)); + }); + } + + return baseUrl.toString(); +} + +function isRateLimitStatus(status: number) { + return status === 403 || status === 429; +} + +async function handleRateLimit(response: Response): Promise { + const resetAt = getRateLimitReset(response); + if (resetAt) { + const waitTime = Math.max(resetAt - Date.now(), 0) + RATE_LIMIT_PADDING_MS; + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${waitTime}ms until reset.`); + await wait(waitTime); + return; + } + + const retryAfter = parseRetryAfter(response.headers.get("Retry-After")); + if (retryAfter !== null) { + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${retryAfter}ms due to Retry-After header.`); + await wait(retryAfter); + return; + } + + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${RETRY_DELAY_MS}ms before retrying.`); + await wait(RETRY_DELAY_MS); +} + +function getRateLimitReset(response: Response): number | null { + const headerValue = response.headers.get("X-RateLimit-Reset"); + if (!headerValue) return null; + + const epochSeconds = Number(headerValue); + if (Number.isNaN(epochSeconds)) { + return null; + } + + return epochSeconds * 1000; +} + +function parseRetryAfter(headerValue: string | null): number | null { + if (!headerValue) return null; + + const numericValue = Number(headerValue); + if (!Number.isNaN(numericValue)) { + return numericValue * 1000; + } + + const retryDate = new Date(headerValue); + const waitTime = retryDate.getTime() - Date.now(); + return Number.isNaN(waitTime) ? null : Math.max(waitTime, 0); +} + +function wait(duration: number) { + return new Promise((resolve) => setTimeout(resolve, duration)); +} diff --git a/lib/snippet-helpers.ts b/lib/snippet-helpers.ts deleted file mode 100644 index fea37db..0000000 --- a/lib/snippet-helpers.ts +++ /dev/null @@ -1,121 +0,0 @@ -import {ILanguage} from "@/shared/types/language"; - -const GITHUB_API_URL = "https://api.github.com"; -const MAX_ATTEMPTS = 3; -const PER_PAGE = 100; -const MIN_FILE_SIZE = 3000; - -interface FileMetadata { - repository: string; - path: string; - commit: string; - sha: string; -} - -interface GitHubSearchResponse { - items?: { - repository: {full_name: string}; - path: string; - url: string; - sha: string; - }[]; -} - -function getToken(): string { - const token = process.env.GITHUB_API_TOKEN; - - if (!token) { - throw new Error("Missing GITHUB_API_TOKEN environment variable"); - } - - return token; -} - -async function waitForGitHubRateLimitReset(token: string): Promise { - const res = await fetch(`${GITHUB_API_URL}/rate_limit`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - if (!res.ok) { - console.error(`Failed to fetch rate limit status: ${res.status} ${res.statusText}`); - return; - } - - const json = (await res.json()) as {resources: {code_search: {reset: number}}}; - const reset = json.resources.code_search.reset * 1000; - const now = Date.now(); - const waitTime = reset - now; - - if (waitTime > 0) { - console.warn(`Rate limited by GitHub. Waiting ${waitTime}ms until reset.`); - await new Promise((resolve) => setTimeout(resolve, waitTime + 3000)); - } -} - -export async function fetchFilesFromGithub(language: ILanguage, page: number): Promise { - const token = getToken(); - const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); - const params = new URLSearchParams({ - q: `language:${language.name} size:>${MIN_FILE_SIZE} ${extensionsFilter}`, - page: page.toString(), - per_page: PER_PAGE.toString(), - }); - - const request = () => - fetch(`${GITHUB_API_URL}/search/code?${params}`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { - const response = await request(); - - if (response.status === 403) { - console.warn(`Received 403 (rate limit) fetching ${language.name} files on page ${page}. Attempt ${attempt}/${MAX_ATTEMPTS}.`); - await waitForGitHubRateLimitReset(token); - continue; - } - - if (!response.ok) { - throw new Error(`GitHub search failed for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}) with status ${response.status}: ${response.statusText}`); - } - - const data = (await response.json()) as GitHubSearchResponse; - const items = data.items ?? []; - - if (items.length > 0) { - console.info(`Fetched ${items.length} files for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}).`); - return items.map((item) => { - return { - repository: item.repository.full_name, - path: item.path, - commit: item.url.split("?ref=")[1], - sha: item.sha, - }; - }); - } - - if (attempt < MAX_ATTEMPTS) { - console.warn(`No files returned for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}). Retrying...`); - } - } - - throw new Error(`No files returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); -} - -export async function getFilesForLanguage(language: ILanguage, {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { - const files: FileMetadata[] = []; - for (let page = 1; page <= pagesPerLanguage; page += 1) { - try { - const file = await fetchFilesFromGithub(language, page); - files.push(...file); - } catch (error) { - throw new Error(`Error fetching files for ${language.name} on page ${page}: ${error}`); - } - } - - return files; -} diff --git a/prisma/seed/files.ts b/prisma/seed/files.ts index de3a2c4..f46d9e5 100644 --- a/prisma/seed/files.ts +++ b/prisma/seed/files.ts @@ -1,7 +1,28 @@ +import {ILanguage} from "@/shared/types/language"; import {PrismaClient} from "@prisma/client"; -import {getFilesForLanguage} from "../../lib/snippet-helpers"; +import {callGitHubApi} from "@lib/github"; import {SUPPORTED_LANGUAGES} from "./languages"; +const PER_PAGE = 100; +const MIN_FILE_SIZE = 3000; +const MAX_ATTEMPTS = 3; + +interface FileMetadata { + repository: string; + path: string; + commit: string; + sha: string; +} + +interface GitHubSearchResponse { + items?: { + repository: {full_name: string}; + path: string; + url: string; + sha: string; + }[]; +} + export async function seedFiles(prisma: PrismaClient) { console.log("Seeding files for supported languages..."); const languages = await prisma.language.findMany({ @@ -60,3 +81,56 @@ async function seedFilesForLanguage(prisma: PrismaClient, languageId: string) { console.log(`Seeded ${files.length} files for language: ${language.name}`); } + +async function getFilesForLanguage(language: ILanguage, {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { + const files = []; + for (let page = 1; page <= pagesPerLanguage; page += 1) { + try { + const pageFiles = await fetchFilesFromGithub(language, page); + files.push(...pageFiles); + } catch (error) { + throw new Error(`Error fetching files for ${language.name} on page ${page}: ${error}`); + } + } + + return files; +} + +async function fetchFilesFromGithub(language: ILanguage, page: number): Promise { + const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); + + const response = await callGitHubApi("/search/code", { + params: { + q: `language:${language.name} size:>${MIN_FILE_SIZE} ${extensionsFilter}`, + page: page.toString(), + per_page: PER_PAGE.toString(), + }, + maxAttempts: MAX_ATTEMPTS, + shouldRetry: async ({response, attempt, maxAttempts}) => { + const data = (await response.json()) as GitHubSearchResponse; + const items = data.items ?? []; + + if (items.length === 0) { + console.warn(`No files returned for ${language.name} on page ${page} (attempt ${attempt}/${maxAttempts}). Retrying...`); + return true; + } + + return false; + }, + }); + + const data = (await response.json()) as GitHubSearchResponse; + const items = data.items ?? []; + + if (items.length === 0) { + throw new Error(`No files returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); + } + + console.info(`Fetched ${items.length} files for ${language.name} on page ${page}.`); + return items.map((item) => ({ + repository: item.repository.full_name, + path: item.path, + commit: item.url.split("?ref=")[1], + sha: item.sha, + })); +} diff --git a/scripts/utils/cli.ts b/scripts/utils/cli.ts deleted file mode 100644 index cd1ecc7..0000000 --- a/scripts/utils/cli.ts +++ /dev/null @@ -1,31 +0,0 @@ -export function getFlagValue(args: string[], flag: string): string | undefined { - const index = args.findIndex((arg) => arg === flag || arg.startsWith(`${flag}=`)); - - if (index === -1) { - return undefined; - } - - const [, value] = args[index].split("="); - - if (value) { - return value; - } - - return args[index + 1]; -} - -export function parseNumberFlag(args: string[], flag: string): number | undefined { - const value = getFlagValue(args, flag); - - if (!value) { - return undefined; - } - - const parsedValue = Number(value); - - if (Number.isNaN(parsedValue)) { - throw new Error(`Invalid value for ${flag}. It must be a number.`); - } - - return parsedValue; -} From c390b124e4c9c7e1b45519cbe2b5c157a5f6017d Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Tue, 18 Nov 2025 19:21:25 +0100 Subject: [PATCH 08/10] fix: fix tabs in between filter --- lib/snippets/processing/filter.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/snippets/processing/filter.ts b/lib/snippets/processing/filter.ts index 8f81186..2d39e4d 100644 --- a/lib/snippets/processing/filter.ts +++ b/lib/snippets/processing/filter.ts @@ -23,8 +23,8 @@ function filterSnippetSpecialCharacters(snippet: string): boolean { } function filterTabsInBetween(snippet: string): boolean { - const regex = /(^|[^\\])(\\t)|(\t)/; - return regex.test(snippet.trim()); + const regex = /^.+\\t.+$/; + return !regex.test(snippet); } export function isValidNode(node: IParser.SyntaxNode) { From c7d4ba3b9d61394a33d6f7f06b373a4a9102e3ab Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Tue, 18 Nov 2025 19:21:59 +0100 Subject: [PATCH 09/10] fix: prevent error on seed when there are no unparsed files --- prisma/seed/snippets.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/prisma/seed/snippets.ts b/prisma/seed/snippets.ts index 826a24b..9b0b73b 100644 --- a/prisma/seed/snippets.ts +++ b/prisma/seed/snippets.ts @@ -25,12 +25,13 @@ export async function seedSnippets(prisma: PrismaClient) { }); if (unparsedFiles.length === 0) { - throw new Error("All files are parsed. No snippets to seed."); + console.log("All files are parsed. No snippets to seed."); + return; } await Promise.all( unparsedFiles.map(async (file) => { - const {repository, path} = file.file; + const {repository, path, languageId} = file.file; const url = `${RAW_GITHUB_BASE_URL}/${repository}/${file.commit}/${path}`; const fileContent = await limit(async () => { @@ -56,11 +57,10 @@ export async function seedSnippets(prisma: PrismaClient) { if (!fileContent) return; - const snippets = getSnippetsFromFile(fileContent, file.file.languageId); + const snippets = getSnippetsFromFile(fileContent, languageId); await Promise.all( snippets.map(async (snippet) => { - console.log(`Inserting snippet with SHA ${snippet.sha} from file version ${file.id}`); try { await prisma.snippet.create({ data: { From 62e4b78f62cec1c6abda63238e52e8ba50885cb3 Mon Sep 17 00:00:00 2001 From: Mattia Cerutti Date: Tue, 18 Nov 2025 19:24:13 +0100 Subject: [PATCH 10/10] feat: add onDelete behavior for foreign keys --- prisma/schema.prisma | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prisma/schema.prisma b/prisma/schema.prisma index d7eae6c..b8024db 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -20,7 +20,7 @@ model File { repository String path String languageId String - language Language @relation(fields: [languageId], references: [id]) + language Language @relation(fields: [languageId], references: [id], onDelete: Cascade) fileVersions FileVersion[] @@unique([repository, path]) @@ -34,7 +34,7 @@ model FileVersion { createdAt DateTime @default(now()) parsedAt DateTime? fileId String - file File @relation(fields: [fileId], references: [id]) + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) snippets Snippet[] @@unique([fileId, sha]) @@ -46,5 +46,5 @@ model Snippet { content String rawContent String fileVersionId String - fileVersion FileVersion @relation(fields: [fileVersionId], references: [id]) + fileVersion FileVersion @relation(fields: [fileVersionId], references: [id], onDelete: Cascade) }