diff --git a/lib/github.ts b/lib/github.ts new file mode 100644 index 0000000..ed14c8c --- /dev/null +++ b/lib/github.ts @@ -0,0 +1,146 @@ +const GITHUB_API_URL = "https://api.github.com"; +const RETRY_DELAY_MS = 1000; +const RATE_LIMIT_PADDING_MS = 3000; + +type GitHubRequestParams = Record; + +export interface GitHubRequestOptions extends RequestInit { + params?: GitHubRequestParams; + maxAttempts?: number; + retry?: boolean; + shouldRetry?: (context: {response: Response; attempt: number; maxAttempts: number; url: string}) => Promise | boolean; +} + +export async function callGitHubApi(path: string, options: GitHubRequestOptions = {}): Promise { + const token = getToken(); + const {params, maxAttempts = 3, retry = true, shouldRetry, headers: customHeaders, ...fetchOptions} = options; + + const url = buildGitHubUrl(path, params); + const headers = new Headers({ + Authorization: `Bearer ${token}`, + Accept: "application/vnd.github+json", + }); + + if (customHeaders) { + new Headers(customHeaders).forEach((value, key) => { + headers.set(key, value); + }); + } + + const attempts = retry ? maxAttempts : 1; + + for (let attempt = 1; attempt <= attempts; attempt += 1) { + const response = await fetch(url, { + ...fetchOptions, + headers, + }); + + if (isRateLimitStatus(response.status)) { + await handleRateLimit(response); + continue; + } + + if (!response.ok) { + if (attempt >= attempts) { + throw new Error(`GitHub API request failed for ${url} (attempt ${attempt}/${attempts}) with status ${response.status}: ${response.statusText}`); + } + + await wait(RETRY_DELAY_MS); + continue; + } + + if (retry && shouldRetry && attempt < attempts) { + const shouldRetryResult = await shouldRetry({ + response: response.clone(), + attempt, + maxAttempts, + url, + }); + + if (shouldRetryResult) { + await wait(RETRY_DELAY_MS); + continue; + } + } + + return response; + } + + throw new Error(`GitHub API request failed after ${attempts} attempts for ${url}`); +} + +function getToken(): string { + const token = process.env.GITHUB_API_TOKEN; + + if (!token) { + throw new Error("Missing GITHUB_API_TOKEN environment variable"); + } + + return token; +} + +function buildGitHubUrl(path: string, params?: GitHubRequestParams): string { + const baseUrl = path.startsWith("http") ? new URL(path) : new URL(path.startsWith("/") ? path : `/${path}`, GITHUB_API_URL); + + if (params) { + Object.entries(params).forEach(([key, value]) => { + if (value === undefined) return; + baseUrl.searchParams.set(key, String(value)); + }); + } + + return baseUrl.toString(); +} + +function isRateLimitStatus(status: number) { + return status === 403 || status === 429; +} + +async function handleRateLimit(response: Response): Promise { + const resetAt = getRateLimitReset(response); + if (resetAt) { + const waitTime = Math.max(resetAt - Date.now(), 0) + RATE_LIMIT_PADDING_MS; + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${waitTime}ms until reset.`); + await wait(waitTime); + return; + } + + const retryAfter = parseRetryAfter(response.headers.get("Retry-After")); + if (retryAfter !== null) { + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${retryAfter}ms due to Retry-After header.`); + await wait(retryAfter); + return; + } + + console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${RETRY_DELAY_MS}ms before retrying.`); + await wait(RETRY_DELAY_MS); +} + +function getRateLimitReset(response: Response): number | null { + const headerValue = response.headers.get("X-RateLimit-Reset"); + if (!headerValue) return null; + + const epochSeconds = Number(headerValue); + if (Number.isNaN(epochSeconds)) { + return null; + } + + return epochSeconds * 1000; +} + +function parseRetryAfter(headerValue: string | null): number | null { + if (!headerValue) return null; + + const numericValue = Number(headerValue); + if (!Number.isNaN(numericValue)) { + return numericValue * 1000; + } + + const retryDate = new Date(headerValue); + const waitTime = retryDate.getTime() - Date.now(); + return Number.isNaN(waitTime) ? null : Math.max(waitTime, 0); +} + +function wait(duration: number) { + return new Promise((resolve) => setTimeout(resolve, duration)); +} diff --git a/lib/snippets/config.ts b/lib/snippets/config.ts new file mode 100644 index 0000000..e9fb8e7 --- /dev/null +++ b/lib/snippets/config.ts @@ -0,0 +1,27 @@ +export const MAX_LINES = 11; +export const MIN_LINES = 3; + +export const MAX_LINE_LENGTH = 72; + +export const MAX_SNIPPET_LENGTH = 350; +export const MIN_SNIPPET_LENGTH = 75; + +export const VALID_NODES = [ + "function_declaration", + "function_definition", + "class_declaration", + "class_definition", + "function_item", + "method_declaration", + "method_definition", + "method_item", + "class_item", + "interface_declaration", + "interface_definition", + "interface_item", + "module", + "switch_statement", + "while_statement", + "for_statement", + "if_statement", +]; diff --git a/lib/snippets/index.ts b/lib/snippets/index.ts new file mode 100644 index 0000000..991eaca --- /dev/null +++ b/lib/snippets/index.ts @@ -0,0 +1,26 @@ +import {extractSnippets} from "./parsing/parser"; +import {filterSnippets} from "./processing/filter"; +import {formatCode} from "./processing/process"; +import {hashSnippet} from "./processing/utils"; + +interface ISnippetOutput { + sha: string; + content: string; + rawContent: string; +} + +export function getSnippetsFromFile(fileContent: string, languageId: string): ISnippetOutput[] { + const extractedSnippets = extractSnippets(fileContent, languageId); + + const formattedSnippets = extractedSnippets + .map((snippet) => ({ + sha: hashSnippet(snippet), + content: formatCode(snippet), + rawContent: snippet, + })) + .filter((s) => s.content !== null) as ISnippetOutput[]; + + const filteredSnippets = formattedSnippets.filter((snippet) => filterSnippets(snippet.content)); + + return filteredSnippets; +} diff --git a/src/features/snippets/logic/parsing/parser-factory.server.ts b/lib/snippets/parsing/parser-factory.ts similarity index 100% rename from src/features/snippets/logic/parsing/parser-factory.server.ts rename to lib/snippets/parsing/parser-factory.ts diff --git a/lib/snippets/parsing/parser.ts b/lib/snippets/parsing/parser.ts new file mode 100644 index 0000000..d967af5 --- /dev/null +++ b/lib/snippets/parsing/parser.ts @@ -0,0 +1,46 @@ +import {getInitialIndentation} from "@lib/snippets/processing/indentation"; +import {getTSParser} from "./parser-factory"; +import IParser from "tree-sitter"; +import {isValidNode} from "@lib/snippets/processing/filter"; + +export function findValidNodes(node: IParser.SyntaxNode): IParser.SyntaxNode[] { + let nodes: IParser.SyntaxNode[] = []; + if (isValidNode(node)) { + nodes.push(node); + } + + node.children.forEach((child) => { + if (child === null) return; + nodes = nodes.concat(findValidNodes(child)); + }); + + return nodes; +} + +export function convertSnippetToText(node: IParser.SyntaxNode, sourceCode: string): string { + const initialIndentation = getInitialIndentation(node.startIndex, sourceCode); + + return initialIndentation + getNodeText(node); +} + +export function extractSnippets(fileContent: string, languageId: string): string[] { + const parser = getTSParser(languageId); + + let parsedCode: IParser.Tree; + + try { + parsedCode = parser.parse(fileContent); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + } catch (error) { + return []; + } + + const validNodes = findValidNodes(parsedCode.rootNode); + const validSnippets = validNodes.map((node) => convertSnippetToText(node, fileContent)); + + return validSnippets; +} + +export function getNodeText(node: IParser.SyntaxNode): string { + return node.text; +} diff --git a/lib/snippets/processing/filter.ts b/lib/snippets/processing/filter.ts new file mode 100644 index 0000000..2d39e4d --- /dev/null +++ b/lib/snippets/processing/filter.ts @@ -0,0 +1,36 @@ +import IParser from "tree-sitter"; +import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH, VALID_NODES} from "../config"; + +function filterLinesNumber(snippet: string): boolean { + const numLines = snippet.split("\n").length; + return numLines <= MAX_LINES && numLines >= MIN_LINES; +} + +function filterLineLength(snippet: string): boolean { + const lines = snippet.split("\n"); + return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH); +} + +function filterSnippetLength(snippet: string): boolean { + return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH; +} + +function filterSnippetSpecialCharacters(snippet: string): boolean { + const regex = /^[\x00-\x7F]+$/; + const test = regex.test(snippet); + + return test; +} + +function filterTabsInBetween(snippet: string): boolean { + const regex = /^.+\\t.+$/; + return !regex.test(snippet); +} + +export function isValidNode(node: IParser.SyntaxNode) { + return VALID_NODES.includes(node.type); +} + +export function filterSnippets(content: string): boolean { + return filterSnippetLength(content) && filterLinesNumber(content) && filterLineLength(content) && filterSnippetSpecialCharacters(content) && filterTabsInBetween(content); +} diff --git a/src/features/snippets/logic/processing/indentation.server.ts b/lib/snippets/processing/indentation.ts similarity index 99% rename from src/features/snippets/logic/processing/indentation.server.ts rename to lib/snippets/processing/indentation.ts index daac2d8..3bf04d7 100644 --- a/src/features/snippets/logic/processing/indentation.server.ts +++ b/lib/snippets/processing/indentation.ts @@ -1,4 +1,4 @@ -import {countInitialWhitespaces, removeInitialCharacters} from "./text.server"; +import {countInitialWhitespaces, removeInitialCharacters} from "./utils"; type IIndentationStyle = {type: "none" | "mixed" | "tab"} | {type: "space"; value: number[]}; export function detectIndentationStyle(snippet: string): IIndentationStyle { diff --git a/lib/snippets/processing/process.ts b/lib/snippets/processing/process.ts new file mode 100644 index 0000000..18c52da --- /dev/null +++ b/lib/snippets/processing/process.ts @@ -0,0 +1,54 @@ +import {adjustIndentationOffset, applyIndentationToEmptyLines, detectIndentationStyle} from "./indentation"; +import {countInitialWhitespaces} from "./utils"; + +function cleanText(text: string): string { + // Remove whitespaces from empty lines + text = text.replace(/(?<=\n)[ \t]+(?=\n)/g, ""); + + return text; +} + +function hasProhibitedCharacters(input: string): boolean { + const regex = /((?:(?![ \t\n])\s)|\\)/; + return regex.test(input); +} + +export function formatCode(snippet: string): string | null { + if (hasProhibitedCharacters(snippet)) { + return null; + } + + // Clean raw snippet + snippet = cleanText(snippet); + + const indentationStyle = detectIndentationStyle(snippet); + + if (indentationStyle.type === "mixed") { + return null; + } + + if (indentationStyle.type === "none") { + return snippet; + } + + // Convert raw snippet indentation style with standardized '\t' indentation + let codeLines = snippet.split("\n").map((line) => { + const initialWhitespaces = countInitialWhitespaces(line); + + const trimmedLine = line.trim(); + + if (indentationStyle.type === "space") { + return "\t".repeat(indentationStyle.value.indexOf(initialWhitespaces)) + trimmedLine; + } + + return "\t".repeat(initialWhitespaces) + trimmedLine; + }); + + // Adjust indentation offset on every line + codeLines = adjustIndentationOffset(codeLines); + + // Applies indentation to empty lines (the minimum indentation of the surrounding lines) + codeLines = applyIndentationToEmptyLines(codeLines); + + return codeLines.join("\n"); +} diff --git a/src/features/snippets/logic/processing/text.server.ts b/lib/snippets/processing/utils.ts similarity index 66% rename from src/features/snippets/logic/processing/text.server.ts rename to lib/snippets/processing/utils.ts index 6004c5a..ecdee49 100644 --- a/src/features/snippets/logic/processing/text.server.ts +++ b/lib/snippets/processing/utils.ts @@ -1,3 +1,5 @@ +import {createHash} from "crypto"; + export function countInitialWhitespaces(text: string): number { let count = 0; for (let i = 0; i < text.length; i++) { @@ -21,3 +23,10 @@ export function removeInitialCharacters(lines: string[], quantity: number): stri return newLines; } + +export function hashSnippet(content: string): string { + // Remove all whitespaces + const newContent = content.replace(/\s+/g, "").trim(); + + return createHash("sha256").update(newContent).digest("hex"); +} diff --git a/package-lock.json b/package-lock.json index c6489cb..743d4c5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "next": "16.0.1", "next-themes": "^0.4.6", "prisma": "^6.5.0", + "prisma-extension-random": "^0.2.2", "rate-limiter-flexible": "^7.0.0", "react": "19.2.0", "react-dom": "19.2.0", @@ -1667,6 +1668,7 @@ "integrity": "sha512-M6w1Ql/BeiGoZmhMdAZUXHu5sz5HubyVcKukbLs3l0ELcQb8hTUJxtGEChhv4SVJ0QJlwtLnwOLgIRQhpsm9dw==", "hasInstallScript": true, "license": "Apache-2.0", + "peer": true, "engines": { "node": ">=18.18" }, @@ -6917,6 +6919,15 @@ } } }, + "node_modules/prisma-extension-random": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/prisma-extension-random/-/prisma-extension-random-0.2.2.tgz", + "integrity": "sha512-tjY5w2pSIgwQh0tDT98KTTYJ0uadUwfbTH+6QbPCfLg+gesgeLZNO4oKv8O6/udx54QvEbyXgNC1LUcjtoypcg==", + "license": "MIT", + "peerDependencies": { + "@prisma/client": "*" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", diff --git a/package.json b/package.json index 69387aa..e0bd155 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "next": "16.0.1", "next-themes": "^0.4.6", "prisma": "^6.5.0", + "prisma-extension-random": "^0.2.2", "rate-limiter-flexible": "^7.0.0", "react": "19.2.0", "react-dom": "19.2.0", diff --git a/prisma/migrations/20250402143241_init/migration.sql b/prisma/migrations/20250402143241_init/migration.sql deleted file mode 100644 index c48a269..0000000 --- a/prisma/migrations/20250402143241_init/migration.sql +++ /dev/null @@ -1,31 +0,0 @@ --- CreateTable -CREATE TABLE "Language" ( - "id" TEXT NOT NULL, - "name" TEXT NOT NULL, - "lastUpdatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, - "extensions" TEXT[], - "highlightAlias" TEXT NOT NULL, - - CONSTRAINT "Language_pkey" PRIMARY KEY ("id") -); - --- CreateTable -CREATE TABLE "Snippet" ( - "url" TEXT NOT NULL, - "languageId" TEXT NOT NULL, - "isValid" BOOLEAN NOT NULL, - - CONSTRAINT "Snippet_pkey" PRIMARY KEY ("url") -); - --- CreateIndex -CREATE UNIQUE INDEX "Language_id_key" ON "Language"("id"); - --- CreateIndex -CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); - --- CreateIndex -CREATE UNIQUE INDEX "Snippet_url_key" ON "Snippet"("url"); - --- AddForeignKey -ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/migrations/20250402153753_add_languageid_index/migration.sql b/prisma/migrations/20250402153753_add_languageid_index/migration.sql deleted file mode 100644 index e42d7ec..0000000 --- a/prisma/migrations/20250402153753_add_languageid_index/migration.sql +++ /dev/null @@ -1,5 +0,0 @@ --- AlterTable -ALTER TABLE "Snippet" ALTER COLUMN "isValid" SET DEFAULT true; - --- CreateIndex -CREATE INDEX "Snippet_languageId_idx" ON "Snippet"("languageId"); diff --git a/prisma/migrations/20251117225653_init/migration.sql b/prisma/migrations/20251117225653_init/migration.sql new file mode 100644 index 0000000..6bd40a1 --- /dev/null +++ b/prisma/migrations/20251117225653_init/migration.sql @@ -0,0 +1,66 @@ +-- CreateTable +CREATE TABLE "Language" ( + "id" TEXT NOT NULL, + "name" TEXT NOT NULL, + "extensions" TEXT[], + "highlightAlias" TEXT NOT NULL, + + CONSTRAINT "Language_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "File" ( + "id" TEXT NOT NULL, + "repository" TEXT NOT NULL, + "path" TEXT NOT NULL, + "languageId" TEXT NOT NULL, + + CONSTRAINT "File_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "FileVersion" ( + "id" TEXT NOT NULL, + "sha" TEXT NOT NULL, + "commit" TEXT NOT NULL, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "parsedAt" TIMESTAMP(3), + "fileId" TEXT NOT NULL, + + CONSTRAINT "FileVersion_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Snippet" ( + "id" TEXT NOT NULL, + "sha" TEXT NOT NULL, + "content" TEXT NOT NULL, + "rawContent" TEXT NOT NULL, + "fileVersionId" TEXT NOT NULL, + + CONSTRAINT "Snippet_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Language_name_key" ON "Language"("name"); + +-- CreateIndex +CREATE INDEX "File_languageId_idx" ON "File"("languageId"); + +-- CreateIndex +CREATE UNIQUE INDEX "File_repository_path_key" ON "File"("repository", "path"); + +-- CreateIndex +CREATE UNIQUE INDEX "FileVersion_fileId_sha_key" ON "FileVersion"("fileId", "sha"); + +-- CreateIndex +CREATE UNIQUE INDEX "Snippet_sha_key" ON "Snippet"("sha"); + +-- AddForeignKey +ALTER TABLE "File" ADD CONSTRAINT "File_languageId_fkey" FOREIGN KEY ("languageId") REFERENCES "Language"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "FileVersion" ADD CONSTRAINT "FileVersion_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE RESTRICT ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Snippet" ADD CONSTRAINT "Snippet_fileVersionId_fkey" FOREIGN KEY ("fileVersionId") REFERENCES "FileVersion"("id") ON DELETE RESTRICT ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index e73530c..b8024db 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -8,19 +8,43 @@ datasource db { } model Language { - id String @id @unique - name String @unique - lastUpdatedAt DateTime @default(now()) + id String @id + name String @unique extensions String[] highlightAlias String - snippets Snippet[] + trackedFiles File[] } -model Snippet { - url String @id @unique - languageId String - isValid Boolean @default(true) - language Language @relation(fields: [languageId], references: [id]) +model File { + id String @id @default(uuid()) + repository String + path String + languageId String + language Language @relation(fields: [languageId], references: [id], onDelete: Cascade) + fileVersions FileVersion[] + @@unique([repository, path]) @@index([languageId]) } + +model FileVersion { + id String @id @default(uuid()) + sha String + commit String + createdAt DateTime @default(now()) + parsedAt DateTime? + fileId String + file File @relation(fields: [fileId], references: [id], onDelete: Cascade) + snippets Snippet[] + + @@unique([fileId, sha]) +} + +model Snippet { + id String @id @default(uuid()) + sha String @unique + content String + rawContent String + fileVersionId String + fileVersion FileVersion @relation(fields: [fileVersionId], references: [id], onDelete: Cascade) +} diff --git a/prisma/seed.ts b/prisma/seed.ts index 94867bb..12cf7b5 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -1,6 +1,8 @@ import dotenv from "dotenv"; import {PrismaClient} from "@prisma/client"; import {seedLanguages} from "./seed/languages"; +import {seedFiles} from "./seed/files"; +import {seedSnippets} from "./seed/snippets"; dotenv.config(); @@ -8,7 +10,8 @@ const prisma = new PrismaClient(); async function main() { await seedLanguages(prisma); - console.log("Languages seeded successfully."); + await seedFiles(prisma); + await seedSnippets(prisma); } main() diff --git a/prisma/seed/files.ts b/prisma/seed/files.ts new file mode 100644 index 0000000..f46d9e5 --- /dev/null +++ b/prisma/seed/files.ts @@ -0,0 +1,136 @@ +import {ILanguage} from "@/shared/types/language"; +import {PrismaClient} from "@prisma/client"; +import {callGitHubApi} from "@lib/github"; +import {SUPPORTED_LANGUAGES} from "./languages"; + +const PER_PAGE = 100; +const MIN_FILE_SIZE = 3000; +const MAX_ATTEMPTS = 3; + +interface FileMetadata { + repository: string; + path: string; + commit: string; + sha: string; +} + +interface GitHubSearchResponse { + items?: { + repository: {full_name: string}; + path: string; + url: string; + sha: string; + }[]; +} + +export async function seedFiles(prisma: PrismaClient) { + console.log("Seeding files for supported languages..."); + const languages = await prisma.language.findMany({ + select: { + id: true, + _count: {select: {trackedFiles: true}}, + }, + }); + + if (languages.length === 0) { + throw new Error("Language table is empty. Run the base seed first: ts-node prisma/seed.ts"); + } + + const missingLanguageIds = languages.filter((language) => language._count.trackedFiles === 0).map((language) => language.id); + + if (missingLanguageIds.length === 0) { + console.log("All languages already have snippets."); + return; + } + + console.log(`Seeding snippets for missing languages: ${missingLanguageIds.join(", ")}`); + + for (const languageId of missingLanguageIds) { + await seedFilesForLanguage(prisma, languageId); + } + + console.log("Finished seeding files for all missing languages."); +} + +async function seedFilesForLanguage(prisma: PrismaClient, languageId: string) { + const language = SUPPORTED_LANGUAGES.find((lang) => lang.id === languageId); + if (!language) { + throw new Error(`Language with ID ${languageId} is not supported.`); + } + + console.log(`Fetching files for language: ${language.name}`); + const files = await getFilesForLanguage(language); + + for (const file of files) { + const insertedFile = await prisma.file.create({ + data: { + repository: file.repository, + path: file.path, + languageId: language.id, + }, + }); + + await prisma.fileVersion.create({ + data: { + sha: file.sha, + commit: file.commit, + fileId: insertedFile.id, + }, + }); + } + + console.log(`Seeded ${files.length} files for language: ${language.name}`); +} + +async function getFilesForLanguage(language: ILanguage, {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { + const files = []; + for (let page = 1; page <= pagesPerLanguage; page += 1) { + try { + const pageFiles = await fetchFilesFromGithub(language, page); + files.push(...pageFiles); + } catch (error) { + throw new Error(`Error fetching files for ${language.name} on page ${page}: ${error}`); + } + } + + return files; +} + +async function fetchFilesFromGithub(language: ILanguage, page: number): Promise { + const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); + + const response = await callGitHubApi("/search/code", { + params: { + q: `language:${language.name} size:>${MIN_FILE_SIZE} ${extensionsFilter}`, + page: page.toString(), + per_page: PER_PAGE.toString(), + }, + maxAttempts: MAX_ATTEMPTS, + shouldRetry: async ({response, attempt, maxAttempts}) => { + const data = (await response.json()) as GitHubSearchResponse; + const items = data.items ?? []; + + if (items.length === 0) { + console.warn(`No files returned for ${language.name} on page ${page} (attempt ${attempt}/${maxAttempts}). Retrying...`); + return true; + } + + return false; + }, + }); + + const data = (await response.json()) as GitHubSearchResponse; + const items = data.items ?? []; + + if (items.length === 0) { + throw new Error(`No files returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); + } + + console.info(`Fetched ${items.length} files for ${language.name} on page ${page}.`); + return items.map((item) => ({ + repository: item.repository.full_name, + path: item.path, + commit: item.url.split("?ref=")[1], + sha: item.sha, + })); +} diff --git a/prisma/seed/languages.ts b/prisma/seed/languages.ts index 1ead7e5..db0d518 100644 --- a/prisma/seed/languages.ts +++ b/prisma/seed/languages.ts @@ -77,4 +77,5 @@ export async function seedLanguages(prisma: PrismaClient) { console.log(`Language seeded: ${languageRecord.name}`); } + console.log("Languages seeded successfully."); } diff --git a/prisma/seed/snippets.ts b/prisma/seed/snippets.ts new file mode 100644 index 0000000..9b0b73b --- /dev/null +++ b/prisma/seed/snippets.ts @@ -0,0 +1,89 @@ +import {getSnippetsFromFile} from "@lib/snippets"; +import {Prisma, PrismaClient} from "@prisma/client"; +import pLimit from "p-limit"; + +const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; +const limit = pLimit(100); + +export async function seedSnippets(prisma: PrismaClient) { + console.log("Seeding snippets for unparsed files.."); + const unparsedFiles = await prisma.fileVersion.findMany({ + select: { + id: true, + file: { + select: { + repository: true, + path: true, + languageId: true, + }, + }, + commit: true, + }, + where: { + parsedAt: null, + }, + }); + + if (unparsedFiles.length === 0) { + console.log("All files are parsed. No snippets to seed."); + return; + } + + await Promise.all( + unparsedFiles.map(async (file) => { + const {repository, path, languageId} = file.file; + const url = `${RAW_GITHUB_BASE_URL}/${repository}/${file.commit}/${path}`; + + const fileContent = await limit(async () => { + const res = await fetch(url).catch((error) => { + console.warn(`Could not fetch file content from ${url}:`, error); + return null; + }); + + if (!res) return null; + + if (!res.ok) { + if (res.status === 404 || res.status === 400) { + await prisma.fileVersion.delete({ + where: {id: file.id}, + }); + console.warn(`Deleted file version ${file.id} due to 404 (file missing).`); + } + return null; + } + + return res.text(); + }); + + if (!fileContent) return; + + const snippets = getSnippetsFromFile(fileContent, languageId); + + await Promise.all( + snippets.map(async (snippet) => { + try { + await prisma.snippet.create({ + data: { + sha: snippet.sha, + content: snippet.content, + rawContent: snippet.rawContent, + fileVersionId: file.id, + }, + }); + } catch (error) { + // Ignore unique constraint violation (since multiple files can produce the same snippet). Upsert is not an option since we are seeding in parallel. + if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") return; + throw error; + } + }) + ); + + await prisma.fileVersion.update({ + where: {id: file.id}, + data: {parsedAt: new Date()}, + }); + }) + ); + + console.log("Finished seeding snippets for all unparsed files."); +} diff --git a/scripts/seed-missing-snippets.ts b/scripts/seed-missing-snippets.ts deleted file mode 100644 index 00c19dc..0000000 --- a/scripts/seed-missing-snippets.ts +++ /dev/null @@ -1,60 +0,0 @@ -import dotenv from "dotenv"; -import {PrismaClient} from "@prisma/client"; -import {parseNumberFlag} from "./utils/cli"; -import {seedLanguageSnippets} from "./utils/snippet-helpers"; - -dotenv.config(); - -const prisma = new PrismaClient(); -const rawArgs = process.argv.slice(2); - -interface CliOptions { - pagesPerLanguage?: number; -} - -function parseCliOptions(): CliOptions { - const pagesPerLanguage = parseNumberFlag(rawArgs, "--pages"); - return {pagesPerLanguage}; -} - -async function seedMissingLanguageSnippets(pagesPerLanguage?: number) { - const languages = await prisma.language.findMany({ - select: { - id: true, - _count: {select: {snippets: true}}, - }, - }); - - if (languages.length === 0) { - console.warn("Language table is empty. Run the base seed first: ts-node prisma/seed.ts"); - return; - } - - const missingLanguageIds = languages.filter((language) => language._count.snippets === 0).map((language) => language.id); - - if (missingLanguageIds.length === 0) { - console.log("All languages already have snippets."); - return; - } - - console.log(`Seeding snippets for missing languages: ${missingLanguageIds.join(", ")}`); - - for (const languageId of missingLanguageIds) { - await seedLanguageSnippets(prisma, languageId, pagesPerLanguage); - } -} - -async function main() { - const options = parseCliOptions(); - await seedMissingLanguageSnippets(options.pagesPerLanguage); -} - -main() - .then(async () => { - await prisma.$disconnect(); - }) - .catch(async (error) => { - console.error(error); - await prisma.$disconnect(); - process.exit(1); - }); diff --git a/scripts/seed-snippets.ts b/scripts/seed-snippets.ts deleted file mode 100644 index f14c866..0000000 --- a/scripts/seed-snippets.ts +++ /dev/null @@ -1,42 +0,0 @@ -import dotenv from "dotenv"; -import {PrismaClient} from "@prisma/client"; -import {getFlagValue, parseNumberFlag} from "./utils/cli"; -import {seedLanguageSnippets as seedLanguageSnippetsHelper} from "./utils/snippet-helpers"; - -dotenv.config(); - -const rawArgs = process.argv.slice(2); - -interface CliOptions { - languageId: string; - pagesPerLanguage?: number; -} - -function parseCliOptions(): CliOptions { - const languageId = getFlagValue(rawArgs, "--language"); - - if (!languageId) { - throw new Error("Missing --language flag. Example: ts-node scripts/seed-snippets.ts --language js"); - } - - const pagesPerLanguage = parseNumberFlag(rawArgs, "--pages"); - - return {languageId, pagesPerLanguage}; -} - -if (require.main === module) { - (async () => { - const prisma = new PrismaClient(); - try { - const options = parseCliOptions(); - await seedLanguageSnippetsHelper(prisma, options.languageId, options.pagesPerLanguage); - await prisma.$disconnect(); - } catch (error) { - console.error(error); - await prisma.$disconnect(); - process.exit(1); - } - })(); -} - -export {fetchLanguageSnippets, seedLanguageSnippets, seedSnippets} from "./utils/snippet-helpers"; diff --git a/scripts/utils/cli.ts b/scripts/utils/cli.ts deleted file mode 100644 index cd1ecc7..0000000 --- a/scripts/utils/cli.ts +++ /dev/null @@ -1,31 +0,0 @@ -export function getFlagValue(args: string[], flag: string): string | undefined { - const index = args.findIndex((arg) => arg === flag || arg.startsWith(`${flag}=`)); - - if (index === -1) { - return undefined; - } - - const [, value] = args[index].split("="); - - if (value) { - return value; - } - - return args[index + 1]; -} - -export function parseNumberFlag(args: string[], flag: string): number | undefined { - const value = getFlagValue(args, flag); - - if (!value) { - return undefined; - } - - const parsedValue = Number(value); - - if (Number.isNaN(parsedValue)) { - throw new Error(`Invalid value for ${flag}. It must be a number.`); - } - - return parsedValue; -} diff --git a/scripts/utils/snippet-helpers.ts b/scripts/utils/snippet-helpers.ts deleted file mode 100644 index 3d3320e..0000000 --- a/scripts/utils/snippet-helpers.ts +++ /dev/null @@ -1,140 +0,0 @@ -import {PrismaClient} from "@prisma/client"; -import {SUPPORTED_LANGUAGES} from "../../prisma/seed/languages"; -import {ILanguage} from "@/shared/types/language"; - -const GITHUB_API_URL = "https://api.github.com"; -const RAW_GITHUB_BASE_URL = "https://raw.githubusercontent.com"; -const MAX_ATTEMPTS = 3; - -interface SnippetData { - repository: {full_name: string}; - url: string; - path: string; -} - -interface GitHubSearchResponse { - items?: SnippetData[]; -} - -function getToken(): string { - const token = process.env.GITHUB_API_TOKEN; - - if (!token) { - throw new Error("Missing GITHUB_API_TOKEN environment variable"); - } - - return token; -} - -async function waitForGitHubRateLimitReset(token: string): Promise { - const res = await fetch(`${GITHUB_API_URL}/rate_limit`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - if (!res.ok) { - console.error(`Failed to fetch rate limit status: ${res.status} ${res.statusText}`); - return; - } - - const json = (await res.json()) as {resources: {code_search: {reset: number}}}; - const reset = json.resources.code_search.reset * 1000; - const now = Date.now(); - const waitTime = reset - now; - - if (waitTime > 0) { - console.warn(`Rate limited by GitHub. Waiting ${waitTime}ms until reset.`); - await new Promise((resolve) => setTimeout(resolve, waitTime + 3000)); - } -} - -function getSnippetRawLink(data: SnippetData): string { - const repository = data.repository.full_name; - const sha = data.url.split("?ref=")[1]; - const path = data.path; - return `${RAW_GITHUB_BASE_URL}/${repository}/${sha}/${path}`; -} - -export async function fetchLanguageSnippets(language: ILanguage, page: number): Promise { - const token = getToken(); - const extensionsFilter = language.extensions.map((ext) => `extension:${ext}`).join(" "); - const params = new URLSearchParams({ - q: `language:${language.name} size:>3000 ${extensionsFilter}`, - page: page.toString(), - per_page: "100", - }); - - const request = () => - fetch(`${GITHUB_API_URL}/search/code?${params}`, { - headers: { - Authorization: `Bearer ${token}`, - }, - }); - - for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { - const response = await request(); - - if (response.status === 403) { - console.warn(`Received 403 (rate limit) fetching ${language.name} snippets on page ${page}. Attempt ${attempt}/${MAX_ATTEMPTS}.`); - await waitForGitHubRateLimitReset(token); - continue; - } - - if (!response.ok) { - throw new Error(`GitHub search failed for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}) with status ${response.status}: ${response.statusText}`); - } - - const data = (await response.json()) as GitHubSearchResponse; - const items = data.items ?? []; - - if (items.length > 0) { - console.info(`Fetched ${items.length} snippets for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}).`); - return items.map((item) => getSnippetRawLink(item)); - } - - if (attempt < MAX_ATTEMPTS) { - console.warn(`No snippets returned for ${language.name} on page ${page} (attempt ${attempt}/${MAX_ATTEMPTS}). Retrying...`); - } - } - - throw new Error(`No snippets returned for ${language.name} on page ${page} after ${MAX_ATTEMPTS} attempts.`); -} - -export async function seedSnippets(prisma: PrismaClient, languages: ILanguage[], {pagesPerLanguage = 10}: {pagesPerLanguage?: number} = {}): Promise { - for (const language of languages) { - for (let page = 1; page <= pagesPerLanguage; page += 1) { - try { - const snippets = await fetchLanguageSnippets(language, page); - - for (const snippet of snippets) { - await prisma.snippet.upsert({ - where: {url: snippet}, - update: {}, - create: { - url: snippet, - language: { - connect: { - id: language.id, - }, - }, - }, - }); - } - } catch (error) { - throw new Error(`Error fetching snippets for ${language.name} on page ${page}: ${error}`); - } - } - } -} - -export async function seedLanguageSnippets(prismaClient: PrismaClient, languageId: string, pagesPerLanguage?: number): Promise { - const language = SUPPORTED_LANGUAGES.find((lang) => lang.id === languageId); - - if (!language) { - throw new Error(`Unsupported language "${languageId}". Supported ids: ${SUPPORTED_LANGUAGES.map((lang) => lang.id).join(", ")}`); - } - - await seedSnippets(prismaClient, [language], {pagesPerLanguage}); - console.log(`Snippets seeded for ${language.name}.`); -} diff --git a/src/features/snippets/config/snippets.server.ts b/src/features/snippets/config/snippets.server.ts index 4e54f85..ac0794b 100644 --- a/src/features/snippets/config/snippets.server.ts +++ b/src/features/snippets/config/snippets.server.ts @@ -26,6 +26,6 @@ export const VALID_NODES = [ "if_statement", ]; -export const MIN_SNIPPETS_PER_LANGUAGE = 10; -export const RANDOM_FILES_FETCHED = 30; -export const MAX_SNIPPETS_FETCH_ATTEMPTS = 20; +export const MIN_SNIPPETS_NUMBER = 10; +export const SNIPPETS_RETRIEVED_PER_QUERY = 30; +export const MAX_GET_SNIPPETS_ATTEMPTS = 10; diff --git a/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts b/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts deleted file mode 100644 index 9a82c07..0000000 --- a/src/features/snippets/infrastructure/adapters/snippet-fetch.server.ts +++ /dev/null @@ -1,15 +0,0 @@ -import {SnippetSourceFile} from "@/features/snippets/types/snippet-source"; - -async function getFileContent(rawUrl: string): Promise { - const response = await fetch(rawUrl); - return response.text(); -} - -export async function getFilesFromUrls(snippetUrls: string[]): Promise { - const filePromises = snippetUrls.map(async (snippetUrl: string) => ({ - url: snippetUrl, - content: await getFileContent(snippetUrl), - })); - - return Promise.all(filePromises); -} diff --git a/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts b/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts index cd8b9e5..c5084ba 100644 --- a/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts +++ b/src/features/snippets/infrastructure/repositories/snippet.repository.server.ts @@ -1,25 +1,20 @@ import {prisma} from "@/shared/db/prisma"; +import {Snippet} from "@prisma/client"; -export async function fetchRandomFiles(languageId: string, quantity: number, alreadyFetched: string[]): Promise { - const alreadyFetchedList = alreadyFetched.length > 0 ? `AND "url" NOT IN (${alreadyFetched.map((_, i) => `$${i + 2}`).join(", ")})` : ""; - - const query = ` - SELECT "url" FROM "Snippet" - WHERE "languageId" = $1 - AND "isValid" = true - ${alreadyFetchedList} - ORDER BY RANDOM() - LIMIT $${alreadyFetched.length + 2}; - `; - - const files = (await prisma.$queryRawUnsafe(query, languageId, ...alreadyFetched, quantity)) as {url: string}[]; - - return files.map((snippet) => snippet.url); -} - -export async function setSnippetAsNonValid(url: string) { - await prisma.snippet.update({ - where: {url}, - data: {isValid: false}, +export async function findRandomSnippets(languageId: string, quantity: number): Promise[]> { + const snippets = await prisma.snippet.findManyRandom(quantity, { + select: { + id: true, + content: true, + }, + where: { + fileVersion: { + file: { + languageId, + }, + }, + }, }); + + return snippets; } diff --git a/src/features/snippets/logic/filter.ts b/src/features/snippets/logic/filter.ts new file mode 100644 index 0000000..aa4674d --- /dev/null +++ b/src/features/snippets/logic/filter.ts @@ -0,0 +1,31 @@ +import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH} from "@/features/snippets/config/snippets.server"; + +function filterLinesNumber(snippet: string): boolean { + const numLines = snippet.split("\n").length; + return numLines <= MAX_LINES && numLines >= MIN_LINES; +} + +function filterLineLength(snippet: string): boolean { + const lines = snippet.split("\n"); + return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH); +} + +function filterSnippetLength(snippet: string): boolean { + return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH; +} + +function filterSnippetSpecialCharacters(snippet: string): boolean { + const regex = /^[\x00-\x7F]+$/; + const test = regex.test(snippet); + + return test; +} + +function filterTabsInBetween(snippet: string): boolean { + const regex = /(^|[^\\])(\\t)|(\t)/; + return regex.test(snippet.trim()); +} + +export function filterSnippets(content: string): boolean { + return filterSnippetLength(content) && filterLinesNumber(content) && filterLineLength(content) && filterSnippetSpecialCharacters(content) && filterTabsInBetween(content); +} diff --git a/src/features/snippets/logic/parsing/snippet-parser.server.ts b/src/features/snippets/logic/parsing/snippet-parser.server.ts index 703a831..bfbadab 100644 --- a/src/features/snippets/logic/parsing/snippet-parser.server.ts +++ b/src/features/snippets/logic/parsing/snippet-parser.server.ts @@ -1,28 +1,6 @@ -import {getTSParser} from "./parser-factory.server"; -import {getInitialIndentation} from "@/features/snippets/logic/processing/indentation.server"; -import {isValidNode} from "@/features/snippets/logic/processing/snippet-filters.server"; +import {getTSParser} from "@lib/snippets/parsing/parser-factory"; import IParser from "tree-sitter"; -export function findValidNodes(node: IParser.SyntaxNode): IParser.SyntaxNode[] { - let nodes: IParser.SyntaxNode[] = []; - if (isValidNode(node)) { - nodes.push(node); - } - - node.children.forEach((child) => { - if (child === null) return; - nodes = nodes.concat(findValidNodes(child)); - }); - - return nodes; -} - -export function convertSnippetToText(node: IParser.SyntaxNode, sourceCode: string): string { - const initialIndentation = getInitialIndentation(node.startIndex, sourceCode); - - return initialIndentation + getNodeText(node); -} - export function extractAutoCompleteDisabledRanges(fileContent: string, languageId: string): {startIndex: number; endIndex: number}[] { const parser = getTSParser(languageId); @@ -71,25 +49,3 @@ export function extractAutoCompleteDisabledRanges(fileContent: string, languageI return ranges; } - -export function extractSnippets(fileContent: string, languageId: string): string[] { - const parser = getTSParser(languageId); - - let parsedCode: IParser.Tree; - - try { - parsedCode = parser.parse(fileContent); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - } catch (error) { - return []; - } - - const validNodes = findValidNodes(parsedCode.rootNode); - const validSnippets = validNodes.map((node) => convertSnippetToText(node, fileContent)); - - return validSnippets; -} - -export function getNodeText(node: IParser.SyntaxNode): string { - return node.text; -} diff --git a/src/features/snippets/logic/processing/snippet-filters.server.ts b/src/features/snippets/logic/processing/snippet-filters.server.ts deleted file mode 100644 index 976b797..0000000 --- a/src/features/snippets/logic/processing/snippet-filters.server.ts +++ /dev/null @@ -1,32 +0,0 @@ -import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH, VALID_NODES} from "@/features/snippets/config/snippets.server"; -import IParser from "tree-sitter"; - -export function filterLinesNumber(snippet: string): boolean { - const numLines = snippet.split("\n").length; - return numLines <= MAX_LINES && numLines >= MIN_LINES; -} - -export function filterLineLength(snippet: string): boolean { - const lines = snippet.split("\n"); - return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH); -} - -export function filterSnippetLength(snippet: string): boolean { - return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH; -} - -export function filterSnippetSpecialCharacters(snippet: string): boolean { - const regex = /^[\x00-\x7F]+$/; - const test = regex.test(snippet); - - return test; -} - -export function filterTabsInBetween(snippet: string): boolean { - const regex = /(^|[^\\])(\\t)|(\t)/; - return regex.test(snippet.trim()); -} - -export function isValidNode(node: IParser.SyntaxNode) { - return VALID_NODES.includes(node.type); -} diff --git a/src/features/snippets/logic/processing/snippet-process.server.ts b/src/features/snippets/logic/processing/snippet-process.server.ts deleted file mode 100644 index c0d0088..0000000 --- a/src/features/snippets/logic/processing/snippet-process.server.ts +++ /dev/null @@ -1,85 +0,0 @@ -import {adjustIndentationOffset, applyIndentationToEmptyLines, detectIndentationStyle} from "./indentation.server"; -import {extractAutoCompleteDisabledRanges, extractSnippets} from "@/features/snippets/logic/parsing/snippet-parser.server"; -import {countInitialWhitespaces} from "./text.server"; -import {filterLineLength, filterLinesNumber, filterSnippetLength, filterSnippetSpecialCharacters, filterTabsInBetween} from "./snippet-filters.server"; -import {ISnippet} from "@/shared/types/snippet.server"; - -function filterSnippets(snippets: string[]): string[] { - return snippets - .filter((snippet) => filterSnippetLength(snippet)) - .filter((snippet) => filterLinesNumber(snippet)) - .filter((snippet) => filterLineLength(snippet)) - .filter((snippet) => filterSnippetSpecialCharacters(snippet)) - .filter((snippet) => filterTabsInBetween(snippet)); -} - -function cleanText(text: string): string { - // Remove whitespaces from empty lines - text = text.replace(/(?<=\n)[ \t]+(?=\n)/g, ""); - - return text; -} - -function hasProhibitedCharacters(input: string): boolean { - const regex = /((?:(?![ \t\n])\s)|\\)/; - return regex.test(input); -} - -function formatCode(snippet: string): string | null { - if (hasProhibitedCharacters(snippet)) { - return null; - } - - // Clean raw snippet - snippet = cleanText(snippet); - - const indentationStyle = detectIndentationStyle(snippet); - - if (indentationStyle.type === "mixed") { - return null; - } - - if (indentationStyle.type === "none") { - return snippet; - } - - // Convert raw snippet indentation style with standardized '\t' indentation - let codeLines = snippet.split("\n").map((line) => { - const initialWhitespaces = countInitialWhitespaces(line); - - const trimmedLine = line.trim(); - - if (indentationStyle.type === "space") { - return "\t".repeat(indentationStyle.value.indexOf(initialWhitespaces)) + trimmedLine; - } - - return "\t".repeat(initialWhitespaces) + trimmedLine; - }); - - // Adjust indentation offset on every line - codeLines = adjustIndentationOffset(codeLines); - - // Applies indentation to empty lines (the minimum indentation of the surrounding lines) - codeLines = applyIndentationToEmptyLines(codeLines); - - return codeLines.join("\n"); -} - -export function processSnippets(fileContent: string, languageId: string): ISnippet[] { - const extractedSnippets = extractSnippets(fileContent, languageId); - - const formattedSnippets = extractedSnippets.map((snippet) => formatCode(snippet)).filter((snippet) => snippet !== null); - - const filteredSnippets = filterSnippets(formattedSnippets); - - const finalSnippets = filteredSnippets.map((snippet) => { - const disabledRanges = extractAutoCompleteDisabledRanges(snippet, languageId); - - return { - content: snippet, - disabledRanges, - }; - }); - - return finalSnippets; -} diff --git a/src/features/snippets/services/get-random-snippets.server.ts b/src/features/snippets/services/get-random-snippets.server.ts index ead613f..f337d8d 100644 --- a/src/features/snippets/services/get-random-snippets.server.ts +++ b/src/features/snippets/services/get-random-snippets.server.ts @@ -1,44 +1,34 @@ -import {MIN_SNIPPETS_PER_LANGUAGE, MAX_SNIPPETS_FETCH_ATTEMPTS, RANDOM_FILES_FETCHED} from "@/features/snippets/config/snippets.server"; -import {fetchRandomFiles as getRandomFiles, setSnippetAsNonValid} from "@/features/snippets/infrastructure/repositories/snippet.repository.server"; -import {getFilesFromUrls} from "@/features/snippets/infrastructure/adapters/snippet-fetch.server"; -import {processSnippets as processFile} from "@/features/snippets/logic/processing/snippet-process.server"; -import type {SnippetSourceFile} from "@/features/snippets/types/snippet-source"; import {ISnippet} from "@/shared/types/snippet.server"; -import {isDev} from "@/core/config/env"; +import {filterSnippets} from "@/features/snippets/logic/filter"; +import {findRandomSnippets} from "@/features/snippets/infrastructure/repositories/snippet.repository.server"; +import {extractAutoCompleteDisabledRanges} from "@/features/snippets/logic/parsing/snippet-parser.server"; +import {MAX_GET_SNIPPETS_ATTEMPTS, MIN_SNIPPETS_NUMBER, SNIPPETS_RETRIEVED_PER_QUERY} from "@/features/snippets/config/snippets.server"; export async function getRandomSnippets(languageId: string): Promise { const snippets: ISnippet[] = []; - const fetchedFilesUrls: string[] = []; let attempts = 0; - while (snippets.length < MIN_SNIPPETS_PER_LANGUAGE && attempts < MAX_SNIPPETS_FETCH_ATTEMPTS) { - const fileUrls = await getRandomFiles(languageId, RANDOM_FILES_FETCHED, fetchedFilesUrls).catch((error) => { - console.error("Error fetching random files:", error); - return []; - }); - - fetchedFilesUrls.push(...fileUrls); + while (snippets.length < MIN_SNIPPETS_NUMBER && attempts < MAX_GET_SNIPPETS_ATTEMPTS) { + const randomSnippets = await findRandomSnippets(languageId, SNIPPETS_RETRIEVED_PER_QUERY); - const fetchedFiles: SnippetSourceFile[] = await getFilesFromUrls(fileUrls); + const filteredSnippets = randomSnippets.filter((s) => filterSnippets(s.content)); - const extractedSnippets = fetchedFiles - .map((file) => { - const snippetsFromFile = processFile(file.content, languageId); + const finalSnippets = filteredSnippets.map((snippet) => { + const disabledRanges = extractAutoCompleteDisabledRanges(snippet.content, languageId); - if (snippetsFromFile.length === 0 && !isDev) { - setSnippetAsNonValid(file.url); - } + return { + content: snippet.content, + disabledRanges, + }; + }); - return snippetsFromFile; - }) - .flat(); + snippets.push(...finalSnippets); - snippets.push(...extractedSnippets); attempts++; } - if (snippets.length < MIN_SNIPPETS_PER_LANGUAGE) { - throw new Error(`Failed to fetch enough snippets in ${MAX_SNIPPETS_FETCH_ATTEMPTS} attempts`); + if (snippets.length < MIN_SNIPPETS_NUMBER) { + throw new Error(`Failed to fetch enough snippets in ${MAX_GET_SNIPPETS_ATTEMPTS} attempts`); } return snippets.sort(() => Math.random() - 0.5); diff --git a/src/features/snippets/types/snippet-source.ts b/src/features/snippets/types/snippet-source.ts deleted file mode 100644 index 7fd10f7..0000000 --- a/src/features/snippets/types/snippet-source.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface SnippetSourceFile { - url: string; - content: string; -} diff --git a/src/shared/db/prisma.ts b/src/shared/db/prisma.ts index 62e15ca..3f2eb73 100644 --- a/src/shared/db/prisma.ts +++ b/src/shared/db/prisma.ts @@ -1,11 +1,22 @@ import {PrismaClient} from "@prisma/client"; +import prismaRandom from "prisma-extension-random"; -const globalForPrisma = globalThis as unknown as {prisma: PrismaClient | undefined}; +const globalForPrisma = globalThis as unknown as { + prisma: ExtendedPrismaClient | undefined; +}; -export const prisma = - globalForPrisma.prisma ?? - new PrismaClient({ +const createPrisma = () => { + const base = new PrismaClient({ log: ["error", "warn"], }); + const extended = base.$extends(prismaRandom()); + return extended; +}; -if (process.env.NODE_ENV !== "production") globalForPrisma.prisma = prisma; +export type ExtendedPrismaClient = ReturnType; + +export const prisma: ExtendedPrismaClient = globalForPrisma.prisma ?? createPrisma(); + +if (process.env.NODE_ENV !== "production") { + globalForPrisma.prisma = prisma; +} diff --git a/tsconfig.json b/tsconfig.json index b575f7d..f946b5b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,6 +25,9 @@ "paths": { "@/*": [ "./src/*" + ], + "@lib/*": [ + "./lib/*" ] } },