feat: refactor architecture to identify snippets

lib/github.ts

@@ -0,0 +1,146 @@
+const GITHUB_API_URL = "https://api.github.com";
+const RETRY_DELAY_MS = 1000;
+const RATE_LIMIT_PADDING_MS = 3000;
+
+type GitHubRequestParams = Record<string, string | number | boolean | undefined>;
+
+export interface GitHubRequestOptions extends RequestInit {
+  params?: GitHubRequestParams;
+  maxAttempts?: number;
+  retry?: boolean;
+  shouldRetry?: (context: {response: Response; attempt: number; maxAttempts: number; url: string}) => Promise<boolean> | boolean;
+}
+
+export async function callGitHubApi(path: string, options: GitHubRequestOptions = {}): Promise<Response> {
+  const token = getToken();
+  const {params, maxAttempts = 3, retry = true, shouldRetry, headers: customHeaders, ...fetchOptions} = options;
+
+  const url = buildGitHubUrl(path, params);
+  const headers = new Headers({
+    Authorization: `Bearer ${token}`,
+    Accept: "application/vnd.github+json",
+  });
+
+  if (customHeaders) {
+    new Headers(customHeaders).forEach((value, key) => {
+      headers.set(key, value);
+    });
+  }
+
+  const attempts = retry ? maxAttempts : 1;
+
+  for (let attempt = 1; attempt <= attempts; attempt += 1) {
+    const response = await fetch(url, {
+      ...fetchOptions,
+      headers,
+    });
+
+    if (isRateLimitStatus(response.status)) {
+      await handleRateLimit(response);
+      continue;
+    }
+
+    if (!response.ok) {
+      if (attempt >= attempts) {
+        throw new Error(`GitHub API request failed for ${url} (attempt ${attempt}/${attempts}) with status ${response.status}: ${response.statusText}`);
+      }
+
+      await wait(RETRY_DELAY_MS);
+      continue;
+    }
+
+    if (retry && shouldRetry && attempt < attempts) {
+      const shouldRetryResult = await shouldRetry({
+        response: response.clone(),
+        attempt,
+        maxAttempts,
+        url,
+      });
+
+      if (shouldRetryResult) {
+        await wait(RETRY_DELAY_MS);
+        continue;
+      }
+    }
+
+    return response;
+  }
+
+  throw new Error(`GitHub API request failed after ${attempts} attempts for ${url}`);
+}
+
+function getToken(): string {
+  const token = process.env.GITHUB_API_TOKEN;
+
+  if (!token) {
+    throw new Error("Missing GITHUB_API_TOKEN environment variable");
+  }
+
+  return token;
+}
+
+function buildGitHubUrl(path: string, params?: GitHubRequestParams): string {
+  const baseUrl = path.startsWith("http") ? new URL(path) : new URL(path.startsWith("/") ? path : `/${path}`, GITHUB_API_URL);
+
+  if (params) {
+    Object.entries(params).forEach(([key, value]) => {
+      if (value === undefined) return;
+      baseUrl.searchParams.set(key, String(value));
+    });
+  }
+
+  return baseUrl.toString();
+}
+
+function isRateLimitStatus(status: number) {
+  return status === 403 || status === 429;
+}
+
+async function handleRateLimit(response: Response): Promise<void> {
+  const resetAt = getRateLimitReset(response);
+  if (resetAt) {
+    const waitTime = Math.max(resetAt - Date.now(), 0) + RATE_LIMIT_PADDING_MS;
+    console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${waitTime}ms until reset.`);
+    await wait(waitTime);
+    return;
+  }
+
+  const retryAfter = parseRetryAfter(response.headers.get("Retry-After"));
+  if (retryAfter !== null) {
+    console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${retryAfter}ms due to Retry-After header.`);
+    await wait(retryAfter);
+    return;
+  }
+
+  console.warn(`Rate limited by GitHub (status ${response.status}). Waiting ${RETRY_DELAY_MS}ms before retrying.`);
+  await wait(RETRY_DELAY_MS);
+}
+
+function getRateLimitReset(response: Response): number | null {
+  const headerValue = response.headers.get("X-RateLimit-Reset");
+  if (!headerValue) return null;
+
+  const epochSeconds = Number(headerValue);
+  if (Number.isNaN(epochSeconds)) {
+    return null;
+  }
+
+  return epochSeconds * 1000;
+}
+
+function parseRetryAfter(headerValue: string | null): number | null {
+  if (!headerValue) return null;
+
+  const numericValue = Number(headerValue);
+  if (!Number.isNaN(numericValue)) {
+    return numericValue * 1000;
+  }
+
+  const retryDate = new Date(headerValue);
+  const waitTime = retryDate.getTime() - Date.now();
+  return Number.isNaN(waitTime) ? null : Math.max(waitTime, 0);
+}
+
+function wait(duration: number) {
+  return new Promise((resolve) => setTimeout(resolve, duration));
+}

lib/snippets/config.ts

@@ -0,0 +1,27 @@
+export const MAX_LINES = 11;
+export const MIN_LINES = 3;
+
+export const MAX_LINE_LENGTH = 72;
+
+export const MAX_SNIPPET_LENGTH = 350;
+export const MIN_SNIPPET_LENGTH = 75;
+
+export const VALID_NODES = [
+  "function_declaration",
+  "function_definition",
+  "class_declaration",
+  "class_definition",
+  "function_item",
+  "method_declaration",
+  "method_definition",
+  "method_item",
+  "class_item",
+  "interface_declaration",
+  "interface_definition",
+  "interface_item",
+  "module",
+  "switch_statement",
+  "while_statement",
+  "for_statement",
+  "if_statement",
+];

lib/snippets/index.ts

@@ -0,0 +1,26 @@
+import {extractSnippets} from "./parsing/parser";
+import {filterSnippets} from "./processing/filter";
+import {formatCode} from "./processing/process";
+import {hashSnippet} from "./processing/utils";
+
+interface ISnippetOutput {
+  sha: string;
+  content: string;
+  rawContent: string;
+}
+
+export function getSnippetsFromFile(fileContent: string, languageId: string): ISnippetOutput[] {
+  const extractedSnippets = extractSnippets(fileContent, languageId);
+
+  const formattedSnippets = extractedSnippets
+    .map((snippet) => ({
+      sha: hashSnippet(snippet),
+      content: formatCode(snippet),
+      rawContent: snippet,
+    }))
+    .filter((s) => s.content !== null) as ISnippetOutput[];
+
+  const filteredSnippets = formattedSnippets.filter((snippet) => filterSnippets(snippet.content));
+
+  return filteredSnippets;
+}

lib/snippets/parsing/parser.ts

@@ -0,0 +1,46 @@
+import {getInitialIndentation} from "@lib/snippets/processing/indentation";
+import {getTSParser} from "./parser-factory";
+import IParser from "tree-sitter";
+import {isValidNode} from "@lib/snippets/processing/filter";
+
+export function findValidNodes(node: IParser.SyntaxNode): IParser.SyntaxNode[] {
+  let nodes: IParser.SyntaxNode[] = [];
+  if (isValidNode(node)) {
+    nodes.push(node);
+  }
+
+  node.children.forEach((child) => {
+    if (child === null) return;
+    nodes = nodes.concat(findValidNodes(child));
+  });
+
+  return nodes;
+}
+
+export function convertSnippetToText(node: IParser.SyntaxNode, sourceCode: string): string {
+  const initialIndentation = getInitialIndentation(node.startIndex, sourceCode);
+
+  return initialIndentation + getNodeText(node);
+}
+
+export function extractSnippets(fileContent: string, languageId: string): string[] {
+  const parser = getTSParser(languageId);
+
+  let parsedCode: IParser.Tree;
+
+  try {
+    parsedCode = parser.parse(fileContent);
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (error) {
+    return [];
+  }
+
+  const validNodes = findValidNodes(parsedCode.rootNode);
+  const validSnippets = validNodes.map((node) => convertSnippetToText(node, fileContent));
+
+  return validSnippets;
+}
+
+export function getNodeText(node: IParser.SyntaxNode): string {
+  return node.text;
+}

lib/snippets/processing/filter.ts

@@ -0,0 +1,36 @@
+import IParser from "tree-sitter";
+import {MAX_LINE_LENGTH, MAX_LINES, MAX_SNIPPET_LENGTH, MIN_LINES, MIN_SNIPPET_LENGTH, VALID_NODES} from "../config";
+
+function filterLinesNumber(snippet: string): boolean {
+  const numLines = snippet.split("\n").length;
+  return numLines <= MAX_LINES && numLines >= MIN_LINES;
+}
+
+function filterLineLength(snippet: string): boolean {
+  const lines = snippet.split("\n");
+  return lines.every((line) => line.trim().length <= MAX_LINE_LENGTH);
+}
+
+function filterSnippetLength(snippet: string): boolean {
+  return snippet.length >= MIN_SNIPPET_LENGTH && snippet.length <= MAX_SNIPPET_LENGTH;
+}
+
+function filterSnippetSpecialCharacters(snippet: string): boolean {
+  const regex = /^[\x00-\x7F]+$/;
+  const test = regex.test(snippet);
+
+  return test;
+}
+
+function filterTabsInBetween(snippet: string): boolean {
+  const regex = /^.+\\t.+$/;
+  return !regex.test(snippet);
+}
+
+export function isValidNode(node: IParser.SyntaxNode) {
+  return VALID_NODES.includes(node.type);
+}
+
+export function filterSnippets(content: string): boolean {
+  return filterSnippetLength(content) && filterLinesNumber(content) && filterLineLength(content) && filterSnippetSpecialCharacters(content) && filterTabsInBetween(content);
+}

src/features/snippets/logic/processing/indentation.server.ts → lib/snippets/processing/indentation.ts

@@ -1,4 +1,4 @@
-import {countInitialWhitespaces, removeInitialCharacters} from "./text.server";
+import {countInitialWhitespaces, removeInitialCharacters} from "./utils";
 
 type IIndentationStyle = {type: "none" | "mixed" | "tab"} | {type: "space"; value: number[]};
 export function detectIndentationStyle(snippet: string): IIndentationStyle {

lib/snippets/processing/process.ts

@@ -0,0 +1,54 @@
+import {adjustIndentationOffset, applyIndentationToEmptyLines, detectIndentationStyle} from "./indentation";
+import {countInitialWhitespaces} from "./utils";
+
+function cleanText(text: string): string {
+  // Remove whitespaces from empty lines
+  text = text.replace(/(?<=\n)[ \t]+(?=\n)/g, "");
+
+  return text;
+}
+
+function hasProhibitedCharacters(input: string): boolean {
+  const regex = /((?:(?![ \t\n])\s)|\\)/;
+  return regex.test(input);
+}
+
+export function formatCode(snippet: string): string | null {
+  if (hasProhibitedCharacters(snippet)) {
+    return null;
+  }
+
+  // Clean raw snippet
+  snippet = cleanText(snippet);
+
+  const indentationStyle = detectIndentationStyle(snippet);
+
+  if (indentationStyle.type === "mixed") {
+    return null;
+  }
+
+  if (indentationStyle.type === "none") {
+    return snippet;
+  }
+
+  // Convert raw snippet indentation style with standardized '\t' indentation
+  let codeLines = snippet.split("\n").map((line) => {
+    const initialWhitespaces = countInitialWhitespaces(line);
+
+    const trimmedLine = line.trim();
+
+    if (indentationStyle.type === "space") {
+      return "\t".repeat(indentationStyle.value.indexOf(initialWhitespaces)) + trimmedLine;
+    }
+
+    return "\t".repeat(initialWhitespaces) + trimmedLine;
+  });
+
+  // Adjust indentation offset on every line
+  codeLines = adjustIndentationOffset(codeLines);
+
+  // Applies indentation to empty lines (the minimum indentation of the surrounding lines)
+  codeLines = applyIndentationToEmptyLines(codeLines);
+
+  return codeLines.join("\n");
+}

src/features/snippets/logic/processing/text.server.ts → lib/snippets/processing/utils.ts

@@ -1,3 +1,5 @@
+import {createHash} from "crypto";
+
 export function countInitialWhitespaces(text: string): number {
   let count = 0;
   for (let i = 0; i < text.length; i++) {
@@ -21,3 +23,10 @@ export function removeInitialCharacters(lines: string[], quantity: number): stri
 
   return newLines;
 }
+
+export function hashSnippet(content: string): string {
+  // Remove all whitespaces
+  const newContent = content.replace(/\s+/g, "").trim();
+
+  return createHash("sha256").update(newContent).digest("hex");
+}