Skip to content

fix: prevent oversized fetch payloads during scraping #280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
YLaxmikanth wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: prevent oversized fetch payloads during scraping #280

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions backend/tests/ragUtilities.fetchLimit.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import { describe, it, expect } from "vitest";
import { safeFetchText } from "../utils/ragUtilities.js";

function createMockResponse({ contentType, contentLength, bodyChunks = [] }) {
const chunks = bodyChunks.map((chunk) =>
typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk,
);
let index = 0;

return {
headers: {
get(name) {
const key = String(name).toLowerCase();
if (key === "content-type") return contentType;
if (key === "content-length") return contentLength;
return null;
},
},
body: {
getReader() {
return {
async read() {
if (index >= chunks.length) {
return { done: true, value: undefined };
}
return { done: false, value: chunks[index++] };
},
async cancel() {
index = chunks.length;
},
};
},
},
};
}

describe("safeFetchText", () => {
it("rejects payloads larger than the configured limit", async () => {
const response = createMockResponse({
contentType: "text/html; charset=utf-8",
contentLength: String(5 * 1024 * 1024 + 1),
bodyChunks: ["<html><body>too big</body></html>"],
});

await expect(safeFetchText(response)).rejects.toThrow("Payload too large");
});

it("rejects unsupported content types", async () => {
const response = createMockResponse({
contentType: "application/octet-stream",
contentLength: "12",
bodyChunks: [new Uint8Array([1, 2, 3])],
});

await expect(safeFetchText(response)).rejects.toThrow("Unsupported content type");
});

it("accepts valid html responses", async () => {
const response = createMockResponse({
contentType: "text/html; charset=utf-8",
contentLength: "31",
bodyChunks: ["<html><body>Hello</body></html>"],
});

await expect(safeFetchText(response)).resolves.toBe("<html><body>Hello</body></html>");
});
});
61 changes: 59 additions & 2 deletions backend/utils/ragUtilities.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ import robotsParser from "robots-parser";

const robotsCache = new Map();
const domainLimiters = new Map();
const MAX_FETCH_SIZE_BYTES = 5 * 1024 * 1024; // 5 MB
const ALLOWED_CONTENT_TYPES = [
"text/html",
"application/xhtml+xml",
"text/plain",
];

let openai;

Expand Down Expand Up @@ -184,6 +190,53 @@ async function fetchTextWithTimeout(url, config) {
}
}

async function safeFetchText(response) {
const contentType = response.headers.get("content-type") || "";

const allowed = ALLOWED_CONTENT_TYPES.some((type) =>
contentType.toLowerCase().includes(type),
);

if (!allowed) {
throw new Error(`Unsupported content type: ${contentType}`);
}

const contentLength = Number(response.headers.get("content-length"));

if (
Number.isFinite(contentLength)
&& contentLength > MAX_FETCH_SIZE_BYTES
) {
throw new Error("Payload too large");
}

const reader = response.body?.getReader();

if (!reader) {
throw new Error("Unable to read response body");
}

const chunks = [];
let totalBytes = 0;

while (true) {
const { done, value } = await reader.read();

if (done) break;

totalBytes += value.length;

if (totalBytes > MAX_FETCH_SIZE_BYTES) {
await reader.cancel();
throw new Error("Payload too large");
}

chunks.push(Buffer.from(value));
}

return Buffer.concat(chunks).toString("utf8");
}

async function fetchRobotsPolicy(origin, config) {
if (!config.respectRobotsTxt) {
return {
Expand Down Expand Up @@ -211,7 +264,10 @@ async function fetchRobotsPolicy(origin, config) {
return { parser: parseRobotsTxt("", robotsUrl), crawlDelayMs: null, failureReason: null };
}

const parser = parseRobotsTxt(await response.text(), robotsUrl);
const parser = parseRobotsTxt(
await safeFetchText(response),
robotsUrl,
);
return {
parser,
crawlDelayMs: getRobotsCrawlDelayMs(parser, config.userAgent),
Expand Down Expand Up @@ -298,7 +354,7 @@ async function fetchCrawlText(urlString) {
throw new Error(`Failed to fetch ${urlString}: HTTP ${response.status}`);
}

return response.text();
return safeFetchText(response);
});
}

Expand Down Expand Up @@ -556,6 +612,7 @@ export {
isValidDocUrl,
scrapeWebpage,
scrapeTitle,
safeFetchText,
generateVectorEmbeddings,
getCrawlConfig,
parseRobotsTxt,
Expand Down