Skip to content

Commit 5a9f64a

Browse files
waleedlatif1waleedlatif1
authored
fix(files): bound HTML parser input before building the DOM (#6423)
* fix(files): bound HTML parser input before building the DOM Rejects HTML documents above a byte and markup-token budget before cheerio builds the document tree, and wires the rejection into the parse route's fail-closed path alongside the existing YAML one. * fix(files): classify HTML extraction RangeErrors as complexity rejections * fix(files): scope the parseFile try to the read so typed rejections propagate * fix(desktop): mock electron in tests that transitively import it url-guard, csp, and telemetry-policy all reach electron through @/main/navigation but never mocked it, so they depend on a working Electron binary download and fail when that install is incomplete.
1 parent 87b7b4b commit 5a9f64a

6 files changed

Lines changed: 198 additions & 5 deletions

File tree

‎apps/desktop/src/main/browser-agent/url-guard.test.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
import { beforeEach, describe, expect, it, vi } from 'vitest'
22

3+
// url-guard pulls in @/main/navigation, which imports electron.
4+
vi.mock('electron', () => import('@/test/electron-mock'))
5+
36
const { mockLookup } = vi.hoisted(() => ({ mockLookup: vi.fn() }))
47

58
// The real resolveHostAddresses runs; only the resolver under it is mocked, so

‎apps/desktop/src/main/csp.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
11
import { beforeEach, describe, expect, it, vi } from 'vitest'
2+
3+
// csp pulls in @/main/navigation, which imports electron.
4+
vi.mock('electron', () => import('@/test/electron-mock'))
5+
26
import { attachCspFallback, DEFAULT_DESKTOP_CSP } from '@/main/csp'
37

48
type HeadersReceivedHandler = (

‎apps/desktop/src/main/telemetry-policy.test.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
import { describe, expect, it } from 'vitest'
1+
import { describe, expect, it, vi } from 'vitest'
2+
3+
// telemetry-policy pulls in @/main/navigation, which imports electron.
4+
vi.mock('electron', () => import('@/test/electron-mock'))
5+
26
import { shouldBlockRequest } from '@/main/telemetry-policy'
37

48
describe('shouldBlockRequest', () => {

‎apps/sim/app/api/files/parse/route.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid'
1313
import { sanitizeUrlForLog } from '@/lib/core/utils/logging'
1414
import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
1515
import { isSupportedFileType, parseFile } from '@/lib/file-parsers'
16+
import { isHtmlComplexityError } from '@/lib/file-parsers/html-parser'
1617
import { isYamlComplexityError } from '@/lib/file-parsers/yaml-parser'
1718
import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
1819
import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
@@ -1047,6 +1048,7 @@ async function handleGenericTextBuffer(
10471048
// Fail closed on a resource-exhaustion rejection instead of silently
10481049
// storing the crafted document as raw text.
10491050
if (isYamlComplexityError(parserError)) throw parserError
1051+
if (isHtmlComplexityError(parserError)) throw parserError
10501052

10511053
logger.warn('Specialized parser failed, falling back to generic parsing:', parserError)
10521054
}

‎apps/sim/lib/file-parsers/html-parser.test.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { rm, writeFile } from 'fs/promises'
5+
import { tmpdir } from 'os'
6+
import { join } from 'path'
7+
import { describe, expect, it } from 'vitest'
8+
import { HtmlComplexityError, HtmlParser } from '@/lib/file-parsers/html-parser'
9+
10+
const parser = new HtmlParser()
11+
12+
describe('HtmlParser', () => {
13+
describe('resource limits', () => {
14+
it('rejects a document above the input byte cap', async () => {
15+
const sparse = Buffer.concat([
16+
Buffer.from('<html><body><p>'),
17+
Buffer.alloc(32 * 1024 * 1024, 0x61),
18+
Buffer.from('</p></body></html>'),
19+
])
20+
21+
await expect(parser.parseBuffer(sparse)).rejects.toThrow(
22+
/above the maximum of 33554432 bytes/
23+
)
24+
})
25+
26+
it('rejects a tag-dense document above the markup-token cap', async () => {
27+
const dense = Buffer.from(`<html><body>${'<p>a</p>'.repeat(300_000)}</body></html>`)
28+
29+
const error = await parser.parseBuffer(dense).catch((e) => e)
30+
31+
expect(error).toBeInstanceOf(HtmlComplexityError)
32+
expect(error.message).toMatch(/exceeds the maximum of 500000 markup tokens/)
33+
})
34+
35+
it('accepts a byte-heavy document whose markup stays under the token cap', async () => {
36+
const paragraph = `<p>${'word '.repeat(200)}</p>`
37+
const buffer = Buffer.from(`<html><body>${paragraph.repeat(2000)}</body></html>`)
38+
39+
const result = await parser.parseBuffer(buffer)
40+
41+
expect(result.content).toContain('word')
42+
})
43+
44+
/**
45+
* Deep nesting overflows the stack inside cheerio's own recursive `.text()`,
46+
* which the pre-parse caps cannot predict. It still has to be classified as
47+
* a resource rejection so callers fail closed rather than fall back to
48+
* storing the document as raw text.
49+
*/
50+
it('preserves the error type through parseFile so callers still fail closed', async () => {
51+
const dense = `<html><body>${'<p>a</p>'.repeat(300_000)}</body></html>`
52+
const path = join(tmpdir(), `html-parser-limits-${process.pid}.html`)
53+
await writeFile(path, dense)
54+
55+
try {
56+
await expect(parser.parseFile(path)).rejects.toBeInstanceOf(HtmlComplexityError)
57+
} finally {
58+
await rm(path, { force: true })
59+
}
60+
})
61+
62+
it('classifies a deep-nesting stack overflow as a complexity rejection', async () => {
63+
const depth = 15_000
64+
const buffer = Buffer.from(
65+
`<html><body>${'<div>'.repeat(depth)}deep${'</div>'.repeat(depth)}</body></html>`
66+
)
67+
68+
await expect(parser.parseBuffer(buffer)).rejects.toThrow(HtmlComplexityError)
69+
})
70+
})
71+
72+
describe('extraction', () => {
73+
it('extracts structured text, headings, links, and metadata', async () => {
74+
const buffer = Buffer.from(
75+
`<html><head><title>Doc</title><meta name="description" content="About"></head>` +
76+
`<body><h1>Title</h1><p>Body text</p>` +
77+
`<ul><li>one</li><li>two</li></ul>` +
78+
`<table><tr><th>h</th></tr><tr><td>c</td></tr></table>` +
79+
`<a href="https://example.com">Example</a>` +
80+
`<script>alert(1)</script></body></html>`
81+
)
82+
83+
const result = await parser.parseBuffer(buffer)
84+
85+
expect(result.metadata?.title).toBe('Doc')
86+
expect(result.metadata?.metaDescription).toBe('About')
87+
expect(result.content).toContain('Title')
88+
expect(result.content).toContain('Body text')
89+
expect(result.content).toContain('• one')
90+
expect(result.content).toContain('| h |')
91+
expect(result.content).toContain('Example (https://example.com)')
92+
expect(result.content).not.toContain('alert(1)')
93+
expect(result.metadata?.headings).toEqual([{ level: 1, text: 'Title' }])
94+
expect(result.metadata?.links).toEqual([{ text: 'Example', href: 'https://example.com' }])
95+
expect(result.metadata?.listCount).toBe(1)
96+
expect(result.metadata?.tableCount).toBe(1)
97+
})
98+
})
99+
})

‎apps/sim/lib/file-parsers/html-parser.ts‎

Lines changed: 85 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,94 @@
11
import { readFile } from 'fs/promises'
22
import { createLogger } from '@sim/logger'
3+
import { getErrorMessage } from '@sim/utils/errors'
34
import * as cheerio from 'cheerio'
45
import type { FileParseResult, FileParser } from '@/lib/file-parsers/types'
56
import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils'
67

78
const logger = createLogger('HtmlParser')
89

10+
/**
11+
* `cheerio.load` retains ~530 bytes of DOM per markup token (`<`) — measured on
12+
* cheerio 1.1.2 at 0.2M/1M/2M tokens (101/504/1008 MB), flat across all three —
13+
* so this bounds one document's tree at roughly 256 MB.
14+
*/
15+
const MAX_HTML_MARKUP_TOKENS = 500_000
16+
17+
/**
18+
* Backstop for markup sparse enough to pass the token cap: bounds the UTF-16
19+
* copy `buffer.toString` allocates and the text nodes the DOM keeps.
20+
*/
21+
const MAX_HTML_INPUT_BYTES = 32 * 1024 * 1024
22+
23+
const MARKUP_TOKEN_BYTE = 0x3c
24+
25+
/**
26+
* Raised when a document exceeds the limits above, so an input rejected on
27+
* resource grounds is not reported as a malformed file.
28+
*/
29+
export class HtmlComplexityError extends Error {
30+
constructor(message: string) {
31+
super(message)
32+
this.name = 'HtmlComplexityError'
33+
}
34+
}
35+
36+
export function isHtmlComplexityError(error: unknown): error is HtmlComplexityError {
37+
return error instanceof HtmlComplexityError
38+
}
39+
40+
function exceedsMarkupTokenLimit(buffer: Buffer): boolean {
41+
let count = 0
42+
let index = buffer.indexOf(MARKUP_TOKEN_BYTE)
43+
44+
while (index !== -1) {
45+
if (++count > MAX_HTML_MARKUP_TOKENS) return true
46+
index = buffer.indexOf(MARKUP_TOKEN_BYTE, index + 1)
47+
}
48+
49+
return false
50+
}
51+
52+
/**
53+
* `cheerio.load` builds the entire parse5 tree before returning, so an outsized
54+
* document has to be rejected on the buffer, before the string copy.
55+
*/
56+
function assertHtmlWithinLimits(buffer: Buffer): void {
57+
if (buffer.length > MAX_HTML_INPUT_BYTES) {
58+
throw new HtmlComplexityError(
59+
`HTML document is ${buffer.length} bytes, above the maximum of ${MAX_HTML_INPUT_BYTES} bytes`
60+
)
61+
}
62+
63+
if (exceedsMarkupTokenLimit(buffer)) {
64+
throw new HtmlComplexityError(
65+
`HTML document exceeds the maximum of ${MAX_HTML_MARKUP_TOKENS} markup tokens`
66+
)
67+
}
68+
}
69+
970
export class HtmlParser implements FileParser {
1071
async parseFile(filePath: string): Promise<FileParseResult> {
72+
let buffer: Buffer
73+
74+
/** Scoped to the read alone so `parseBuffer`'s typed rejections reach callers intact. */
1175
try {
1276
if (!filePath) {
1377
throw new Error('No file path provided')
1478
}
1579

16-
const buffer = await readFile(filePath)
17-
return this.parseBuffer(buffer)
80+
buffer = await readFile(filePath)
1881
} catch (error) {
1982
logger.error('HTML file error:', error)
20-
throw new Error(`Failed to parse HTML file: ${(error as Error).message}`)
83+
throw new Error(`Failed to parse HTML file: ${getErrorMessage(error, 'Unknown error')}`)
2184
}
85+
86+
return this.parseBuffer(buffer)
2287
}
2388

2489
async parseBuffer(buffer: Buffer): Promise<FileParseResult> {
90+
assertHtmlWithinLimits(buffer)
91+
2592
try {
2693
logger.info('Parsing HTML buffer, size:', buffer.length)
2794

@@ -72,8 +139,22 @@ export class HtmlParser implements FileParser {
72139
},
73140
}
74141
} catch (error) {
142+
/**
143+
* Every `RangeError` reachable here is resource exhaustion the pre-parse
144+
* caps cannot predict: a stack overflow inside cheerio's recursive
145+
* `.text()` on deeply nested markup, or an over-long string from joining
146+
* the extracted parts. Both must stay fail-closed rather than degrade to
147+
* the route's raw-text fallback.
148+
*/
149+
if (error instanceof RangeError) {
150+
logger.warn('HTML document exhausted parser resources:', error)
151+
throw new HtmlComplexityError(
152+
`HTML document could not be extracted within resource limits: ${error.message}`
153+
)
154+
}
155+
75156
logger.error('HTML buffer parsing error:', error)
76-
throw new Error(`Failed to parse HTML buffer: ${(error as Error).message}`)
157+
throw new Error(`Failed to parse HTML buffer: ${getErrorMessage(error, 'Unknown error')}`)
77158
}
78159
}
79160

0 commit comments

Comments
 (0)