Update behavior of read-files to truncate after 100k chars

Author: jahooma

sdk/src/__tests__/read-files.test.ts

@@ -186,12 +186,13 @@ describe('getFiles', () => {
   })
 
   describe('file too large', () => {
-    test('should return TOO_LARGE for files over 1MB', async () => {
+    test('should truncate files over 100k chars to 1k chars with message', async () => {
+      const largeContent = 'x'.repeat(101_000) // 101k chars - over limit
       const mockFs = createMockFs({
         files: {
           '/project/large.bin': {
-            content: 'x',
-            size: 2 * 1024 * 1024, // 2MB
+            content: largeContent,
+            size: largeContent.length,
           },
         },
       })
@@ -202,28 +203,75 @@ describe('getFiles', () => {
         fs: mockFs,
       })
 
-      expect(result['large.bin']).toContain(FILE_READ_STATUS.TOO_LARGE)
-      expect(result['large.bin']).toContain('2.00MB')
+      // Should contain first 1k chars
+      expect(result['large.bin']).toContain('x'.repeat(1000))
+      // Should contain truncation message
+      expect(result['large.bin']).toContain('FILE_TOO_LARGE')
+      expect(result['large.bin']).toContain('101,000 chars')
+    })
+
+    test('should read files at exactly 100k chars', async () => {
+      const exactly100kContent = 'x'.repeat(100_000) // exactly 100k chars
+      const mockFs = createMockFs({
+        files: {
+          '/project/exactly100k.bin': {
+            content: exactly100kContent,
+            size: exactly100kContent.length,
+          },
+        },
+      })
+
+      const result = await getFiles({
+        filePaths: ['exactly100k.bin'],
+        cwd: '/project',
+        fs: mockFs,
+      })
+
+      // Should be read fully (no truncation message)
+      expect(result['exactly100k.bin']).toBe(exactly100kContent)
+      expect(result['exactly100k.bin']).not.toContain('FILE_TOO_LARGE')
     })
 
-    test('should read files exactly at 1MB limit', async () => {
-      const oneMBContent = 'x'.repeat(1024 * 1024)
+    test('should reject files over 10MB without reading them', async () => {
       const mockFs = createMockFs({
         files: {
-          '/project/exactly1mb.bin': {
-            content: oneMBContent,
-            size: 1024 * 1024, // exactly 1MB
+          '/project/huge.bin': {
+            content: 'x',
+            size: 15 * 1024 * 1024, // 15MB
           },
         },
       })
 
       const result = await getFiles({
-        filePaths: ['exactly1mb.bin'],
+        filePaths: ['huge.bin'],
         cwd: '/project',
         fs: mockFs,
       })
 
-      expect(result['exactly1mb.bin']).toBe(oneMBContent)
+      expect(result['huge.bin']).toContain(FILE_READ_STATUS.TOO_LARGE)
+      expect(result['huge.bin']).toContain('15.0MB')
+    })
+
+    test('should read files just under 100k chars', async () => {
+      const justUnder100k = 'x'.repeat(99_000) // under limit
+      const mockFs = createMockFs({
+        files: {
+          '/project/underlimit.bin': {
+            content: justUnder100k,
+            size: justUnder100k.length,
+          },
+        },
+      })
+
+      const result = await getFiles({
+        filePaths: ['underlimit.bin'],
+        cwd: '/project',
+        fs: mockFs,
+      })
+
+      // Should be read fully (no truncation message)
+      expect(result['underlimit.bin']).toBe(justUnder100k)
+      expect(result['underlimit.bin']).not.toContain('FILE_TOO_LARGE')
     })
   })
 
@@ -347,18 +395,6 @@ describe('getFiles', () => {
         },
       })
 
-      // Need to also make stat fail with same error
-      const originalStat = mockFs.stat
-      Object.assign(mockFs, {
-        stat: async (filePath: PathLike) => {
-          const pathStr = String(filePath)
-          if (pathStr === '/project/broken.ts') {
-            throw createNodeError('Permission denied', 'EACCES')
-          }
-          return originalStat(pathStr)
-        },
-      })
-
       const result = await getFiles({
         filePaths: ['broken.ts'],
         cwd: '/project',

sdk/src/tools/read-files.ts

@@ -28,7 +28,11 @@ export async function getFiles(params: {
   const hasCustomFilter = fileFilter !== undefined
 
   const result: Record<string, string | null> = {}
-  const MAX_FILE_SIZE = 1024 * 1024 // 1MB in bytes
+  const MAX_FILE_BYTES = 10 * 1024 * 1024 // 10MB - skip reading entirely
+  const MAX_CHARS = 100_000 // 100k characters threshold
+  const TRUNCATE_TO_CHARS = 1_000 // Show first 1k chars when over limit
+  const numFmt = new Intl.NumberFormat('en-US')
+  const fmtNum = (n: number) => numFmt.format(n)
 
   for (const filePath of filePaths) {
     if (!filePath) {
@@ -68,13 +72,27 @@ export async function getFiles(params: {
     }
 
     try {
+      // Safety check: skip reading files over 10MB to avoid OOM
       const stats = await fs.stat(fullPath)
-      if (stats.size > MAX_FILE_SIZE) {
+      if (stats.size > MAX_FILE_BYTES) {
         result[relativePath] =
           FILE_READ_STATUS.TOO_LARGE +
-          ` [${(stats.size / (1024 * 1024)).toFixed(2)}MB]`
+          ` [${(stats.size / (1024 * 1024)).toFixed(1)}MB exceeds 10MB limit. Use code_search or glob to find specific content.]`
+        continue
+      }
+
+      const content = await fs.readFile(fullPath, 'utf8')
+
+      if (content.length > MAX_CHARS) {
+        const truncated = content.slice(0, TRUNCATE_TO_CHARS)
+        result[relativePath] =
+          truncated +
+          '\n\n[FILE_TOO_LARGE: This file is ' +
+          fmtNum(content.length) +
+          ' chars, exceeding the 100k char limit. Only the first ' +
+          fmtNum(TRUNCATE_TO_CHARS) +
+          ' chars are shown. Use other tools to read sections of the file.]'
       } else {
-        const content = await fs.readFile(fullPath, 'utf8')
         // Prepend TEMPLATE marker for example files
         result[relativePath] = isExampleFile
           ? FILE_READ_STATUS.TEMPLATE + '\n' + content