From f42991670ace48a08aab9a3b74eb779e51b2e41b Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Sun, 9 Aug 2026 23:37:47 +0530 Subject: [PATCH 1/9] feat(intelligent-assistant): add screenshot capture utility with sensitive data redaction Signed-off-by: its-mitesh-kumar Co-authored-by: Cursor --- .../.changeset/swift-foxes-glow.md | 6 + .../src/service/router.test.ts | 2 +- .../src/service/validation.test.ts | 83 ++++- .../src/service/validation.ts | 28 +- .../plugins/intelligent-assistant/config.d.ts | 27 ++ .../intelligent-assistant/package.json | 1 + .../utils/__tests__/screen-capture.test.ts | 310 ++++++++++++++++++ .../__tests__/sensitive-data-redactor.test.ts | 276 ++++++++++++++++ .../src/utils/screen-capture.ts | 197 +++++++++++ .../src/utils/sensitive-data-redactor.ts | 144 ++++++++ workspaces/intelligent-assistant/yarn.lock | 45 +++ 11 files changed, 1111 insertions(+), 8 deletions(-) create mode 100644 workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts diff --git a/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md new file mode 100644 index 00000000000..336e29b3a1a --- /dev/null +++ b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md @@ -0,0 +1,6 @@ +--- +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': minor +--- + +Add screenshot capture utility with WebP-first format, sensitive data redaction, and performance guardrails for deep context awareness diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts index 536ef2a7275..23ad741a1fb 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts @@ -1826,7 +1826,7 @@ describe('intelligent-assistant router tests', () => { expect(response.statusCode).toEqual(400); expect(response.body.error).toContain( - 'This model does not support JPEG images', + 'This model does not support images', ); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts index 55904037903..0bf1c5d8ee4 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts @@ -358,7 +358,8 @@ describe('validateAttachmentsForModel', () => { expect(statusMock).toHaveBeenCalledWith(400); expect(jsonMock).toHaveBeenCalledWith({ - error: 'Image attachment does not contain a valid JPEG file', + error: + 'Image attachment does not contain a valid image file (JPEG or WebP)', }); expect(mockNext).not.toHaveBeenCalled(); }); @@ -381,6 +382,86 @@ describe('validateAttachmentsForModel', () => { expect(mockNext).toHaveBeenCalled(); expect(statusMock).not.toHaveBeenCalled(); }); + + it('should accept image with valid WebP magic bytes', () => { + // WebP format: RIFF....WEBP + const webpBytes = Buffer.alloc(12); + webpBytes.write('RIFF', 0, 'ascii'); + webpBytes.writeUInt32LE(0, 4); // file size placeholder + webpBytes.write('WEBP', 8, 'ascii'); + const VALID_WEBP_B64 = webpBytes.toString('base64'); + + mockReq.body = { + model: 'gpt-4', + provider: 'openai', + attachments: [ + { + attachment_type: 'image', + content_type: 'image/webp', + content: VALID_WEBP_B64, + }, + ], + }; + + callValidate(); + + expect(mockNext).toHaveBeenCalled(); + expect(statusMock).not.toHaveBeenCalled(); + }); + + it('should accept WebP image with data URL prefix', () => { + const webpBytes = Buffer.alloc(12); + webpBytes.write('RIFF', 0, 'ascii'); + webpBytes.writeUInt32LE(100, 4); + webpBytes.write('WEBP', 8, 'ascii'); + const VALID_WEBP_B64 = webpBytes.toString('base64'); + + mockReq.body = { + model: 'gpt-4', + provider: 'openai', + attachments: [ + { + attachment_type: 'image', + content_type: 'image/webp', + content: `data:image/webp;base64,${VALID_WEBP_B64}`, + }, + ], + }; + + callValidate(); + + expect(mockNext).toHaveBeenCalled(); + expect(statusMock).not.toHaveBeenCalled(); + }); + + it('should reject PNG image (neither JPEG nor WebP)', () => { + // PNG magic bytes + const pngBytes = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, + ]); + const PNG_B64 = pngBytes.toString('base64'); + + mockReq.body = { + model: 'gpt-4', + provider: 'openai', + attachments: [ + { + attachment_type: 'image', + content_type: 'image/png', + content: PNG_B64, + }, + ], + }; + + callValidate(); + + expect(statusMock).toHaveBeenCalledWith(400); + expect(jsonMock).toHaveBeenCalledWith({ + error: + 'Image attachment does not contain a valid image file (JPEG or WebP)', + }); + expect(mockNext).not.toHaveBeenCalled(); + }); }); describe('JSON content validation', () => { diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts index edc4db00812..1d1838b59c1 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts @@ -25,6 +25,9 @@ import { import { Attachments, QueryRequestBody } from './types'; const JPEG_MAGIC = [0xff, 0xd8, 0xff]; +const WEBP_RIFF = [0x52, 0x49, 0x46, 0x46]; // "RIFF" +const WEBP_MARKER = [0x57, 0x45, 0x42, 0x50]; // "WEBP" at offset 8 +const WEBP_MIN_BYTES = 12; function extractBase64(content: string): string { const commaIndex = content.indexOf(','); @@ -34,12 +37,25 @@ function extractBase64(content: string): string { return content; } -function hasValidJpegMagicBytes(content: string): boolean { +function hasValidImageMagicBytes(content: string): boolean { const bytes = Buffer.from(extractBase64(content), 'base64'); - return ( + + if ( bytes.length >= JPEG_MAGIC.length && JPEG_MAGIC.every((b, i) => bytes[i] === b) - ); + ) { + return true; + } + + if ( + bytes.length >= WEBP_MIN_BYTES && + WEBP_RIFF.every((b, i) => bytes[i] === b) && + WEBP_MARKER.every((b, i) => bytes[i + 8] === b) + ) { + return true; + } + + return false; } function isValidJson(content: string): boolean { @@ -68,9 +84,9 @@ function validateAttachments(attachments: Array): string | null { if ( attachment.attachment_type === 'image' && attachment.content && - !hasValidJpegMagicBytes(attachment.content) + !hasValidImageMagicBytes(attachment.content) ) { - return 'Image attachment does not contain a valid JPEG file'; + return 'Image attachment does not contain a valid image file (JPEG or WebP)'; } if ( @@ -179,7 +195,7 @@ export const validateAttachmentsForModel = ( if (!supportsVision) { return res.status(400).json({ error: - 'This model does not support JPEG images. Please select a vision-capable model.', + 'This model does not support images. Please select a vision-capable model.', model, }); } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts index 66ff84ae5b1..e0db69cf425 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts @@ -68,5 +68,32 @@ export interface Config { provider_id: string; }; }; + /** + * Configuration for Screen Context (deep context awareness) + * @visibility frontend + */ + 'screen-context'?: { + /** + * Enable/disable the Screen Context feature. + * When enabled, the assistant can capture the user's current RHDH screen + * to provide context-aware responses. + * @default false + * @visibility frontend + */ + enabled?: boolean; + /** + * Screenshot capture configuration + * @visibility frontend + */ + screenshots?: { + /** + * Enable/disable screenshot capture within Screen Context. + * Admin can disable screenshots org-wide while keeping DOM text context. + * @default true + * @visibility frontend + */ + enabled?: boolean; + }; + }; }; } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/package.json b/workspaces/intelligent-assistant/plugins/intelligent-assistant/package.json index 77653a4466d..113d3d964b8 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/package.json +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/package.json @@ -84,6 +84,7 @@ "@red-hat-developer-hub/backstage-plugin-intelligent-assistant-common": "workspace:^", "@red-hat-developer-hub/backstage-plugin-theme": "^0.14.10", "@tanstack/react-query": "^5.59.15", + "html2canvas-pro": "^1.5.0", "monaco-editor": "^0.55.0", "react-dropzone": "^14.4.0", "react-use": "^17.2.4" diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts new file mode 100644 index 00000000000..e06804d3daf --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts @@ -0,0 +1,310 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import html2canvas from 'html2canvas-pro'; + +import { captureScreenshot } from '../screen-capture'; +import { sanitizeClonedDom } from '../sensitive-data-redactor'; + +jest.mock('html2canvas-pro'); +jest.mock('../sensitive-data-redactor', () => ({ + sanitizeClonedDom: jest.fn(), +})); + +const mockHtml2canvas = html2canvas as jest.MockedFunction; +const mockSanitize = sanitizeClonedDom as jest.MockedFunction< + typeof sanitizeClonedDom +>; + +function createMockCanvas( + width: number, + height: number, + webpSupport = true, +): HTMLCanvasElement { + const canvas = document.createElement('canvas'); + Object.defineProperty(canvas, 'width', { value: width, writable: true }); + Object.defineProperty(canvas, 'height', { value: height, writable: true }); + + canvas.toDataURL = jest.fn((type?: string, quality?: unknown) => { + if (type === 'image/webp' && webpSupport) { + return `data:image/webp;base64,WEBP_${quality}_DATA`; + } + if (type === 'image/webp' && !webpSupport) { + return `data:image/png;base64,PNG_FALLBACK`; + } + if (type === 'image/jpeg') { + return `data:image/jpeg;base64,JPEG_${quality}_DATA`; + } + return 'data:image/png;base64,DEFAULT'; + }); + + canvas.getContext = jest.fn().mockReturnValue({ + drawImage: jest.fn(), + }); + + return canvas; +} + +describe('captureScreenshot', () => { + let rootElement: HTMLDivElement; + + beforeEach(() => { + jest.clearAllMocks(); + + rootElement = document.createElement('div'); + rootElement.id = 'root'; + document.body.appendChild(rootElement); + + Object.defineProperty(document, 'visibilityState', { + value: 'visible', + writable: true, + configurable: true, + }); + + (window as any).requestIdleCallback = (cb: Function) => { + cb(); + return 0; + }; + + HTMLCanvasElement.prototype.getContext = jest.fn().mockReturnValue({ + drawImage: jest.fn(), + }) as any; + HTMLCanvasElement.prototype.toDataURL = jest.fn( + (type?: string, quality?: unknown) => { + if (type === 'image/webp') { + return `data:image/webp;base64,SCALED_WEBP_${quality}_DATA`; + } + if (type === 'image/jpeg') { + return `data:image/jpeg;base64,SCALED_JPEG_${quality}_DATA`; + } + return 'data:image/png;base64,DEFAULT'; + }, + ); + }); + + afterEach(() => { + if (document.body.contains(rootElement)) { + document.body.removeChild(rootElement); + } + delete (window as any).requestIdleCallback; + }); + + it('should return WebP when browser supports it', async () => { + const mockCanvas = createMockCanvas(800, 600, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot(); + + expect(result).toEqual( + expect.objectContaining({ + success: true, + contentType: 'image/webp', + base64: 'WEBP_0.7_DATA', + width: 800, + height: 600, + }), + ); + expect((result as any).captureTimeMs).toBeGreaterThanOrEqual(0); + }); + + it('should fall back to JPEG when WebP is not supported', async () => { + const mockCanvas = createMockCanvas(800, 600, false); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot(); + + expect(result).toEqual( + expect.objectContaining({ + success: true, + contentType: 'image/jpeg', + base64: 'JPEG_0.7_DATA', + }), + ); + }); + + it('should return CaptureError when tab is hidden', async () => { + Object.defineProperty(document, 'visibilityState', { + value: 'hidden', + writable: true, + configurable: true, + }); + + const result = await captureScreenshot(); + + expect(result).toEqual({ success: false, error: 'Tab not visible' }); + expect(mockHtml2canvas).not.toHaveBeenCalled(); + }); + + it('should return CaptureError on timeout', async () => { + mockHtml2canvas.mockImplementation( + () => + new Promise(resolve => { + setTimeout(() => resolve(createMockCanvas(800, 600)), 5000); + }), + ); + + const result = await captureScreenshot({ timeoutMs: 50 }); + + expect(result).toEqual({ + success: false, + error: 'Capture timeout exceeded', + }); + }); + + it('should return CaptureError when #root is not found', async () => { + document.body.removeChild(rootElement); + + const result = await captureScreenshot(); + + expect(result).toEqual({ + success: false, + error: 'Root element (#root) not found', + }); + + document.body.appendChild(rootElement); + }); + + it('should call sanitizeClonedDom in onclone', async () => { + const mockCanvas = createMockCanvas(800, 600); + mockHtml2canvas.mockImplementation(async (_el, options) => { + if (options?.onclone) { + (options.onclone as Function)(document); + } + return mockCanvas; + }); + + await captureScreenshot(); + + expect(mockSanitize).toHaveBeenCalledWith(document); + }); + + it('should exclude elements with data-screen-capture-exclude', async () => { + const mockCanvas = createMockCanvas(800, 600); + let ignoreElementsFn: ((el: Element) => boolean) | undefined; + + mockHtml2canvas.mockImplementation(async (_el, options) => { + ignoreElementsFn = options?.ignoreElements as + ((el: Element) => boolean) | undefined; + return mockCanvas; + }); + + await captureScreenshot(); + + expect(ignoreElementsFn).toBeDefined(); + + const excludedElement = document.createElement('div'); + excludedElement.setAttribute('data-screen-capture-exclude', ''); + expect(ignoreElementsFn!(excludedElement)).toBe(true); + + const normalElement = document.createElement('div'); + expect(ignoreElementsFn!(normalElement)).toBe(false); + }); + + it('should exclude elements with pf-chatbot class', async () => { + const mockCanvas = createMockCanvas(800, 600); + let ignoreElementsFn: ((el: Element) => boolean) | undefined; + + mockHtml2canvas.mockImplementation(async (_el, options) => { + ignoreElementsFn = options?.ignoreElements as + ((el: Element) => boolean) | undefined; + return mockCanvas; + }); + + await captureScreenshot(); + + const chatElement = document.createElement('div'); + chatElement.classList.add('pf-chatbot'); + expect(ignoreElementsFn!(chatElement)).toBe(true); + }); + + it('should scale down canvas when width exceeds maxWidth', async () => { + const mockCanvas = createMockCanvas(2560, 1440, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot({ maxWidth: 1280 }); + + expect(result).toEqual( + expect.objectContaining({ + success: true, + width: 1280, + height: 720, + contentType: 'image/webp', + }), + ); + }); + + it('should not scale canvas when width is within maxWidth', async () => { + const mockCanvas = createMockCanvas(1024, 768, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot({ maxWidth: 1280 }); + + expect(result).toEqual( + expect.objectContaining({ + success: true, + width: 1024, + height: 768, + }), + ); + }); + + it('should use custom quality parameter', async () => { + const mockCanvas = createMockCanvas(800, 600, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot({ quality: 0.5 }); + + expect(result).toEqual( + expect.objectContaining({ + success: true, + base64: 'WEBP_0.5_DATA', + }), + ); + }); + + it('should return CaptureError when html2canvas throws', async () => { + mockHtml2canvas.mockRejectedValue(new Error('Canvas rendering failed')); + + const result = await captureScreenshot(); + + expect(result).toEqual({ + success: false, + error: 'Canvas rendering failed', + }); + }); + + it('should use setTimeout fallback when requestIdleCallback is unavailable', async () => { + delete (window as any).requestIdleCallback; + + const mockCanvas = createMockCanvas(800, 600, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot(); + + expect(result.success).toBe(true); + }); + + it('should report captureTimeMs', async () => { + const mockCanvas = createMockCanvas(800, 600, true); + mockHtml2canvas.mockResolvedValue(mockCanvas); + + const result = await captureScreenshot(); + + expect(result.success).toBe(true); + expect((result as any).captureTimeMs).toBeGreaterThanOrEqual(0); + expect(typeof (result as any).captureTimeMs).toBe('number'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts new file mode 100644 index 00000000000..81174740e0c --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts @@ -0,0 +1,276 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + isSensitiveElement, + redactText, + SAFE_LABEL_EXCLUSIONS, + sanitizeClonedDom, + SECRET_PATTERNS, + SENSITIVE_LABEL_PATTERN, +} from '../sensitive-data-redactor'; + +describe('SECRET_PATTERNS', () => { + it('should match GitHub PATs', () => { + expect('ghp_abcdefghijklmnopqrstuvwxyz0123456789').toMatch( + SECRET_PATTERNS[0], + ); + }); + + it('should match GitLab PATs', () => { + expect('glpat-abcdefghijklmnopqrst').toMatch(SECRET_PATTERNS[4]); + }); + + it('should match AWS Access Key IDs', () => { + expect('AKIAIOSFODNN7EXAMPLE').toMatch(SECRET_PATTERNS[6]); + }); + + it('should match JWTs', () => { + const jwt = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + expect(jwt).toMatch(SECRET_PATTERNS[9]); + }); +}); + +describe('SENSITIVE_LABEL_PATTERN', () => { + it('should match labels containing "secret"', () => { + expect(SENSITIVE_LABEL_PATTERN.test('Secret value')).toBe(true); + }); + + it('should match labels containing "token"', () => { + expect(SENSITIVE_LABEL_PATTERN.test('Personal Token')).toBe(true); + }); + + it('should match labels containing "password"', () => { + expect(SENSITIVE_LABEL_PATTERN.test('Enter password')).toBe(true); + }); + + it('should match labels containing "api key"', () => { + expect(SENSITIVE_LABEL_PATTERN.test('API Key')).toBe(true); + }); + + it('should not match unrelated labels', () => { + expect(SENSITIVE_LABEL_PATTERN.test('Username')).toBe(false); + expect(SENSITIVE_LABEL_PATTERN.test('Email')).toBe(false); + }); +}); + +describe('SAFE_LABEL_EXCLUSIONS', () => { + it('should match "token usage" as safe', () => { + expect(SAFE_LABEL_EXCLUSIONS.test('Token usage: 500')).toBe(true); + }); + + it('should match "token count" as safe', () => { + expect(SAFE_LABEL_EXCLUSIONS.test('Token count')).toBe(true); + }); + + it('should not match actual sensitive labels', () => { + expect(SAFE_LABEL_EXCLUSIONS.test('Access token')).toBe(false); + }); +}); + +describe('redactText', () => { + it('should replace GitHub PATs with [REDACTED]', () => { + const text = 'My token is ghp_abcdefghijklmnopqrstuvwxyz0123456789 here'; + expect(redactText(text)).toBe('My token is [REDACTED] here'); + }); + + it('should replace JWTs with [REDACTED]', () => { + const jwt = + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + const text = `Bearer ${jwt}`; + const result = redactText(text); + expect(result).not.toContain('eyJ'); + expect(result).toContain('[REDACTED]'); + }); + + it('should replace AWS access key IDs with [REDACTED]', () => { + const text = 'aws_access_key_id = AKIAIOSFODNN7EXAMPLE'; + expect(redactText(text)).toBe('aws_access_key_id = [REDACTED]'); + }); + + it('should replace Bearer tokens with [REDACTED]', () => { + const text = + 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.rTCH8cLoGxAm_xw68z-zXVKi9ie6xJn9tnVWjd_9ftE'; + const result = redactText(text); + expect(result).toContain('[REDACTED]'); + expect(result).not.toContain('Bearer eyJ'); + }); + + it('should not alter normal text', () => { + const text = 'This is a normal sentence with no secrets.'; + expect(redactText(text)).toBe(text); + }); + + it('should handle multiple secrets in one string', () => { + const text = + 'Key1: ghp_abcdefghijklmnopqrstuvwxyz0123456789 Key2: AKIAIOSFODNN7EXAMPLE'; + const result = redactText(text); + expect(result).toBe('Key1: [REDACTED] Key2: [REDACTED]'); + }); + + it('should handle empty string', () => { + expect(redactText('')).toBe(''); + }); +}); + +describe('isSensitiveElement', () => { + let doc: Document; + + beforeEach(() => { + doc = document.implementation.createHTMLDocument('test'); + }); + + it('should detect password inputs', () => { + const input = doc.createElement('input'); + input.type = 'password'; + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should detect elements with sensitive aria-label', () => { + const input = doc.createElement('input'); + input.setAttribute('aria-label', 'Access token'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should detect elements with associated sensitive label', () => { + const label = doc.createElement('label'); + label.setAttribute('for', 'secret-field'); + label.textContent = 'Secret value'; + doc.body.appendChild(label); + + const input = doc.createElement('input'); + input.id = 'secret-field'; + doc.body.appendChild(input); + + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should detect elements inside a sensitive parent label', () => { + const label = doc.createElement('label'); + label.textContent = 'API Key'; + + const input = doc.createElement('input'); + label.appendChild(input); + doc.body.appendChild(label); + + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should exclude safe labels like "token usage"', () => { + const input = doc.createElement('input'); + input.setAttribute('aria-label', 'Token usage: 500'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(false); + }); + + it('should return false for normal text inputs', () => { + const input = doc.createElement('input'); + input.type = 'text'; + input.setAttribute('aria-label', 'Username'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(false); + }); +}); + +describe('sanitizeClonedDom', () => { + let doc: Document; + + beforeEach(() => { + doc = document.implementation.createHTMLDocument('test'); + }); + + it('should mask password input values', () => { + const input = doc.createElement('input'); + input.type = 'password'; + input.value = 'my-secret-password'; + doc.body.appendChild(input); + + sanitizeClonedDom(doc); + + expect(input.value).toBe('••••••••'); + }); + + it('should mask inputs with sensitive labels', () => { + const label = doc.createElement('label'); + label.setAttribute('for', 'token-input'); + label.textContent = 'Bearer Token'; + doc.body.appendChild(label); + + const input = doc.createElement('input'); + input.id = 'token-input'; + input.type = 'text'; + input.value = 'ghp_abcdefghijklmnopqrstuvwxyz0123456789'; + doc.body.appendChild(input); + + sanitizeClonedDom(doc); + + expect(input.value).toBe('••••••••'); + }); + + it('should redact secrets in text nodes', () => { + const span = doc.createElement('span'); + span.textContent = 'Token: ghp_abcdefghijklmnopqrstuvwxyz0123456789'; + doc.body.appendChild(span); + + sanitizeClonedDom(doc); + + expect(span.textContent).toBe('Token: [REDACTED]'); + }); + + it('should handle visibility-toggle secrets', () => { + const container = doc.createElement('div'); + container.className = 'Box'; + + const pre = doc.createElement('pre'); + pre.textContent = 'super-secret-value-123'; + container.appendChild(pre); + + const button = doc.createElement('button'); + button.setAttribute('aria-label', 'Hide secret value'); + container.appendChild(button); + + doc.body.appendChild(container); + + sanitizeClonedDom(doc); + + expect(pre.textContent).toBe('••••••••'); + }); + + it('should not modify normal text', () => { + const p = doc.createElement('p'); + p.textContent = 'Hello, this is a normal paragraph.'; + doc.body.appendChild(p); + + sanitizeClonedDom(doc); + + expect(p.textContent).toBe('Hello, this is a normal paragraph.'); + }); + + it('should not modify non-sensitive inputs', () => { + const input = doc.createElement('input'); + input.type = 'text'; + input.setAttribute('aria-label', 'Search'); + input.value = 'my search query'; + doc.body.appendChild(input); + + sanitizeClonedDom(doc); + + expect(input.value).toBe('my search query'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts new file mode 100644 index 00000000000..075f68bbf64 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts @@ -0,0 +1,197 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import html2canvas from 'html2canvas-pro'; + +import { sanitizeClonedDom } from './sensitive-data-redactor'; + +const DEFAULT_QUALITY = 0.7; +const DEFAULT_MAX_WIDTH = 1280; +const DEFAULT_TIMEOUT_MS = 3000; +const DEFAULT_EXCLUDE_SELECTOR = '[data-screen-capture-exclude]'; + +export interface CaptureOptions { + /** Image quality 0-1 (default: 0.7) */ + quality?: number; + /** CSS selector for elements to exclude (default: '[data-screen-capture-exclude]') */ + excludeSelector?: string; + /** Max output width in pixels for compression (default: 1280) */ + maxWidth?: number; + /** Maximum time allowed for capture in ms (default: 3000) */ + timeoutMs?: number; +} + +export interface CaptureResult { + success: true; + /** Base64-encoded image without the data URI prefix */ + base64: string; + /** MIME type: 'image/webp' or 'image/jpeg' */ + contentType: string; + width: number; + height: number; + /** Time taken for the capture in milliseconds */ + captureTimeMs: number; +} + +export interface CaptureError { + success: false; + error: string; +} + +export type CaptureResponse = CaptureResult | CaptureError; + +function stripDataUriPrefix(dataUri: string): string { + const commaIdx = dataUri.indexOf(','); + return commaIdx !== -1 ? dataUri.slice(commaIdx + 1) : dataUri; +} + +function waitForIdle(): Promise { + return new Promise(resolve => { + if ('requestIdleCallback' in window) { + (window as Window).requestIdleCallback(() => resolve(), { timeout: 100 }); + } else { + setTimeout(resolve, 0); + } + }); +} + +function scaleCanvas( + canvas: HTMLCanvasElement, + maxWidth: number, +): HTMLCanvasElement { + if (canvas.width <= maxWidth) { + return canvas; + } + + const scaleFactor = maxWidth / canvas.width; + const scaledWidth = maxWidth; + const scaledHeight = Math.round(canvas.height * scaleFactor); + + const scaledCanvas = document.createElement('canvas'); + scaledCanvas.width = scaledWidth; + scaledCanvas.height = scaledHeight; + + const ctx = scaledCanvas.getContext('2d'); + if (ctx) { + ctx.drawImage(canvas, 0, 0, scaledWidth, scaledHeight); + } + + return scaledCanvas; +} + +function negotiateFormat( + canvas: HTMLCanvasElement, + quality: number, +): { dataUri: string; contentType: string } { + const webpDataUri = canvas.toDataURL('image/webp', quality); + if (webpDataUri.startsWith('data:image/webp')) { + return { dataUri: webpDataUri, contentType: 'image/webp' }; + } + const jpegDataUri = canvas.toDataURL('image/jpeg', quality); + return { dataUri: jpegDataUri, contentType: 'image/jpeg' }; +} + +async function doCapture( + options: Required, +): Promise { + const targetElement = document.querySelector('#root'); + if (!targetElement) { + return { success: false, error: 'Root element (#root) not found' }; + } + + const start = window.performance.now(); + + const canvas = await html2canvas(targetElement as HTMLElement, { + useCORS: true, + allowTaint: false, + logging: false, + onclone: (clonedDocument: Document) => { + sanitizeClonedDom(clonedDocument); + }, + ignoreElements: (element: Element) => { + if (element.hasAttribute('data-screen-capture-exclude')) { + return true; + } + if (element.classList.contains('pf-chatbot')) { + return true; + } + const selector = options.excludeSelector; + if (selector !== DEFAULT_EXCLUDE_SELECTOR && element.matches(selector)) { + return true; + } + return false; + }, + }); + + const scaledCanvas = scaleCanvas(canvas, options.maxWidth); + const { dataUri, contentType } = negotiateFormat( + scaledCanvas, + options.quality, + ); + + const captureTimeMs = Math.round(window.performance.now() - start); + + return { + success: true, + base64: stripDataUriPrefix(dataUri), + contentType, + width: scaledCanvas.width, + height: scaledCanvas.height, + captureTimeMs, + }; +} + +/** + * Captures a screenshot of the current RHDH viewport, excluding the + * Lightspeed chat panel and redacting sensitive data. + * + * Uses WebP format when the browser supports it, falling back to JPEG. + * Includes performance guardrails: visibility check, idle deferral, and timeout. + */ +export async function captureScreenshot( + options?: CaptureOptions, +): Promise { + if (document.visibilityState === 'hidden') { + return { success: false, error: 'Tab not visible' }; + } + + await waitForIdle(); + + const resolvedOptions: Required = { + quality: options?.quality ?? DEFAULT_QUALITY, + excludeSelector: options?.excludeSelector ?? DEFAULT_EXCLUDE_SELECTOR, + maxWidth: options?.maxWidth ?? DEFAULT_MAX_WIDTH, + timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }; + + try { + const result = await Promise.race([ + doCapture(resolvedOptions), + new Promise(resolve => + setTimeout( + () => resolve({ success: false, error: 'Capture timeout exceeded' }), + resolvedOptions.timeoutMs, + ), + ), + ]); + return result; + } catch (err) { + return { + success: false, + error: err instanceof Error ? err.message : 'Unknown capture error', + }; + } +} diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts new file mode 100644 index 00000000000..b2eeb8bb79a --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts @@ -0,0 +1,144 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const REDACTED = '[REDACTED]'; +const MASK = '••••••••'; + +/** + * Regex patterns matching known secret/token formats. + * Each pattern uses the global flag for replaceAll behavior. + */ +export const SECRET_PATTERNS: RegExp[] = [ + /ghp_[a-zA-Z0-9]{36,}/g, + /ghs_[a-zA-Z0-9]{36,}/g, + /gho_[a-zA-Z0-9]{36,}/g, + /github_pat_[a-zA-Z0-9_]{22,}/g, + /glpat-[a-zA-Z0-9\-_]{20,}/g, + /sk-[a-zA-Z0-9]{32,}/g, + /AKIA[0-9A-Z]{16}/g, + /xoxb-[0-9]+-[A-Za-z0-9]+/g, + /xoxp-[0-9]+-[A-Za-z0-9]+/g, + /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]*/g, + /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g, + /Bearer\s+[A-Za-z0-9\-._~+/]{20,}=*/g, +]; + +/** + * Label text patterns that indicate a field holds sensitive content. + */ +export const SENSITIVE_LABEL_PATTERN = + /secret|token|password|api.?key|credential|private.?key|access.?key/i; + +/** + * Label patterns that are safe false-positives and should NOT trigger redaction. + */ +export const SAFE_LABEL_EXCLUSIONS = + /token usage|token count|token limit|token budget/i; + +/** + * Scans text for known secret patterns and replaces them with [REDACTED]. + * Used by DOM extraction to sanitize extracted text before sending. + */ +export function redactText(text: string): string { + let result = text; + for (const pattern of SECRET_PATTERNS) { + pattern.lastIndex = 0; + result = result.replace(pattern, REDACTED); + } + return result; +} + +/** + * Determines if an input/textarea element is likely holding a secret, + * based on its type, aria-label, or associated label element. + */ +export function isSensitiveElement(el: Element, doc: Document): boolean { + if ((el as HTMLInputElement).type === 'password') { + return true; + } + + const ariaLabel = el.getAttribute('aria-label') || ''; + if ( + SENSITIVE_LABEL_PATTERN.test(ariaLabel) && + !SAFE_LABEL_EXCLUSIONS.test(ariaLabel) + ) { + return true; + } + + const id = el.getAttribute('id'); + if (id) { + const label = doc.querySelector(`label[for="${id}"]`); + if (label?.textContent && SENSITIVE_LABEL_PATTERN.test(label.textContent)) { + return !SAFE_LABEL_EXCLUSIONS.test(label.textContent); + } + } + + const parentLabel = el.closest('label'); + if ( + parentLabel?.textContent && + SENSITIVE_LABEL_PATTERN.test(parentLabel.textContent) + ) { + return !SAFE_LABEL_EXCLUSIONS.test(parentLabel.textContent); + } + + return false; +} + +/** + * Sanitizes a cloned DOM before it is rendered to canvas. + * Used in html2canvas onclone callback. + * Mutates the clone — safe because it's not the live DOM. + */ +export function sanitizeClonedDom(clonedDoc: Document): void { + // 1. Mask all sensitive inputs/textareas (password fields + label-detected) + clonedDoc.querySelectorAll('input, textarea').forEach(el => { + if ( + (el as HTMLInputElement).type === 'password' || + isSensitiveElement(el, clonedDoc) + ) { + (el as HTMLInputElement).value = MASK; + } + }); + + // 2. Regex scan all text nodes for token patterns + const walker = clonedDoc.createTreeWalker( + clonedDoc.body, + NodeFilter.SHOW_TEXT, + ); + let node = walker.nextNode(); + while (node) { + if (node.textContent) { + const redacted = redactText(node.textContent); + if (redacted !== node.textContent) { + node.textContent = redacted; + } + } + node = walker.nextNode(); + } + + // 3. Handle visibility-toggle secrets (e.g. Akeyless show/hide pattern) + clonedDoc + .querySelectorAll('[aria-label*="Hide secret"], [aria-label*="Hide value"]') + .forEach(btn => { + const container = btn.closest('[class*="Box"], [class*="flex"]'); + if (container) { + const textEl = container.querySelector('pre, code, span, p'); + if (textEl?.textContent && textEl.textContent !== MASK) { + textEl.textContent = MASK; + } + } + }); +} diff --git a/workspaces/intelligent-assistant/yarn.lock b/workspaces/intelligent-assistant/yarn.lock index e98f385d40b..55f1b3ab222 100644 --- a/workspaces/intelligent-assistant/yarn.lock +++ b/workspaces/intelligent-assistant/yarn.lock @@ -9406,6 +9406,7 @@ __metadata: "@testing-library/jest-dom": "npm:^6.0.0" "@testing-library/react": "npm:^15.0.0" "@testing-library/user-event": "npm:14.6.1" + html2canvas-pro: "npm:^1.5.0" monaco-editor: "npm:^0.55.0" prettier: "npm:3.9.6" react: "npm:^18.0.0" @@ -15206,6 +15207,13 @@ __metadata: languageName: node linkType: hard +"base64-arraybuffer@npm:^1.0.2": + version: 1.0.2 + resolution: "base64-arraybuffer@npm:1.0.2" + checksum: 10c0/3acac95c70f9406e87a41073558ba85b6be9dbffb013a3d2a710e3f2d534d506c911847d5d9be4de458af6362c676de0a5c4c2d7bdf4def502d00b313368e72f + languageName: node + linkType: hard + "base64-js@npm:^1.0.2, base64-js@npm:^1.3.0, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" @@ -17098,6 +17106,15 @@ __metadata: languageName: node linkType: hard +"css-line-break@npm:^2.1.0": + version: 2.1.0 + resolution: "css-line-break@npm:2.1.0" + dependencies: + utrie: "npm:^1.0.2" + checksum: 10c0/b2222d99d5daf7861ecddc050244fdce296fad74b000dcff6bdfb1eb16dc2ef0b9ffe2c1c965e3239bd05ebe9eadb6d5438a91592fa8648d27a338e827cf9048 + languageName: node + linkType: hard + "css-loader@npm:^6.5.1": version: 6.11.0 resolution: "css-loader@npm:6.11.0" @@ -21359,6 +21376,16 @@ __metadata: languageName: node linkType: hard +"html2canvas-pro@npm:^1.5.0": + version: 1.6.7 + resolution: "html2canvas-pro@npm:1.6.7" + dependencies: + css-line-break: "npm:^2.1.0" + text-segmentation: "npm:^1.0.3" + checksum: 10c0/bc8f8ca563a3d33dbce301c9edfb69b912e095dcba0576af27251af0ed0776f3b5f54ba1af9b585b476b3d1411045e4bbccf38ec08e8b3f6e7e1059ee3963171 + languageName: node + linkType: hard + "htmlparser2@npm:^6.1.0": version: 6.1.0 resolution: "htmlparser2@npm:6.1.0" @@ -32942,6 +32969,15 @@ __metadata: languageName: node linkType: hard +"text-segmentation@npm:^1.0.3": + version: 1.0.3 + resolution: "text-segmentation@npm:1.0.3" + dependencies: + utrie: "npm:^1.0.2" + checksum: 10c0/8b9ae8524e3a332371060d0ca62f10ad49a13e954719ea689a6c3a8b8c15c8a56365ede2bb91c322fb0d44b6533785f0da603e066b7554d052999967fb72d600 + languageName: node + linkType: hard + "text-table@npm:0.2.0, text-table@npm:^0.2.0": version: 0.2.0 resolution: "text-table@npm:0.2.0" @@ -34436,6 +34472,15 @@ __metadata: languageName: node linkType: hard +"utrie@npm:^1.0.2": + version: 1.0.2 + resolution: "utrie@npm:1.0.2" + dependencies: + base64-arraybuffer: "npm:^1.0.2" + checksum: 10c0/eaffe645bd81a39e4bc3abb23df5895e9961dbdd49748ef3b173529e8b06ce9dd1163e9705d5309a1c61ee41ffcb825e2043bc0fd1659845ffbdf4b1515dfdb4 + languageName: node + linkType: hard + "uuid@npm:^10.0.0": version: 10.0.0 resolution: "uuid@npm:10.0.0" From 4473536dc3d889e958ac6c3ec17cfef746d48b0d Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Sun, 9 Aug 2026 23:46:50 +0530 Subject: [PATCH 2/9] updating the changeset Signed-off-by: its-mitesh-kumar --- workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md index 336e29b3a1a..db3fd7b31e1 100644 --- a/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md +++ b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md @@ -1,6 +1,6 @@ --- '@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor -'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': minor +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': patch --- Add screenshot capture utility with WebP-first format, sensitive data redaction, and performance guardrails for deep context awareness From eb7d83167c6a8e72cb7c8901338f31374fd8fa19 Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Mon, 10 Aug 2026 11:31:42 +0530 Subject: [PATCH 3/9] dynamically importing html2canvas-pro Signed-off-by: its-mitesh-kumar --- .../__tests__/sensitive-data-redactor.test.ts | 23 ---------------- .../src/utils/screen-capture.ts | 26 ++++++++++++------- .../src/utils/sensitive-data-redactor.ts | 11 ++++---- 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts index 81174740e0c..d0e0b8ebd2c 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts @@ -19,32 +19,9 @@ import { redactText, SAFE_LABEL_EXCLUSIONS, sanitizeClonedDom, - SECRET_PATTERNS, SENSITIVE_LABEL_PATTERN, } from '../sensitive-data-redactor'; -describe('SECRET_PATTERNS', () => { - it('should match GitHub PATs', () => { - expect('ghp_abcdefghijklmnopqrstuvwxyz0123456789').toMatch( - SECRET_PATTERNS[0], - ); - }); - - it('should match GitLab PATs', () => { - expect('glpat-abcdefghijklmnopqrst').toMatch(SECRET_PATTERNS[4]); - }); - - it('should match AWS Access Key IDs', () => { - expect('AKIAIOSFODNN7EXAMPLE').toMatch(SECRET_PATTERNS[6]); - }); - - it('should match JWTs', () => { - const jwt = - 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; - expect(jwt).toMatch(SECRET_PATTERNS[9]); - }); -}); - describe('SENSITIVE_LABEL_PATTERN', () => { it('should match labels containing "secret"', () => { expect(SENSITIVE_LABEL_PATTERN.test('Secret value')).toBe(true); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts index 075f68bbf64..e77cfd009f2 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts @@ -14,8 +14,6 @@ * limitations under the License. */ -import html2canvas from 'html2canvas-pro'; - import { sanitizeClonedDom } from './sensitive-data-redactor'; const DEFAULT_QUALITY = 0.7; @@ -112,6 +110,8 @@ async function doCapture( return { success: false, error: 'Root element (#root) not found' }; } + const { default: html2canvas } = await import('html2canvas-pro'); + const start = window.performance.now(); const canvas = await html2canvas(targetElement as HTMLElement, { @@ -128,9 +128,14 @@ async function doCapture( if (element.classList.contains('pf-chatbot')) { return true; } - const selector = options.excludeSelector; - if (selector !== DEFAULT_EXCLUDE_SELECTOR && element.matches(selector)) { - return true; + if (options.excludeSelector !== DEFAULT_EXCLUDE_SELECTOR) { + try { + if (element.matches(options.excludeSelector)) { + return true; + } + } catch { + // Invalid selector — skip rather than crash capture + } } return false; }, @@ -177,15 +182,16 @@ export async function captureScreenshot( timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, }; + let timeoutId: ReturnType; try { const result = await Promise.race([ doCapture(resolvedOptions), - new Promise(resolve => - setTimeout( + new Promise(resolve => { + timeoutId = setTimeout( () => resolve({ success: false, error: 'Capture timeout exceeded' }), resolvedOptions.timeoutMs, - ), - ), + ); + }), ]); return result; } catch (err) { @@ -193,5 +199,7 @@ export async function captureScreenshot( success: false, error: err instanceof Error ? err.message : 'Unknown capture error', }; + } finally { + clearTimeout(timeoutId!); } } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts index b2eeb8bb79a..5a07dae1e80 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts @@ -80,7 +80,7 @@ export function isSensitiveElement(el: Element, doc: Document): boolean { const id = el.getAttribute('id'); if (id) { - const label = doc.querySelector(`label[for="${id}"]`); + const label = doc.querySelector(`label[for="${CSS.escape(id)}"]`); if (label?.textContent && SENSITIVE_LABEL_PATTERN.test(label.textContent)) { return !SAFE_LABEL_EXCLUSIONS.test(label.textContent); } @@ -105,10 +105,7 @@ export function isSensitiveElement(el: Element, doc: Document): boolean { export function sanitizeClonedDom(clonedDoc: Document): void { // 1. Mask all sensitive inputs/textareas (password fields + label-detected) clonedDoc.querySelectorAll('input, textarea').forEach(el => { - if ( - (el as HTMLInputElement).type === 'password' || - isSensitiveElement(el, clonedDoc) - ) { + if (isSensitiveElement(el, clonedDoc)) { (el as HTMLInputElement).value = MASK; } }); @@ -131,7 +128,9 @@ export function sanitizeClonedDom(clonedDoc: Document): void { // 3. Handle visibility-toggle secrets (e.g. Akeyless show/hide pattern) clonedDoc - .querySelectorAll('[aria-label*="Hide secret"], [aria-label*="Hide value"]') + .querySelectorAll( + '[aria-label*="Hide secret" i], [aria-label*="Hide value" i]', + ) .forEach(btn => { const container = btn.closest('[class*="Box"], [class*="flex"]'); if (container) { From 29cf8b41b92a453fedacbd777f1d214dffe29aad Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Mon, 10 Aug 2026 12:18:48 +0530 Subject: [PATCH 4/9] fix(intelligent-assistant): address code review findings - Fall back to original canvas when getContext('2d') returns null - Add CSS.escape() to prevent selector injection in label lookup - Remove redundant password type check (already in isSensitiveElement) - Wrap element.matches() in try/catch for invalid selectors - Use case-insensitive CSS attribute selectors for aria-label matching - Convert html2canvas-pro to dynamic import for bundle size optimization - Clear timeout timer in finally block to prevent resource leak - Replace as-any casts with expect.objectContaining in tests - Remove fragile index-based SECRET_PATTERNS tests - Add JSDoc warning about stateful /g regexes Signed-off-by: its-mitesh-kumar Co-authored-by: Cursor --- .../src/utils/__tests__/screen-capture.test.ts | 11 +++++++---- .../intelligent-assistant/src/utils/screen-capture.ts | 9 +++++++-- .../src/utils/sensitive-data-redactor.ts | 2 ++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts index e06804d3daf..a2c4a717bd2 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts @@ -115,9 +115,9 @@ describe('captureScreenshot', () => { base64: 'WEBP_0.7_DATA', width: 800, height: 600, + captureTimeMs: expect.any(Number), }), ); - expect((result as any).captureTimeMs).toBeGreaterThanOrEqual(0); }); it('should fall back to JPEG when WebP is not supported', async () => { @@ -303,8 +303,11 @@ describe('captureScreenshot', () => { const result = await captureScreenshot(); - expect(result.success).toBe(true); - expect((result as any).captureTimeMs).toBeGreaterThanOrEqual(0); - expect(typeof (result as any).captureTimeMs).toBe('number'); + expect(result).toEqual( + expect.objectContaining({ + success: true, + captureTimeMs: expect.any(Number), + }), + ); }); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts index e77cfd009f2..bf6b9c0586a 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts @@ -83,9 +83,10 @@ function scaleCanvas( scaledCanvas.height = scaledHeight; const ctx = scaledCanvas.getContext('2d'); - if (ctx) { - ctx.drawImage(canvas, 0, 0, scaledWidth, scaledHeight); + if (!ctx) { + return canvas; } + ctx.drawImage(canvas, 0, 0, scaledWidth, scaledHeight); return scaledCanvas; } @@ -182,6 +183,10 @@ export async function captureScreenshot( timeoutMs: options?.timeoutMs ?? DEFAULT_TIMEOUT_MS, }; + // When the timeout wins the race, doCapture continues running in the background + // since html2canvas-pro does not support AbortController-based cancellation. + // The timeout prevents the caller from waiting indefinitely; background work + // completes harmlessly and its result is discarded. let timeoutId: ReturnType; try { const result = await Promise.race([ diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts index 5a07dae1e80..3e0b1b8973c 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts @@ -20,6 +20,8 @@ const MASK = '••••••••'; /** * Regex patterns matching known secret/token formats. * Each pattern uses the global flag for replaceAll behavior. + * WARNING: These regexes are stateful (/g). Reset lastIndex before calling + * .test() or .exec() directly. Prefer using redactText() instead. */ export const SECRET_PATTERNS: RegExp[] = [ /ghp_[a-zA-Z0-9]{36,}/g, From 5dec1cc41e29cbb1b9f856f39c21887e54af1876 Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Thu, 13 Aug 2026 17:05:40 +0530 Subject: [PATCH 5/9] addressing the comments Signed-off-by: its-mitesh-kumar Co-authored-by: Cursor --- .../.changeset/swift-foxes-glow.md | 3 +- .../src/service/router.test.ts | 2 +- .../src/service/validation.test.ts | 83 +------- .../src/service/validation.ts | 28 +-- .../utils/__tests__/screen-capture.test.ts | 52 ++--- .../__tests__/sensitive-data-redactor.test.ts | 197 ++++++++++++++++-- .../src/utils/screen-capture.ts | 23 +- .../src/utils/sensitive-data-redactor.ts | 48 +++-- 8 files changed, 242 insertions(+), 194 deletions(-) diff --git a/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md index db3fd7b31e1..7f6de2ce99c 100644 --- a/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md +++ b/workspaces/intelligent-assistant/.changeset/swift-foxes-glow.md @@ -1,6 +1,5 @@ --- '@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor -'@red-hat-developer-hub/backstage-plugin-intelligent-assistant-backend': patch --- -Add screenshot capture utility with WebP-first format, sensitive data redaction, and performance guardrails for deep context awareness +Add screenshot capture utility with sensitive data redaction and performance guardrails for deep context awareness diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts index 23ad741a1fb..536ef2a7275 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/router.test.ts @@ -1826,7 +1826,7 @@ describe('intelligent-assistant router tests', () => { expect(response.statusCode).toEqual(400); expect(response.body.error).toContain( - 'This model does not support images', + 'This model does not support JPEG images', ); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts index 0bf1c5d8ee4..55904037903 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.test.ts @@ -358,8 +358,7 @@ describe('validateAttachmentsForModel', () => { expect(statusMock).toHaveBeenCalledWith(400); expect(jsonMock).toHaveBeenCalledWith({ - error: - 'Image attachment does not contain a valid image file (JPEG or WebP)', + error: 'Image attachment does not contain a valid JPEG file', }); expect(mockNext).not.toHaveBeenCalled(); }); @@ -382,86 +381,6 @@ describe('validateAttachmentsForModel', () => { expect(mockNext).toHaveBeenCalled(); expect(statusMock).not.toHaveBeenCalled(); }); - - it('should accept image with valid WebP magic bytes', () => { - // WebP format: RIFF....WEBP - const webpBytes = Buffer.alloc(12); - webpBytes.write('RIFF', 0, 'ascii'); - webpBytes.writeUInt32LE(0, 4); // file size placeholder - webpBytes.write('WEBP', 8, 'ascii'); - const VALID_WEBP_B64 = webpBytes.toString('base64'); - - mockReq.body = { - model: 'gpt-4', - provider: 'openai', - attachments: [ - { - attachment_type: 'image', - content_type: 'image/webp', - content: VALID_WEBP_B64, - }, - ], - }; - - callValidate(); - - expect(mockNext).toHaveBeenCalled(); - expect(statusMock).not.toHaveBeenCalled(); - }); - - it('should accept WebP image with data URL prefix', () => { - const webpBytes = Buffer.alloc(12); - webpBytes.write('RIFF', 0, 'ascii'); - webpBytes.writeUInt32LE(100, 4); - webpBytes.write('WEBP', 8, 'ascii'); - const VALID_WEBP_B64 = webpBytes.toString('base64'); - - mockReq.body = { - model: 'gpt-4', - provider: 'openai', - attachments: [ - { - attachment_type: 'image', - content_type: 'image/webp', - content: `data:image/webp;base64,${VALID_WEBP_B64}`, - }, - ], - }; - - callValidate(); - - expect(mockNext).toHaveBeenCalled(); - expect(statusMock).not.toHaveBeenCalled(); - }); - - it('should reject PNG image (neither JPEG nor WebP)', () => { - // PNG magic bytes - const pngBytes = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - ]); - const PNG_B64 = pngBytes.toString('base64'); - - mockReq.body = { - model: 'gpt-4', - provider: 'openai', - attachments: [ - { - attachment_type: 'image', - content_type: 'image/png', - content: PNG_B64, - }, - ], - }; - - callValidate(); - - expect(statusMock).toHaveBeenCalledWith(400); - expect(jsonMock).toHaveBeenCalledWith({ - error: - 'Image attachment does not contain a valid image file (JPEG or WebP)', - }); - expect(mockNext).not.toHaveBeenCalled(); - }); }); describe('JSON content validation', () => { diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts index 1d1838b59c1..edc4db00812 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant-backend/src/service/validation.ts @@ -25,9 +25,6 @@ import { import { Attachments, QueryRequestBody } from './types'; const JPEG_MAGIC = [0xff, 0xd8, 0xff]; -const WEBP_RIFF = [0x52, 0x49, 0x46, 0x46]; // "RIFF" -const WEBP_MARKER = [0x57, 0x45, 0x42, 0x50]; // "WEBP" at offset 8 -const WEBP_MIN_BYTES = 12; function extractBase64(content: string): string { const commaIndex = content.indexOf(','); @@ -37,25 +34,12 @@ function extractBase64(content: string): string { return content; } -function hasValidImageMagicBytes(content: string): boolean { +function hasValidJpegMagicBytes(content: string): boolean { const bytes = Buffer.from(extractBase64(content), 'base64'); - - if ( + return ( bytes.length >= JPEG_MAGIC.length && JPEG_MAGIC.every((b, i) => bytes[i] === b) - ) { - return true; - } - - if ( - bytes.length >= WEBP_MIN_BYTES && - WEBP_RIFF.every((b, i) => bytes[i] === b) && - WEBP_MARKER.every((b, i) => bytes[i + 8] === b) - ) { - return true; - } - - return false; + ); } function isValidJson(content: string): boolean { @@ -84,9 +68,9 @@ function validateAttachments(attachments: Array): string | null { if ( attachment.attachment_type === 'image' && attachment.content && - !hasValidImageMagicBytes(attachment.content) + !hasValidJpegMagicBytes(attachment.content) ) { - return 'Image attachment does not contain a valid image file (JPEG or WebP)'; + return 'Image attachment does not contain a valid JPEG file'; } if ( @@ -195,7 +179,7 @@ export const validateAttachmentsForModel = ( if (!supportsVision) { return res.status(400).json({ error: - 'This model does not support images. Please select a vision-capable model.', + 'This model does not support JPEG images. Please select a vision-capable model.', model, }); } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts index a2c4a717bd2..3023ebd55d5 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/screen-capture.test.ts @@ -29,22 +29,12 @@ const mockSanitize = sanitizeClonedDom as jest.MockedFunction< typeof sanitizeClonedDom >; -function createMockCanvas( - width: number, - height: number, - webpSupport = true, -): HTMLCanvasElement { +function createMockCanvas(width: number, height: number): HTMLCanvasElement { const canvas = document.createElement('canvas'); Object.defineProperty(canvas, 'width', { value: width, writable: true }); Object.defineProperty(canvas, 'height', { value: height, writable: true }); canvas.toDataURL = jest.fn((type?: string, quality?: unknown) => { - if (type === 'image/webp' && webpSupport) { - return `data:image/webp;base64,WEBP_${quality}_DATA`; - } - if (type === 'image/webp' && !webpSupport) { - return `data:image/png;base64,PNG_FALLBACK`; - } if (type === 'image/jpeg') { return `data:image/jpeg;base64,JPEG_${quality}_DATA`; } @@ -84,9 +74,6 @@ describe('captureScreenshot', () => { }) as any; HTMLCanvasElement.prototype.toDataURL = jest.fn( (type?: string, quality?: unknown) => { - if (type === 'image/webp') { - return `data:image/webp;base64,SCALED_WEBP_${quality}_DATA`; - } if (type === 'image/jpeg') { return `data:image/jpeg;base64,SCALED_JPEG_${quality}_DATA`; } @@ -102,8 +89,8 @@ describe('captureScreenshot', () => { delete (window as any).requestIdleCallback; }); - it('should return WebP when browser supports it', async () => { - const mockCanvas = createMockCanvas(800, 600, true); + it('should return JPEG format', async () => { + const mockCanvas = createMockCanvas(800, 600); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot(); @@ -111,8 +98,8 @@ describe('captureScreenshot', () => { expect(result).toEqual( expect.objectContaining({ success: true, - contentType: 'image/webp', - base64: 'WEBP_0.7_DATA', + contentType: 'image/jpeg', + base64: 'JPEG_0.7_DATA', width: 800, height: 600, captureTimeMs: expect.any(Number), @@ -120,21 +107,6 @@ describe('captureScreenshot', () => { ); }); - it('should fall back to JPEG when WebP is not supported', async () => { - const mockCanvas = createMockCanvas(800, 600, false); - mockHtml2canvas.mockResolvedValue(mockCanvas); - - const result = await captureScreenshot(); - - expect(result).toEqual( - expect.objectContaining({ - success: true, - contentType: 'image/jpeg', - base64: 'JPEG_0.7_DATA', - }), - ); - }); - it('should return CaptureError when tab is hidden', async () => { Object.defineProperty(document, 'visibilityState', { value: 'hidden', @@ -231,7 +203,7 @@ describe('captureScreenshot', () => { }); it('should scale down canvas when width exceeds maxWidth', async () => { - const mockCanvas = createMockCanvas(2560, 1440, true); + const mockCanvas = createMockCanvas(2560, 1440); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot({ maxWidth: 1280 }); @@ -241,13 +213,13 @@ describe('captureScreenshot', () => { success: true, width: 1280, height: 720, - contentType: 'image/webp', + contentType: 'image/jpeg', }), ); }); it('should not scale canvas when width is within maxWidth', async () => { - const mockCanvas = createMockCanvas(1024, 768, true); + const mockCanvas = createMockCanvas(1024, 768); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot({ maxWidth: 1280 }); @@ -262,7 +234,7 @@ describe('captureScreenshot', () => { }); it('should use custom quality parameter', async () => { - const mockCanvas = createMockCanvas(800, 600, true); + const mockCanvas = createMockCanvas(800, 600); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot({ quality: 0.5 }); @@ -270,7 +242,7 @@ describe('captureScreenshot', () => { expect(result).toEqual( expect.objectContaining({ success: true, - base64: 'WEBP_0.5_DATA', + base64: 'JPEG_0.5_DATA', }), ); }); @@ -289,7 +261,7 @@ describe('captureScreenshot', () => { it('should use setTimeout fallback when requestIdleCallback is unavailable', async () => { delete (window as any).requestIdleCallback; - const mockCanvas = createMockCanvas(800, 600, true); + const mockCanvas = createMockCanvas(800, 600); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot(); @@ -298,7 +270,7 @@ describe('captureScreenshot', () => { }); it('should report captureTimeMs', async () => { - const mockCanvas = createMockCanvas(800, 600, true); + const mockCanvas = createMockCanvas(800, 600); mockHtml2canvas.mockResolvedValue(mockCanvas); const result = await captureScreenshot(); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts index d0e0b8ebd2c..f3a491d6b97 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/sensitive-data-redactor.test.ts @@ -60,33 +60,159 @@ describe('SAFE_LABEL_EXCLUSIONS', () => { }); describe('redactText', () => { - it('should replace GitHub PATs with [REDACTED]', () => { - const text = 'My token is ghp_abcdefghijklmnopqrstuvwxyz0123456789 here'; - expect(redactText(text)).toBe('My token is [REDACTED] here'); + // --- GitHub tokens --- + it('should replace GitHub classic PATs (ghp_)', () => { + const text = 'token: ghp_ABCDEFghijklmnopqrstuvwxyz0123456789'; + expect(redactText(text)).toBe('token: [REDACTED]'); }); - it('should replace JWTs with [REDACTED]', () => { + it('should replace GitHub server tokens (ghs_)', () => { + const text = 'ghs_ABCDEFghijklmnopqrstuvwxyz0123456789'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace GitHub OAuth tokens (gho_)', () => { + const text = 'gho_ABCDEFghijklmnopqrstuvwxyz0123456789'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace GitHub user-to-server tokens (ghu_)', () => { + const text = 'ghu_ABCDEFghijklmnopqrstuvwxyz0123456789'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace GitHub refresh tokens (ghr_)', () => { + const text = 'ghr_ABCDEFghijklmnopqrstuvwxyz0123456789'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace GitHub fine-grained PATs (github_pat_)', () => { + const text = + 'github_pat_11ABCDEF22_abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOP'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + // --- GitLab PAT --- + it('should replace GitLab PATs (glpat-)', () => { + const text = 'GITLAB_TOKEN=glpat-AAAABBBBCCCCDDDDEEEE'; + expect(redactText(text)).toBe('GITLAB_TOKEN=[REDACTED]'); + }); + + // --- OpenAI / Anthropic --- + it('should replace OpenAI keys (old format)', () => { + const text = 'sk-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKL'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace OpenAI keys with hyphens (new format)', () => { + const text = + 'key: sk-proj-abc123-def456-ghi789-jklmnopqrstuvwxyz0123456789abc'; + expect(redactText(text)).toBe('key: [REDACTED]'); + }); + + // --- AWS --- + it('should replace AWS access key IDs (AKIA)', () => { + const text = 'aws_access_key_id = AKIAIOSFODNN7EXAMPLE'; + expect(redactText(text)).toBe('aws_access_key_id = [REDACTED]'); + }); + + // --- Slack tokens (all 5 prefixes) --- + it('should replace Slack bot tokens (xoxb-)', () => { + const text = 'SLACK_TOKEN=xoxb-123456789012-1234567890123'; + expect(redactText(text)).toBe('SLACK_TOKEN=[REDACTED]'); + }); + + it('should replace Slack user tokens (xoxp-)', () => { + const text = 'token: xoxp-123456789012-abcdef1234567890'; + expect(redactText(text)).toBe('token: [REDACTED]'); + }); + + it('should replace Slack app-level tokens (xoxa-)', () => { + const text = 'xoxa-2-123456'; + expect(redactText(text)).toBe('[REDACTED]'); + }); + + it('should replace Slack refresh tokens (xoxr-)', () => { + const text = 'refresh: xoxr-123456-abcdef7890'; + expect(redactText(text)).toBe('refresh: [REDACTED]'); + }); + + it('should replace Slack session tokens (xoxs-)', () => { + const text = 'session: xoxs-987654-ghijkl1234'; + expect(redactText(text)).toBe('session: [REDACTED]'); + }); + + // --- Slack webhooks --- + it('should replace Slack incoming webhook URLs', () => { + const text = + 'url: https://hooks.slack.com/services/T024F9JRE/B024F9JRE/abc123def456ghi789'; + expect(redactText(text)).toBe('url: [REDACTED]'); + }); + + // --- npm tokens --- + it('should replace npm access tokens (npm_)', () => { + const text = 'NPM_TOKEN=npm_AbCdEfGhIjKlMnOpQrStUvWxYz0123456789ab'; + expect(redactText(text)).toBe('NPM_TOKEN=[REDACTED]'); + }); + + // --- Google API keys --- + it('should replace Google API keys (AIza)', () => { + const text = 'GOOGLE_API_KEY=AIzaSyA1bcDeFgHiJkLmNoPqRsTuVwXyZ012345'; + expect(redactText(text)).toBe('GOOGLE_API_KEY=[REDACTED]'); + }); + + // --- Azure AD client secrets --- + it('should replace Azure AD client secrets (7Q~ format)', () => { + const text = 'AZURE_SECRET=abc7Q~abcdefghijklmnopqrstuvwxyz01234'; + expect(redactText(text)).toBe('AZURE_SECRET=[REDACTED]'); + }); + + it('should replace Azure AD client secrets (8Q~ format)', () => { + const text = 'secret: XYZ8Q~ABCDEFGHIJKLMNOPQRSTUVWXYZabcde'; + expect(redactText(text)).toBe('secret: [REDACTED]'); + }); + + // --- JWTs --- + it('should replace JWTs', () => { const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; - const text = `Bearer ${jwt}`; - const result = redactText(text); + const result = redactText(jwt); expect(result).not.toContain('eyJ'); expect(result).toContain('[REDACTED]'); }); - it('should replace AWS access key IDs with [REDACTED]', () => { - const text = 'aws_access_key_id = AKIAIOSFODNN7EXAMPLE'; - expect(redactText(text)).toBe('aws_access_key_id = [REDACTED]'); + // --- PEM private keys --- + it('should replace PEM private keys', () => { + const text = + '-----BEGIN RSA PRIVATE KEY-----\nMIIBogIBAAJBALRiMLAH\n-----END RSA PRIVATE KEY-----'; + expect(redactText(text)).toBe('[REDACTED]'); }); - it('should replace Bearer tokens with [REDACTED]', () => { + // --- Basic auth in URIs --- + it('should replace Basic auth in URIs', () => { + const text = 'db: mongodb://admin:SuperSecret123@db.example.com:27017/mydb'; + const result = redactText(text); + expect(result).toContain('[REDACTED]'); + expect(result).not.toContain('SuperSecret123'); + }); + + it('should replace postgres connection strings with credentials', () => { + const text = 'postgres://dbuser:p4ssw0rd@localhost:5432/catalog'; + const result = redactText(text); + expect(result).toContain('[REDACTED]'); + expect(result).not.toContain('p4ssw0rd'); + }); + + // --- Bearer tokens --- + it('should replace Bearer tokens', () => { const text = - 'Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.rTCH8cLoGxAm_xw68z-zXVKi9ie6xJn9tnVWjd_9ftE'; + 'Authorization: Bearer ya29.a0AfH6SMBxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'; const result = redactText(text); expect(result).toContain('[REDACTED]'); - expect(result).not.toContain('Bearer eyJ'); + expect(result).not.toContain('ya29'); }); + // --- General --- it('should not alter normal text', () => { const text = 'This is a normal sentence with no secrets.'; expect(redactText(text)).toBe(text); @@ -94,7 +220,7 @@ describe('redactText', () => { it('should handle multiple secrets in one string', () => { const text = - 'Key1: ghp_abcdefghijklmnopqrstuvwxyz0123456789 Key2: AKIAIOSFODNN7EXAMPLE'; + 'Key1: ghp_ABCDEFghijklmnopqrstuvwxyz0123456789 Key2: AKIAIOSFODNN7EXAMPLE'; const result = redactText(text); expect(result).toBe('Key1: [REDACTED] Key2: [REDACTED]'); }); @@ -102,6 +228,16 @@ describe('redactText', () => { it('should handle empty string', () => { expect(redactText('')).toBe(''); }); + + it('should not false-positive on URLs without credentials', () => { + const text = 'Visit https://example.com/api/v1/users'; + expect(redactText(text)).toBe(text); + }); + + it('should not false-positive on UUIDs', () => { + const text = '550e8400-e29b-41d4-a716-446655440000'; + expect(redactText(text)).toBe(text); + }); }); describe('isSensitiveElement', () => { @@ -156,6 +292,27 @@ describe('isSensitiveElement', () => { expect(isSensitiveElement(input, doc)).toBe(false); }); + it('should detect elements with sensitive name attribute', () => { + const input = doc.createElement('input'); + input.setAttribute('name', 'api_token'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should detect elements with sensitive placeholder', () => { + const input = doc.createElement('input'); + input.setAttribute('placeholder', 'Enter your secret key'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(true); + }); + + it('should detect elements with sensitive title', () => { + const input = doc.createElement('input'); + input.setAttribute('title', 'Access token'); + doc.body.appendChild(input); + expect(isSensitiveElement(input, doc)).toBe(true); + }); + it('should return false for normal text inputs', () => { const input = doc.createElement('input'); input.type = 'text'; @@ -239,7 +396,7 @@ describe('sanitizeClonedDom', () => { expect(p.textContent).toBe('Hello, this is a normal paragraph.'); }); - it('should not modify non-sensitive inputs', () => { + it('should not modify non-sensitive inputs without tokens', () => { const input = doc.createElement('input'); input.type = 'text'; input.setAttribute('aria-label', 'Search'); @@ -250,4 +407,16 @@ describe('sanitizeClonedDom', () => { expect(input.value).toBe('my search query'); }); + + it('should redact token pasted in a non-sensitive input', () => { + const input = doc.createElement('input'); + input.type = 'text'; + input.setAttribute('aria-label', 'Search'); + input.value = 'find ghp_abcdefghijklmnopqrstuvwxyz0123456789 in results'; + doc.body.appendChild(input); + + sanitizeClonedDom(doc); + + expect(input.value).toBe('find [REDACTED] in results'); + }); }); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts index bf6b9c0586a..9aaad771946 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts @@ -36,7 +36,7 @@ export interface CaptureResult { success: true; /** Base64-encoded image without the data URI prefix */ base64: string; - /** MIME type: 'image/webp' or 'image/jpeg' */ + /** MIME type: 'image/jpeg' */ contentType: string; width: number; height: number; @@ -91,18 +91,6 @@ function scaleCanvas( return scaledCanvas; } -function negotiateFormat( - canvas: HTMLCanvasElement, - quality: number, -): { dataUri: string; contentType: string } { - const webpDataUri = canvas.toDataURL('image/webp', quality); - if (webpDataUri.startsWith('data:image/webp')) { - return { dataUri: webpDataUri, contentType: 'image/webp' }; - } - const jpegDataUri = canvas.toDataURL('image/jpeg', quality); - return { dataUri: jpegDataUri, contentType: 'image/jpeg' }; -} - async function doCapture( options: Required, ): Promise { @@ -143,17 +131,14 @@ async function doCapture( }); const scaledCanvas = scaleCanvas(canvas, options.maxWidth); - const { dataUri, contentType } = negotiateFormat( - scaledCanvas, - options.quality, - ); + const dataUri = scaledCanvas.toDataURL('image/jpeg', options.quality); const captureTimeMs = Math.round(window.performance.now() - start); return { success: true, base64: stripDataUriPrefix(dataUri), - contentType, + contentType: 'image/jpeg', width: scaledCanvas.width, height: scaledCanvas.height, captureTimeMs, @@ -164,7 +149,7 @@ async function doCapture( * Captures a screenshot of the current RHDH viewport, excluding the * Lightspeed chat panel and redacting sensitive data. * - * Uses WebP format when the browser supports it, falling back to JPEG. + * Outputs JPEG format. * Includes performance guardrails: visibility check, idle deferral, and timeout. */ export async function captureScreenshot( diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts index 3e0b1b8973c..6efa02f2724 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/sensitive-data-redactor.ts @@ -24,17 +24,32 @@ const MASK = '••••••••'; * .test() or .exec() directly. Prefer using redactText() instead. */ export const SECRET_PATTERNS: RegExp[] = [ - /ghp_[a-zA-Z0-9]{36,}/g, - /ghs_[a-zA-Z0-9]{36,}/g, - /gho_[a-zA-Z0-9]{36,}/g, + // GitHub tokens (classic PAT, server-to-server, OAuth, fine-grained, user-to-server, refresh) + /gh[pousr]_[a-zA-Z0-9]{36,}/g, /github_pat_[a-zA-Z0-9_]{22,}/g, + // GitLab PAT /glpat-[a-zA-Z0-9\-_]{20,}/g, - /sk-[a-zA-Z0-9]{32,}/g, + // OpenAI / Anthropic API keys + /sk-[a-zA-Z0-9-]{32,}/g, + // AWS access key IDs /AKIA[0-9A-Z]{16}/g, - /xoxb-[0-9]+-[A-Za-z0-9]+/g, - /xoxp-[0-9]+-[A-Za-z0-9]+/g, + // Slack tokens (app, bot, refresh, session, user) + /xox[abrsp]-[0-9]+-[A-Za-z0-9]+/g, + // Slack incoming webhooks + /https:\/\/hooks\.slack\.com\/services\/T[A-Za-z0-9]+\/B[A-Za-z0-9]+\/[A-Za-z0-9]+/g, + // npm access tokens + /npm_[A-Za-z0-9]{36,}/g, + // Google API keys + /AIza[A-Za-z0-9_-]{35}/g, + // Azure AD client secrets (v2 format: 3 chars + 7Q~ or 8Q~ + 31 chars) + /[\w~.-]{3}[78]Q~[\w~.-]{31}/g, + // JWTs /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]*/g, + // PEM private keys /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g, + // Basic auth in URIs (e.g. https://user:pass@host) + /\w+:\/\/[^/\s:]+:[^/\s@]+@[^/\s]+/g, + // Bearer tokens /Bearer\s+[A-Za-z0-9\-._~+/]{20,}=*/g, ]; @@ -72,12 +87,11 @@ export function isSensitiveElement(el: Element, doc: Document): boolean { return true; } - const ariaLabel = el.getAttribute('aria-label') || ''; - if ( - SENSITIVE_LABEL_PATTERN.test(ariaLabel) && - !SAFE_LABEL_EXCLUSIONS.test(ariaLabel) - ) { - return true; + for (const attr of ['aria-label', 'name', 'title', 'placeholder']) { + const val = el.getAttribute(attr) || ''; + if (SENSITIVE_LABEL_PATTERN.test(val) && !SAFE_LABEL_EXCLUSIONS.test(val)) { + return true; + } } const id = el.getAttribute('id'); @@ -106,13 +120,19 @@ export function isSensitiveElement(el: Element, doc: Document): boolean { */ export function sanitizeClonedDom(clonedDoc: Document): void { // 1. Mask all sensitive inputs/textareas (password fields + label-detected) + // 2. Regex-scan all remaining input/textarea values for token patterns clonedDoc.querySelectorAll('input, textarea').forEach(el => { if (isSensitiveElement(el, clonedDoc)) { (el as HTMLInputElement).value = MASK; + } else { + const inp = el as HTMLInputElement; + if (inp.value) { + inp.value = redactText(inp.value); + } } }); - // 2. Regex scan all text nodes for token patterns + // 3. Regex scan all text nodes for token patterns const walker = clonedDoc.createTreeWalker( clonedDoc.body, NodeFilter.SHOW_TEXT, @@ -128,7 +148,7 @@ export function sanitizeClonedDom(clonedDoc: Document): void { node = walker.nextNode(); } - // 3. Handle visibility-toggle secrets (e.g. Akeyless show/hide pattern) + // 4. Handle visibility-toggle secrets (e.g. Akeyless show/hide pattern) clonedDoc .querySelectorAll( '[aria-label*="Hide secret" i], [aria-label*="Hide value" i]', From e80882177fe703dc87141f8d9430b8b538ed1208 Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Fri, 14 Aug 2026 01:06:03 +0530 Subject: [PATCH 6/9] reduce default capture timeout from 3s to 1s Co-authored-by: Cursor --- .../plugins/intelligent-assistant/src/utils/screen-capture.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts index 9aaad771946..07626b557b7 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/screen-capture.ts @@ -18,7 +18,7 @@ import { sanitizeClonedDom } from './sensitive-data-redactor'; const DEFAULT_QUALITY = 0.7; const DEFAULT_MAX_WIDTH = 1280; -const DEFAULT_TIMEOUT_MS = 3000; +const DEFAULT_TIMEOUT_MS = 1000; const DEFAULT_EXCLUDE_SELECTOR = '[data-screen-capture-exclude]'; export interface CaptureOptions { From c56f822b846054b187df04fd941fccbb0bf46f3f Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Fri, 14 Aug 2026 04:22:32 +0530 Subject: [PATCH 7/9] feat(intelligent-assistant): add DOM text extraction for deep context awareness Implements RHIDP-14317 - extracts structured page content (headings, tables, alerts, body text) from the visible RHDH page and sends it as a text/plain attachment alongside user messages when screen context is enabled. Key decisions over the PoC (innerHTML) approach: - No token waste: strips HTML tags/attributes that add zero semantic value - Priority-based extraction: alerts > headings > tables > body text - Configurable char budget (default 8000) to cap token usage - Zero backend changes: uses existing attachment pipeline Co-authored-by: Cursor --- .../plugins/intelligent-assistant/config.d.ts | 21 ++ .../src/components/LightSpeedChat.tsx | 31 +- .../src/utils/__tests__/dom-extractor.test.ts | 354 ++++++++++++++++++ .../src/utils/dom-extractor.ts | 272 ++++++++++++++ 4 files changed, 677 insertions(+), 1 deletion(-) create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts create mode 100644 workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts index e0db69cf425..8fef1bfd691 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts @@ -94,6 +94,27 @@ export interface Config { */ enabled?: boolean; }; + /** + * DOM text extraction configuration + * @visibility frontend + */ + 'dom-extraction'?: { + /** + * Enable/disable DOM text extraction within Screen Context. + * When enabled, extracts structured page content (headings, tables, alerts, text) + * and sends it as context alongside the user's message. + * @default true + * @visibility frontend + */ + enabled?: boolean; + /** + * Maximum characters to extract from the page. + * Higher values provide richer context but consume more LLM tokens. + * @default 8000 + * @visibility frontend + */ + maxChars?: number; + }; }; }; } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx index 95ec0df97dd..fb73101eec6 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx @@ -111,6 +111,7 @@ import { useTranslation } from '../hooks/useTranslation'; import { useWelcomePrompts } from '../hooks/useWelcomePrompts'; import { ConversationSummary, NotebookSession } from '../types'; import { getAttachments } from '../utils/attachment-utils'; +import { extractPageContext } from '../utils/dom-extractor'; import { ChatbotFootnoteWithIcon, getCategorizeMessages, @@ -660,6 +661,19 @@ export const LightspeedChat = ({ const notebooksEnabled = configApi.getOptionalBoolean('intelligent-assistant.notebooks.enabled') ?? false; + const screenContextEnabled = + configApi.getOptionalBoolean( + 'intelligent-assistant.screen-context.enabled', + ) ?? false; + const domExtractionEnabled = + configApi.getOptionalBoolean( + 'intelligent-assistant.screen-context.dom-extraction.enabled', + ) ?? true; + const domExtractionMaxChars = + configApi.getOptionalNumber( + 'intelligent-assistant.screen-context.dom-extraction.maxChars', + ) ?? 8000; + const notebooksRouteMatch = useMatch(`${LIGHTSPEED_PATH}/notebooks`); const notebookViewRouteMatch = useMatch( `${LIGHTSPEED_PATH}/notebooks/:notebookId`, @@ -1136,7 +1150,22 @@ export const LightspeedChat = ({ prompt: message.toString(), }), ); - handleInputPrompt(message.toString(), getAttachments(fileContents)); + const allAttachments = getAttachments(fileContents); + + if (screenContextEnabled && domExtractionEnabled) { + const domContext = extractPageContext({ + maxChars: domExtractionMaxChars, + }); + if (domContext) { + allAttachments.push({ + attachment_type: 'configuration', + content_type: 'text/plain', + content: domContext, + }); + } + } + + handleInputPrompt(message.toString(), allAttachments); setIsSendButtonDisabled(true); setFileContents([]); setDraftMessage(''); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts new file mode 100644 index 00000000000..a8986dc441d --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts @@ -0,0 +1,354 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { extractPageContext } from '../dom-extractor'; +import { redactText } from '../sensitive-data-redactor'; + +jest.mock('../sensitive-data-redactor', () => ({ + redactText: jest.fn(text => text), +})); + +const mockRedactText = redactText as jest.MockedFunction; + +function setRootHtml(html: string) { + const root = document.getElementById('root'); + if (root) { + root.innerHTML = html; + } else { + const el = document.createElement('div'); + el.id = 'root'; + el.innerHTML = html; + document.body.appendChild(el); + } +} + +describe('extractPageContext', () => { + beforeEach(() => { + jest.clearAllMocks(); + document.body.innerHTML = ''; + }); + + it('should return empty string when #root is not present', () => { + const result = extractPageContext(); + expect(result).toBe(''); + }); + + it('should include page header with pathname and document title', () => { + setRootHtml('

Hello world

'); + const result = extractPageContext(); + + expect(result).toContain('Page:'); + expect(result).toContain('Document:'); + }); + + it('should extract alerts', () => { + setRootHtml(` +
Pipeline failed: build-123
+

Other content

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Alerts'); + expect(result).toContain('- Pipeline failed: build-123'); + }); + + it('should extract headings with hierarchy', () => { + setRootHtml(` +

Overview

+

Dependencies

+

Runtime

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Headings'); + expect(result).toContain('- Overview'); + expect(result).toContain(' - Dependencies'); + expect(result).toContain(' - Runtime'); + }); + + it('should extract tables in pipe-delimited format', () => { + setRootHtml(` + + + + +
NameStatus
my-serviceRunning
my-dbHealthy
+ `); + const result = extractPageContext(); + + expect(result).toContain('## Tables'); + expect(result).toContain('| Name | Status |'); + expect(result).toContain('| my-service | Running |'); + expect(result).toContain('| my-db | Healthy |'); + }); + + it('should extract body text via TreeWalker', () => { + setRootHtml(` +
+ Component: my-service + Owner: team-platform +
+ `); + const result = extractPageContext(); + + expect(result).toContain('## Content'); + expect(result).toContain('Component: my-service'); + expect(result).toContain('Owner: team-platform'); + }); + + it('should remove noise elements before extraction', () => { + setRootHtml(` +
Chat UI content
+ + + SVG text +

Visible content

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Chat UI content'); + expect(result).not.toContain('var x = 1'); + expect(result).not.toContain('color: red'); + expect(result).not.toContain('SVG text'); + expect(result).toContain('Visible content'); + }); + + it('should remove elements matching custom excludeSelector', () => { + setRootHtml(` +
Sidebar nav
+

Main content

+ `); + const result = extractPageContext({ excludeSelector: '.sidebar' }); + + expect(result).not.toContain('Sidebar nav'); + expect(result).toContain('Main content'); + }); + + it('should respect maxChars budget and truncate', () => { + const longContent = 'A'.repeat(500); + setRootHtml(` +

Title

+

${longContent}

+ `); + const result = extractPageContext({ maxChars: 200 }); + + expect(result.length).toBeLessThanOrEqual(200); + }); + + it('should limit table rows to MAX_TABLE_ROWS (50)', () => { + const rows = Array.from( + { length: 100 }, + (_, i) => `Row ${i}`, + ).join(''); + setRootHtml(`${rows}
`); + const result = extractPageContext(); + + expect(result).toContain('| Row 0 |'); + expect(result).toContain('| Row 49 |'); + expect(result).not.toContain('| Row 50 |'); + }); + + it('should limit tables to MAX_TABLES (10)', () => { + const tables = Array.from( + { length: 15 }, + (_, i) => `
Table ${i}
`, + ).join(''); + setRootHtml(tables); + const result = extractPageContext(); + + expect(result).toContain('| Table 0 |'); + expect(result).toContain('| Table 9 |'); + expect(result).not.toContain('| Table 10 |'); + }); + + it('should call redactText on the final output', () => { + mockRedactText.mockImplementation(text => + text.replace(/secret-token/g, '[REDACTED]'), + ); + setRootHtml('

My secret-token is here

'); + const result = extractPageContext(); + + expect(mockRedactText).toHaveBeenCalled(); + expect(result).toContain('[REDACTED]'); + expect(result).not.toContain('secret-token'); + }); + + it('should deduplicate identical text nodes', () => { + setRootHtml(` + Repeated text + Repeated text + Unique text + `); + const result = extractPageContext(); + + const matches = result.match(/Repeated text/g) || []; + expect(matches.length).toBe(1); + expect(result).toContain('Unique text'); + }); + + it('should collapse whitespace in extracted text', () => { + setRootHtml(` +

Multiple spaces and + newlines

+ `); + const result = extractPageContext(); + + expect(result).toContain('Multiple spaces and newlines'); + }); + + it('should remove aria-hidden elements', () => { + setRootHtml(` + +

Visible to screen reader

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Hidden decorative content'); + expect(result).toContain('Visible to screen reader'); + }); + + it('should handle empty #root gracefully', () => { + setRootHtml(''); + const result = extractPageContext(); + + expect(result).toContain('Page:'); + }); + + it('should remove alerts from clone after extraction to avoid duplication in body', () => { + setRootHtml(` +
Error message
+

Regular content

+ `); + const result = extractPageContext(); + + const alertMatches = result.match(/Error message/g) || []; + expect(alertMatches.length).toBe(1); + }); + + it('should remove headings from clone after extraction to avoid duplication in body', () => { + setRootHtml(` +

Page Title

+

Content below title

+ `); + const result = extractPageContext(); + + const titleMatches = result.match(/Page Title/g) || []; + expect(titleMatches.length).toBe(1); + }); + + it('should extract plugin name from BUI header toolbar', () => { + setRootHtml(` +

Settings

+

Page content

+ `); + const result = extractPageContext(); + + expect(result).toContain('Plugin: Settings'); + }); + + it('should extract page title from BUI header title', () => { + setRootHtml(` +

Red Hat Catalog

+

Catalog items

+ `); + const result = extractPageContext(); + + expect(result).toContain('Title: Red Hat Catalog'); + }); + + it('should extract active tab from aria-selected tab', () => { + setRootHtml(` +
+ Tree + Text + Detailed +
+

Tab content

+ `); + const result = extractPageContext(); + + expect(result).toContain('Tab: Text'); + }); + + it('should not include Plugin/Title/Tab lines when elements are absent', () => { + setRootHtml('

Simple page

'); + const result = extractPageContext(); + + expect(result).not.toContain('Plugin:'); + expect(result).not.toContain('Title:'); + expect(result).not.toContain('Tab:'); + }); + + it('should extract MuiAlert-root elements as alerts', () => { + setRootHtml(` +
Deployment warning: resource quota exceeded
+

Other content

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Alerts'); + expect(result).toContain('- Deployment warning: resource quota exceeded'); + }); + + it('should remove tables from clone to avoid duplication in body', () => { + setRootHtml(` +
Table data
+

Paragraph content

+ `); + const result = extractPageContext(); + + const tableDataMatches = result.match(/Table data/g) || []; + expect(tableDataMatches.length).toBe(1); + }); + + it('should remove data-screen-capture-exclude elements', () => { + setRootHtml(` +
Excluded widget
+

Included content

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Excluded widget'); + expect(result).toContain('Included content'); + }); + + it('should extract all sections together in priority order', () => { + setRootHtml(` +

Catalog

+

Red Hat Catalog

+
+ Overview +
+
Warning: 2 pipelines failing
+

Components

+
NameOwner
my-svcteam-a
+

Description of the catalog page

+ `); + const result = extractPageContext(); + + expect(result).toContain('Plugin: Catalog'); + expect(result).toContain('Title: Red Hat Catalog'); + expect(result).toContain('Tab: Overview'); + expect(result).toContain('## Alerts'); + expect(result).toContain('Warning: 2 pipelines failing'); + expect(result).toContain('## Headings'); + expect(result).toContain('Components'); + expect(result).toContain('## Tables'); + expect(result).toContain('| my-svc | team-a |'); + expect(result).toContain('## Content'); + expect(result).toContain('Description of the catalog page'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts new file mode 100644 index 00000000000..f61f01c7fa4 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts @@ -0,0 +1,272 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { redactText } from './sensitive-data-redactor'; + +const DEFAULT_MAX_CHARS = 8000; +const MAX_TABLE_ROWS = 50; +const MAX_TABLES = 10; + +const NOISE_SELECTOR = [ + '.pf-chatbot', + 'script', + 'style', + 'noscript', + 'svg', + 'canvas', + 'img', + 'video', + 'iframe', + '[aria-hidden="true"]', + '[data-screen-capture-exclude]', +].join(', '); + +export interface DomExtractionOptions { + /** Maximum characters in the output (default: 8000) */ + maxChars?: number; + /** Additional CSS selector for elements to exclude */ + excludeSelector?: string; +} + +/** + * Extracts structured page context from the current RHDH viewport. + * Returns a plain text string suitable for LLM consumption as an attachment. + * + * Extraction priority: header > alerts > headings > tables > body text. + * Stops appending when maxChars budget is reached. + */ +export function extractPageContext(options?: DomExtractionOptions): string { + const maxChars = options?.maxChars ?? DEFAULT_MAX_CHARS; + + const root = document.querySelector('#root'); + if (!root) { + return ''; + } + + const clone = root.cloneNode(true) as HTMLElement; + + // Remove noise elements + clone.querySelectorAll(NOISE_SELECTOR).forEach(el => el.remove()); + if (options?.excludeSelector) { + clone.querySelectorAll(options.excludeSelector).forEach(el => el.remove()); + } + + const sections: string[] = []; + let charCount = 0; + + const appendSection = (content: string): boolean => { + if (!content.trim()) return true; + if (charCount + content.length > maxChars) { + const remaining = maxChars - charCount; + if (remaining > 50) { + sections.push(content.slice(0, remaining)); + charCount = maxChars; + } + return false; + } + sections.push(content); + charCount += content.length; + return true; + }; + + // 1. Header -- always included + const headerLines = [`Page: ${window.location.pathname}`]; + + const pluginName = document + .querySelector('.bui-PluginHeaderToolbarName') + ?.textContent?.trim(); + if (pluginName) { + headerLines.push(`Plugin: ${pluginName}`); + } + + const pageTitle = document + .querySelector('.bui-HeaderTitle') + ?.textContent?.trim(); + if (pageTitle) { + headerLines.push(`Title: ${pageTitle}`); + } + + const activeTab = document + .querySelector('[role="tab"][aria-selected="true"]') + ?.textContent?.trim(); + if (activeTab) { + headerLines.push(`Tab: ${activeTab}`); + } + + headerLines.push(`Document: ${document.title}`); + + const header = headerLines.join('\n'); + if (!appendSection(header)) { + return redactText(sections.join('\n\n')); + } + + // 2. Alerts / Toasts + const alertText = extractAlerts(clone); + if (alertText && !appendSection(alertText)) { + return redactText(sections.join('\n\n')); + } + + // 3. Headings + const headingsText = extractHeadings(clone); + if (headingsText && !appendSection(headingsText)) { + return redactText(sections.join('\n\n')); + } + + // 4. Tables + const tablesText = extractTables(clone); + if (tablesText && !appendSection(tablesText)) { + return redactText(sections.join('\n\n')); + } + + // 5. Body text (TreeWalker) -- fills remaining budget + const bodyText = extractBodyText(clone, maxChars - charCount); + if (bodyText) { + appendSection(bodyText); + } + + // 6. TechDocs Shadow DOM (if present on the live DOM) + const shadowText = extractShadowDomContent(); + if (shadowText) { + appendSection(shadowText); + } + + const result = sections.join('\n\n'); + return result ? redactText(result) : ''; +} + +function extractAlerts(root: HTMLElement): string { + const alerts = root.querySelectorAll( + '[role="alert"], [aria-live="assertive"], [class*="MuiAlert-root"]', + ); + if (alerts.length === 0) return ''; + + const lines: string[] = ['## Alerts']; + alerts.forEach(alert => { + const text = collapseWhitespace(alert.textContent || ''); + if (text) { + lines.push(`- ${text}`); + } + alert.remove(); + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractHeadings(root: HTMLElement): string { + const headings = root.querySelectorAll('h1, h2, h3, h4'); + if (headings.length === 0) return ''; + + const lines: string[] = ['## Headings']; + headings.forEach(heading => { + const level = parseInt(heading.tagName[1], 10); + const indent = ' '.repeat(level - 1); + const text = collapseWhitespace(heading.textContent || ''); + if (text) { + lines.push(`${indent}- ${text}`); + } + heading.remove(); + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractTables(root: HTMLElement): string { + const tables = root.querySelectorAll('table'); + if (tables.length === 0) return ''; + + const output: string[] = ['## Tables']; + let tableCount = 0; + + tables.forEach(table => { + if (tableCount >= MAX_TABLES) { + table.remove(); + return; + } + + const rows = table.querySelectorAll('tr'); + const tableLines: string[] = []; + let rowCount = 0; + + rows.forEach(row => { + if (rowCount >= MAX_TABLE_ROWS) return; + const cells = row.querySelectorAll('th, td'); + const cellTexts = Array.from(cells).map(cell => + collapseWhitespace(cell.textContent || ''), + ); + if (cellTexts.some(t => t)) { + tableLines.push(`| ${cellTexts.join(' | ')} |`); + } + rowCount++; + }); + + if (tableLines.length > 0) { + output.push(tableLines.join('\n')); + } + + table.remove(); + tableCount++; + }); + + return output.length > 1 ? output.join('\n') : ''; +} + +function extractBodyText(root: HTMLElement, budget: number): string { + if (budget <= 0) return ''; + + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT); + const lines: string[] = ['## Content']; + let length = lines[0].length; + const seen = new Set(); + + let node = walker.nextNode(); + while (node) { + const text = collapseWhitespace(node.textContent || ''); + if (text && text.length > 1 && !seen.has(text)) { + seen.add(text); + const lineLength = text.length + 1; // +1 for newline + if (length + lineLength > budget) { + break; + } + lines.push(text); + length += lineLength; + } + node = walker.nextNode(); + } + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractShadowDomContent(): string { + const shadowHosts = document.querySelectorAll('[class*="shadowDom" i]'); + if (shadowHosts.length === 0) return ''; + + const lines: string[] = ['## Documentation']; + + shadowHosts.forEach(host => { + if (host.shadowRoot) { + const text = collapseWhitespace(host.shadowRoot.textContent || ''); + if (text) { + lines.push(text.slice(0, 2000)); + } + } + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} From 470eb9d4db7a9c241fdc40dc9c6fcca2ee7da71c Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Fri, 14 Aug 2026 04:25:49 +0530 Subject: [PATCH 8/9] enable screen-context Signed-off-by: its-mitesh-kumar --- workspaces/intelligent-assistant/app-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workspaces/intelligent-assistant/app-config.yaml b/workspaces/intelligent-assistant/app-config.yaml index a1fbee7d43b..2ae34eea2a4 100644 --- a/workspaces/intelligent-assistant/app-config.yaml +++ b/workspaces/intelligent-assistant/app-config.yaml @@ -37,6 +37,8 @@ intelligent-assistant: # auth: dcr # - name: test-mcp-server # token: ${MCP_TOKEN_1} + screen-context: + enabled: true notebooks: enabled: ${NOTEBOOKS_ENABLED:-false} queryDefaults: From d64df67466fb9c7056ac75083352d552b6dea348 Mon Sep 17 00:00:00 2001 From: its-mitesh-kumar Date: Fri, 14 Aug 2026 18:32:09 +0530 Subject: [PATCH 9/9] chore: add changeset for DOM text extraction feature Co-authored-by: Cursor --- .../intelligent-assistant/.changeset/dom-text-extraction.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 workspaces/intelligent-assistant/.changeset/dom-text-extraction.md diff --git a/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md b/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md new file mode 100644 index 00000000000..bf8d2922f1c --- /dev/null +++ b/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor +--- + +Add DOM text extraction for deep context awareness — extracts structured page content (headings, tables, alerts, text) and sends as context alongside user messages