diff --git a/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md b/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md new file mode 100644 index 00000000000..bf8d2922f1c --- /dev/null +++ b/workspaces/intelligent-assistant/.changeset/dom-text-extraction.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-intelligent-assistant': minor +--- + +Add DOM text extraction for deep context awareness — extracts structured page content (headings, tables, alerts, text) and sends as context alongside user messages diff --git a/workspaces/intelligent-assistant/app-config.yaml b/workspaces/intelligent-assistant/app-config.yaml index a1fbee7d43b..2ae34eea2a4 100644 --- a/workspaces/intelligent-assistant/app-config.yaml +++ b/workspaces/intelligent-assistant/app-config.yaml @@ -37,6 +37,8 @@ intelligent-assistant: # auth: dcr # - name: test-mcp-server # token: ${MCP_TOKEN_1} + screen-context: + enabled: true notebooks: enabled: ${NOTEBOOKS_ENABLED:-false} queryDefaults: diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts index e0db69cf425..8fef1bfd691 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/config.d.ts @@ -94,6 +94,27 @@ export interface Config { */ enabled?: boolean; }; + /** + * DOM text extraction configuration + * @visibility frontend + */ + 'dom-extraction'?: { + /** + * Enable/disable DOM text extraction within Screen Context. + * When enabled, extracts structured page content (headings, tables, alerts, text) + * and sends it as context alongside the user's message. + * @default true + * @visibility frontend + */ + enabled?: boolean; + /** + * Maximum characters to extract from the page. + * Higher values provide richer context but consume more LLM tokens. + * @default 8000 + * @visibility frontend + */ + maxChars?: number; + }; }; }; } diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx index 95ec0df97dd..fb73101eec6 100644 --- a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/components/LightSpeedChat.tsx @@ -111,6 +111,7 @@ import { useTranslation } from '../hooks/useTranslation'; import { useWelcomePrompts } from '../hooks/useWelcomePrompts'; import { ConversationSummary, NotebookSession } from '../types'; import { getAttachments } from '../utils/attachment-utils'; +import { extractPageContext } from '../utils/dom-extractor'; import { ChatbotFootnoteWithIcon, getCategorizeMessages, @@ -660,6 +661,19 @@ export const LightspeedChat = ({ const notebooksEnabled = configApi.getOptionalBoolean('intelligent-assistant.notebooks.enabled') ?? false; + const screenContextEnabled = + configApi.getOptionalBoolean( + 'intelligent-assistant.screen-context.enabled', + ) ?? false; + const domExtractionEnabled = + configApi.getOptionalBoolean( + 'intelligent-assistant.screen-context.dom-extraction.enabled', + ) ?? true; + const domExtractionMaxChars = + configApi.getOptionalNumber( + 'intelligent-assistant.screen-context.dom-extraction.maxChars', + ) ?? 8000; + const notebooksRouteMatch = useMatch(`${LIGHTSPEED_PATH}/notebooks`); const notebookViewRouteMatch = useMatch( `${LIGHTSPEED_PATH}/notebooks/:notebookId`, @@ -1136,7 +1150,22 @@ export const LightspeedChat = ({ prompt: message.toString(), }), ); - handleInputPrompt(message.toString(), getAttachments(fileContents)); + const allAttachments = getAttachments(fileContents); + + if (screenContextEnabled && domExtractionEnabled) { + const domContext = extractPageContext({ + maxChars: domExtractionMaxChars, + }); + if (domContext) { + allAttachments.push({ + attachment_type: 'configuration', + content_type: 'text/plain', + content: domContext, + }); + } + } + + handleInputPrompt(message.toString(), allAttachments); setIsSendButtonDisabled(true); setFileContents([]); setDraftMessage(''); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts new file mode 100644 index 00000000000..a8986dc441d --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/__tests__/dom-extractor.test.ts @@ -0,0 +1,354 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { extractPageContext } from '../dom-extractor'; +import { redactText } from '../sensitive-data-redactor'; + +jest.mock('../sensitive-data-redactor', () => ({ + redactText: jest.fn(text => text), +})); + +const mockRedactText = redactText as jest.MockedFunction; + +function setRootHtml(html: string) { + const root = document.getElementById('root'); + if (root) { + root.innerHTML = html; + } else { + const el = document.createElement('div'); + el.id = 'root'; + el.innerHTML = html; + document.body.appendChild(el); + } +} + +describe('extractPageContext', () => { + beforeEach(() => { + jest.clearAllMocks(); + document.body.innerHTML = ''; + }); + + it('should return empty string when #root is not present', () => { + const result = extractPageContext(); + expect(result).toBe(''); + }); + + it('should include page header with pathname and document title', () => { + setRootHtml('

Hello world

'); + const result = extractPageContext(); + + expect(result).toContain('Page:'); + expect(result).toContain('Document:'); + }); + + it('should extract alerts', () => { + setRootHtml(` +
Pipeline failed: build-123
+

Other content

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Alerts'); + expect(result).toContain('- Pipeline failed: build-123'); + }); + + it('should extract headings with hierarchy', () => { + setRootHtml(` +

Overview

+

Dependencies

+

Runtime

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Headings'); + expect(result).toContain('- Overview'); + expect(result).toContain(' - Dependencies'); + expect(result).toContain(' - Runtime'); + }); + + it('should extract tables in pipe-delimited format', () => { + setRootHtml(` + + + + +
NameStatus
my-serviceRunning
my-dbHealthy
+ `); + const result = extractPageContext(); + + expect(result).toContain('## Tables'); + expect(result).toContain('| Name | Status |'); + expect(result).toContain('| my-service | Running |'); + expect(result).toContain('| my-db | Healthy |'); + }); + + it('should extract body text via TreeWalker', () => { + setRootHtml(` +
+ Component: my-service + Owner: team-platform +
+ `); + const result = extractPageContext(); + + expect(result).toContain('## Content'); + expect(result).toContain('Component: my-service'); + expect(result).toContain('Owner: team-platform'); + }); + + it('should remove noise elements before extraction', () => { + setRootHtml(` +
Chat UI content
+ + + SVG text +

Visible content

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Chat UI content'); + expect(result).not.toContain('var x = 1'); + expect(result).not.toContain('color: red'); + expect(result).not.toContain('SVG text'); + expect(result).toContain('Visible content'); + }); + + it('should remove elements matching custom excludeSelector', () => { + setRootHtml(` + +

Main content

+ `); + const result = extractPageContext({ excludeSelector: '.sidebar' }); + + expect(result).not.toContain('Sidebar nav'); + expect(result).toContain('Main content'); + }); + + it('should respect maxChars budget and truncate', () => { + const longContent = 'A'.repeat(500); + setRootHtml(` +

Title

+

${longContent}

+ `); + const result = extractPageContext({ maxChars: 200 }); + + expect(result.length).toBeLessThanOrEqual(200); + }); + + it('should limit table rows to MAX_TABLE_ROWS (50)', () => { + const rows = Array.from( + { length: 100 }, + (_, i) => `Row ${i}`, + ).join(''); + setRootHtml(`${rows}
`); + const result = extractPageContext(); + + expect(result).toContain('| Row 0 |'); + expect(result).toContain('| Row 49 |'); + expect(result).not.toContain('| Row 50 |'); + }); + + it('should limit tables to MAX_TABLES (10)', () => { + const tables = Array.from( + { length: 15 }, + (_, i) => `
Table ${i}
`, + ).join(''); + setRootHtml(tables); + const result = extractPageContext(); + + expect(result).toContain('| Table 0 |'); + expect(result).toContain('| Table 9 |'); + expect(result).not.toContain('| Table 10 |'); + }); + + it('should call redactText on the final output', () => { + mockRedactText.mockImplementation(text => + text.replace(/secret-token/g, '[REDACTED]'), + ); + setRootHtml('

My secret-token is here

'); + const result = extractPageContext(); + + expect(mockRedactText).toHaveBeenCalled(); + expect(result).toContain('[REDACTED]'); + expect(result).not.toContain('secret-token'); + }); + + it('should deduplicate identical text nodes', () => { + setRootHtml(` + Repeated text + Repeated text + Unique text + `); + const result = extractPageContext(); + + const matches = result.match(/Repeated text/g) || []; + expect(matches.length).toBe(1); + expect(result).toContain('Unique text'); + }); + + it('should collapse whitespace in extracted text', () => { + setRootHtml(` +

Multiple spaces and + newlines

+ `); + const result = extractPageContext(); + + expect(result).toContain('Multiple spaces and newlines'); + }); + + it('should remove aria-hidden elements', () => { + setRootHtml(` + +

Visible to screen reader

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Hidden decorative content'); + expect(result).toContain('Visible to screen reader'); + }); + + it('should handle empty #root gracefully', () => { + setRootHtml(''); + const result = extractPageContext(); + + expect(result).toContain('Page:'); + }); + + it('should remove alerts from clone after extraction to avoid duplication in body', () => { + setRootHtml(` +
Error message
+

Regular content

+ `); + const result = extractPageContext(); + + const alertMatches = result.match(/Error message/g) || []; + expect(alertMatches.length).toBe(1); + }); + + it('should remove headings from clone after extraction to avoid duplication in body', () => { + setRootHtml(` +

Page Title

+

Content below title

+ `); + const result = extractPageContext(); + + const titleMatches = result.match(/Page Title/g) || []; + expect(titleMatches.length).toBe(1); + }); + + it('should extract plugin name from BUI header toolbar', () => { + setRootHtml(` +

Settings

+

Page content

+ `); + const result = extractPageContext(); + + expect(result).toContain('Plugin: Settings'); + }); + + it('should extract page title from BUI header title', () => { + setRootHtml(` +

Red Hat Catalog

+

Catalog items

+ `); + const result = extractPageContext(); + + expect(result).toContain('Title: Red Hat Catalog'); + }); + + it('should extract active tab from aria-selected tab', () => { + setRootHtml(` +
+ Tree + Text + Detailed +
+

Tab content

+ `); + const result = extractPageContext(); + + expect(result).toContain('Tab: Text'); + }); + + it('should not include Plugin/Title/Tab lines when elements are absent', () => { + setRootHtml('

Simple page

'); + const result = extractPageContext(); + + expect(result).not.toContain('Plugin:'); + expect(result).not.toContain('Title:'); + expect(result).not.toContain('Tab:'); + }); + + it('should extract MuiAlert-root elements as alerts', () => { + setRootHtml(` +
Deployment warning: resource quota exceeded
+

Other content

+ `); + const result = extractPageContext(); + + expect(result).toContain('## Alerts'); + expect(result).toContain('- Deployment warning: resource quota exceeded'); + }); + + it('should remove tables from clone to avoid duplication in body', () => { + setRootHtml(` +
Table data
+

Paragraph content

+ `); + const result = extractPageContext(); + + const tableDataMatches = result.match(/Table data/g) || []; + expect(tableDataMatches.length).toBe(1); + }); + + it('should remove data-screen-capture-exclude elements', () => { + setRootHtml(` +
Excluded widget
+

Included content

+ `); + const result = extractPageContext(); + + expect(result).not.toContain('Excluded widget'); + expect(result).toContain('Included content'); + }); + + it('should extract all sections together in priority order', () => { + setRootHtml(` +

Catalog

+

Red Hat Catalog

+
+ Overview +
+
Warning: 2 pipelines failing
+

Components

+
NameOwner
my-svcteam-a
+

Description of the catalog page

+ `); + const result = extractPageContext(); + + expect(result).toContain('Plugin: Catalog'); + expect(result).toContain('Title: Red Hat Catalog'); + expect(result).toContain('Tab: Overview'); + expect(result).toContain('## Alerts'); + expect(result).toContain('Warning: 2 pipelines failing'); + expect(result).toContain('## Headings'); + expect(result).toContain('Components'); + expect(result).toContain('## Tables'); + expect(result).toContain('| my-svc | team-a |'); + expect(result).toContain('## Content'); + expect(result).toContain('Description of the catalog page'); + }); +}); diff --git a/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts new file mode 100644 index 00000000000..f61f01c7fa4 --- /dev/null +++ b/workspaces/intelligent-assistant/plugins/intelligent-assistant/src/utils/dom-extractor.ts @@ -0,0 +1,272 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { redactText } from './sensitive-data-redactor'; + +const DEFAULT_MAX_CHARS = 8000; +const MAX_TABLE_ROWS = 50; +const MAX_TABLES = 10; + +const NOISE_SELECTOR = [ + '.pf-chatbot', + 'script', + 'style', + 'noscript', + 'svg', + 'canvas', + 'img', + 'video', + 'iframe', + '[aria-hidden="true"]', + '[data-screen-capture-exclude]', +].join(', '); + +export interface DomExtractionOptions { + /** Maximum characters in the output (default: 8000) */ + maxChars?: number; + /** Additional CSS selector for elements to exclude */ + excludeSelector?: string; +} + +/** + * Extracts structured page context from the current RHDH viewport. + * Returns a plain text string suitable for LLM consumption as an attachment. + * + * Extraction priority: header > alerts > headings > tables > body text. + * Stops appending when maxChars budget is reached. + */ +export function extractPageContext(options?: DomExtractionOptions): string { + const maxChars = options?.maxChars ?? DEFAULT_MAX_CHARS; + + const root = document.querySelector('#root'); + if (!root) { + return ''; + } + + const clone = root.cloneNode(true) as HTMLElement; + + // Remove noise elements + clone.querySelectorAll(NOISE_SELECTOR).forEach(el => el.remove()); + if (options?.excludeSelector) { + clone.querySelectorAll(options.excludeSelector).forEach(el => el.remove()); + } + + const sections: string[] = []; + let charCount = 0; + + const appendSection = (content: string): boolean => { + if (!content.trim()) return true; + if (charCount + content.length > maxChars) { + const remaining = maxChars - charCount; + if (remaining > 50) { + sections.push(content.slice(0, remaining)); + charCount = maxChars; + } + return false; + } + sections.push(content); + charCount += content.length; + return true; + }; + + // 1. Header -- always included + const headerLines = [`Page: ${window.location.pathname}`]; + + const pluginName = document + .querySelector('.bui-PluginHeaderToolbarName') + ?.textContent?.trim(); + if (pluginName) { + headerLines.push(`Plugin: ${pluginName}`); + } + + const pageTitle = document + .querySelector('.bui-HeaderTitle') + ?.textContent?.trim(); + if (pageTitle) { + headerLines.push(`Title: ${pageTitle}`); + } + + const activeTab = document + .querySelector('[role="tab"][aria-selected="true"]') + ?.textContent?.trim(); + if (activeTab) { + headerLines.push(`Tab: ${activeTab}`); + } + + headerLines.push(`Document: ${document.title}`); + + const header = headerLines.join('\n'); + if (!appendSection(header)) { + return redactText(sections.join('\n\n')); + } + + // 2. Alerts / Toasts + const alertText = extractAlerts(clone); + if (alertText && !appendSection(alertText)) { + return redactText(sections.join('\n\n')); + } + + // 3. Headings + const headingsText = extractHeadings(clone); + if (headingsText && !appendSection(headingsText)) { + return redactText(sections.join('\n\n')); + } + + // 4. Tables + const tablesText = extractTables(clone); + if (tablesText && !appendSection(tablesText)) { + return redactText(sections.join('\n\n')); + } + + // 5. Body text (TreeWalker) -- fills remaining budget + const bodyText = extractBodyText(clone, maxChars - charCount); + if (bodyText) { + appendSection(bodyText); + } + + // 6. TechDocs Shadow DOM (if present on the live DOM) + const shadowText = extractShadowDomContent(); + if (shadowText) { + appendSection(shadowText); + } + + const result = sections.join('\n\n'); + return result ? redactText(result) : ''; +} + +function extractAlerts(root: HTMLElement): string { + const alerts = root.querySelectorAll( + '[role="alert"], [aria-live="assertive"], [class*="MuiAlert-root"]', + ); + if (alerts.length === 0) return ''; + + const lines: string[] = ['## Alerts']; + alerts.forEach(alert => { + const text = collapseWhitespace(alert.textContent || ''); + if (text) { + lines.push(`- ${text}`); + } + alert.remove(); + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractHeadings(root: HTMLElement): string { + const headings = root.querySelectorAll('h1, h2, h3, h4'); + if (headings.length === 0) return ''; + + const lines: string[] = ['## Headings']; + headings.forEach(heading => { + const level = parseInt(heading.tagName[1], 10); + const indent = ' '.repeat(level - 1); + const text = collapseWhitespace(heading.textContent || ''); + if (text) { + lines.push(`${indent}- ${text}`); + } + heading.remove(); + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractTables(root: HTMLElement): string { + const tables = root.querySelectorAll('table'); + if (tables.length === 0) return ''; + + const output: string[] = ['## Tables']; + let tableCount = 0; + + tables.forEach(table => { + if (tableCount >= MAX_TABLES) { + table.remove(); + return; + } + + const rows = table.querySelectorAll('tr'); + const tableLines: string[] = []; + let rowCount = 0; + + rows.forEach(row => { + if (rowCount >= MAX_TABLE_ROWS) return; + const cells = row.querySelectorAll('th, td'); + const cellTexts = Array.from(cells).map(cell => + collapseWhitespace(cell.textContent || ''), + ); + if (cellTexts.some(t => t)) { + tableLines.push(`| ${cellTexts.join(' | ')} |`); + } + rowCount++; + }); + + if (tableLines.length > 0) { + output.push(tableLines.join('\n')); + } + + table.remove(); + tableCount++; + }); + + return output.length > 1 ? output.join('\n') : ''; +} + +function extractBodyText(root: HTMLElement, budget: number): string { + if (budget <= 0) return ''; + + const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT); + const lines: string[] = ['## Content']; + let length = lines[0].length; + const seen = new Set(); + + let node = walker.nextNode(); + while (node) { + const text = collapseWhitespace(node.textContent || ''); + if (text && text.length > 1 && !seen.has(text)) { + seen.add(text); + const lineLength = text.length + 1; // +1 for newline + if (length + lineLength > budget) { + break; + } + lines.push(text); + length += lineLength; + } + node = walker.nextNode(); + } + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function extractShadowDomContent(): string { + const shadowHosts = document.querySelectorAll('[class*="shadowDom" i]'); + if (shadowHosts.length === 0) return ''; + + const lines: string[] = ['## Documentation']; + + shadowHosts.forEach(host => { + if (host.shadowRoot) { + const text = collapseWhitespace(host.shadowRoot.textContent || ''); + if (text) { + lines.push(text.slice(0, 2000)); + } + } + }); + + return lines.length > 1 ? lines.join('\n') : ''; +} + +function collapseWhitespace(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +}