From ffc0fc1838228aeb05ac57588a75a08093e007b3 Mon Sep 17 00:00:00 2001 From: 0xbbjoker <0xbbjoker@proton.me> Date: Wed, 15 Oct 2025 14:58:47 +0900 Subject: [PATCH] feat: add KNOWLEDGE_ALLOW_PDF environment variable --- README.md | 5 + package.json | 2 +- src/config.ts | 22 +- src/docs-loader.ts | 16 +- src/frontend/ui/knowledge-tab.tsx | 1040 ++++++++++++++++------------- src/index.ts | 3 +- src/routes.ts | 48 +- src/service.ts | 3 +- 8 files changed, 670 insertions(+), 469 deletions(-) diff --git a/README.md b/README.md index 69d718c..270b7b4 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ The plugin can read almost any document: - **Documents:** `.pdf`, `.doc`, `.docx` - **Code Files:** `.js`, `.ts`, `.py`, `.java`, `.cpp`, `.html`, `.css` and many more +> **Security Note:** PDF support can be disabled by setting `KNOWLEDGE_ALLOW_PDF=false` in your `.env` file if you prefer not to process PDF files. + ## 💬 Using the Web Interface The plugin includes a web interface for managing documents! @@ -137,8 +139,11 @@ OPENROUTER_API_KEY=your-openrouter-api-key ```env LOAD_DOCS_ON_STARTUP=true # Auto-load from docs folder KNOWLEDGE_PATH=/custom/path # Custom document path (default: ./docs) +KNOWLEDGE_ALLOW_PDF=false # Disable PDF support (default: true) ``` +**Note:** To disable PDF uploads for security reasons, set `KNOWLEDGE_ALLOW_PDF=false` in your `.env` file. + ### Embedding Configuration ```env diff --git a/package.json b/package.json index 8eabd4c..5798819 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@elizaos/plugin-knowledge", "description": "Plugin for Knowledge", - "version": "1.5.11", + "version": "1.5.12", "type": "module", "main": "dist/index.js", "module": "dist/index.js", diff --git a/src/config.ts b/src/config.ts index 88442c7..02da018 100644 --- a/src/config.ts +++ b/src/config.ts @@ -2,12 +2,30 @@ import { ModelConfig, ModelConfigSchema, ProviderRateLimits } from './types.ts'; import z from 'zod'; import { logger, IAgentRuntime } from '@elizaos/core'; -const parseBooleanEnv = (value: any): boolean => { +const parseBooleanEnv = (value: any, defaultValue: boolean = false): boolean => { if (typeof value === 'boolean') return value; if (typeof value === 'string') return value.toLowerCase() === 'true'; - return false; // Default to false if undefined or other type + if (value === undefined || value === null) return defaultValue; + return defaultValue; }; +/** + * Checks if PDF support is enabled + * @param runtime The agent runtime to get settings from + * @returns true if PDF support is enabled, false otherwise + */ +export function isPdfAllowed(runtime?: IAgentRuntime): boolean { + const getSetting = (key: string, defaultValue?: string) => { + if (runtime) { + return runtime.getSetting(key) || process.env[key] || defaultValue; + } + return process.env[key] || defaultValue; + }; + + // Default to true (enabled) if not specified + return parseBooleanEnv(getSetting('KNOWLEDGE_ALLOW_PDF'), true); +} + /** * Validates the model configuration using runtime settings * @param runtime The agent runtime to get settings from diff --git a/src/docs-loader.ts b/src/docs-loader.ts index 450421f..7f8c8e3 100644 --- a/src/docs-loader.ts +++ b/src/docs-loader.ts @@ -4,6 +4,7 @@ import * as path from 'path'; import { KnowledgeService } from './service.ts'; import { AddKnowledgeOptions } from './types.ts'; import { isBinaryContentType } from './utils.ts'; +import { isPdfAllowed } from './config.ts'; /** * Get the knowledge path from runtime settings, environment, or default to ./docs @@ -37,7 +38,8 @@ export async function loadDocsFromPath( service: KnowledgeService, agentId: UUID, worldId?: UUID, - knowledgePath?: string + knowledgePath?: string, + runtime?: any ): Promise<{ total: number; successful: number; failed: number }> { const docsPath = getKnowledgePath(knowledgePath); @@ -48,6 +50,12 @@ export async function loadDocsFromPath( logger.info(`Loading documents from: ${docsPath}`); + // Check if PDF support is enabled + const pdfAllowed = isPdfAllowed(runtime); + if (!pdfAllowed) { + logger.warn('PDF support is disabled via KNOWLEDGE_ALLOW_PDF setting'); + } + // Get all files recursively const files = getAllFiles(docsPath); @@ -80,6 +88,12 @@ export async function loadDocsFromPath( continue; } + // Skip PDF files if PDF support is disabled + if (contentType === 'application/pdf' && !pdfAllowed) { + logger.debug(`Skipping PDF file (PDF support disabled): ${filePath}`); + continue; + } + // Read file const fileBuffer = fs.readFileSync(filePath); diff --git a/src/frontend/ui/knowledge-tab.tsx b/src/frontend/ui/knowledge-tab.tsx index e17ce0a..86150aa 100644 --- a/src/frontend/ui/knowledge-tab.tsx +++ b/src/frontend/ui/knowledge-tab.tsx @@ -42,21 +42,109 @@ const cn = (...classes: (string | undefined | null | false)[]) => { return classes.filter(Boolean).join(' '); }; -// Temporary toast implementation -const useToast = () => ({ - toast: ({ - title, - description, - variant, - }: { - title: string; - description: string; - variant?: string; - }) => { - console.log(`Toast: ${title} - ${description} (${variant || 'default'})`); - // TODO: Implement proper toast functionality +// Toast state management +const toastState = { + toasts: [] as Array<{ id: string; title: string; description: string; variant?: string }>, + listeners: new Set<() => void>(), + addToast(toast: { title: string; description: string; variant?: string }) { + const id = Date.now().toString() + Math.random().toString(36); + this.toasts.push({ ...toast, id }); + this.notify(); + // Auto-remove after 5 seconds + setTimeout(() => { + this.removeToast(id); + }, 5000); }, -}); + removeToast(id: string) { + this.toasts = this.toasts.filter((t) => t.id !== id); + this.notify(); + }, + subscribe(listener: () => void) { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }, + notify() { + this.listeners.forEach((listener) => listener()); + }, +}; + +// Toast hook +const useToast = () => { + const [, forceUpdate] = useState({}); + + useEffect(() => { + const unsubscribe = toastState.subscribe(() => { + forceUpdate({}); + }); + return unsubscribe; + }, []); + + return { + toast: (toast: { title: string; description: string; variant?: string }) => { + toastState.addToast(toast); + }, + }; +}; + +// Toast container component +const ToastContainer = () => { + const [, forceUpdate] = useState({}); + + useEffect(() => { + const unsubscribe = toastState.subscribe(() => { + forceUpdate({}); + }); + return unsubscribe; + }, []); + + return ( +
+ {toastState.toasts.map((toast) => ( +
+
+
+

{toast.title}

+

{toast.description}

+
+ +
+
+ ))} +
+ ); +}; // Simple Dialog components for now const Dialog = ({ @@ -727,8 +815,28 @@ export function KnowledgeTab({ agentId }: { agentId: UUID }) { }); if (!result.ok) { - const error = await result.text(); - throw new Error(error); + // Try to parse error response as JSON + let errorMessage = `Upload failed: ${result.statusText}`; + try { + const contentType = result.headers.get('content-type'); + if (contentType && contentType.includes('application/json')) { + const errorData = await result.json(); + if (errorData.error?.message) { + errorMessage = errorData.error.message; + } else if (errorData.error?.code === 'PDF_DISABLED') { + errorMessage = 'PDF uploads are currently disabled. Please contact your administrator to enable PDF support.'; + } + } else { + const errorText = await result.text(); + if (errorText) { + errorMessage = errorText; + } + } + } catch (e) { + // If parsing fails, keep the default error message + console.error('Error parsing URL upload error response:', e); + } + throw new Error(errorMessage); } const data = await result.json(); @@ -781,7 +889,28 @@ export function KnowledgeTab({ agentId }: { agentId: UUID }) { }); if (!response.ok) { - throw new Error(`Upload failed: ${response.statusText}`); + // Try to parse error response as JSON + let errorMessage = `Upload failed: ${response.statusText}`; + try { + const contentType = response.headers.get('content-type'); + if (contentType && contentType.includes('application/json')) { + const errorData = await response.json(); + if (errorData.error?.message) { + errorMessage = errorData.error.message; + } else if (errorData.error?.code === 'PDF_DISABLED') { + errorMessage = 'PDF uploads are currently disabled. Please contact your administrator to enable PDF support.'; + } + } else { + const errorText = await response.text(); + if (errorText) { + errorMessage = errorText; + } + } + } catch (e) { + // If parsing fails, keep the default error message + console.error('Error parsing upload error response:', e); + } + throw new Error(errorMessage); } const result = await response.json(); @@ -1205,407 +1334,465 @@ export function KnowledgeTab({ agentId }: { agentId: UUID }) { const isDocumentFocused = viewMode === 'graph' && selectedDocumentForGraph && !showSearch; return ( -
-
-
-

Knowledge

-

- {showSearch - ? 'Searching knowledge fragments' - : viewMode === 'list' - ? `Viewing ${documentCount} document${documentCount !== 1 ? 's' : ''}` - : isDocumentFocused - ? `Inspecting document with ${fragmentCount} fragment${fragmentCount !== 1 ? 's' : ''}` - : `Viewing ${documentCount} document${documentCount !== 1 ? 's' : ''}`} -

-
-
- + <> + +
+
+
+

Knowledge

+

+ {showSearch + ? 'Searching knowledge fragments' + : viewMode === 'list' + ? `Viewing ${documentCount} document${documentCount !== 1 ? 's' : ''}` + : isDocumentFocused + ? `Inspecting document with ${fragmentCount} fragment${fragmentCount !== 1 ? 's' : ''}` + : `Viewing ${documentCount} document${documentCount !== 1 ? 's' : ''}`} +

+
+
+ +
-
- - {/* Search Panel */} - {showSearch && ( -
-
-
- -

- Search your knowledge base using semantic vector search. Adjust the similarity - threshold to control how closely results must match your query. -

-
-
-
- setSearchQuery(e.target.value)} - onKeyDown={(e: React.KeyboardEvent) => { - if (e.key === 'Enter' && searchQuery.trim()) { - e.preventDefault(); - handleSearch(); - } - }} - className="flex-1" - /> - + {/* Search Panel */} + {showSearch && ( +
+
+
+ +

+ Search your knowledge base using semantic vector search. Adjust the similarity + threshold to control how closely results must match your query. +

-
-
- - - {searchThreshold.toFixed(2)} ({Math.round(searchThreshold * 100)}% match) - +
+
+ setSearchQuery(e.target.value)} + onKeyDown={(e: React.KeyboardEvent) => { + if (e.key === 'Enter' && searchQuery.trim()) { + e.preventDefault(); + handleSearch(); + } + }} + className="flex-1" + /> +
- setSearchThreshold(parseFloat(e.target.value))} - className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer dark:bg-gray-700" - /> -
- 0% (least similar) - 100% (exact match) + +
+
+ + + {searchThreshold.toFixed(2)} ({Math.round(searchThreshold * 100)}% match) + +
+ setSearchThreshold(parseFloat(e.target.value))} + className="w-full h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer dark:bg-gray-700" + /> +
+ 0% (least similar) + 100% (exact match) +
-
- )} + )} - {/* Dialog for URL upload */} - {showUrlDialog && ( - - - - Import from URL - - Enter one or more URLs of PDF, text, or other files to import into the knowledge - base. - - - -
-
- setUrlInput(e.target.value)} - disabled={isUrlUploading} - className="flex-1" - onKeyDown={(e: React.KeyboardEvent) => { - if (e.key === 'Enter' && urlInput.trim()) { - e.preventDefault(); - addUrlToList(); - } - }} - /> + {/* Dialog for URL upload */} + {showUrlDialog && ( + + + + Import from URL + + Enter one or more URLs of PDF, text, or other files to import into the knowledge + base. + + + +
+
+ setUrlInput(e.target.value)} + disabled={isUrlUploading} + className="flex-1" + onKeyDown={(e: React.KeyboardEvent) => { + if (e.key === 'Enter' && urlInput.trim()) { + e.preventDefault(); + addUrlToList(); + } + }} + /> + +
+ + {urlError && ( +
+ {urlError} +
+ )} + + {urls.length > 0 && ( +
+

URLs to import ({urls.length})

+
+ {urls.map((url, index) => ( +
+ {url} + +
+ ))} +
+
+ )} +
+ + -
+ + + +
+ )} - {urlError && ( -
- {urlError} + {/* Existing input for file upload */} + + +
+ {showSearch ? ( +
+ {isSearching && ( +
+
)} - - {urls.length > 0 && ( -
-

URLs to import ({urls.length})

-
- {urls.map((url, index) => ( + {searchError && !isSearching && ( +
+
+ {searchError} +
+
+ )} + {searchResults.length > 0 && !isSearching && ( +
+

+ Found {searchResults.length} result{searchResults.length !== 1 ? 's' : ''} +

+
+ {searchResults.map((result, index) => (
setViewingContent(result)} > - {url} -
+

+ {result.content?.text || 'No content'} +

+
+
+ {result.metadata?.position !== undefined && ( + Fragment #{result.metadata.position} + )} +
+
+ - - + > + + + + Read more +
+
))}
)} -
- - - - - - - - )} - - {/* Existing input for file upload */} - - -
- {showSearch ? ( -
- {isSearching && ( -
- -
- )} - {searchError && !isSearching && ( -
-
- {searchError} -
-
- )} - {searchResults.length > 0 && !isSearching && ( -
-

- Found {searchResults.length} result{searchResults.length !== 1 ? 's' : ''} -

-
- {searchResults.map((result, index) => ( -
setViewingContent(result)} - > -
-
- - {(result.similarity * 100).toFixed(1)}% match - -
-
-

- {result.content?.text || 'No content'} -

-
-
- {result.metadata?.position !== undefined && ( - Fragment #{result.metadata.position} - )} -
-
- - - - - Read more -
-
-
- ))} -
-
- )} - {!isSearching && searchResults.length === 0 && !searchError && ( -
-
- -

Enter a query to search your knowledge base.

-
-
- )} -
- ) : memories.length === 0 ? ( - - ) : viewMode === 'graph' ? ( -
-
- - {viewMode === 'graph' && graphLoading && selectedDocumentForGraph && ( -
-
- - Loading document fragments... + {!isSearching && searchResults.length === 0 && !searchError && ( +
+
+ +

Enter a query to search your knowledge base.

)}
- - {/* Display details of selected node */} - {selectedMemory && ( + ) : memories.length === 0 ? ( + + ) : viewMode === 'graph' ? ( +
- + + {viewMode === 'graph' && graphLoading && selectedDocumentForGraph && ( +
+
+ + Loading document fragments... +
+
+ )}
- )} -
- ) : ( -
- {/* Filename filter search bar */} -
- setFilenameFilter(e.target.value)} - className="w-full text-sm" - /> + + {/* Display details of selected node */} + {selectedMemory && ( +
+ +
+ )}
+ ) : ( +
+ {/* Filename filter search bar */} +
+ setFilenameFilter(e.target.value)} + className="w-full text-sm" + /> +
-
-
- {visibleMemories.map((memory, index) => ( - - ))} +
+
+ {visibleMemories.map((memory, index) => ( + + ))} +
+ {hasMoreToLoad && }
- {hasMoreToLoad && }
-
- )} -
+ )} +
- {viewingContent && ( - setViewingContent(null)}> - - -
-
- - {(viewingContent.metadata as MemoryMetadata)?.title || 'Document Content'} - - - {(viewingContent.metadata as MemoryMetadata)?.filename || 'Knowledge document'} - + {viewingContent && ( + setViewingContent(null)}> + + +
+
+ + {(viewingContent.metadata as MemoryMetadata)?.title || 'Document Content'} + + + {(viewingContent.metadata as MemoryMetadata)?.filename || 'Knowledge document'} + +
+ {(() => { + const metadata = viewingContent.metadata as MemoryMetadata; + const contentType = metadata?.contentType || ''; + const fileExt = metadata?.fileExt?.toLowerCase() || ''; + const isPdf = contentType === 'application/pdf' || fileExt === 'pdf'; + + if (isPdf) { + return ( +
+ + + {Math.round(pdfZoom * 100)}% + + + +
+ ); + } + return null; + })()}
+ +
{(() => { const metadata = viewingContent.metadata as MemoryMetadata; const contentType = metadata?.contentType || ''; const fileExt = metadata?.fileExt?.toLowerCase() || ''; const isPdf = contentType === 'application/pdf' || fileExt === 'pdf'; - if (isPdf) { + if (isPdf && viewingContent.content?.text) { + // For PDFs, the content.text contains base64 data + // Validate base64 content before creating data URL + const base64Content = viewingContent.content.text.trim(); + + if (!base64Content) { + // Show error message if no content available + return ( +
+
+ + + +

PDF Content Unavailable

+

The PDF content could not be loaded.

+
+
+ ); + } + + // Create a data URL for the PDF + const pdfDataUrl = `data:application/pdf;base64,${base64Content}`; + return ( -
- - - {Math.round(pdfZoom * 100)}% - - - +