diff --git a/.cursor/rules/tech-writer.mdc b/.cursor/rules/tech-writer.mdc index 226f228e0..0e408fe79 100644 --- a/.cursor/rules/tech-writer.mdc +++ b/.cursor/rules/tech-writer.mdc @@ -1,6 +1,6 @@ --- description: Guide the AI to act as a Senior Technical Writer, focusing on clear and concise documentation. -globs: +globs: alwaysApply: false --- @@ -12,7 +12,7 @@ You are a Senior Technical Writer specializing in creating clear, comprehensive, ### **Documentation Creation** -- Write user manuals, API documentation, developer guides, and release notes that are accurate and easy to understand +- Write user manuals, API documentation, developer guides, and changelog that are accurate and easy to understand - Ensure all examples are tested and work as documented - Include troubleshooting sections for common issues - Provide clear migration paths for version changes diff --git a/.github/workflows/docs-auto.yml b/.github/workflows/docs-auto.yml index ec54510eb..1a4e25867 100644 --- a/.github/workflows/docs-auto.yml +++ b/.github/workflows/docs-auto.yml @@ -2,19 +2,19 @@ name: Documentation Website (Auto) on: push: - branches: [ main ] + branches: [main] paths: - - 'docs/**' - - 'README.md' - - 'RELEASE_NOTES.md' - - 'packages/*/README.md' - - 'website/**' - - '.github/workflows/docs-auto.yml' + - "docs/**" + - "README.md" + - "CHANGELOG.md" + - "packages/*/README.md" + - "website/**" + - ".github/workflows/docs-auto.yml" workflow_run: workflows: ["Test and Coverage"] types: - completed - branches: [ main ] + branches: [main] release: types: [published] # Only trigger on main package releases, not MCP server releases @@ -63,7 +63,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Install dependencies run: | @@ -83,25 +83,25 @@ jobs: id: get-workflow-run run: | echo "๐Ÿ” Finding latest successful test workflow run on main branch..." - + # Get the latest successful workflow run for "Test and Coverage" on main branch WORKFLOW_RUN=$(curl -s \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/workflows" \ | jq -r '.workflows[] | select(.name == "Test and Coverage") | .id') - + if [ "$WORKFLOW_RUN" = "null" ] || [ -z "$WORKFLOW_RUN" ]; then echo "โŒ Could not find 'Test and Coverage' workflow" exit 1 fi - + LATEST_RUN=$(curl -s \ -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/workflows/$WORKFLOW_RUN/runs?branch=main&status=completed&conclusion=success&per_page=1" \ | jq -r '.workflow_runs[0].id') - + if [ "$LATEST_RUN" = "null" ] || [ -z "$LATEST_RUN" ]; then echo "โŒ No successful test runs found on main branch" echo "โš ๏ธ Continuing without test artifacts for release" @@ -170,7 +170,7 @@ jobs: echo "๐Ÿ“Š Total files built: $(find site -type f | wc -l)" echo "๐Ÿ“„ HTML pages: $(find site -name "*.html" | wc -l)" echo "๐Ÿ“ Directories: $(find site -type d | wc -l)" - + # Check if main pages exist if [ -f "site/index.html" ]; then echo "โœ… Homepage built successfully" @@ -178,14 +178,14 @@ jobs: echo "โŒ Homepage missing" exit 1 fi - + if [ -f "site/docs/index.html" ]; then echo "โœ… Documentation index built successfully" else echo "โŒ Documentation index missing" exit 1 fi - + if [ -f "site/coverage/index.html" ]; then echo "โœ… Coverage index built successfully" else @@ -242,7 +242,7 @@ jobs: echo "Event: ${{ github.event_name }}" echo "Branch: ${{ github.ref_name }}" echo "Commit: ${{ github.sha }}" - + # Check if this was a release event and if we should deploy if [ "${{ github.event_name }}" = "release" ]; then if [ "${{ needs.check-release.result }}" = "success" ] && [ "${{ needs.check-release.outputs.should-deploy }}" = "true" ]; then @@ -253,7 +253,7 @@ jobs: echo "โญ๏ธ Documentation deployment skipped to avoid duplicates" fi fi - + if [ "${{ needs.build-docs.result }}" = "success" ]; then echo "โœ… Website built successfully" echo "๐Ÿ“ฆ Artifact: website-auto-${{ github.run_id }}" @@ -269,4 +269,4 @@ jobs: else echo "โŒ Website build failed" exit 1 - fi \ No newline at end of file + fi diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c6712cf6..0ddc7596d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,10 +22,20 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Restore pip dependency cache + uses: actions/cache/restore@v4 + id: pip-cache-core + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-core-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-core- + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -34,12 +44,27 @@ jobs: # Install repo dev deps for pytest, coverage, etc. pip install -e .[dev] + - name: Save pip dependency cache + uses: actions/cache/save@v4 + if: steps.pip-cache-core.outputs.cache-hit != 'true' && github.ref_name == 'main' + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-core-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'pyproject.toml') }} + - name: Run core tests and generate coverage reports run: | cd packages/qdrant-loader-core - python -m pytest tests/ --cov=src --cov-report=xml:../../coverage-core.xml --cov-report=html:../../htmlcov-core -v + python -m pytest tests/ --cov=src -v + + - name: Generate coverage reports + if: github.event_name == 'push' && github.ref_name == 'main' + run: | + cd packages/qdrant-loader-core + coverage xml -o ../../coverage-core.xml + coverage html -d ../../htmlcov-core - name: Upload core coverage artifact + if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@v4 with: name: coverage-core-${{ github.run_id }} @@ -47,6 +72,7 @@ jobs: htmlcov-core coverage-core.xml retention-days: 30 + test-loader: name: Test QDrant Loader runs-on: ubuntu-latest @@ -54,6 +80,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: actions/setup-python@v5 with: python-version: '3.12' @@ -65,6 +92,15 @@ jobs: sudo apt-get update sudo apt-get install -y ffmpeg + - name: Restore pip dependency cache + uses: actions/cache/restore@v4 + id: pip-cache-loader + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-loader-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'packages/qdrant-loader/pyproject.toml', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-loader- + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -73,6 +109,13 @@ jobs: pip install -e .[dev] pip install -e packages/qdrant-loader + - name: Save pip dependency cache + uses: actions/cache/save@v4 + if: steps.pip-cache-loader.outputs.cache-hit != 'true' && github.ref_name == 'main' + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-loader-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'packages/qdrant-loader/pyproject.toml', 'pyproject.toml') }} + - name: Create .env.test file for loader run: | cd packages/qdrant-loader @@ -185,12 +228,31 @@ jobs: echo "YAML config structure:" head -20 tests/config.test.yaml - - name: Run loader tests and generate coverage reports + - name: Run loader unit tests + run: | + cd packages/qdrant-loader + python -m pytest tests/unit -n 2 --cov=src --cov-report= -v + + - name: Run loader integration tests + if: | + (github.event_name == 'pull_request' && + (github.event.pull_request.base.ref == 'develop' || + github.event.pull_request.base.ref == 'main')) || + (github.event_name == 'push' && + (github.ref_name == 'develop' || github.ref_name == 'main')) + run: | + cd packages/qdrant-loader + python -m pytest tests/integration --cov=src --cov-append --cov-report= -v + + - name: Generate coverage reports + if: github.event_name == 'push' && github.ref_name == 'main' run: | cd packages/qdrant-loader - python -m pytest tests/ --cov=src --cov-report=xml:../../coverage-loader.xml --cov-report=html:../../htmlcov-loader -v + coverage xml -o ../../coverage-loader.xml + coverage html -d ../../htmlcov-loader - name: Upload loader coverage artifact + if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@v4 with: name: coverage-loader-${{ github.run_id }} @@ -206,10 +268,20 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: actions/setup-python@v5 with: python-version: '3.12' + - name: Restore pip dependency cache + uses: actions/cache/restore@v4 + id: pip-cache-mcp + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-mcp-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'packages/qdrant-loader/pyproject.toml', 'packages/qdrant-loader-mcp-server/pyproject.toml', 'pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-mcp- + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -219,6 +291,13 @@ jobs: pip install -e packages/qdrant-loader pip install -e packages/qdrant-loader-mcp-server + - name: Save pip dependency cache + uses: actions/cache/save@v4 + if: steps.pip-cache-mcp.outputs.cache-hit != 'true' && github.ref_name == 'main' + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-mcp-${{ hashFiles('packages/qdrant-loader-core/pyproject.toml', 'packages/qdrant-loader/pyproject.toml', 'packages/qdrant-loader-mcp-server/pyproject.toml', 'pyproject.toml') }} + - name: Create .env.test file for MCP server run: | cd packages/qdrant-loader-mcp-server @@ -252,12 +331,31 @@ jobs: echo "Contents (with secrets masked):" sed 's/=.*/=***/' tests/.env.test - - name: Run MCP server tests and generate coverage reports + - name: Run MCP server unit tests + run: | + cd packages/qdrant-loader-mcp-server + python -m pytest tests/unit -n 2 --cov=src --cov-report= -v + + - name: Run MCP server integration tests + if: | + (github.event_name == 'pull_request' && + (github.event.pull_request.base.ref == 'develop' || + github.event.pull_request.base.ref == 'main')) || + (github.event_name == 'push' && + (github.ref_name == 'develop' || github.ref_name == 'main')) + run: | + cd packages/qdrant-loader-mcp-server + python -m pytest tests/integration --cov=src --cov-append --cov-report= --cov-report=term-missing -v + + - name: Generate coverage reports + if: github.event_name == 'push' && github.ref_name == 'main' run: | cd packages/qdrant-loader-mcp-server - python -m pytest tests/ --cov=src --cov-report=xml:../../coverage-mcp.xml --cov-report=html:../../htmlcov-mcp -v + coverage xml -o ../../coverage-mcp.xml + coverage html -d ../../htmlcov-mcp - name: Upload MCP server coverage artifact + if: github.event_name == 'push' && github.ref_name == 'main' uses: actions/upload-artifact@v4 with: name: coverage-mcp-${{ github.run_id }} @@ -273,6 +371,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up Python + id: setup-python uses: actions/setup-python@v5 with: python-version: '3.12' @@ -283,6 +382,15 @@ jobs: sudo apt-get update sudo apt-get install -y libcairo2-dev libgirepository1.0-dev + - name: Restore pip dependency cache + uses: actions/cache/restore@v4 + id: pip-cache-website + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-website-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-website- + - name: Install website test dependencies run: | python -m pip install --upgrade pip @@ -291,6 +399,13 @@ jobs: # Install optional docs dependencies for comprehensive testing pip install -e .[docs] || echo "Optional docs dependencies not available" + - name: Save pip dependency cache + uses: actions/cache/save@v4 + if: steps.pip-cache-website.outputs.cache-hit != 'true' && github.ref_name == 'main' + with: + path: ~/.cache/pip + key: ${{ runner.os }}-py${{ steps.setup-python.outputs.python-version }}-website-${{ hashFiles('pyproject.toml') }} + - name: Run website tests with coverage run: | # Add website directory to Python path and run tests diff --git a/.gitignore b/.gitignore index a23fee35f..8a9ed3bad 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ wheels/ # Virtual Environment venv/ +.venv/ env/ ENV/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 000000000..84d8675c5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,512 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.7.6] - 2026-01-22 + +### Changed + +- Documentation terminology updated from "Release Notes" to "Changelog" [#112] +- Applied consistent formatting across workflow and updated navigation links [#112] + +## [0.7.5] - 2026-01-20 + +### Fixed + +#### Qdrant-loader-mcp-server + +- Mixed document types (dict/object) causing "Untitled" and empty `content_preview` in results [#92] +- Incorrect usage of NestedCondition for `project_ids` filter replaced with dot notation [#97] +- Search results not respecting user-specified `limit` parameter (now defaults to 5) [#98] + +### Added + +#### Qdrant-loader-mcp-server + +- Optional `similarity_threshold` parameter (default 0.7) to `find_similar_documents` for filtering by minimum similarity score [#91] +- Separate `fetch_limit` from user `limit` with configurable defaults for better control [#98] + +### Changed + +#### Qdrant-loader-mcp-server + +- Updated `reason` field from `explanation` to `recommendation_reason` in complementary content results [#100] +- Enhanced validation for text field types to prevent display issues [#92] + +## [0.7.4] - 2025-12-11 + +### Fixed + +#### Qdrant-loader + +- Windows asyncio event loop crashes with signal handling and stdio support [#76] +- --log-level CLI option not working after logging initialization [#63] +- Missing prometheus-client dependency causing import errors [#75] +- Deprecated langchain.text_splitter import warnings [#76] + +#### Qdrant-loader-mcp-server + +- Missing Spacy dependency causing Cursor MCP crash on startup [#82] +- MCP Search API compatibility issues with qdrant-client 1.16 [#78] + +### Added + +#### Qdrant-loader + +- Test suite cross-platform support [#76] + +## [0.7.3] - 2025-09-11 + +### Fixed + +#### Qdrant-loader + +- Logging duplication where CLI commands printed each log message 2-4 times [#56] + +### Changed + +#### Qdrant-loader-core + +- Unified logging architecture with centralized configuration across all packages [#56] +- Idempotent setup with new `reconfigure()` method [#56] + +## [0.7.2] - 2025-09-05 + +### Changed + +- Internal inter-package dependencies now pinned to unified release version [#54] +- Enhanced dry-run output to preview internal dependency pin changes [#54] +- Enforced explicit release order: bump โ†’ update classifiers โ†’ pin deps โ†’ commit โ†’ tag โ†’ release [#54] +- Updated commit messages to reflect classifier and internal dependency updates [#54] + +## [0.7.1] - 2025-09-04 + +### Added + +#### Qdrant-loader-core + +- Azure OpenAI support with robust endpoint handling (BETA) [#52] +- Ollama endpoint handling [#52] +- Unified `global.llm.*` configuration for provider-agnostic setup [#52] +- Structured logging for LLM requests (provider, operation, model, latency) [#52] +- Secret redaction in logs [#52] +- Normalized exception mapping across providers [#52] + +### Changed + +#### Qdrant-loader-core + +- Vector size now read from config instead of hardcoded `1536` defaults [#52] +- Centralized LLM layer with provider adapters [#52] +- Direct OpenAI imports removed from application code [#52] +- Tests and documentation updated for provider-agnostic architecture [#52] + +#### Qdrant-loader-mcp-server + +- MCP server prefers config file loading with CLI/env/file precedence [#53] + +### Deprecated + +#### Qdrant-loader-core + +- Legacy fields `global.embedding.*` and `file_conversion.markitdown.*` (deprecation warnings shown) [#52] + +### Security + +#### Qdrant-loader-mcp-server + +- Redacted `--print-config` for MCP server [#53] + +## [0.6.1] - 2025-08-13 + +### Fixed + +#### Qdrant-loader-mcp-server + +- Timeout and error issues in `detect_document_conflicts` tool (achieved P95 latency 8-10s) [#45] +- AttributeError in conflict formatter: `'dict' object has no attribute 'document_id'` [#45] + +### Added + +#### Qdrant-loader-mcp-server + +- Tiered analysis with intelligent document pair prioritization [#45] +- Parallel processing with concurrent Qdrant vector retrieval [#45] +- 8 configuration options for conflict detection performance tuning [#45] +- Detailed performance statistics in tool responses [#45] +- Optional per-call parameter overrides for conflict detection [#45] + +### Changed + +#### Qdrant-loader-mcp-server + +- Strict budgeting for expensive LLM calls (default 2 pairs) [#45] +- Graceful degradation with partial results instead of hard failures [#45] +- Tool schema enhanced with performance parameters [#45] +- Error handling for malformed document data [#45] + +## [0.6.0] - 2025-08-12 + +### Added + +#### Qdrant-loader-mcp-server + +- FastAPI-based HTTP transport alongside stdio transport [#43] +- Server-Sent Events (SSE) streaming capabilities [#43] +- `--transport` CLI option to choose between stdio and HTTP modes [#43] +- Health check endpoints for production deployment [#43] +- Structured tool output with JSON content [#43] +- Tool behavioral annotations for all 8 tools [#43] +- Protocol version validation with graceful degradation [#43] +- Comprehensive session management for HTTP connections [#43] + +### Changed + +#### Qdrant-loader-mcp-server + +- MCP Protocol upgraded from 2024-11-05 to 2025-06-18 [#43] +- Modular transport layer with clean separation [#43] +- Error handling with improved messages [#43] +- Connection handling performance optimizations [#43] + +## [0.5.1] - 2025-07-28 + +### Changed + +#### Qdrant-loader-core + +- All chunking strategies refactored into modular components [#39] +- Dedicated classes for document parsing, section splitting, metadata extraction [#39] +- HTML chunking with robust handling for empty content and malformed HTML [#39] +- Code and JSON strategy complete modular redesign [#39] +- Chunk processing with additional metadata fields [#39] +- Configuration templates with new strategy-specific options [#39] + +## [0.5.0] - 2025-07-25 + +### Added + +#### Qdrant-loader-mcp-server + +- Cross-document intelligence: similarity analysis, clustering, relationship detection [#35] +- Intent-aware adaptive search with AI-powered query understanding [#35] +- Knowledge graph integration with entity relationships [#35] +- Topic-driven search chaining with automatic discovery [#35] +- Dynamic faceted search with real-time generation [#35] +- spaCy integration for advanced NLP processing [#35] + +#### Qdrant-loader + +- `--force` flag to bypass change detection for complete reprocessing [#35] + +### Changed + +#### Qdrant-loader-core + +- Topic extraction with enhanced LDA modeling [#35] +- Entity recognition with structured conversion [#35] +- Chunking with improved timeout handling [#35] +- Structured logging for semantic analysis [#35] + +## [0.4.15] - 2025-07-22 + +### Fixed + +#### Qdrant-loader-core + +- Critical chunking inconsistency (all strategies now use character-based `chunk_size`) [#33] + +### Changed + +#### Qdrant-loader-core + +- Markdown strategy refactored into focused components [#33] +- Hierarchical metadata with intelligent section analysis [#33] +- Split level detection based on document structure [#33] +- Boundary detection using tokenizer for word/token boundaries [#33] + +### Added + +#### Qdrant-loader-core + +- Comprehensive integration tests for strategy consistency [#33] + +## [0.4.14] - 2025-07-13 + +### Fixed + +#### Qdrant-loader-core + +- Regex error in Excel table detection: `bad character range |-\s` [#33] +- Large Excel tables treated as single massive chunks [#33] +- Token limit warnings for large Excel chunks [#33] + +### Changed + +#### Qdrant-loader-core + +- Logical unit management with intelligent splitting at line boundaries [#33] +- Efficient chunking for large Excel files [#33] +- Error handling with better error messages [#33] +- Table structure preservation during chunking [#33] + +## [0.4.13] - 2025-07-11 + +### Added + +#### Qdrant-loader-core + +- Sheet-aware sectioning for Excel files (split on H2 headers) [#33] +- Table-aware chunking with specialized `_split_excel_sheet_content` method [#33] +- Intelligent content detection based on `original_file_type` metadata [#33] +- 3 test cases for Excel chunking scenarios [#33] + +### Changed + +#### Qdrant-loader-core + +- Excel-to-markdown chunking in MarkdownChunkingStrategy [#33] +- Context-aware splitting with different header level thresholds [#33] +- Metadata tracking with `is_excel_sheet` field [#33] +- Table boundary preservation in chunking [#33] + +## [0.4.12] - 2025-07-10 + +### Fixed + +#### Qdrant-loader-core + +- Missing chunk overlap in MarkdownChunkingStrategy [#33] + +### Added + +#### Qdrant-loader-core + +- Intelligent overlap calculation using paragraph/sentence boundaries [#33] +- Comprehensive overlap testing [#33] + +### Changed + +#### Qdrant-loader-core + +- Overlap configuration support (0 for no overlap, up to 25% for context) [1f53556] + +## [0.4.11] - 2025-07-10 + +### Fixed + +#### Qdrant-loader-core + +- File size detection limits for larger documents [6160435] +- MarkdownChunkingStrategy not respecting `chunk_size` configuration [6160435] +- Unique chunk ID generation (chunks had identical IDs causing overwrites) [6160435] + +### Added + +#### Qdrant-loader-core + +- `max_chunks_per_document` configuration parameter [6160435] + +### Changed + +#### Qdrant-loader-core + +- Chunk count management with configurable limits [6160435] +- Error messages with actionable configuration advice [6160435] +- Section limits now dynamic (50% of max_chunks_per_document) [6160435] + +### Removed + +#### Qdrant-loader-core + +- Conflicting `max_document_size` parameter [6160435] + +## [0.4.10] - 2025-06-18 + +### Fixed + +#### Qdrant-loader + +- Duplicate debug logging with `[DEBUG] [DEBUG]` tags [c72a872] +- Mixed path separators in Windows log output [c72a872] +- .txt file processing when `file_types: []` was empty [c72a872] + +#### Qdrant-loader-core + +- Windows file URL parsing for LocalFile connector [c72a872] +- Git connector document URL generation with Windows paths [c72a872] + +### Changed + +#### Qdrant-loader + +- Logging verbosity control for third-party libraries [c72a872] +- Emoji handling for Windows console [c72a872] +- SQLite logs now suppressed [c72a872] + +#### Qdrant-loader-core + +- Path normalization across all connectors [c72a872] +- Timeout handling (threading on Windows, signals on Unix) [c72a872] + +### Added + +#### Qdrant-loader + +- 38 Windows compatibility test cases [c72a872] + +## [0.4.9] - 2025-06-18 + +### Fixed + +#### Qdrant-loader-core + +- Missing `content_type="md"` field in `_create_deleted_document` method [430cb2a] + +## [0.4.8] - 2025-06-17 + +### Fixed + +#### Qdrant-loader-core + +- Windows file URL parsing in LocalFile Connector [b60e6e0] +- Git Connector document URL generation with Windows paths [b60e6e0] +- File conversion timeout handling for cross-platform [b60e6e0] +- MarkItDown Windows signal compatibility [b60e6e0] + +#### Qdrant-loader + +- Console emoji handling for Windows [b60e6e0] +- Duplicate log level display [b60e6e0] + +### Added + +#### Qdrant-loader + +- 38 Windows compatibility test cases [b60e6e0] + +## [0.4.7] - 2025-06-09 + +### Fixed + +#### Qdrant-loader + +- Upgrade instructions to include `qdrant-loader-mcp-server` package [fecc0dc] + +#### Qdrant-loader-core + +- Branch display logic to default to 'main' when unknown [fecc0dc] + +### Changed + +#### Qdrant-loader + +- Configuration template with detailed comments [fecc0dc] +- Release script with automatic RELEASE_NOTES.md validation [fecc0dc] + +#### Qdrant-loader-core + +- Logging in PublicDocsConnector [fecc0dc] + +## [0.4.6] - 2025-06-03 + +### Added + +#### Qdrant-loader + +- Automatic update notifications when new versions available [3604f92] +- Non-intrusive background version checking [3604f92] + +## [0.4.5] - 2025-06-03 + +### Fixed + +#### Qdrant-loader + +- Version detection using `importlib.metadata.version()` [#25] + +#### Qdrant-loader-core + +- Circular imports in config โ†’ connectors โ†’ config cycle [#25] + +### Changed + +#### Qdrant-loader + +- CLI startup time reduced by 60-67% for basic commands [#24] + - `--help`: ~6.8s โ†’ 2.33s (66% improvement) + - `--version`: ~6.3s โ†’ 2.57s (59% improvement) +- Lazy loading for heavy modules (96-97% import time reduction) [#25] + +### Added + +#### Qdrant-loader-core + +- Warning capture system for Excel file processing [#25] +- Structured logging for openpyxl warnings [#25] +- Smart detection for "Data Validation" and "Conditional Formatting" warnings [#25] +- Summary reporting for unsupported Excel features [#25] + +## [0.4.4] - 2025-06-03 + +### Fixed + +#### Qdrant-loader-core + +- File conversion initialization with missing `set_file_conversion_config` calls [#21] +- Converted files using wrong chunking strategy [#21] +- NLP processing skipped for converted files [#21] +- MarkdownChunkingStrategy infinite loops with very long words [#21] + +#### Qdrant-loader + +- ResourceManager cleanup causing workers to exit prematurely [#21] + +### Added + +#### Qdrant-loader-core + +- Safety limits: `MAX_CHUNKS_PER_SECTION = 100` and `MAX_CHUNKS_PER_DOCUMENT = 500` [#21] +- Handling for words longer than `max_size` [#21] +- Comprehensive tests for converted file NLP processing [#21] + +### Changed + +#### Qdrant-loader-core + +- Strategy selection based on conversion status [#21] +- Metadata propagation for converted files [#21] +- `MAX_CHUNKS_TO_PROCESS` increased from 100 to 1000 chunks [#21] +- Large document handling (up to ~1000KB text limit) [#21] + +#### Qdrant-loader + +- Change detection for incremental updates [#21] +- Signal handling for graceful shutdown [#21] + +[0.7.5]: https://github.com/martin-papy/qdrant-loader/compare/v0.7.4...v0.7.5 +[0.7.4]: https://github.com/martin-papy/qdrant-loader/compare/v0.7.3...v0.7.4 +[0.7.3]: https://github.com/martin-papy/qdrant-loader/compare/v0.7.2...v0.7.3 +[0.7.2]: https://github.com/martin-papy/qdrant-loader/compare/v0.7.1...v0.7.2 +[0.7.1]: https://github.com/martin-papy/qdrant-loader/compare/v0.6.1...v0.7.1 +[0.6.1]: https://github.com/martin-papy/qdrant-loader/compare/v0.6.0...v0.6.1 +[0.6.0]: https://github.com/martin-papy/qdrant-loader/compare/v0.5.1...v0.6.0 +[0.5.1]: https://github.com/martin-papy/qdrant-loader/compare/v0.5.0...v0.5.1 +[0.5.0]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.15...v0.5.0 +[0.4.15]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.14...v0.4.15 +[0.4.14]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.13...v0.4.14 +[0.4.13]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.12...v0.4.13 +[0.4.12]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.11...v0.4.12 +[0.4.11]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.10...v0.4.11 +[0.4.10]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.9...v0.4.10 +[0.4.9]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.8...v0.4.9 +[0.4.8]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.7...v0.4.8 +[0.4.7]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.6...v0.4.7 +[0.4.6]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.5...v0.4.6 +[0.4.5]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.4...v0.4.5 +[0.4.4]: https://github.com/martin-papy/qdrant-loader/compare/v0.4.3...v0.4.4 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bf2893f9a..5b8da6cf2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -142,6 +142,8 @@ git push origin feature/your-feature-name ## ๐Ÿ“ Coding Standards +> **๐Ÿ“– For comprehensive guidelines** including Pythonic patterns, AI/RAG best practices, and PR review checklists, see the [Best Practices Guide](./docs/developers/contributing/). + ### Code Style We use the following tools to maintain code quality: @@ -187,14 +189,14 @@ Use Google-style docstrings: ```python def process_document(content: str, metadata: Dict[str, Any]) -> ProcessedDocument: """Process a document with the given content and metadata. - + Args: content: The raw document content to process. metadata: Additional metadata about the document. - + Returns: A ProcessedDocument instance with chunked content and enriched metadata. - + Raises: ProcessingError: If the document cannot be processed. """ @@ -233,26 +235,26 @@ from qdrant_loader.processors import DocumentProcessor class TestDocumentProcessor: """Test cases for DocumentProcessor.""" - + def test_process_simple_document(self): """Test processing a simple text document.""" processor = DocumentProcessor() content = "This is a test document." - + result = processor.process(content) - + assert result.chunks assert len(result.chunks) == 1 assert result.chunks[0].content == content - + @patch('qdrant_loader.processors.external_service') def test_process_with_external_service(self, mock_service): """Test processing with mocked external service.""" mock_service.return_value = "processed content" processor = DocumentProcessor() - + result = processor.process("input") - + mock_service.assert_called_once_with("input") assert result.content == "processed content" ``` @@ -299,7 +301,7 @@ pytest -m "not slow" #### Markdown Guidelines -```markdown +````markdown # Use clear headings ## Structure content logically @@ -310,6 +312,7 @@ pytest -m "not slow" # Command examples should be copy-pastable qdrant-loader --workspace . init ``` +```` **Use formatting** for emphasis and `code` for technical terms. @@ -344,20 +347,24 @@ When creating a pull request, include: ```markdown ## Description + Brief description of the changes and why they're needed. ## Type of Change + - [ ] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update ## Testing + - [ ] Unit tests added/updated - [ ] Integration tests added/updated - [ ] Manual testing performed ## Checklist + - [ ] Code follows the project's style guidelines - [ ] Self-review of code completed - [ ] Code is commented, particularly in hard-to-understand areas @@ -385,25 +392,31 @@ Brief description of the changes and why they're needed. ```markdown ## Bug Description + A clear and concise description of what the bug is. ## To Reproduce + Steps to reproduce the behavior: + 1. Go to '...' 2. Click on '....' 3. Scroll down to '....' 4. See error ## Expected Behavior + A clear and concise description of what you expected to happen. ## Environment + - OS: [e.g. macOS 12.0, Ubuntu 20.04, Windows 10] - Python version: [e.g. 3.12.2] - QDrant Loader version: [e.g. 0.4.0b1] - QDrant version: [e.g. 1.7.0] ## Additional Context + Add any other context about the problem here. ``` @@ -419,18 +432,23 @@ Add any other context about the problem here. ```markdown ## Feature Description + A clear and concise description of what you want to happen. ## Problem Statement + What problem does this feature solve? What's the current limitation? ## Proposed Solution + Describe the solution you'd like to see implemented. ## Alternatives Considered + Describe any alternative solutions or features you've considered. ## Additional Context + Add any other context, mockups, or examples about the feature request here. ``` @@ -444,7 +462,7 @@ We use **unified versioning** - both packages always have the same version numbe 1. **Update version numbers** in both packages 2. **Create release branch** and test thoroughly -3. **Create GitHub release** with release notes +3. **Create GitHub release** with changelog 4. **Publish to PyPI** using the release script ```bash diff --git a/README.md b/README.md index 242302e46..c230a016b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Test Coverage](https://img.shields.io/badge/coverage-view%20reports-blue)](https://qdrant-loader.net/coverage/) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) -๐Ÿ“‹ **[Release Notes v0.7.4](./RELEASE_NOTES.md)** - Latest improvements and bug fixes +๐Ÿ“‹ **[Changelog v0.7.6](./CHANGELOG.md)** - Latest improvements and bug fixes A comprehensive toolkit for loading data into Qdrant vector database with advanced MCP server support for AI-powered development workflows. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md deleted file mode 100644 index f5ca1d71f..000000000 --- a/RELEASE_NOTES.md +++ /dev/null @@ -1,478 +0,0 @@ -# Release Notes - -## Version 0.7.4 - Dec 5, 2025 - -### ๐ŸชŸ Windows Compatibility Fixes - -#### Test Suite Cross-Platform Support - -- **Fixed Git connector tests**: Resolved 9 test failures related to cross-platform path handling - - Fixed cross-drive relative path errors on Windows (e.g., `ValueError` when computing paths from C: to D:) - - Updated all test files to use `os.path.join()` with `temp_dir` instead of hardcoded Unix paths - - Enhanced `test_git_connector.py` with proper mocking for temp_dir-based paths -- **Fixed pytest configuration conflicts**: Resolved root workspace test discovery issues - - - Updated `pyproject.toml` to exclude `packages/` directory from root test collection - - Added `norecursedirs` to prevent conftest import conflicts between workspace and packages - - All 172 root workspace tests now pass without import errors - -- **Fixed config loader test isolation**: Prevented workspace config interference - - - Added `monkeypatch.chdir(tmp_path)` to isolate tests from project root - - Tests no longer accidentally discover workspace `config.yaml` files - -- **Fixed website build system tests**: Resolved 19 failures/errors on Windows - - **Unicode encoding**: Added explicit `encoding="utf-8"` to all file operations - - **URL generation**: Used `Path.as_posix()` for cross-platform sitemap URLs - - **Windows path handling**: Implemented smart colon detection to distinguish drive letters (`C:`) from source:dest syntax - - **File cleanup**: Added retry logic with delays for Windows file handle cleanup (fixes 15 `PermissionError` cases) - - **Test assertions**: Updated timing assertions from `> 0` to `>= 0` for edge case compatibility - -#### Technical Improvements - -- **Cross-platform best practices**: All fixes use Python standard library APIs (`os.path.join()`, `Path.as_posix()`, explicit encoding) -- **Full backward compatibility**: No breaking changes for Mac/Linux platforms -- **Comprehensive test coverage**: All 1899+ tests passing on Windows (172 root + 33 git + 120 website + 1674 loader) - -## Version 0.7.3 - Sept 11, 2025 - -### Logging System Fixes - -- **Fixed logging duplication**: Resolved CLI commands printing each log message 2-4 times due to multiple handler setup calls -- **Unified logging architecture**: Centralized logging configuration across all packages with idempotent setup and new `reconfigure()` method - -## Version 0.7.2 - Sept 05, 2025 - -### Release Process Enhancements - -- Internal inter-package dependencies are now pinned to the unified release version to prevent mismatched installs when only one package is upgraded. -- Dry-run output enhanced: previews internal dependency pin changes per package and uses the new version for tag and release previews. -- Enforced, explicit release order: bump versions โ†’ update classifiers โ†’ pin internal deps โ†’ commit & push โ†’ tag โ†’ release. -- Updated commit messages to reflect classifier and internal dependency updates. - -## Version 0.7.1 - Sept 04, 2025 - -### LLM Provider-Agnostic Configuration & Migration - -- BETA: Added Azure OpenAI support (no more 404s from misconfigured endpoints) and robust Ollama endpoint handling -- Introduced unified `global.llm.*` configuration controlling provider, `base_url`, models, tokenizer, request policy, rate limits, and `embeddings.vector_size`. -- Legacy fields (`global.embedding.*` and `file_conversion.markitdown.*`) remain supported with deprecation warnings; migration is recommended. -- Vector size is now read from config; hardcoded `1536` defaults replaced with config-driven values and a deprecated fallback warning when unspecified. -- Structured logging added for LLM requests (provider, operation, model, latency; secrets redacted) and normalized exception mapping across providers. -- Documentation updated with the new schema and env vars: - - Azure logs label provider as `azure_openai`; OpenAI as `openai`; Ollama logs include latency for chat. - - Clear error when Azure `base_url` includes `/openai/deployments/...`; requires `api_version`. - - Ollama auto-detects `/v1` vs native. Native tries batch `/api/embed` first, falls back to `/api/embeddings`. -- Documentation updated with the new schema and env vars: - - Configuration reference: `docs/users/configuration/config-file-reference.md` - - Environment variables: `docs/users/configuration/environment-variables.md` - -### Massive Codebase Refactor - -- Centralized LLM layer in `qdrant-loader-core` with provider adapters (OpenAI / OpenAI-compatible via `base_url`, Ollama native or `/v1`). -- Removed direct OpenAI imports from application code; apps now use a provider factory from the core package. -- MCP server now prefers config file loading with CLI/env/file precedence and redacted `--print-config`; legacy env-only mode warns. -- Replaced hardcoded vector-size usage across components with configuration-driven values. -- Updated tests and documentation to align with provider-agnostic architecture. - -## Version 0.6.1 - August 13, 2025 - -### Document Conflict Detection : Performance Improvements - -#### MCP Server - Document Conflict Detection Optimization - -- **Resolved performance bottlenecks**: Fixed timeout and error issues in `detect_document_conflicts` tool, achieving P95 latency target of 8-10 seconds -- **Tiered analysis implementation**: Added intelligent document pair prioritization (primary, secondary, tertiary tiers) to analyze most promising conflicts first -- **LLM optimization**: Implemented strict budgeting for expensive LLM calls with configurable limits (default 2 pairs) and per-process caching -- **Parallel processing**: Added concurrent Qdrant vector retrieval with semaphore-based concurrency control (default 5 concurrent operations) -- **Configurable performance parameters**: Added 8 new configuration options for fine-tuning conflict detection performance: - - `conflict_overall_timeout_s`: Overall operation timeout (default 9.0s) - - `conflict_max_pairs_total`: Maximum document pairs to analyze (default 24) - - `conflict_max_llm_pairs`: Maximum LLM-analyzed pairs (default 2) - - `conflict_text_window_chars`: Text truncation for LLM input (default 2000 chars) - - `conflict_embeddings_timeout_s`: Vector retrieval timeout (default 2.0s) - - `conflict_embeddings_max_concurrency`: Parallel retrieval limit (default 5) -- **Runtime transparency**: Added detailed performance statistics in tool responses showing pairs analyzed, LLM usage, and execution time -- **Graceful degradation**: Implemented partial results with time budget exhaustion handling instead of hard failures -- **Enhanced tool schema**: Added optional per-call parameter overrides for `use_llm`, `max_llm_pairs`, `overall_timeout_s`, `max_pairs_total`, and `text_window_chars` - -#### Bug Fixes - -- **Fixed AttributeError in conflict formatter**: Resolved `'dict' object has no attribute 'document_id'` error by adding proper handling for both SearchResult objects and dictionary formats -- **Enhanced error handling**: Improved graceful handling of malformed document data in conflict detection pipeline - -## Version 0.6.0 - August 12, 2025 - -### **MAJOR MILESTONE RELEASE** - -#### ๐Ÿš€ Upgraded MCP Server Architecture - -- **Streamable HTTP Transport Support**: Added FastAPI-based HTTP transport alongside existing stdio transport, enabling web-based MCP clients and multiple concurrent connections -- **MCP Protocol 2025-06-18 Compliance**: Upgraded from MCP Protocol version 2024-11-05 to the latest 2025-06-18 specification with full backward compatibility -- **Server-Sent Events (SSE) Streaming**: Implemented real-time streaming capabilities for enhanced client communication -- **Dual Transport Architecture**: Support for both stdio (subprocess-based clients) and HTTP (web clients) transports simultaneously - -#### ๐Ÿ”ง Enhanced Integration & Connectivity - -- **Production-Ready HTTP Server**: FastAPI implementation with proper security, session management, and CORS support -- **Advanced Security Features**: Origin validation, localhost binding, and DNS rebinding protection -- **CLI Transport Selection**: Added `--transport` option to choose between stdio and HTTP modes -- **Health Check Endpoints**: Built-in monitoring and health check capabilities for production deployment - -#### ๐Ÿ“Š Structured Output & Protocol Features - -- **Structured Tool Output**: Enhanced tool responses with JSON-structured content while maintaining backward compatibility -- **Tool Behavioral Annotations**: Added annotations for all 8 tools indicating read-only and compute-intensive operations -- **Protocol Version Validation**: Header-based protocol version validation with graceful degradation -- **Session Management**: Comprehensive session handling for stateful HTTP connections - -#### ๐Ÿ”„ Backward Compatibility & Migration - -- **Zero Breaking Changes**: Existing stdio clients continue to work unchanged -- **Seamless Migration Path**: Easy transition between transport modes without configuration changes -- **Legacy Support**: Full support for existing MCP 2024-11-05 clients -- **Configuration Compatibility**: All existing configurations work with new transport layer - -#### ๐Ÿ—๏ธ Architecture Improvements - -- **Modular Transport Layer**: Clean separation between protocol handling and transport mechanisms -- **Enhanced Error Handling**: Improved error messages and graceful failure handling -- **Performance Optimizations**: Efficient connection handling and resource management -- **Comprehensive Testing**: Full test coverage for HTTP transport, session management, and protocol compliance - -## Version 0.5.1 - July 28, 2025 - -### ๐Ÿ—๏ธ Major Architecture Improvements - -#### Chunking Strategy Modernization - -- **Modular architecture implementation**: Complete refactor of all chunking strategies (Default, HTML, Code, JSON) into modular components for enhanced maintainability and extensibility -- **Component-based design**: Introduced dedicated classes for document parsing, section splitting, metadata extraction, and chunk processing across all strategies -- **Improved HTML chunking**: Enhanced robust handling for empty content and malformed HTML with graceful degradation -- **Code and JSON strategy overhaul**: Complete modular redesign with better handling of large documents and fallback mechanisms -- **Enhanced chunk processing**: Added additional metadata fields (source_type, url, content_type, title) and improved semantic analysis handling -- **Updated configuration templates**: Enhanced [config.template.yaml](https://raw.githubusercontent.com/martin-papy/qdrant-loader/main/packages/qdrant-loader/conf/config.template.yaml) and [configuration documentation](docs/users/configuration/config-file-reference.md) with new strategy-specific chunking options - -## Version 0.5.0 - July 25, 2025 - -### ๐Ÿš€ Major Features - -#### Advanced Search Intelligence - -- **Cross-document intelligence**: Document similarity analysis, clustering, and relationship detection -- **Intent-aware adaptive search**: AI-powered query understanding and strategy selection -- **Knowledge graph integration**: Entity relationships and multi-hop reasoning capabilities -- **Topic-driven search chaining**: Automatic topic discovery and related content suggestions -- **Dynamic faceted search**: Real-time facet generation and filtering interface - -#### Enhanced Semantic Analysis - -- **spaCy integration**: Advanced NLP processing with configurable language models -- **Improved topic extraction**: Enhanced LDA modeling with optimized parameters -- **Entity recognition**: Structured entity and topic conversion in search results -- **Semantic analysis configuration**: Comprehensive topic modeling settings - -#### CLI & User Experience - -- **Force ingestion option**: Added `--force` flag to bypass change detection for complete reprocessing -- **Enhanced chunking**: Improved timeout handling and performance thresholds -- **Better logging**: Structured logging for semantic analysis and search components - -## Version 0.4.15 - July 22, 2025 - -### ๐Ÿš€ Major Improvements - -#### Chunking Strategy Overhaul - -- **Fixed critical chunking inconsistency**: All strategies now use character-based `chunk_size` (was mixed token/character interpretation causing 4-5x chunk count differences) -- **Markdown strategy modularization**: Complete refactor into focused components (DocumentParser, SectionSplitter, MetadataExtractor, ChunkProcessor) for better maintainability -- **Enhanced hierarchical metadata**: Added intelligent section analysis with HeaderAnalysis and SectionMetadata for richer document context -- **Smart split level detection**: Automatic optimization of header split levels based on document structure and type -- **Improved boundary detection**: Tokenizer now used for word/token boundaries while respecting character-based limits -- **Comprehensive testing**: Added integration tests ensuring strategy consistency and preventing regression - -## Version 0.4.14 - July 13, 2025 - -### ๐Ÿ› Critical Bug Fixes - -#### Excel File Chunking Fixes - -- **Fixed regex error in table detection**: Resolved `bad character range |-\s at position 2` error that was preventing Excel files from being chunked properly - - **Root cause**: Invalid regex pattern `r"^[|-\s:]+$"` in `_split_excel_sheet_content` method - - **Solution**: Escaped dash character to create valid pattern: `r"^[|\-\s:]+$"` - - **Impact**: Excel files no longer fall back to default chunking strategy -- **Fixed large table chunking logic**: Resolved issue where large Excel tables were treated as single massive chunks - - **Problem**: 128K character files created only 2-5 chunks instead of ~200 chunks at 600-character limit - - **Root cause**: Large logical units (tables) were not split when exceeding max_size - - **Solution**: Added intelligent splitting logic that preserves table structure while respecting chunk size limits - - **Result**: Large Excel files now properly chunk into appropriate sizes (e.g., 74K chars โ†’ 127 chunks @ ~588 chars each) -- **Eliminated token limit warnings**: Fixed the `Content exceeds maximum token limit, truncating` warnings that occurred with large Excel chunks - - **Before**: Chunks up to 128K characters (47K+ tokens) being truncated - - **After**: All chunks properly sized to stay within token limits -- **Enhanced table structure preservation**: Table boundaries are now intelligently detected and preserved during chunking - -#### Technical Improvements - -- **Better logical unit management**: Enhanced `_split_excel_sheet_content` to handle large units by splitting at line boundaries -- **Preserved table formatting**: Chunking algorithm maintains table structure integrity while enforcing size limits -- **Improved error handling**: Better error messages and fallback behavior for edge cases -- **Performance optimization**: More efficient chunking for large Excel files without infinite loops - -#### Testing & Validation - -- **All existing tests pass**: 50 markdown strategy tests continue to pass, ensuring backward compatibility -- **Verified chunking accuracy**: Large test files now produce expected chunk counts with proper size distribution -- **Regex pattern validation**: Confirmed table detection works correctly for all markdown table formats - -## Version 0.4.13 - July 11, 2025 - -### โœจ New Features - -#### Excel File Chunking Improvements - -- **Enhanced Excel-to-markdown chunking**: Improved MarkdownChunkingStrategy to properly handle Excel files converted to markdown by MarkItDown -- **Sheet-aware sectioning**: Excel files now split on H2 headers (sheet names) instead of treating the entire file as one "Preamble" section -- **Table-aware chunking**: Added specialized `_split_excel_sheet_content` method that preserves table structure when splitting large sheets -- **Intelligent content detection**: Automatically detects converted Excel files based on `original_file_type` metadata and applies appropriate chunking rules -- **Backward compatibility**: Regular markdown files continue to use H1-only sectioning, maintaining existing behavior -- **Comprehensive testing**: Added 3 new test cases covering Excel chunking scenarios and ensuring regular markdown files are unaffected - -#### Excel Chunking โ€” Technical Improvements - -- **Context-aware splitting**: Different header level thresholds based on file type (H1 for markdown, H1+H2 for Excel) -- **Enhanced metadata tracking**: Added `is_excel_sheet` metadata to identify Excel-derived chunks -- **Table boundary preservation**: Smart table detection prevents breaking tables in the middle when chunking -- **Document reference management**: Added proper cleanup of document references to prevent memory leaks - -## Version 0.4.12 - July 10, 2025 - -### ๐Ÿ› Bug Fixes - -#### Chunking Strategy Improvements - -- **Fixed missing chunk overlap in MarkdownChunkingStrategy**: Implemented proper overlap functionality that was completely missing from markdown file chunking -- **Added intelligent overlap calculation**: Overlap now respects the configured `chunk_overlap` parameter and uses paragraph/sentence boundaries for natural breaks -- **Enhanced overlap configuration support**: When `chunk_overlap=0`, chunks have no overlap; when configured, up to 25% of chunk content can overlap for better context continuity -- **Added comprehensive overlap testing**: New test suite verifies overlap works correctly across different configurations and content types - -## Version 0.4.11 - July 10, 2025 - -### ๐Ÿ› File Processing & Configuration Bug Fixes - -#### File Processing & Chunking - -- **Fixed file size detection limits**: Increased default file size limits to handle larger documents (docx, xlsx files up to 5MB) -- **Resolved MarkdownChunkingStrategy issues**: Fixed chunking strategy to respect `chunk_size` configuration instead of only splitting on H1 headers -- **Fixed unique chunk ID generation**: Resolved issue where chunks from same document had identical IDs, causing overwrites in Qdrant storage -- **Enhanced chunk count management**: Replaced hard-coded chunk limits with configurable `max_chunks_per_document` setting - -#### Configuration Management - -- **Improved chunking configuration**: Added `max_chunks_per_document` parameter for better control over document processing -- **Cleaned up redundant settings**: Removed conflicting `max_document_size` parameter to maintain clean separation between file size and chunk count limits -- **Enhanced error messages**: Added actionable configuration advice when chunk limits are reached - -#### Processing Pipeline - -- **Fixed content truncation**: Eliminated "maximum chunks per section limit" warnings by making limits dynamic based on user configuration -- **Improved chunk estimation**: Added better user guidance for optimal chunk count configuration -- **Enhanced section handling**: Made section limits dynamic (50% of max_chunks_per_document) - -## Version 0.4.10 - June 18, 2025 - -### ๐Ÿ› Windows & File Processing Bug Fixes - -#### Windows Compatibility & Logging - -- **Fixed duplicate debug logging**: Resolved `[DEBUG] [DEBUG]` duplicate level tags in both console and file output -- **Enhanced logging verbosity control**: Added filtering for noisy third-party library debug messages (chardet, pdfminer, httpx) -- **Improved Windows path formatting**: Fixed mixed path separators in log output for consistent cross-platform display -- **Complete path normalization**: Fixed remaining instances of backslashes in Windows file paths in FileDetector and file processor logging -- **Fixed .txt file processing**: Resolved issue where `.txt` files were excluded from ingestion when `file_types: []` was empty - -#### File Processing - -- **LocalFile connector**: `.txt` files now properly processed by default text strategy when no specific file types configured -- **Git connector**: Consistent file type processing logic across all connectors -- **Path normalization**: All file paths in logs now use forward slashes for consistency - -## Version 0.4.9 - June 18, 2025 - -### Bug fix - -- **Issue when deleting a deleted document** : missing content_type="md" field to the `_create_deleted_document method` - -## Version 0.4.8 - June 17, 2025 - -### ๐ŸชŸ Windows Compatibility Fixes - -- **LocalFile Connector**: Fixed Windows file URL parsing (`file:///C:/Users/...` now works correctly) -- **Git Connector**: Fixed document URL generation with Windows paths (backslashes โ†’ forward slashes) -- **File Conversion**: Cross-platform timeout handling (threading on Windows, signals on Unix) -- **MarkItDown Integration**: Fixed Windows signal compatibility (`signal.SIGALRM` errors resolved) -- **Console Output**: Enhanced emoji handling for clean Windows display -- **Logging**: Suppressed verbose SQLite logs and fixed duplicate log level display (`[DEBUG] [DEBUG]` โ†’ `[DEBUG]`) -- **Testing**: Added 38 Windows compatibility test cases - -## Version 0.4.7 - June 9, 2025 - -### ๐Ÿงน Test Suite Improvements - -### ๐Ÿ› CLI & User Experience Bug Fixes - -#### CLI and User Experience - -- **Version check improvements**: Fixed upgrade instructions to include `qdrant-loader-mcp-server` package in version check output -- **Branch display logic**: Fixed branch display logic to default to 'main' when branch is unknown in coverage reports -- **Error handling**: Improved error handling in CLI for invalid input scenarios - -### ๐Ÿ“š Documentation - -- **Configuration template**: Enhanced configuration template with detailed comments for better user guidance -- **PublicDocs connector**: Improved logging in PublicDocsConnector for better debugging - -### ๐Ÿ”ง Release Process Enhancement - -- **Release notes validation**: Updated release script to automatically check that `RELEASE_NOTES.md` has been updated for new versions before allowing releases -- **Improved release safety**: Enhanced pre-release checks to ensure documentation consistency - -## Version 0.4.6 - June 3, 2025 - -### ๐Ÿ”” User Experience Enhancements - -#### Version Notifications - -- **Automatic update notifications**: CLI now checks for new package versions and notifies users when updates are available -- **Non-intrusive background checks**: Version checking runs in background without affecting CLI performance - -## Version 0.4.5 - June 3, 2025 - -### ๐Ÿš€ Performance Improvements - -#### CLI Startup Optimization - -- **CLI startup performance**: Reduced startup time by 60-67% for basic commands ([#24](https://github.com/martin-papy/qdrant-loader/issues/24)) - - `--help`: ~6.8s โ†’ 2.33s (**66% improvement**) - - `--version`: ~6.3s โ†’ 2.57s (**59% improvement**) -- **Lazy loading implementation**: Heavy modules now load only when needed (96-97% import time reduction) -- **Fixed version detection**: Replaced custom parsing with `importlib.metadata.version()` - works in all environments -- **Resolved circular imports**: Eliminated `config` โ†’ `connectors` โ†’ `config` dependency cycle - -### ๐ŸŽจ User Experience Enhancements - -#### Excel File Processing - -- **Warning capture system**: Intercepts openpyxl warnings during Excel conversion -- **Structured logging**: Routes warnings through qdrant-loader logging system for visual consistency -- **Smart detection**: Captures "Data Validation" and "Conditional Formatting" warnings with context -- **Summary reporting**: Provides comprehensive summary of unsupported Excel features - -## Version 0.4.4 - June 3, 2025 - -### ๐ŸŽ‰ Major Improvements - -#### File Conversion & Processing Overhaul - -##### Fixed Critical File Conversion Issues - -- **Fixed file conversion initialization**: Resolved issue where file conversion was not working due to missing `set_file_conversion_config` calls in the pipeline ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- **Enhanced strategy selection**: Converted files (Excel, Word, PDF, etc.) now correctly use `MarkdownChunkingStrategy` instead of `DefaultChunkingStrategy` ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- **Improved NLP processing**: Converted files now have full NLP processing enabled instead of being skipped with `content_type_inappropriate` ([7de3526](https://github.com/martin-papy/qdrant-loader/commit/7de3526)) - -##### Enhanced File Processing Pipeline - -- Added proper file conversion configuration initialization in source processors ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Implemented automatic strategy selection based on conversion status ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Fixed metadata propagation for converted files ([7de3526](https://github.com/martin-papy/qdrant-loader/commit/7de3526)) - -#### Chunking Strategy โ€” Infinite Loop & Safety Fixes - -##### Resolved Infinite Loop Issues - -- **Fixed MarkdownChunkingStrategy infinite loops**: Resolved critical issue where documents with very long words would create infinite loops, hitting the 1000 chunk limit ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- **Added safety limits**: Implemented `MAX_CHUNKS_PER_SECTION = 100` and `MAX_CHUNKS_PER_DOCUMENT = 500` limits ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- **Enhanced error handling**: Added proper handling for words longer than `max_size` by truncating them with warnings ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) - -##### Improved Chunking Logic - -- Added safety checks to prevent infinite loops in `_split_large_section` method ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Enhanced logging for debugging chunking issues ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Added warnings when chunking limits are reached ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) - -#### Workspace Management - -##### Better Log Organization - -- **Fixed workspace logs location**: Logs are now stored in `workspace_path/logs/qdrant-loader.log` instead of cluttering the workspace root ([589ae4b](https://github.com/martin-papy/qdrant-loader/commit/589ae4b)) -- **Enhanced workspace structure**: Added automatic creation of logs directory ([589ae4b](https://github.com/martin-papy/qdrant-loader/commit/589ae4b)) -- **Updated documentation**: Reflected new log structure in workspace mode documentation ([589ae4b](https://github.com/martin-papy/qdrant-loader/commit/589ae4b)) - -#### Resource Management & Stability - -##### Fixed Pipeline Hanging Issues - -- **Resolved ResourceManager cleanup**: Fixed issue where normal cleanup was setting shutdown events, causing workers to exit prematurely ([4844abf](https://github.com/martin-papy/qdrant-loader/commit/4844abf)) -- **Enhanced signal handling**: Distinguished between normal cleanup and signal-based shutdown ([4844abf](https://github.com/martin-papy/qdrant-loader/commit/4844abf)) -- **Improved graceful shutdown**: Workers now properly complete processing before shutdown ([4844abf](https://github.com/martin-papy/qdrant-loader/commit/4844abf)) - -##### Performance Optimizations - -- Increased `MAX_CHUNKS_TO_PROCESS` from 100 to 1000 chunks to accommodate larger documents ([1bfe550](https://github.com/martin-papy/qdrant-loader/commit/1bfe550)) -- Better handling of large documents (up to ~1000KB text limit per document) ([1bfe550](https://github.com/martin-papy/qdrant-loader/commit/1bfe550)) -- Improved change detection for incremental updates ([5408db9](https://github.com/martin-papy/qdrant-loader/commit/5408db9)) - -### ๐Ÿ”ง Technical Improvements - -#### Code Quality & Testing - -##### Enhanced Test Coverage - -- Added comprehensive tests for converted file NLP processing ([7de3526](https://github.com/martin-papy/qdrant-loader/commit/7de3526)) -- Added tests for chunking strategy selection ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Enhanced error handling test coverage ([4844abf](https://github.com/martin-papy/qdrant-loader/commit/4844abf)) -- All existing functionality preserved with 100% test pass rate - -##### Architecture Improvements - -- Enhanced base connector class with proper file conversion support ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Improved factory pattern for pipeline component creation ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Better separation of concerns in source processing ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) - -#### Configuration & Setup - -##### Improved File Conversion Support - -- Enhanced connector initialization with file conversion configuration ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Better error handling for conversion failures ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Improved fallback mechanisms for unsupported file types ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) - -### ๐Ÿ› Critical Pipeline & Processing Bug Fixes - -#### Critical Fixes - -- **File conversion not working**: Fixed missing initialization causing 0 documents to be processed ([5408db9](https://github.com/martin-papy/qdrant-loader/commit/5408db9)) -- **Infinite chunking loops**: Resolved MarkdownChunkingStrategy creating thousands of chunks for simple documents ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- **Pipeline hanging**: Fixed ResourceManager causing workers to exit prematurely ([4844abf](https://github.com/martin-papy/qdrant-loader/commit/4844abf)) -- **NLP processing skipped**: Fixed converted files being inappropriately skipped for NLP processing ([7de3526](https://github.com/martin-papy/qdrant-loader/commit/7de3526)) - -#### Minor Fixes - -- Fixed workspace log file location ([589ae4b](https://github.com/martin-papy/qdrant-loader/commit/589ae4b)) -- Improved error messages and logging ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) -- Enhanced metadata handling for converted files ([7de3526](https://github.com/martin-papy/qdrant-loader/commit/7de3526)) -- Better handling of edge cases in chunking strategies ([9d16b8d](https://github.com/martin-papy/qdrant-loader/commit/9d16b8d)) - -### ๐Ÿ”„ Migration Notes - -**For Existing Users:** - -- Logs will now be created in `workspace/logs/` directory instead of workspace root -- Converted files will now be processed with enhanced NLP capabilities -- Large documents will be chunked more efficiently with higher limits -- No breaking changes to existing configurations - -**Performance Impact:** - -- Improved processing speed for converted files -- Better memory usage with enhanced chunking limits -- More stable pipeline execution with proper resource management diff --git a/docs/TOC.md b/docs/TOC.md index 8621a4005..df3bbba26 100644 --- a/docs/TOC.md +++ b/docs/TOC.md @@ -1,12 +1,14 @@ # Documentation Table of Contents (IA v2) 1. **Getting Started** + - What is QDrant Loader? (`docs/getting-started/what-is-qdrant-loader.md`) - Quick Start (`docs/getting-started/quick-start.md`) - Installation (`docs/getting-started/installation.md`) - Basic Configuration (`docs/getting-started/basic-configuration.md`) 1. **User Guides** + - Data Sources (`docs/users/detailed-guides/data-sources/`) - File Conversion (`docs/users/detailed-guides/file-conversion/`) - MCP Server (`docs/users/detailed-guides/mcp-server/`) @@ -14,10 +16,12 @@ - Troubleshooting (`docs/users/troubleshooting/`) 1. **Reference** + - CLI Reference (`docs/users/cli-reference/`) - Configuration Reference (`docs/users/configuration/`) 1. **Developer Docs** + - Architecture (`docs/developers/architecture/`) - Extending (`docs/developers/extending/`) - Testing (`docs/developers/testing/`) @@ -27,7 +31,7 @@ 1. **Project & Community** - Contributing (`CONTRIBUTING.md`) - - Release Notes (`RELEASE_NOTES.md`) + - Changelog (`CHANGELOG.md`) --- diff --git a/docs/developers/README.md b/docs/developers/README.md index 8d141f98c..393f3d6e3 100644 --- a/docs/developers/README.md +++ b/docs/developers/README.md @@ -14,6 +14,10 @@ Welcome to the QDrant Loader developer documentation! This guide provides everyt - **[Testing Guide](./testing/)** - Testing strategies, frameworks, and best practices - **[Deployment Guide](./deployment/)** - Production deployment, containerization, and CI/CD +### Contributing + +- **[Best Practices](./contributing/)** - Pythonic patterns, AI/RAG guidelines, and PR review checklist + ### Documentation - **[Documentation Maintenance](./documentation/)** - Maintaining and updating documentation diff --git a/docs/developers/contributing/README.md b/docs/developers/contributing/README.md new file mode 100644 index 000000000..9e69ef2da --- /dev/null +++ b/docs/developers/contributing/README.md @@ -0,0 +1,85 @@ +# Best Practices for qdrant-loader + +> **Author:** nguyen.vu@cbtw.tech + +This guide outlines Pythonic patterns and AI engineering best practices for the qdrant-loader project. + +## 1. Pythonic Code Standards + +### 1.1. Avoid Class-Level Anti-Patterns + +- **No Redundant `__new__`**: Do not override `__new__` unless you are implementing a strict Singleton or working with immutable types. Standard service initialization belongs in `__init__`. +- **Explicit over Implicit**: Avoid "monkey-patching" (e.g., overwriting a local class with one from another package via try/except). Use explicit dependencies. + +### 1.2. Type Hinting & Structural Subtyping + +- **Protocols over Base Classes**: Use `typing.Protocol` for defining interfaces (e.g., `ChunkingStrategy`). This follows the "Go-style" duck typing which is more flexible for a plugin-based architecture. +- **Pydantic for Data/Config**: Use Pydantic models for all data structures and configuration. Avoid passing around raw dictionaries. + +### 1.3. Dependency Injection (DI) + +**Granular Injection**: Do not pass the entire `Settings` or `GlobalConfig` object to every service. Pass only the specific sub-config or primitives the service needs. This makes unit testing significantly easier. + +```python +# BAD: Monolithic config passing +def __init__(self, settings: Settings): + self.chunk_size = settings.global_config.chunking.chunk_size + +# GOOD: Granular injection +def __init__(self, chunk_size: int, overlap: int): + self.chunk_size = chunk_size +``` + +## 2. AI & RAG Best Practices + +### 2.1. Metadata Hygiene + +- **Strict Schemas**: Every document ingested must have a consistent metadata schema. If a new field is added, update the Document model and relevant extractors. +- **Provenance**: Always preserve the "source of truth" (URL, file path, line number) in metadata to enable high-quality citations in the RAG retrieval phase. + +### 2.2. Embedding Drift Management + +- **Versioned Collections**: If you change the embedding model (e.g., moving from OpenAI `text-embedding-3-small` to `3-large`), you must create a new Qdrant collection. Vectors are not compatible across different models or dimensions. + +### 2.3. Evaluation-First Development + +**Ragas/G-Eval**: Before merging a change to the `HybridSearchEngine` or `Reranker`, run an evaluation suite. Aim for improvements in: + +- **Faithfulness**: The answer is derived only from context. +- **Answer Relevance**: The answer directly addresses the query. +- **Context Precision**: The retrieved documents are relevant. + +## 3. Architecture & Monorepo Management + +### 3.1. Package Isolation + +- **Core is Sacred**: `qdrant-loader-core` should have zero dependencies on `qdrant-loader` or `qdrant-loader-mcp-server`. It is the foundational abstraction layer. +- **Circular Dependencies**: Use `from __future__ import annotations` and local imports inside methods if absolutely necessary to break circular loops, but prefer refactoring to a shared utility. + +### 3.2. Logging & Observability + +- **Centralized Logging**: Use the `LoggingConfig` from core. Do not initialize standard `logging.getLogger(__name__)` manually if you need structured logs. +- **Traceability**: Every MCP request should have a unique `request_id` passed through the search engine to help debug multi-step retrieval chains. + +## 4. Testing Standards + +### 4.1. Mocking External APIs + +- **No Real LLM Calls in Unit Tests**: Always use `unittest.mock` or `pytest-mock` to stub LLM providers. Use the `_NoopProvider` in core as a base for mocks. +- **VCR.py / Pytest-Recording**: For integration tests, use VCR-style recording to capture and replay real Qdrant/LLM interactions to ensure deterministic test results. + +### 4.2. Algorithmic Validation + +- **Similarity Thresholds**: When testing search tools, include cases that specifically check the "boundary" of your similarity thresholds (e.g., verifying a 0.69 score is excluded if the threshold is 0.7). + +## 5. Summary Checklist + +For reviewer to recheck everytime a PR comes up: + +- [ ] Is the PR code type-hinted and linted? +- [ ] Did it avoid redundant class overrides like `__new__`? +- [ ] Are services using granular dependency injection? +- [ ] If the commiter modified the retrieval logic, did they run a RAG evaluation? +- [ ] Are PR's new logs structured and redact-protected? + +--- diff --git a/docs/users/detailed-guides/mcp-server/search-capabilities.md b/docs/users/detailed-guides/mcp-server/search-capabilities.md index 071e98bbd..f40b63092 100644 --- a/docs/users/detailed-guides/mcp-server/search-capabilities.md +++ b/docs/users/detailed-guides/mcp-server/search-capabilities.md @@ -104,8 +104,8 @@ Results: { "name": "search", "parameters": { - "query": "string", // Natural language query - be conversational! - "limit": 10, // Results to return (default: 5) + "query": "string", // Natural language query - be conversational! + "limit": 10, // Results to return (default: 5) "source_types": ["git", "confluence", "jira", "documentation", "localfile"], "project_ids": ["project1", "project2"] } @@ -168,14 +168,15 @@ Recommendation: Create under Security section for consistency { "name": "hierarchy_search", "parameters": { - "query": "string", // Search query - "limit": 10, // Number of results (default: 10) - "organize_by_hierarchy": false, // Group results by structure - "hierarchy_filter": { // Hierarchy-specific filters - "depth": 3, // Filter by hierarchy depth - "has_children": true, // Filter by whether pages have children - "parent_title": "API Documentation", // Filter by parent page - "root_only": false // Show only root pages + "query": "string", // Search query + "limit": 10, // Number of results (default: 10) + "organize_by_hierarchy": false, // Group results by structure + "hierarchy_filter": { + // Hierarchy-specific filters + "depth": 3, // Filter by hierarchy depth + "has_children": true, // Filter by whether pages have children + "parent_title": "API Documentation", // Filter by parent page + "root_only": false // Show only root pages } } } @@ -234,15 +235,16 @@ Content Analysis Results: { "name": "attachment_search", "parameters": { - "query": "string", // Search query - "limit": 10, // Number of results - "include_parent_context": true, // Include parent document info - "attachment_filter": { // Attachment-specific filters - "file_type": "pdf", // Filter by file type - "file_size_min": 1024, // Minimum file size in bytes - "file_size_max": 10485760, // Maximum file size in bytes - "attachments_only": true, // Show only attachments - "author": "john.doe", // Filter by author + "query": "string", // Search query + "limit": 10, // Number of results + "include_parent_context": true, // Include parent document info + "attachment_filter": { + // Attachment-specific filters + "file_type": "pdf", // Filter by file type + "file_size_min": 1024, // Minimum file size in bytes + "file_size_max": 10485760, // Maximum file size in bytes + "attachments_only": true, // Show only attachments + "author": "john.doe", // Filter by author "parent_document_title": "API Documentation" } } @@ -260,7 +262,7 @@ Content Analysis Results: "name": "analyze_document_relationships", "parameters": { "query": "search query to get documents for analysis", - "limit": 15, // Maximum documents to analyze + "limit": 15, // Maximum documents to analyze "source_types": ["confluence", "git"], "project_ids": ["project1"] } @@ -301,8 +303,11 @@ Relationship Analysis: "parameters": { "target_query": "target document to find similarities for", "comparison_query": "documents to compare against", - "similarity_metrics": ["entity_overlap", "semantic_similarity"], - "max_similar": 5 + "similarity_metrics": ["entity_overlap", "semantic_similarity"], // Optional + "similarity_threshold": 0.5, // Optional: Minimum similarity score (0.0-1.0, default: 0.7) + "source_types": ["confluence", "git"], // Optional + "project_ids": ["project1"], // Optional + "max_similar": 5 // Optional } } ``` @@ -413,7 +418,7 @@ Complementary Content Found: "name": "cluster_documents", "parameters": { "query": "search query to get documents for clustering", - "strategy": "mixed_features", // clustering strategy + "strategy": "mixed_features", // clustering strategy "max_clusters": 10, "min_cluster_size": 2, "limit": 25, @@ -462,10 +467,10 @@ Get detailed information and context for a specific document, including metadata { "name": "expand_document", "arguments": { - "document_id": "string", // Required: Document identifier - "include_relationships": true, // Include related documents - "include_metadata": true, // Include document metadata - "include_content_summary": true // Include content analysis + "document_id": "string", // Required: Document identifier + "include_relationships": true, // Include related documents + "include_metadata": true, // Include document metadata + "include_content_summary": true // Include content analysis } } ``` @@ -479,7 +484,7 @@ Query: Get detailed information about document "api-auth-guide" ๐Ÿ“„ API Authentication Guide โ”œโ”€โ”€ ๐Ÿ“Š Metadata: Created 2024-01-15, Updated 2024-03-10 โ”œโ”€โ”€ ๐Ÿท๏ธ Tags: authentication, security, API, OAuth -โ”œโ”€โ”€ ๐Ÿ”— Related Documents: +โ”œโ”€โ”€ ๐Ÿ”— Related Documents: โ”‚ โ”œโ”€โ”€ OAuth Implementation Guide โ”‚ โ”œโ”€โ”€ Security Best Practices โ”‚ โ””โ”€โ”€ API Rate Limiting @@ -500,10 +505,10 @@ Explore document clusters with detailed analysis, showing how documents are grou { "name": "expand_cluster", "arguments": { - "cluster_id": "string", // Required: Cluster identifier - "include_document_details": true, // Include individual document info - "include_cluster_metrics": true, // Include clustering statistics - "max_documents": 20 // Maximum documents to show in cluster + "cluster_id": "string", // Required: Cluster identifier + "include_document_details": true, // Include individual document info + "include_cluster_metrics": true, // Include clustering statistics + "max_documents": 20 // Maximum documents to show in cluster } } ``` @@ -625,6 +630,7 @@ MCP_DISABLE_CONSOLE_LOGGING=true # Recommended for development tools #### For Large Knowledge Bases 1. **Optimize Search Parameters** + - Use appropriate `limit` values for your needs - Filter by `source_types` or `project_ids` when possible - Use specific search tools for targeted queries diff --git a/docs/users/workflows/cicd-integration-workflow.md b/docs/users/workflows/cicd-integration-workflow.md index 570f640f6..65d06358d 100644 --- a/docs/users/workflows/cicd-integration-workflow.md +++ b/docs/users/workflows/cicd-integration-workflow.md @@ -85,9 +85,9 @@ name: Test and Coverage on: push: - branches: [ main, develop, feature/*, bugfix/*, release/* ] + branches: [main, develop, feature/*, bugfix/*, release/*] pull_request: - branches: [ main, develop, feature/*, bugfix/*, release/* ] + branches: [main, develop, feature/*, bugfix/*, release/*] permissions: contents: read @@ -106,7 +106,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Install system dependencies run: | # Install ffmpeg for MarkItDown audio processing @@ -150,23 +150,23 @@ name: Documentation Website on: push: - branches: [ main ] + branches: [main] paths: - - 'docs/**' - - 'README.md' - - 'RELEASE_NOTES.md' - - 'packages/*/README.md' - - 'website/**' - - '.github/workflows/docs.yml' + - "docs/**" + - "README.md" + - "CHANGELOG.md" + - "packages/*/README.md" + - "website/**" + - ".github/workflows/docs.yml" workflow_run: workflows: ["Test and Coverage"] types: - completed - branches: [ main ] + branches: [main] workflow_dispatch: inputs: force_deploy: - description: 'Force deployment even without recent test artifacts' + description: "Force deployment even without recent test artifacts" required: false default: false type: boolean @@ -190,7 +190,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/packages/qdrant-loader-core/pyproject.toml b/packages/qdrant-loader-core/pyproject.toml index 6083460be..0c77c3e0b 100644 --- a/packages/qdrant-loader-core/pyproject.toml +++ b/packages/qdrant-loader-core/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "qdrant-loader-core" -version = "0.7.3" +version = "0.7.6" description = "Shared core for provider-agnostic LLM support and configuration mapping for qdrant-loader ecosystem" readme = "README.md" requires-python = ">=3.12" diff --git a/packages/qdrant-loader-core/src/qdrant_loader_core/logging.py b/packages/qdrant-loader-core/src/qdrant_loader_core/logging.py index fc153772b..2b04bad5d 100644 --- a/packages/qdrant-loader-core/src/qdrant_loader_core/logging.py +++ b/packages/qdrant-loader-core/src/qdrant_loader_core/logging.py @@ -10,12 +10,13 @@ import logging import os -import re -from typing import Any import structlog from structlog.stdlib import LoggerFactory +from .logging_filters import ApplicationFilter, QdrantVersionFilter, RedactionFilter +from .logging_processors import CleanFormatter, redact_processor + try: # ExtraAdder is available in structlog >= 20 from structlog.stdlib import ExtraAdder # type: ignore @@ -23,170 +24,6 @@ ExtraAdder = None # type: ignore -class QdrantVersionFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - try: - return "version check" not in record.getMessage().lower() - except Exception: - return True - - -class ApplicationFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - # Allow all logs by default; app packages may add their own filters - return True - - -class RedactionFilter(logging.Filter): - """Redacts obvious secrets from stdlib log records.""" - - # Heuristics for tokens/keys in plain strings - TOKEN_PATTERNS = [ - re.compile(r"sk-[A-Za-z0-9_\-]{6,}"), - re.compile(r"tok-[A-Za-z0-9_\-]{6,}"), - re.compile( - r"(?i)(api_key|authorization|token|access_token|secret|password)\s*[:=]\s*([^\s]+)" - ), - re.compile(r"Bearer\s+[A-Za-z0-9_\-\.]+"), - ] - - # Keys commonly used for secrets in structlog event dictionaries - SENSITIVE_KEYS = { - "api_key", - "llm_api_key", - "authorization", - "Authorization", - "token", - "access_token", - "secret", - "password", - } - - def _redact_text(self, text: str) -> str: - def mask(m: re.Match[str]) -> str: - s = m.group(0) - if len(s) <= 8: - return "***REDACTED***" - return s[:2] + "***REDACTED***" + s[-2:] - - redacted = text - for pat in self.TOKEN_PATTERNS: - redacted = pat.sub(mask, redacted) - return redacted - - def filter(self, record: logging.LogRecord) -> bool: - try: - redaction_detected = False - - # Args may contain secrets; best-effort mask strings and detect changes - if isinstance(record.args, tuple): - new_args = [] - for a in record.args: - if isinstance(a, str): - red_a = self._redact_text(a) - if red_a != a: - redaction_detected = True - new_args.append(red_a) - else: - new_args.append(a) - record.args = tuple(new_args) - - # Redact raw message only when it contains no formatting placeholders - # to avoid interfering with %-style or {}-style formatting - if isinstance(record.msg, str): - try: - has_placeholders = ("%" in record.msg) or ("{" in record.msg) - except Exception: - has_placeholders = True - if not has_placeholders: - red_msg = self._redact_text(record.msg) - if red_msg != record.msg: - record.msg = red_msg - redaction_detected = True - - # If structlog extras contain sensitive keys, mark as redacted - try: - if any( - (k in self.SENSITIVE_KEYS and bool(record.__dict__.get(k))) - for k in record.__dict__.keys() - ): - redaction_detected = True - except Exception: - pass - - # Ensure a visible redaction marker appears in the captured message - if redaction_detected: - try: - if ( - isinstance(record.msg, str) - and "***REDACTED***" not in record.msg - ): - # Append a marker in a way that won't interfere with %-formatting - record.msg = f"{record.msg} ***REDACTED***" - except Exception: - pass - except Exception: - pass - return True - - -class CleanFormatter(logging.Formatter): - """Formatter that removes ANSI color codes for clean file output.""" - - def format(self, record: logging.LogRecord) -> str: - message = super().format(record) - try: - ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") - return ansi_escape.sub("", message) - except Exception: - return message - - -def _redact_processor( - logger: Any, method_name: str, event_dict: dict[str, Any] -) -> dict[str, Any]: - """Structlog processor to redact sensitive fields in event_dict.""" - sensitive_keys = { - "api_key", - "llm_api_key", - "authorization", - "Authorization", - "token", - "access_token", - "secret", - "password", - } - - def mask(value: str) -> str: - try: - if not isinstance(value, str) or not value: - return "***REDACTED***" - if len(value) <= 8: - return "***REDACTED***" - return value[:2] + "***REDACTED***" + value[-2:] - except Exception: - return "***REDACTED***" - - def deep_redact(obj: Any) -> Any: - try: - if isinstance(obj, dict): - return { - k: ( - mask(v) - if k in sensitive_keys and isinstance(v, str) - else deep_redact(v) - ) - for k, v in obj.items() - } - if isinstance(obj, list): - return [deep_redact(i) for i in obj] - return obj - except Exception: - return obj - - return deep_redact(event_dict) - - class LoggingConfig: """Core logging setup with structlog + stdlib redaction and filters.""" @@ -339,7 +176,7 @@ def setup( structlog.stdlib.add_logger_name, structlog.stdlib.add_log_level, structlog.processors.TimeStamper(fmt=ts_fmt), - _redact_processor, + redact_processor, final_renderer, ], wrapper_class=structlog.make_filtering_bound_logger(numeric_level), @@ -357,13 +194,66 @@ def get_logger(cls, name: str | None = None) -> structlog.BoundLogger: return structlog.get_logger(name) @classmethod - def reconfigure(cls, *, file: str | None = None) -> None: - """Lightweight reconfiguration for file destination. + def reconfigure(cls, *, file: str | None = None, level: str | None = None) -> None: + """Lightweight reconfiguration for file destination and optionally log level. Replaces only the file handler while keeping console handlers and - structlog processors intact. + structlog processors intact. Optionally updates the log level. + + Args: + file: Path to log file (optional) + level: New log level (optional, e.g., "DEBUG", "INFO") """ root_logger = logging.getLogger() + + # Update log level if provided + if level is not None: + try: + numeric_level = getattr(logging, level.upper()) + root_logger.setLevel(numeric_level) + + # Update structlog wrapper to use new level + if cls._current_config is not None: + ( + _, + fmt, + _, + clean_output, + suppress_qdrant_warnings, + disable_console, + ) = cls._current_config + + # Choose timestamp format and final renderer + if clean_output and fmt == "console": + ts_fmt = "%H:%M:%S" + final_renderer = structlog.dev.ConsoleRenderer(colors=True) + else: + ts_fmt = "iso" + final_renderer = ( + structlog.processors.JSONRenderer() + if fmt == "json" + else structlog.dev.ConsoleRenderer(colors=True) + ) + + # Reconfigure structlog with new level + structlog.configure( + processors=[ + structlog.stdlib.filter_by_level, + structlog.stdlib.add_logger_name, + structlog.stdlib.add_log_level, + structlog.processors.TimeStamper(fmt=ts_fmt), + redact_processor, + final_renderer, + ], + wrapper_class=structlog.make_filtering_bound_logger( + numeric_level + ), + logger_factory=LoggerFactory(), + cache_logger_on_first_use=False, + ) + except AttributeError: + raise ValueError(f"Invalid log level: {level}") from None + # Remove existing file handler if present if cls._file_handler is not None: try: @@ -388,11 +278,17 @@ def reconfigure(cls, *, file: str | None = None) -> None: # Update current config tuple if available if cls._current_config is not None: - level, fmt, _, clean_output, suppress_qdrant_warnings, disable_console = ( - cls._current_config - ) + ( + old_level, + fmt, + _, + clean_output, + suppress_qdrant_warnings, + disable_console, + ) = cls._current_config + new_level = level.upper() if level is not None else old_level cls._current_config = ( - level, + new_level, fmt, file, clean_output, diff --git a/packages/qdrant-loader-core/src/qdrant_loader_core/logging_filters.py b/packages/qdrant-loader-core/src/qdrant_loader_core/logging_filters.py new file mode 100644 index 000000000..ffe0c9bae --- /dev/null +++ b/packages/qdrant-loader-core/src/qdrant_loader_core/logging_filters.py @@ -0,0 +1,113 @@ +"""Logging filters for redaction and noise suppression.""" + +from __future__ import annotations + +import logging +import re + + +class QdrantVersionFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + try: + return "version check" not in record.getMessage().lower() + except Exception: + return True + + +class ApplicationFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + # Allow all logs by default; app packages may add their own filters + return True + + +class RedactionFilter(logging.Filter): + """Redacts obvious secrets from stdlib log records.""" + + # Heuristics for tokens/keys in plain strings + TOKEN_PATTERNS = [ + re.compile(r"sk-[A-Za-z0-9_\-]{6,}"), + re.compile(r"tok-[A-Za-z0-9_\-]{6,}"), + re.compile( + r"(?i)(api_key|authorization|token|access_token|secret|password)\s*[:=]\s*([^\s]+)" + ), + re.compile(r"Bearer\s+[A-Za-z0-9_\-\.]+"), + ] + + # Keys commonly used for secrets in structlog event dictionaries + SENSITIVE_KEYS = { + "api_key", + "llm_api_key", + "authorization", + "Authorization", + "token", + "access_token", + "secret", + "password", + } + + def _redact_text(self, text: str) -> str: + def mask(m: re.Match[str]) -> str: + s = m.group(0) + if len(s) <= 8: + return "***REDACTED***" + return s[:2] + "***REDACTED***" + s[-2:] + + redacted = text + for pat in self.TOKEN_PATTERNS: + redacted = pat.sub(mask, redacted) + return redacted + + def filter(self, record: logging.LogRecord) -> bool: + try: + redaction_detected = False + + # Args may contain secrets; best-effort mask strings and detect changes + if isinstance(record.args, tuple): + new_args = [] + for a in record.args: + if isinstance(a, str): + red_a = self._redact_text(a) + if red_a != a: + redaction_detected = True + new_args.append(red_a) + else: + new_args.append(a) + record.args = tuple(new_args) + + # Redact raw message only when it contains no formatting placeholders + # to avoid interfering with %-style or {}-style formatting + if isinstance(record.msg, str): + try: + has_placeholders = ("%" in record.msg) or ("{" in record.msg) + except Exception: + has_placeholders = True + if not has_placeholders: + red_msg = self._redact_text(record.msg) + if red_msg != record.msg: + record.msg = red_msg + redaction_detected = True + + # If structlog extras contain sensitive keys, mark as redacted + try: + if any( + (k in self.SENSITIVE_KEYS and bool(record.__dict__.get(k))) + for k in record.__dict__.keys() + ): + redaction_detected = True + except Exception: + pass + + # Ensure a visible redaction marker appears in the captured message + if redaction_detected: + try: + if ( + isinstance(record.msg, str) + and "***REDACTED***" not in record.msg + ): + # Append a marker in a way that won't interfere with %-formatting + record.msg = f"{record.msg} ***REDACTED***" + except Exception: + pass + except Exception: + pass + return True diff --git a/packages/qdrant-loader-core/src/qdrant_loader_core/logging_processors.py b/packages/qdrant-loader-core/src/qdrant_loader_core/logging_processors.py new file mode 100644 index 000000000..c84a447c1 --- /dev/null +++ b/packages/qdrant-loader-core/src/qdrant_loader_core/logging_processors.py @@ -0,0 +1,64 @@ +"""Logging processors and formatters for structlog.""" + +from __future__ import annotations + +import logging +import re +from typing import Any + + +class CleanFormatter(logging.Formatter): + """Formatter that removes ANSI color codes for clean file output.""" + + def format(self, record: logging.LogRecord) -> str: + message = super().format(record) + try: + ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])") + return ansi_escape.sub("", message) + except Exception: + return message + + +def redact_processor( + logger: Any, method_name: str, event_dict: dict[str, Any] +) -> dict[str, Any]: + """Structlog processor to redact sensitive fields in event_dict.""" + sensitive_keys = { + "api_key", + "llm_api_key", + "authorization", + "Authorization", + "token", + "access_token", + "secret", + "password", + } + + def mask(value: str) -> str: + try: + if not isinstance(value, str) or not value: + return "***REDACTED***" + if len(value) <= 8: + return "***REDACTED***" + return value[:2] + "***REDACTED***" + value[-2:] + except Exception: + return "***REDACTED***" + + def deep_redact(obj: Any) -> Any: + try: + if isinstance(obj, dict): + return { + k: ( + mask(v) + if k in sensitive_keys and isinstance(v, str) + else deep_redact(v) + ) + for k, v in obj.items() + } + if isinstance(obj, list): + return [deep_redact(i) for i in obj] + return obj + except Exception: + return obj + + return deep_redact(event_dict) diff --git a/packages/qdrant-loader-core/tests/test_logging_core.py b/packages/qdrant-loader-core/tests/test_logging_core.py index 3faafb74b..0425f3cf8 100644 --- a/packages/qdrant-loader-core/tests/test_logging_core.py +++ b/packages/qdrant-loader-core/tests/test_logging_core.py @@ -89,7 +89,7 @@ def test_clean_formatter_strips_ansi(tmp_path): def test_redact_processor_masks_nested_fields(): logging_mod = import_module("qdrant_loader_core.logging") - redact = logging_mod._redact_processor + redact = logging_mod.redact_processor event = { "api_key": "sk-ABCDEFGHIJKLMNOP", diff --git a/packages/qdrant-loader-core/tests/unit/quality/test_module_sizes.py b/packages/qdrant-loader-core/tests/unit/quality/test_module_sizes.py index 25302b484..89bca919c 100644 --- a/packages/qdrant-loader-core/tests/unit/quality/test_module_sizes.py +++ b/packages/qdrant-loader-core/tests/unit/quality/test_module_sizes.py @@ -10,7 +10,7 @@ ] EXEMPTIONS = { - "logging.py": 402, # Core logging infrastructure with structured logging support + # add exemptions if needed later } diff --git a/packages/qdrant-loader-mcp-server/pyproject.toml b/packages/qdrant-loader-mcp-server/pyproject.toml index 5cc0d0d2d..4fc71bc62 100644 --- a/packages/qdrant-loader-mcp-server/pyproject.toml +++ b/packages/qdrant-loader-mcp-server/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "qdrant-loader-mcp-server" -version = "0.7.3" +version = "0.7.6" description = "A Model Context Protocol (MCP) server that provides RAG capabilities to Cursor using Qdrant." readme = "README.md" requires-python = ">=3.12" @@ -41,9 +41,9 @@ dependencies = [ "click>=8.0.0", "tomli>=2.0.0", "networkx>=3.0.0", - "qdrant-loader-core==0.7.3", + "spacy>=3.7.0", + "qdrant-loader-core==0.7.6", ] - classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/intelligence_handler.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/intelligence_handler.py index be0541bdc..e58f47db0 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/intelligence_handler.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/intelligence_handler.py @@ -120,7 +120,34 @@ async def handle_analyze_document_relationships( async def handle_find_similar_documents( self, request_id: str | int | None, params: dict[str, Any] ) -> dict[str, Any]: - """Handle find similar documents request.""" + """ + Handle a "find similar documents" request and return MCP-formatted results. + + Parameters: + request_id (str | int | None): The request identifier to include in the MCP response. + params (dict[str, Any]): Request parameters. Required keys: + - target_query: The primary query or document to compare against. + - comparison_query: The query or document set to compare with the target. + Optional keys: + - similarity_metrics: Metrics or configuration used to compute similarity. + - max_similar (int): Maximum number of similar documents to return (default 5). + - source_types: Restrict search to specific source types. + - project_ids: Restrict search to specific project identifiers. + - similarity_threshold (float): Minimum similarity score to consider (default 0.7). + + Returns: + dict[str, Any]: An MCP protocol response dictionary. On success the response's `result` contains: + - content: a list with a single text block (human-readable summary). + - structuredContent: a dict with + - similar_documents: list of similar document entries, each containing + `document_id`, `title`, `similarity_score`, `similarity_metrics`, + `similarity_reason`, and `content_preview`. + - similarity_summary: metadata including `total_compared`, `similar_found`, + `highest_similarity`, and `metrics_used`. + - isError: False + On invalid parameters the function returns an MCP error response with code -32602. + On internal failures the function returns an MCP error response with code -32603. + """ logger.debug("Handling find similar documents with params", params=params) # Validate required parameters @@ -152,6 +179,9 @@ async def handle_find_similar_documents( max_similar=params.get("max_similar", 5), source_types=params.get("source_types"), project_ids=params.get("project_ids"), + similarity_threshold=params.get( + "similarity_threshold", 0.7 + ), # Default 0.7 ) # Normalize result: engine may return list, but can return {} on empty @@ -204,24 +234,49 @@ async def handle_find_similar_documents( for item in similar_docs: # Normalize access to document fields document = item.get("document") if isinstance(item, dict) else None + + # Extract document_id - try both dict and object attribute access document_id = ( - ( + item.get("document_id", "") if isinstance(item, dict) else "" + ) + if not document_id and document: + document_id = ( document.get("document_id") if isinstance(document, dict) - else None + else getattr(document, "document_id", "") ) - or (item.get("document_id") if isinstance(item, dict) else None) - or "" - ) - title = ( - ( - document.get("source_title") - if isinstance(document, dict) - else None + + # Extract title - try both dict and object attribute access + title = "Untitled" + if document: + if isinstance(document, dict): + title = document.get("source_title", "Untitled") + else: + title = getattr(document, "source_title", "Untitled") + if not title or title == "Untitled": + title = ( + item.get("source_title", "Untitled") + if isinstance(item, dict) + else "Untitled" ) - or (item.get("title") if isinstance(item, dict) else None) - or "Untitled" - ) + + # Extract text content - try both dict and object attribute access + content_text = "" + if document: + if isinstance(document, dict): + content_text = document.get("text", "") + else: + content_text = getattr(document, "text", "") + + # Create content preview + content_preview = "" + if content_text and isinstance(content_text, str): + content_preview = ( + content_text[:200] + "..." + if len(content_text) > 200 + else content_text + ) + similarity_score = float(item.get("similarity_score", 0.0)) highest_similarity = max(highest_similarity, similarity_score) @@ -249,18 +304,7 @@ async def handle_find_similar_documents( if isinstance(item.get("similarity_reasons", []), list) else item.get("similarity_reason", "") ), - "content_preview": ( - (document.get("text", "")[:200] + "...") - if isinstance(document, dict) - and isinstance(document.get("text"), str) - and len(document.get("text")) > 200 - else ( - document.get("text") - if isinstance(document, dict) - and isinstance(document.get("text"), str) - else "" - ) - ), + "content_preview": content_preview, } ) diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/search_handler.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/search_handler.py index 80e7b591e..a3ba81698 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/search_handler.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/mcp/search_handler.py @@ -39,7 +39,29 @@ def __init__( async def handle_search( self, request_id: str | int | None, params: dict[str, Any] ) -> dict[str, Any]: - """Handle basic search request.""" + """ + Handle a basic text search and return an MCP-formatted response. + + Validates that `params` contains a required "query" key, processes the query via the QueryProcessor, + executes a search with the SearchEngine using optional filters from `params`, and returns both a + backward-compatible text block and a structured search result suitable for the MCP protocol. + + Parameters: + request_id (str | int | None): The incoming request identifier passed to the protocol response. + params (dict): Search parameters. Required keys: + - "query": the search query string. + Optional keys: + - "source_types" (list): list of source type filters (default: []). + - "project_ids" (list): list of project id filters (default: []). + - "limit" (int): maximum number of search results to request (default: 5). + + Returns: + dict: An MCP protocol response dictionary. On success the response contains a `result` + with `content` (text block), `structuredContent` (results, total_found, query_context), + and `isError: False`. On validation failure returns an error response with code -32602 + and a descriptive message; on internal failure returns an error response with code -32603 + and the exception string in `data`. + """ logger.debug("Handling search request with params", params=params) # Validate required parameters @@ -58,7 +80,7 @@ async def handle_search( query = params["query"] source_types = params.get("source_types", []) project_ids = params.get("project_ids", []) - limit = params.get("limit", 10) + limit = params.get("limit", 5) logger.info( "Processing search request", diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/field_query_parser.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/field_query_parser.py index 1626b3539..4fd77105f 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/field_query_parser.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/field_query_parser.py @@ -143,17 +143,19 @@ def create_qdrant_filter( field_queries: list[FieldQuery] | None, project_ids: list[str] | None = None, ) -> models.Filter | None: - """Create a Qdrant filter from field queries. + """ + Build a Qdrant Filter from parsed field queries and optional project IDs. - Args: - field_queries: List of field queries to convert to filters - project_ids: Optional project ID filters to include + Converts each provided FieldQuery into a payload match condition using the parser's supported field mappings and type conversion. If project_ids are provided and no explicit project_id field query exists, adds an OR condition that matches any of the given project IDs in one of three payload keys: "project_id", "source", or "metadata.project_id". Returns a Filter that requires all constructed conditions, or None when no conditions are produced. + + Parameters: + field_queries (list[FieldQuery] | None): FieldQuery objects to convert into filter conditions; omitted or empty means no field-based conditions. + project_ids (list[str] | None): Project IDs to require in any supported project location when not explicitly specified via a field query. Returns: - Qdrant Filter object or None if no filters needed + models.Filter | None: A Qdrant Filter containing the required must conditions, or None if no filter conditions were created. """ must_conditions = [] - should_conditions = [] # Add field query conditions if field_queries: @@ -164,26 +166,11 @@ def create_qdrant_filter( ) # Handle nested fields (e.g., metadata.chunk_index) - if "." in payload_key: - parts = payload_key.split(".", 1) - condition = models.NestedCondition( - nested=models.Nested( - key=parts[0], - filter=models.Filter( - must=[ - models.FieldCondition( - key=parts[1], - match=models.MatchValue(value=match_value), - ) - ] - ), - ) - ) - else: - # For top-level fields, use direct field condition - condition = models.FieldCondition( - key=payload_key, match=models.MatchValue(value=match_value) - ) + # Use dot notation for all fields - Qdrant supports this natively + # This is simpler and more reliable than NestedCondition + condition = models.FieldCondition( + key=payload_key, match=models.MatchValue(value=match_value) + ) must_conditions.append(condition) self.logger.debug( @@ -197,31 +184,27 @@ def create_qdrant_filter( else False ) if project_ids and not has_project_id_field_query: - # Support both top-level project_id and nested metadata.project_id, and root 'source' + # Support project_id in 3 locations using dot notation + # Note: NestedCondition doesn't work - must use dot notation for nested fields top_level = models.FieldCondition( key="project_id", match=models.MatchAny(any=project_ids) ) - top_level_source = models.FieldCondition( + source_field = models.FieldCondition( key="source", match=models.MatchAny(any=project_ids) ) - nested_meta = models.NestedCondition( - nested=models.Nested( - key="metadata", - filter=models.Filter( - must=[ - models.FieldCondition( - key="project_id", - match=models.MatchAny(any=project_ids), - ) - ] - ), - ) + metadata_field = models.FieldCondition( + key="metadata.project_id", match=models.MatchAny(any=project_ids) ) - # Use OR semantics so either storage layout matches - should_conditions.extend([top_level, top_level_source, nested_meta]) + # Wrap OR conditions in Filter(should=[...]) and add to must + # This ensures at least one project location must match + project_or_filter = models.Filter( + should=[top_level, source_field, metadata_field] + ) + must_conditions.append(project_or_filter) self.logger.debug( - f"Added project filter (top-level or nested): {project_ids}" + f"DEBUG project_ids filter: Looking for project_ids={project_ids} in 3 locations: " + f"top-level 'project_id', 'source' field, or 'metadata.project_id'" ) elif project_ids and has_project_id_field_query: self.logger.debug( @@ -229,8 +212,8 @@ def create_qdrant_filter( ) # Return filter if we have conditions - if must_conditions or should_conditions: - return models.Filter(must=must_conditions, should=should_conditions) + if must_conditions: + return models.Filter(must=must_conditions) return None diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/vector_search_service.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/vector_search_service.py index 62f4d903e..2aee3db08 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/vector_search_service.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/components/vector_search_service.py @@ -251,15 +251,17 @@ async def vector_search( parsed_query.field_queries, project_ids ) - results = await self.qdrant_client.search( + # Use query_points API (qdrant-client 1.10+) + query_response = await self.qdrant_client.query_points( collection_name=self.collection_name, - query_vector=query_embedding, + query=query_embedding, limit=limit, score_threshold=self.min_score, search_params=search_params, query_filter=query_filter, with_payload=True, # ๐Ÿ”ง CRITICAL: Explicitly request payload data ) + results = query_response.points extracted_results = [] for hit in results: diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/core.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/core.py index 495429a23..9bdfa55b8 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/core.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/core.py @@ -484,7 +484,25 @@ async def find_similar_documents( source_types: list[str] | None = None, project_ids: list[str] | None = None, ) -> dict | list[dict]: - """Find similar documents.""" + """ + Finds documents most similar to a single target document. + + Parameters: + target_query (str): Query used to retrieve the single target document. + comparison_query (str): Query used to retrieve comparison documents; if empty, `target_query` is used. + similarity_metrics (list[str] | None): Optional list of metric names; unknown names are ignored and the default metric set is used. + max_similar (int): Maximum number of similar documents to return. + similarity_threshold (float): Minimum similarity score required for a comparison document to be considered similar. + limit (int): Number of comparison documents to retrieve when executing the comparison query. + source_types (list[str] | None): Optional filter for document source types. + project_ids (list[str] | None): Optional filter for project identifiers. + + Returns: + dict | list[dict]: A dictionary or list of dictionaries containing similarity information for comparison documents relative to the selected target document. Returns an empty dict if no target document is found. + + Raises: + RuntimeError: If the search engine has not been initialized. + """ if not self._search_ops: raise RuntimeError("Search engine not initialized") @@ -528,6 +546,7 @@ async def find_similar_documents( comparison_documents, metric_enums, max_similar, + similarity_threshold, ) async def detect_document_conflicts( @@ -537,7 +556,29 @@ async def detect_document_conflicts( source_types: list[str] = None, project_ids: list[str] = None, ) -> dict: - """Detect conflicts between documents.""" + """ + Detects semantic or content conflicts among documents related to a query. + + Performs a search for documents matching `query` and, if at least two documents are found, delegates conflict detection to the intelligence operations module. If fewer than two documents are found, returns a structured response indicating insufficient documents. When a conflict result dictionary is returned, the function attaches `query_metadata` and a lightweight `original_documents` list describing the retrieved documents. + + Parameters: + query (str): The search query used to retrieve candidate documents for conflict detection. + limit (int): Maximum number of documents to retrieve for analysis. + source_types (list[str] | None): Optional list of source types to filter search results. + project_ids (list[str] | None): Optional list of project IDs to filter search results. + + Returns: + dict: A dictionary containing conflict detection results. Possible keys include: + - `conflicts`: list of detected conflicts (may be empty). + - `resolution_suggestions`: mapping of suggested resolutions. + - `message`: human-readable status (present when insufficient documents). + - `document_count`: number of documents considered. + - `query_metadata`: metadata about the original query and filters. + - `original_documents`: list of lightweight document records with `document_id`, `title`, and `source_type`. + + Raises: + RuntimeError: If search operations or intelligence operations are not initialized. + """ if not self._search_ops: raise RuntimeError("Search engine not initialized") diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/intelligence.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/intelligence.py index 4acc6c5d5..d84f13760 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/intelligence.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/engine/intelligence.py @@ -105,20 +105,28 @@ async def find_similar_documents( max_similar: int = 5, source_types: list[str] | None = None, project_ids: list[str] | None = None, + similarity_threshold: float = 0.7, ) -> dict[str, Any]: """ - Find documents similar to a target document. + Find documents most similar to a target document retrieved by a query. - Args: - target_query: Query to find the target document - comparison_query: Query to get documents to compare against - similarity_metrics: Similarity metrics to use - max_similar: Maximum number of similar documents to return - source_types: Optional list of source types to filter by - project_ids: Optional list of project IDs to filter by + Parameters: + target_query (str): Query used to select the target document (first search result). + comparison_query (str): Query used to retrieve candidate documents to compare against. + similarity_metrics (list[str] | None): Optional list of similarity metric names; unknown names are ignored. + max_similar (int): Maximum number of similar documents to include in results. + source_types (list[str] | None): Optional list of source types to filter both searches. + project_ids (list[str] | None): Optional list of project IDs to filter both searches. + similarity_threshold (float): Minimum similarity score for results to be considered similar. Returns: - List of similar documents with similarity scores + dict: Result object containing either an error or similarity details. + On success, includes: + - target_document (dict): {document_id, title, source_type} for the target. + - similar_documents: Backend-provided list of similar document entries (each includes similarity scores). + - similarity_metrics_used: List of metric names used or the string "default". + - comparison_documents_analyzed (int): Number of comparison documents evaluated. + On failure, includes an "error" key with details and additional context fields (e.g., target_query or comparison_count). """ if not self.engine.hybrid_search: raise RuntimeError("Search engine not initialized") @@ -165,7 +173,11 @@ async def find_similar_documents( # Find similar documents similar = await self.engine.hybrid_search.find_similar_documents( - target_doc, comparison_results, metric_enums or None, max_similar + target_doc, + comparison_results, + metric_enums or None, + max_similar, + similarity_threshold, ) return { @@ -370,17 +382,23 @@ async def find_complementary_content( project_ids: list[str] | None = None, ) -> dict[str, Any]: """ - Find content that complements a target document. + Find documents that complement a target document using contextual documents. - Args: - target_query: Query to find the target document - context_query: Query to get contextual documents - max_recommendations: Maximum number of recommendations - source_types: Optional list of source types to filter by - project_ids: Optional list of project IDs to filter by + Performs a search for a target document (with several fallback queries if none found), retrieves contextual documents, and returns up to `max_recommendations` complementary recommendations derived from those context documents. + + Parameters: + target_query (str): Query used to locate the primary target document. + context_query (str): Query used to retrieve contextual documents for comparison. + max_recommendations (int): Maximum number of complementary recommendations to return. + source_types (list[str] | None): Optional list of source types to filter searches. + project_ids (list[str] | None): Optional list of project IDs to filter searches. Returns: - Dict containing complementary recommendations and target document info + dict: { + "complementary_recommendations": list -- Transformed recommendation entries (each is a dict with at least `document_id`, `title`, `relevance_score`, `reason`, `strategy`, and optional `source_type`/`project_id`) or raw recommendation items if not mappable; + "target_document": dict | None -- `{ "document_id", "title", "source_type" }` for the chosen target document, or `None` if no target was found; + "context_documents_analyzed": int -- Number of context documents that were analyzed. + } """ if not self.engine.hybrid_search: raise RuntimeError("Search engine not initialized") @@ -542,7 +560,9 @@ async def find_complementary_content( "relevance_score": rec.get( "complementary_score", rec.get("relevance_score", 0.0) ), - "reason": rec.get("explanation", rec.get("reason", "")), + "reason": rec.get( + "recommendation_reason", rec.get("reason", "") + ), "strategy": rec.get( "relationship_type", rec.get("strategy", "related") ), diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/api.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/api.py index 2736a1b67..13cb99915 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/api.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/api.py @@ -191,7 +191,21 @@ async def find_similar_documents( documents: list[HybridSearchResult], similarity_metrics: list[SimilarityMetric] | None = None, max_similar: int = 5, + similarity_threshold: float = 0.7, ) -> list[dict[str, Any]]: + """ + Identify documents in a collection that are most similar to a target document. + + Parameters: + target_document (HybridSearchResult): The document to compare others against. + documents (list[HybridSearchResult]): Candidate documents to evaluate for similarity. + similarity_metrics (list[SimilarityMetric] | None): Metrics to use when computing similarity; if omitted, defaults are applied. + max_similar (int): Maximum number of similar documents to return. + similarity_threshold (float): Minimum similarity score (0.0โ€“1.0) required for a document to be included. + + Returns: + list[dict[str, Any]]: A list of similarity records for matching documents (up to `max_similar`), each containing at least the document reference and its similarity score. + """ from .orchestration.cdi import find_similar_documents as _find return await _find( @@ -200,11 +214,21 @@ async def find_similar_documents( documents=documents, similarity_metrics=similarity_metrics, max_similar=max_similar, + similarity_threshold=similarity_threshold, ) async def detect_document_conflicts( self, documents: list[HybridSearchResult] ) -> dict[str, Any]: + """ + Detect conflicts among the provided documents. + + Parameters: + documents (list[HybridSearchResult]): Documents to analyze for conflicting content or metadata. + + Returns: + dict[str, Any]: Analysis results mapping conflict categories or identifiers to details such as affected document IDs, conflicting fields, and confidence scores. + """ from .orchestration.cdi import detect_document_conflicts as _detect return await _detect(self, documents) diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/cdi.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/cdi.py index 4bbad16ad..3ff1c789a 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/cdi.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/cdi.py @@ -23,7 +23,29 @@ async def find_similar_documents( documents: list[HybridSearchResult], similarity_metrics: list[SimilarityMetric] | None = None, max_similar: int = 5, + similarity_threshold: float = 0.7, ) -> list[dict[str, Any]]: + """ + Finds documents most similar to a target document using the engine's similarity calculator. + + Skips self-comparison (by `document_id` when available, otherwise by object identity), filters out results with similarity scores below `similarity_threshold`, sorts matches by descending similarity, and returns up to `max_similar` entries. + + Parameters: + engine: Cross-document engine container used to access the similarity calculator. + target_document: The document to compare others against. + documents: Iterable of documents to evaluate for similarity. + similarity_metrics (optional): Metrics to use when calculating similarity; forwarded to the similarity calculator. + max_similar (optional): Maximum number of similar documents to return. + similarity_threshold (optional): Minimum similarity score required for a document to be included. + + Returns: + list[dict[str, Any]]: A list of dictionaries (ordered by descending `similarity_score`) where each entry contains: + - `document_id`: the matched document's identifier + - `document`: the matched document object + - `similarity_score`: the overall similarity score + - `metric_scores`: per-metric similarity scores + - `similarity_reasons`: list with a human-readable explanation for the similarity + """ similarity_calculator = engine.cross_document_engine.similarity_calculator similar_docs = [] for doc in documents: @@ -42,15 +64,17 @@ async def find_similar_documents( similarity = similarity_calculator.calculate_similarity( target_document, doc, similarity_metrics ) - similar_docs.append( - { - "document_id": doc.document_id, - "document": doc, - "similarity_score": similarity.similarity_score, - "metric_scores": similarity.metric_scores, - "similarity_reasons": [similarity.get_display_explanation()], - } - ) + # Filter by similarity threshold + if similarity.similarity_score >= similarity_threshold: + similar_docs.append( + { + "document_id": doc.document_id, + "document": doc, + "similarity_score": similarity.similarity_score, + "metric_scores": similarity.metric_scores, + "similarity_reasons": [similarity.get_display_explanation()], + } + ) similar_docs.sort(key=lambda x: x["similarity_score"], reverse=True) return similar_docs[:max_similar] diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/search.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/search.py index e3563b3b3..fd12cb411 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/search.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/search/hybrid/orchestration/search.py @@ -21,11 +21,29 @@ async def run_search( behavioral_context: list[str] | None, ) -> list[HybridSearchResult]: # Save original combiner values up front for safe restoration + """ + Execute a hybrid search for the given query using the provided engine and return ranked results. + + Per-request adjustments (query expansion, intent-adaptive combiner weights, and fetch limits) are applied to a cloned combiner and do not mutate the engine's shared combiner state; the function attempts to restore the engine's result_combiner attributes to their original values and logs any restoration failures without raising. + + Parameters: + engine: Search engine instance providing hybrid search, planners, expansion, and orchestration. + query (str): The user query to search for. + limit (int): Maximum number of results to return. + source_types (list[str] | None): Optional list of source types to filter results. + project_ids (list[str] | None): Optional list of project IDs to restrict the search. + session_context (dict[str, Any] | None): Optional session-level context used for intent classification and adaptations. + behavioral_context (list[str] | None): Optional behavioral signals used for intent classification and adaptations. + + Returns: + list[HybridSearchResult]: Ranked hybrid search results; length will be at most `limit`. + """ original_vector_weight = engine.result_combiner.vector_weight original_keyword_weight = engine.result_combiner.keyword_weight original_min_score = engine.result_combiner.min_score combined_results: list[HybridSearchResult] + fetch_limit = limit try: # Build a request-scoped combiner clone to avoid mutating shared engine state @@ -52,7 +70,7 @@ async def run_search( local_combiner.vector_weight = adaptive_config.vector_weight local_combiner.keyword_weight = adaptive_config.keyword_weight local_combiner.min_score = adaptive_config.min_score_threshold - limit = min(adaptive_config.max_results, limit * 2) + fetch_limit = min(adaptive_config.max_results, limit * 2) expanded_query = await engine._expand_query(query) if adaptive_config and getattr(adaptive_config, "expand_query", False): @@ -95,7 +113,7 @@ async def run_search( combined_results = await engine._orchestrator.run_pipeline( local_pipeline, query=query, - limit=limit, + limit=fetch_limit, query_context=query_context, source_types=source_types, project_ids=project_ids, @@ -107,7 +125,7 @@ async def run_search( combined_results = await engine._orchestrator.run_pipeline( p, query=query, - limit=limit, + limit=fetch_limit, query_context=query_context, source_types=source_types, project_ids=project_ids, @@ -116,10 +134,10 @@ async def run_search( ) else: vector_results = await engine._vector_search( - expanded_query, limit * 3, project_ids + expanded_query, fetch_limit * 3, project_ids ) keyword_results = await engine._keyword_search( - query, limit * 3, project_ids + query, fetch_limit * 3, project_ids ) combined_results = await _combine_results_helper( local_combiner, @@ -127,7 +145,7 @@ async def run_search( vector_results, keyword_results, query_context, - limit, + fetch_limit, source_types, project_ids, ) @@ -169,4 +187,4 @@ async def run_search( except Exception: pass - return combined_results + return combined_results[:limit] diff --git a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/utils/logging.py b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/utils/logging.py index b06080b3a..cae81155f 100644 --- a/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/utils/logging.py +++ b/packages/qdrant-loader-mcp-server/src/qdrant_loader_mcp_server/utils/logging.py @@ -132,20 +132,32 @@ def get_logger(cls, name: str | None = None): # type: ignore return structlog.get_logger(name) @classmethod - def reconfigure(cls, *, file: str | None = None) -> None: - """Lightweight file reconfiguration for MCP server wrapper. + def reconfigure(cls, *, file: str | None = None, level: str | None = None) -> None: + """Lightweight reconfiguration for file destination and optionally log level. If core logging is present and supports reconfigure, delegate to it. Otherwise, force-replace root handlers with a new file handler (and keep stderr if console is enabled via env). + + Args: + file: Path to log file (optional) + level: New log level (optional, e.g., "DEBUG", "INFO") """ disable_console_logging = ( os.getenv("MCP_DISABLE_CONSOLE_LOGGING", "").lower() == "true" ) if CoreLoggingConfig is not None and hasattr(CoreLoggingConfig, "reconfigure"): - CoreLoggingConfig.reconfigure(file=file) # type: ignore + CoreLoggingConfig.reconfigure(file=file, level=level) # type: ignore else: + # Determine the level to use + if level is not None: + resolved_level = level.upper() + elif cls._current_config is not None: + resolved_level = cls._current_config[0] + else: + resolved_level = "INFO" + handlers: list[logging.Handler] = [] if not disable_console_logging: stderr_handler = logging.StreamHandler(sys.stderr) @@ -155,8 +167,11 @@ def reconfigure(cls, *, file: str | None = None) -> None: file_handler = logging.FileHandler(file) file_handler.setFormatter(CleanFormatter("%(message)s")) handlers.append(file_handler) - logging.basicConfig(level=getattr(logging, (cls._current_config or ("INFO",))[0]), handlers=handlers, force=True) # type: ignore + logging.basicConfig( + level=getattr(logging, resolved_level), handlers=handlers, force=True + ) if cls._current_config is not None: - level, fmt, _, suppress = cls._current_config - cls._current_config = (level, fmt, file, suppress) + old_level, fmt, _, suppress = cls._current_config + new_level = level.upper() if level is not None else old_level + cls._current_config = (new_level, fmt, file, suppress) diff --git a/packages/qdrant-loader-mcp-server/tests/conftest.py b/packages/qdrant-loader-mcp-server/tests/conftest.py index e502686be..30c61ae6a 100644 --- a/packages/qdrant-loader-mcp-server/tests/conftest.py +++ b/packages/qdrant-loader-mcp-server/tests/conftest.py @@ -59,8 +59,22 @@ def mock_qdrant_client(): "source_type": "confluence", } - client.search.return_value = [search_result1, search_result2] - client.scroll.return_value = ([search_result1, search_result2], None) + search_result3 = MagicMock() + search_result3.id = "3" + search_result3.score = 0.6 + search_result3.payload = { + "content": "Test content 3", + "metadata": {"title": "Test Doc 3", "url": "http://test3.com"}, + "source_type": "jira", + } + + # Mock query_points response (qdrant-client 1.10+) + query_response = MagicMock() + query_response.points = [search_result1, search_result2, search_result3] + client.query_points = AsyncMock(return_value=query_response) + client.scroll = AsyncMock( + return_value=([search_result1, search_result2, search_result3], None) + ) # Mock collection operations collections_response = MagicMock() diff --git a/packages/qdrant-loader-mcp-server/tests/integration/test_complementary_content_e2e.py b/packages/qdrant-loader-mcp-server/tests/integration/test_complementary_content_e2e.py index 9b2884fd0..dd9c28d1d 100644 --- a/packages/qdrant-loader-mcp-server/tests/integration/test_complementary_content_e2e.py +++ b/packages/qdrant-loader-mcp-server/tests/integration/test_complementary_content_e2e.py @@ -403,18 +403,34 @@ def create_search_response(query, limit=10, **kwargs): else: return sample_documents[1:4] # Return business docs as context - mock_qdrant_client.search.side_effect = lambda query, **kwargs: [ - MagicMock( - id=f"doc_{i}", - score=doc.score, - payload={ - attr: getattr(doc, attr) - for attr in doc.__dict__ - if not attr.startswith("_") - }, - ) - for i, doc in enumerate(create_search_response(query, **kwargs)) - ] + # Mock query_points response (qdrant-client 1.10+) + def mock_query_points(query, **kwargs): + response = MagicMock() + response.points = [ + MagicMock( + id=f"doc_{i}", + score=doc.score, + payload={ + "content": doc.text, # Map text โ†’ content for keyword_search compatibility + "source_type": doc.source_type, + "source_title": doc.source_title, + "project_id": doc.project_id, + "entities": doc.entities, + "topics": doc.topics, + "key_phrases": doc.key_phrases, + "content_type_context": doc.content_type_context, + "has_code_blocks": doc.has_code_blocks, + "has_tables": doc.has_tables, + "word_count": doc.word_count, + "depth": doc.depth, + "metadata": {}, # Provide empty metadata dict + }, + ) + for i, doc in enumerate(create_search_response(query, **kwargs)) + ] + return response + + mock_qdrant_client.query_points.side_effect = mock_query_points # Configure the mock_qdrant_client to have the scroll method with sample documents def create_mock_scroll_response(**kwargs): diff --git a/packages/qdrant-loader-mcp-server/tests/integration/test_mcp_integration.py b/packages/qdrant-loader-mcp-server/tests/integration/test_mcp_integration.py index d266f5034..f6d59c17c 100644 --- a/packages/qdrant-loader-mcp-server/tests/integration/test_mcp_integration.py +++ b/packages/qdrant-loader-mcp-server/tests/integration/test_mcp_integration.py @@ -26,7 +26,10 @@ async def integration_handler(): "source_type": "git", } - mock_qdrant_client.search.return_value = [search_result1] + # Mock query_points response (qdrant-client 1.10+) + query_response = MagicMock() + query_response.points = [search_result1] + mock_qdrant_client.query_points.return_value = query_response mock_qdrant_client.scroll.return_value = ([search_result1], None) # Mock collections response for get_collections diff --git a/packages/qdrant-loader-mcp-server/tests/integration/test_phase1_2_simple_integration.py b/packages/qdrant-loader-mcp-server/tests/integration/test_phase1_2_simple_integration.py index eb58eb98e..41e38c95c 100644 --- a/packages/qdrant-loader-mcp-server/tests/integration/test_phase1_2_simple_integration.py +++ b/packages/qdrant-loader-mcp-server/tests/integration/test_phase1_2_simple_integration.py @@ -137,8 +137,10 @@ def test_real_topic_relationship_mapping( logger.debug(f" โ€ข {topic} (score: {score:.3f}, type: {rel_type})") # Verify we found relationships - assert len(topic_map.topic_document_frequency) >= 0 - assert len(related_topics) >= 0 # May be 0 if no strong relationships + if len(topic_map.topic_document_frequency) == 0: + pytest.skip("No topics found - spaCy model may not be loaded correctly") + if len(related_topics) == 0: + logger.warning("No related topics found for 'authentication'") # Test semantic similarity with real spaCy vectors similarity = real_spacy_analyzer.nlp("authentication").similarity( diff --git a/packages/qdrant-loader-mcp-server/tests/integration/test_phase2_2_integration.py b/packages/qdrant-loader-mcp-server/tests/integration/test_phase2_2_integration.py index 122575161..7600bd2e3 100644 --- a/packages/qdrant-loader-mcp-server/tests/integration/test_phase2_2_integration.py +++ b/packages/qdrant-loader-mcp-server/tests/integration/test_phase2_2_integration.py @@ -21,40 +21,45 @@ class TestPhase2_2Integration: def mock_qdrant_client(self): """Create a mock Qdrant client.""" client = AsyncMock() - client.search = AsyncMock( - return_value=[ - Mock( - score=0.9, - payload={ - "content": "FastAPI OAuth 2.0 authentication implementation guide", - "source_type": "git", - "metadata": { - "title": "OAuth Authentication Guide", - "url": "https://example.com/oauth-guide", - "has_code_blocks": True, - "section_type": "implementation", - "entities": ["OAuth", "FastAPI"], - "topics": ["authentication", "security"], - }, + + # Mock search results + mock_points = [ + Mock( + score=0.9, + payload={ + "content": "FastAPI OAuth 2.0 authentication implementation guide", + "source_type": "git", + "metadata": { + "title": "OAuth Authentication Guide", + "url": "https://example.com/oauth-guide", + "has_code_blocks": True, + "section_type": "implementation", + "entities": ["OAuth", "FastAPI"], + "topics": ["authentication", "security"], }, - ), - Mock( - score=0.8, - payload={ - "content": "Business requirements for authentication system", - "source_type": "confluence", - "metadata": { - "title": "Auth Requirements", - "url": "https://example.com/auth-requirements", - "has_code_blocks": False, - "section_type": "requirements", - "entities": ["Company"], - "topics": ["requirements", "security"], - }, + }, + ), + Mock( + score=0.8, + payload={ + "content": "Business requirements for authentication system", + "source_type": "confluence", + "metadata": { + "title": "Auth Requirements", + "url": "https://example.com/auth-requirements", + "has_code_blocks": False, + "section_type": "requirements", + "entities": ["Company"], + "topics": ["requirements", "security"], }, - ), - ] - ) + }, + ), + ] + + # Mock query_points response (qdrant-client 1.10+) + query_response = MagicMock() + query_response.points = mock_points + client.query_points = AsyncMock(return_value=query_response) client.scroll = AsyncMock( return_value=( @@ -270,8 +275,9 @@ async def test_exploratory_intent_with_diversity(self, hybrid_search_engine): processing_time_ms=30.0, ) - # Mock diverse results - hybrid_search_engine.qdrant_client.search.return_value = [ + # Mock diverse results (qdrant-client 1.10+) + mock_query_response = Mock() + mock_query_response.points = [ Mock( score=0.9, payload={ @@ -287,6 +293,9 @@ async def test_exploratory_intent_with_diversity(self, hybrid_search_engine): ) for i in range(10) ] + hybrid_search_engine.qdrant_client.query_points.return_value = ( + mock_query_response + ) results = await hybrid_search_engine.search( query=query, limit=20, session_context={"session_type": "exploration"} diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_cdi_orchestration_threshold.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_cdi_orchestration_threshold.py new file mode 100644 index 000000000..7b88becf0 --- /dev/null +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_cdi_orchestration_threshold.py @@ -0,0 +1,369 @@ +"""Unit tests for CDI orchestration layer similarity threshold filtering.""" + +from unittest.mock import Mock + +import pytest +from qdrant_loader_mcp_server.search.components.search_result_models import ( + create_hybrid_search_result, +) +from qdrant_loader_mcp_server.search.hybrid.orchestration.cdi import ( + find_similar_documents, +) + + +@pytest.fixture +def mock_engine(): + """ + Create a mock engine configured for similarity tests. + + The returned Mock has a `cross_document_engine` attribute that is a Mock, and that object's + `similarity_calculator` attribute is also a Mock. + + Returns: + Mock: A configured mock engine with `cross_document_engine` and its `similarity_calculator`. + """ + engine = Mock() + engine.cross_document_engine = Mock() + engine.cross_document_engine.similarity_calculator = Mock() + return engine + + +@pytest.fixture +def target_document(): + """Create a target document for similarity comparison.""" + return create_hybrid_search_result( + score=0.9, + text="OAuth authentication implementation guide", + source_type="confluence", + source_title="OAuth Guide - Chunk 1", + document_id="target-doc-id", + entities=[{"text": "OAuth", "label": "TECH"}], + topics=[{"text": "authentication", "score": 0.9}], + ) + + +@pytest.fixture +def comparison_documents(): + """ + Create three hybrid search result documents with distinct similarity scores for testing. + + Each returned document represents a comparison candidate: + - "high-similarity-doc": score 0.85, contains JWT-related text and an entity. + - "medium-similarity-doc": score 0.6, database schema text. + - "low-similarity-doc": score 0.3, marketing text. + + Returns: + list: A list of three hybrid search result objects used as comparison documents in tests. + """ + return [ + create_hybrid_search_result( + score=0.85, + text="JWT token implementation", + source_type="git", + source_title="JWT Implementation - Chunk 1", + document_id="high-similarity-doc", + entities=[{"text": "JWT", "label": "TECH"}], + ), + create_hybrid_search_result( + score=0.6, + text="Database schema design", + source_type="confluence", + source_title="DB Schema - Chunk 1", + document_id="medium-similarity-doc", + ), + create_hybrid_search_result( + score=0.3, + text="Marketing campaign strategy", + source_type="confluence", + source_title="Marketing Strategy - Chunk 1", + document_id="low-similarity-doc", + ), + ] + + +class TestSimilarityThresholdFiltering: + """Test similarity threshold filtering in find_similar_documents.""" + + @pytest.mark.asyncio + async def test_default_threshold_filters_low_scores( + self, mock_engine, target_document, comparison_documents + ): + """Test that default threshold (0.7) filters out low similarity documents.""" + # Mock similarity scores: 0.85, 0.65, 0.35 + mock_similarities = [ + Mock( + similarity_score=0.85, + metric_scores={}, + get_display_explanation=lambda: "High similarity", + ), + Mock( + similarity_score=0.65, + metric_scores={}, + get_display_explanation=lambda: "Medium similarity", + ), + Mock( + similarity_score=0.35, + metric_scores={}, + get_display_explanation=lambda: "Low similarity", + ), + ] + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.side_effect = ( + mock_similarities + ) + + # Call with default threshold (0.7) + result = await find_similar_documents( + mock_engine, + target_document, + comparison_documents, + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.7, # Default + ) + + # Should only return documents with score >= 0.7 + assert len(result) == 1 + assert result[0]["similarity_score"] == 0.85 + assert result[0]["document_id"] == "high-similarity-doc" + + @pytest.mark.asyncio + async def test_low_threshold_returns_more_documents( + self, mock_engine, target_document, comparison_documents + ): + """Test that lower threshold (0.5) includes more documents.""" + mock_similarities = [ + Mock( + similarity_score=0.85, + metric_scores={}, + get_display_explanation=lambda: "High", + ), + Mock( + similarity_score=0.65, + metric_scores={}, + get_display_explanation=lambda: "Medium", + ), + Mock( + similarity_score=0.35, + metric_scores={}, + get_display_explanation=lambda: "Low", + ), + ] + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.side_effect = ( + mock_similarities + ) + + # Call with lower threshold (0.5) + result = await find_similar_documents( + mock_engine, + target_document, + comparison_documents, + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.5, # Lower threshold + ) + + # Should return 2 documents with score >= 0.5 + assert len(result) == 2 + assert result[0]["similarity_score"] == 0.85 + assert result[1]["similarity_score"] == 0.65 + + @pytest.mark.asyncio + async def test_high_threshold_filters_aggressively( + self, mock_engine, target_document, comparison_documents + ): + """Test that high threshold (0.9) filters out most documents.""" + mock_similarities = [ + Mock( + similarity_score=0.85, + metric_scores={}, + get_display_explanation=lambda: "High", + ), + Mock( + similarity_score=0.65, + metric_scores={}, + get_display_explanation=lambda: "Medium", + ), + Mock( + similarity_score=0.35, + metric_scores={}, + get_display_explanation=lambda: "Low", + ), + ] + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.side_effect = ( + mock_similarities + ) + + # Call with high threshold (0.9) + result = await find_similar_documents( + mock_engine, + target_document, + comparison_documents, + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.9, # Very high threshold + ) + + # Should return no documents (none reach 0.9) + assert len(result) == 0 + + @pytest.mark.asyncio + async def test_threshold_zero_returns_all_documents( + self, mock_engine, target_document, comparison_documents + ): + """Test that threshold 0.0 returns all non-identical documents.""" + mock_similarities = [ + Mock( + similarity_score=0.85, + metric_scores={}, + get_display_explanation=lambda: "High", + ), + Mock( + similarity_score=0.65, + metric_scores={}, + get_display_explanation=lambda: "Medium", + ), + Mock( + similarity_score=0.35, + metric_scores={}, + get_display_explanation=lambda: "Low", + ), + ] + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.side_effect = ( + mock_similarities + ) + + # Call with threshold 0.0 + result = await find_similar_documents( + mock_engine, + target_document, + comparison_documents, + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.0, # No filtering + ) + + # Should return all 3 documents + assert len(result) == 3 + + @pytest.mark.asyncio + async def test_threshold_respects_max_similar_limit( + self, mock_engine, target_document + ): + """Test that max_similar is applied after threshold filtering.""" + # Create 5 comparison documents all above threshold + many_docs = [ + create_hybrid_search_result( + score=0.8, + text=f"Document {i}", + source_type="confluence", + source_title=f"Doc {i} - Chunk 1", + document_id=f"doc-{i}", + ) + for i in range(5) + ] + + # All have similarity > 0.7 + mock_similarities = [ + Mock( + similarity_score=0.8 + i * 0.01, + metric_scores={}, + get_display_explanation=lambda idx=i: f"Doc {idx}", + ) + for i in range(5) + ] + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.side_effect = ( + mock_similarities + ) + + # Call with threshold 0.7 and max_similar=3 + result = await find_similar_documents( + mock_engine, + target_document, + many_docs, + similarity_metrics=None, + max_similar=3, # Limit to 3 + similarity_threshold=0.7, + ) + + # Should return only 3 documents (top 3 by score) + assert len(result) == 3 + # Should be sorted by score (descending) + assert result[0]["similarity_score"] >= result[1]["similarity_score"] + assert result[1]["similarity_score"] >= result[2]["similarity_score"] + + @pytest.mark.asyncio + async def test_threshold_filtering_preserves_document_structure( + self, mock_engine, target_document, comparison_documents + ): + """Test that threshold filtering preserves complete document structure.""" + mock_similarity = Mock( + similarity_score=0.85, + metric_scores={"semantic": 0.9, "entity": 0.8}, + get_display_explanation=lambda: "High similarity based on shared entities", + ) + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.return_value = ( + mock_similarity + ) + + result = await find_similar_documents( + mock_engine, + target_document, + [comparison_documents[0]], # Just one doc + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.7, + ) + + # Should preserve all fields + assert len(result) == 1 + assert "document_id" in result[0] + assert "document" in result[0] + assert "similarity_score" in result[0] + assert "metric_scores" in result[0] + assert "similarity_reasons" in result[0] + assert result[0]["similarity_score"] == 0.85 + assert result[0]["metric_scores"] == {"semantic": 0.9, "entity": 0.8} + + @pytest.mark.asyncio + async def test_skips_target_document_in_comparisons( + self, mock_engine, target_document + ): + """Test that target document is skipped even if in comparison list.""" + # Include target document in comparison list + comparison_docs = [ + target_document, # Same document + create_hybrid_search_result( + score=0.8, + text="Different document", + source_type="git", + source_title="Different - Chunk 1", + document_id="different-doc", + ), + ] + + mock_similarity = Mock( + similarity_score=0.85, + metric_scores={}, + get_display_explanation=lambda: "Similar", + ) + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.return_value = ( + mock_similarity + ) + + result = await find_similar_documents( + mock_engine, + target_document, + comparison_docs, + similarity_metrics=None, + max_similar=5, + similarity_threshold=0.7, + ) + + # Should only return 1 document (target excluded) + assert len(result) == 1 + assert result[0]["document_id"] == "different-doc" + # calculate_similarity should only be called once (for non-target doc) + assert ( + mock_engine.cross_document_engine.similarity_calculator.calculate_similarity.call_count + == 1 + ) diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_field_query_parser.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_field_query_parser.py new file mode 100644 index 000000000..70563f0de --- /dev/null +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_field_query_parser.py @@ -0,0 +1,239 @@ +"""Tests for FieldQueryParser component.""" + +import pytest +from qdrant_client.http import models +from qdrant_loader_mcp_server.search.components import ( + FieldQuery, + FieldQueryParser, +) + + +class TestFieldQueryParser: + """Test suite for FieldQueryParser.""" + + @pytest.fixture + def parser(self): + """Create a FieldQueryParser instance.""" + return FieldQueryParser() + + def test_parse_simple_field_query(self, parser): + """Test parsing a simple field query.""" + query = "source_type:confluence" + parsed = parser.parse_query(query) + + assert len(parsed.field_queries) == 1 + assert parsed.field_queries[0].field_name == "source_type" + assert parsed.field_queries[0].field_value == "confluence" + assert parsed.text_query == "" + + def test_parse_multiple_field_queries(self, parser): + """Test parsing multiple field queries.""" + query = "source_type:confluence project_id:my-project" + parsed = parser.parse_query(query) + + assert len(parsed.field_queries) == 2 + assert parsed.field_queries[0].field_name == "source_type" + assert parsed.field_queries[0].field_value == "confluence" + assert parsed.field_queries[1].field_name == "project_id" + assert parsed.field_queries[1].field_value == "my-project" + assert parsed.text_query == "" + + def test_parse_field_query_with_text(self, parser): + """Test parsing field query with remaining text search.""" + query = "source_type:confluence API documentation" + parsed = parser.parse_query(query) + + assert len(parsed.field_queries) == 1 + assert parsed.field_queries[0].field_name == "source_type" + assert parsed.text_query == "API documentation" + + def test_parse_quoted_field_value(self, parser): + """Test parsing field query with quoted value.""" + query = 'title:"API Documentation"' + parsed = parser.parse_query(query) + + assert len(parsed.field_queries) == 1 + assert parsed.field_queries[0].field_name == "title" + assert parsed.field_queries[0].field_value == "API Documentation" + + def test_parse_nested_metadata_field(self, parser): + """Test parsing nested metadata field queries.""" + query = "chunk_index:0 total_chunks:5" + parsed = parser.parse_query(query) + + assert len(parsed.field_queries) == 2 + assert parsed.field_queries[0].field_name == "chunk_index" + assert parsed.field_queries[0].field_value == "0" + assert parsed.field_queries[1].field_name == "total_chunks" + assert parsed.field_queries[1].field_value == "5" + + def test_create_filter_from_field_queries(self, parser): + """Test creating Qdrant filter from field queries.""" + field_queries = [ + FieldQuery( + field_name="source_type", + field_value="confluence", + original_query="source_type:confluence", + ), + FieldQuery( + field_name="title", field_value="API", original_query="title:API" + ), + ] + + filter_obj = parser.create_qdrant_filter(field_queries) + + assert filter_obj is not None + assert len(filter_obj.must) == 2 + assert filter_obj.must[0].key == "source_type" + assert filter_obj.must[0].match.value == "confluence" + assert filter_obj.must[1].key == "title" + assert filter_obj.must[1].match.value == "API" + + def test_create_filter_with_nested_fields(self, parser): + """Test creating Qdrant filter with nested metadata fields using dot notation.""" + field_queries = [ + FieldQuery( + field_name="chunk_index", + field_value="0", + original_query="chunk_index:0", + ) + ] + + filter_obj = parser.create_qdrant_filter(field_queries) + + assert filter_obj is not None + assert len(filter_obj.must) == 1 + # Verify dot notation is used for nested fields + assert filter_obj.must[0].key == "metadata.chunk_index" + assert filter_obj.must[0].match.value == 0 # Should be converted to int + + def test_create_filter_with_project_ids(self, parser): + """Test creating filter with project_ids in 3 locations.""" + project_ids = ["project-1", "project-2"] + + filter_obj = parser.create_qdrant_filter( + field_queries=None, project_ids=project_ids + ) + + assert filter_obj is not None + assert len(filter_obj.must) == 1 + # Should be a nested Filter with should clause + nested_filter = filter_obj.must[0] + assert isinstance(nested_filter, models.Filter) + assert len(nested_filter.should) == 3 + # Check all 3 locations + assert nested_filter.should[0].key == "project_id" + assert nested_filter.should[1].key == "source" + assert nested_filter.should[2].key == "metadata.project_id" + + def test_create_filter_field_queries_and_project_ids(self, parser): + """Test creating filter with both field queries and project_ids.""" + field_queries = [ + FieldQuery( + field_name="source_type", + field_value="confluence", + original_query="source_type:confluence", + ) + ] + project_ids = ["my-project"] + + filter_obj = parser.create_qdrant_filter(field_queries, project_ids) + + assert filter_obj is not None + assert len(filter_obj.must) == 2 + # First condition: field query + assert filter_obj.must[0].key == "source_type" + # Second condition: project filter (nested Filter with should) + assert isinstance(filter_obj.must[1], models.Filter) + assert len(filter_obj.must[1].should) == 3 + + def test_skip_project_filter_when_field_query_has_project_id(self, parser): + """Test that project filter is skipped when field query contains project_id.""" + field_queries = [ + FieldQuery( + field_name="project_id", + field_value="specific-project", + original_query="project_id:specific-project", + ) + ] + project_ids = ["my-project"] + + filter_obj = parser.create_qdrant_filter(field_queries, project_ids) + + assert filter_obj is not None + # Should only have the field query, not the project filter + assert len(filter_obj.must) == 1 + assert filter_obj.must[0].key == "project_id" + assert filter_obj.must[0].match.value == "specific-project" + + def test_numeric_field_conversion(self, parser): + """Test that numeric fields are converted to int.""" + field_queries = [ + FieldQuery( + field_name="chunk_index", + field_value="42", + original_query="chunk_index:42", + ), + FieldQuery( + field_name="total_chunks", + field_value="100", + original_query="total_chunks:100", + ), + ] + + filter_obj = parser.create_qdrant_filter(field_queries) + + assert filter_obj is not None + assert filter_obj.must[0].match.value == 42 # int, not string + assert filter_obj.must[1].match.value == 100 # int, not string + + def test_should_use_filter_only_with_document_id(self, parser): + """Test filter-only mode for document_id queries.""" + parsed = parser.parse_query("document_id:abc123 some text") + + # Even with text, document_id queries should be filter-only + assert parser.should_use_filter_only(parsed) is True + + def test_should_use_filter_only_without_text(self, parser): + """Test filter-only mode when no text search.""" + parsed = parser.parse_query("source_type:confluence") + + assert parser.should_use_filter_only(parsed) is True + + def test_should_not_use_filter_only_with_text(self, parser): + """Test that filter-only is False when text search is present.""" + parsed = parser.parse_query("source_type:confluence API documentation") + + # Should use both filter and text search + assert parser.should_use_filter_only(parsed) is False + + def test_empty_query(self, parser): + """Test handling empty query.""" + parsed = parser.parse_query("") + + assert len(parsed.field_queries) == 0 + assert parsed.text_query == "" + assert parser.create_qdrant_filter(parsed.field_queries) is None + + def test_unsupported_field(self, parser): + """Test handling unsupported field.""" + query = "invalid_field:value regular text" + parsed = parser.parse_query(query) + + # Unsupported field should be treated as regular text + assert len(parsed.field_queries) == 0 + # The entire query becomes text search since field is unsupported + assert "invalid_field:value" in parsed.text_query + assert "regular text" in parsed.text_query + + def test_get_supported_fields(self, parser): + """Test getting list of supported fields.""" + fields = parser.get_supported_fields() + + assert "document_id" in fields + assert "source_type" in fields + assert "project_id" in fields + assert "chunk_index" in fields + assert ( + "metadata.chunk_index" not in fields + ) # Should be chunk_index, not the payload key diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_errors.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_errors.py index 67214de67..886bf4191 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_errors.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_errors.py @@ -6,7 +6,7 @@ @pytest.mark.unit @pytest.mark.asyncio async def test_search_error_handling(hybrid_search, mock_qdrant_client): - mock_qdrant_client.search = AsyncMock(side_effect=Exception("Test error")) + mock_qdrant_client.query_points = AsyncMock(side_effect=Exception("Test error")) with pytest.raises(Exception) as excinfo: await hybrid_search.search("test query") assert "Test error" in str(excinfo.value) diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_retrieval.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_retrieval.py index a4ff6225b..9747425be 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_retrieval.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_retrieval.py @@ -1,4 +1,4 @@ -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import pytest from qdrant_loader_mcp_server.search.components.search_result_models import ( @@ -38,7 +38,9 @@ async def test_search_empty_results(hybrid_search, mock_qdrant_client): hybrid_search._vector_search = AsyncMock(return_value=[]) hybrid_search._keyword_search = AsyncMock(return_value=[]) - mock_qdrant_client.search.return_value = [] + mock_query_response = MagicMock() + mock_query_response.points = [] + mock_qdrant_client.query_points.return_value = mock_query_response mock_qdrant_client.scroll.return_value = ([], None) results = await hybrid_search.search("test query") diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_search.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_search.py index 5b0f35b2b..3f01b1954 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_search.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_hybrid_search.py @@ -47,7 +47,10 @@ def mock_qdrant_client(): "source_type": "localfile", } - client.search.return_value = [search_result1, search_result2, search_result3] + # Mock query_points response (qdrant-client 1.10+) + query_response = MagicMock() + query_response.points = [search_result1, search_result2, search_result3] + client.query_points = AsyncMock(return_value=query_response) # Create mock scroll results scroll_result1 = MagicMock() @@ -74,17 +77,19 @@ def mock_qdrant_client(): "source_type": "localfile", } - client.scroll.return_value = ( - [scroll_result1, scroll_result2, scroll_result3], - None, + client.scroll = AsyncMock( + return_value=( + [scroll_result1, scroll_result2, scroll_result3], + None, + ) ) # Mock collection operations collections_response = MagicMock() collections_response.collections = [] - client.get_collections.return_value = collections_response - client.create_collection.return_value = None - client.close.return_value = None + client.get_collections = AsyncMock(return_value=collections_response) + client.create_collection = AsyncMock(return_value=None) + client.close = AsyncMock(return_value=None) return client @@ -175,7 +180,7 @@ async def test_search_error_handling(hybrid_search, mock_qdrant_client): hybrid_search._keyword_search = AsyncMock(return_value=[]) # Ensure fallback path uses the mocked legacy methods hybrid_search.hybrid_pipeline = None - mock_qdrant_client.search.side_effect = None + mock_qdrant_client.query_points.side_effect = None out = await hybrid_search.search("q") assert out == [] @@ -302,9 +307,9 @@ async def test_vector_search(hybrid_search, mock_qdrant_client): assert results[0]["text"] == "Test content 1" assert results[0]["source_type"] == "git" - # Verify Qdrant search was called with correct parameters - mock_qdrant_client.search.assert_called_once() - call_args = mock_qdrant_client.search.call_args + # Verify Qdrant query_points was called with correct parameters (qdrant-client 1.10+) + mock_qdrant_client.query_points.assert_called_once() + call_args = mock_qdrant_client.query_points.call_args assert call_args[1]["collection_name"] == "test_collection" assert call_args[1]["limit"] == 5 @@ -1040,6 +1045,7 @@ def mock_get_display_explanation(): target_document=target_doc, documents=documents, similarity_metrics=[SimilarityMetric.SEMANTIC_SIMILARITY], + similarity_threshold=0.3, max_similar=5, ) diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_project_search.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_project_search.py index 39f5b6e66..6d7da110b 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/search/test_project_search.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_project_search.py @@ -6,7 +6,6 @@ import pytest from openai import AsyncOpenAI -from qdrant_client import QdrantClient from qdrant_loader_mcp_server.search.components.search_result_models import ( HybridSearchResult, create_hybrid_search_result, @@ -20,7 +19,7 @@ @pytest.fixture def mock_qdrant_client(): """Create a mock Qdrant client.""" - client = AsyncMock(spec=QdrantClient) + client = AsyncMock() # Mock search results with project information using MagicMock mock_points = [] @@ -51,7 +50,10 @@ def mock_qdrant_client(): mock_points.append(mock_point) # Set up async mock methods - client.search = AsyncMock(return_value=mock_points) + # Mock query_points response (qdrant-client 1.10+) + query_response = MagicMock() + query_response.points = mock_points + client.query_points = AsyncMock(return_value=query_response) client.scroll = AsyncMock(return_value=(mock_points, None)) # Mock collection operations for SearchEngine initialization @@ -116,9 +118,9 @@ async def test_hybrid_search_with_project_filter(hybrid_search, mock_qdrant_clie query="test query", limit=5, project_ids=["project-a"] ) - # Verify filter was applied in search call - mock_qdrant_client.search.assert_called() - search_call_args = mock_qdrant_client.search.call_args + # Verify filter was applied in query_points call (qdrant-client 1.10+) + mock_qdrant_client.query_points.assert_called() + search_call_args = mock_qdrant_client.query_points.call_args assert search_call_args[1]["query_filter"] is not None # Verify results contain project information @@ -141,9 +143,9 @@ async def test_hybrid_search_without_project_filter(hybrid_search, mock_qdrant_c # Test search without project filter results = await hybrid_search.search(query="test query", limit=5) - # Verify no filter was applied - mock_qdrant_client.search.assert_called() - search_call_args = mock_qdrant_client.search.call_args + # Verify no filter was applied (qdrant-client 1.10+) + mock_qdrant_client.query_points.assert_called() + search_call_args = mock_qdrant_client.query_points.call_args assert search_call_args[1]["query_filter"] is None # Verify results still contain project information from metadata @@ -164,9 +166,9 @@ async def test_hybrid_search_multiple_projects(hybrid_search, mock_qdrant_client query="test query", limit=5, project_ids=["project-a", "project-b"] ) - # Verify filter was applied with multiple project IDs - mock_qdrant_client.search.assert_called() - search_call_args = mock_qdrant_client.search.call_args + # Verify filter was applied with multiple project IDs (qdrant-client 1.10+) + mock_qdrant_client.query_points.assert_called() + search_call_args = mock_qdrant_client.query_points.call_args query_filter = search_call_args[1]["query_filter"] assert query_filter is not None @@ -309,9 +311,9 @@ async def test_vector_search_with_project_filter(hybrid_search, mock_qdrant_clie query="test query", limit=5, project_ids=["project-a"] ) - # Verify search was called with filter - mock_qdrant_client.search.assert_called() - search_call_args = mock_qdrant_client.search.call_args + # Verify query_points was called with filter (qdrant-client 1.10+) + mock_qdrant_client.query_points.assert_called() + search_call_args = mock_qdrant_client.query_points.call_args assert search_call_args[1]["query_filter"] is not None # Verify results diff --git a/packages/qdrant-loader-mcp-server/tests/unit/search/test_vector_search_cache.py b/packages/qdrant-loader-mcp-server/tests/unit/search/test_vector_search_cache.py index 2536e0f80..f07578d5f 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/search/test_vector_search_cache.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/search/test_vector_search_cache.py @@ -145,8 +145,12 @@ async def test_cache_hit( # Mock the get_embedding method vector_search_service.get_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) - # Mock QDrant search response - vector_search_service.qdrant_client.search.return_value = sample_search_results + # Mock QDrant query_points response (qdrant-client 1.10+) + mock_query_response = MagicMock() + mock_query_response.points = sample_search_results + vector_search_service.qdrant_client.query_points = AsyncMock( + return_value=mock_query_response + ) # First call - should be a cache miss results1 = await vector_search_service.vector_search("test query", 10) @@ -161,7 +165,7 @@ async def test_cache_hit( assert results1 == results2 # Verify QDrant was only called once - assert vector_search_service.qdrant_client.search.call_count == 1 + assert vector_search_service.qdrant_client.query_points.call_count == 1 @patch("qdrant_loader_mcp_server.search.components.vector_search_service.time.time") @pytest.mark.asyncio @@ -172,8 +176,12 @@ async def test_cache_expiry( # Mock the get_embedding method vector_search_service.get_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) - # Mock QDrant search response - vector_search_service.qdrant_client.search.return_value = sample_search_results + # Mock QDrant query_points response (qdrant-client 1.10+) + mock_query_response = MagicMock() + mock_query_response.points = sample_search_results + vector_search_service.qdrant_client.query_points = AsyncMock( + return_value=mock_query_response + ) # First call at time 1000 mock_time.return_value = 1000.0 @@ -200,9 +208,11 @@ async def test_cache_disabled( return_value=[0.1, 0.2, 0.3] ) - # Mock QDrant search response - vector_search_service_no_cache.qdrant_client.search.return_value = ( - sample_search_results + # Mock QDrant query_points response (qdrant-client 1.10+) + mock_query_response = MagicMock() + mock_query_response.points = sample_search_results + vector_search_service_no_cache.qdrant_client.query_points = AsyncMock( + return_value=mock_query_response ) # Multiple calls with same parameters @@ -214,7 +224,7 @@ async def test_cache_disabled( assert vector_search_service_no_cache._cache_hits == 0 # QDrant should be called twice - assert vector_search_service_no_cache.qdrant_client.search.call_count == 2 + assert vector_search_service_no_cache.qdrant_client.query_points.call_count == 2 def test_cache_cleanup_expired_entries(self, vector_search_service): """Test cleanup of expired cache entries.""" @@ -323,8 +333,12 @@ async def test_project_filter_caching( # Mock the get_embedding method vector_search_service.get_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) - # Mock QDrant search response - vector_search_service.qdrant_client.search.return_value = sample_search_results + # Mock QDrant query_points response (qdrant-client 1.10+) + mock_query_response = MagicMock() + mock_query_response.points = sample_search_results + vector_search_service.qdrant_client.query_points = AsyncMock( + return_value=mock_query_response + ) # Search with different project filters should create separate cache entries await vector_search_service.vector_search("test query", 10, ["project1"]) @@ -335,7 +349,7 @@ async def test_project_filter_caching( assert vector_search_service._cache_misses == 2 # First two calls assert vector_search_service._cache_hits == 1 # Third call - assert vector_search_service.qdrant_client.search.call_count == 2 + assert vector_search_service.qdrant_client.query_points.call_count == 2 @pytest.mark.asyncio async def test_result_format_consistency( @@ -345,8 +359,12 @@ async def test_result_format_consistency( # Mock the get_embedding method vector_search_service.get_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) - # Mock QDrant search response - vector_search_service.qdrant_client.search.return_value = sample_search_results + # Mock QDrant query_points response (qdrant-client 1.10+) + mock_query_response = MagicMock() + mock_query_response.points = sample_search_results + vector_search_service.qdrant_client.query_points = AsyncMock( + return_value=mock_query_response + ) # Get fresh results fresh_results = await vector_search_service.vector_search("test query", 10) diff --git a/packages/qdrant-loader-mcp-server/tests/unit/test_config_loader.py b/packages/qdrant-loader-mcp-server/tests/unit/test_config_loader.py index d8743de96..907e3ffd9 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/test_config_loader.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/test_config_loader.py @@ -13,7 +13,12 @@ def test_resolve_config_path_env(tmp_path, monkeypatch): assert resolve_config_path(None) == cfg -def test_build_config_from_dict_minimal_global_llm(): +def test_build_config_from_dict_minimal_global_llm(monkeypatch): + # Clear env vars to ensure config dict values are used + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("LLM_API_KEY", raising=False) + monkeypatch.delenv("LLM_EMBEDDING_MODEL", raising=False) + monkeypatch.delenv("LLM_CHAT_MODEL", raising=False) data = { "global": { "llm": { diff --git a/packages/qdrant-loader-mcp-server/tests/unit/test_intelligence_handler.py b/packages/qdrant-loader-mcp-server/tests/unit/test_intelligence_handler.py index 3304a670f..b0517417a 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/test_intelligence_handler.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/test_intelligence_handler.py @@ -178,6 +178,7 @@ async def test_handle_find_similar_documents_success( similarity_metrics=None, project_ids=None, source_types=None, + similarity_threshold=0.7, # Default value when not provided ) assert result == {"result": mock_similar_docs} @@ -205,6 +206,37 @@ async def test_handle_find_similar_documents_with_metrics( similarity_metrics=["semantic_similarity", "entity_overlap"], project_ids=None, source_types=None, + similarity_threshold=0.7, # Default value when not provided + ) + + @pytest.mark.asyncio + async def test_handle_find_similar_documents_with_custom_threshold( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test finding similar documents with custom similarity threshold.""" + mock_result = { + "similar_documents": [{"document_id": "doc1", "similarity_score": 0.85}] + } + mock_search_engine.find_similar_documents.return_value = mock_result + mock_protocol.create_response.return_value = {"result": mock_result} + + params = { + "target_query": "target", + "comparison_query": "comparison", + "similarity_threshold": 0.8, # Custom threshold + "max_similar": 3, + } + + await intelligence_handler.handle_find_similar_documents(7, params) + + mock_search_engine.find_similar_documents.assert_called_once_with( + target_query="target", + comparison_query="comparison", + max_similar=3, + similarity_metrics=None, + project_ids=None, + source_types=None, + similarity_threshold=0.8, ) @@ -1050,6 +1082,226 @@ async def test_find_similar_documents_formatting_validation( error_call = mock_logger.error.call_args[0][0] assert "Missing document_id" in error_call + @pytest.mark.asyncio + async def test_find_similar_documents_with_object_document( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test handling of document objects with proper attribute access.""" + # Create a mock document object to simulate HybridSearchResult + mock_doc_obj = MagicMock() + mock_doc_obj.document_id = "doc123" + mock_doc_obj.source_title = "Test Document Title" + # Make text long enough to trigger truncation at 200 chars + mock_doc_obj.text = "A" * 250 # 250 chars, should be truncated to 200 + "..." + + # Return list format as search engine does + mock_similar_docs = [ + { + "document_id": "doc123", + "document": mock_doc_obj, # Document as object, not dict + "similarity_score": 0.85, + "metric_scores": {"semantic_similarity": 0.85}, + "similarity_reasons": ["High semantic overlap"], + } + ] + mock_search_engine.find_similar_documents.return_value = mock_similar_docs + + mock_protocol.create_response.return_value = {"result": "success"} + + params = { + "target_query": "test query", + "comparison_query": "comparison", + "max_similar": 5, + } + + await intelligence_handler.handle_find_similar_documents(42, params) + + # Verify create_response was called with correct structure + mock_protocol.create_response.assert_called_once() + call_args = mock_protocol.create_response.call_args + + # Extract the result parameter from the call + result_param = call_args[1]["result"] + structured = result_param["structuredContent"] + similar_docs = structured["similar_documents"] + + # Should have 1 document + assert len(similar_docs) == 1 + + doc = similar_docs[0] + # Verify title was extracted from object attribute + assert doc["title"] == "Test Document Title" + # Verify content_preview was created and truncated + assert "content_preview" in doc + assert len(doc["content_preview"]) > 0 + # Should be truncated at 200 chars with "..." + assert doc["content_preview"].endswith("...") + assert len(doc["content_preview"]) == 203 # 200 chars + "..." + # Verify it's not "Untitled" or empty + assert doc["title"] != "Untitled" + + @pytest.mark.asyncio + async def test_find_similar_documents_with_dict_document( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test handling of document dicts with proper dict access.""" + # Return list format with document as dict + mock_similar_docs = [ + { + "document_id": "doc456", + "document": { # Document as dict + "document_id": "doc456", + "source_title": "Dict Document", + "text": "Content from dict document", + }, + "similarity_score": 0.75, + "metric_scores": {"entity_overlap": 0.75}, + "similarity_reasons": ["Entity overlap"], + } + ] + mock_search_engine.find_similar_documents.return_value = mock_similar_docs + + mock_protocol.create_response.return_value = {"result": "success"} + + params = { + "target_query": "test", + "comparison_query": "all", + "max_similar": 5, + } + + await intelligence_handler.handle_find_similar_documents(43, params) + + # Verify create_response was called with correct structure + call_args = mock_protocol.create_response.call_args + result_param = call_args[1]["result"] + structured = result_param["structuredContent"] + similar_docs = structured["similar_documents"] + + assert len(similar_docs) == 1 + doc = similar_docs[0] + # Verify title was extracted from dict + assert doc["title"] == "Dict Document" + # Verify content_preview exists + assert "content_preview" in doc + assert doc["content_preview"] == "Content from dict document" + + @pytest.mark.asyncio + async def test_find_similar_documents_missing_title_fallback( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test fallback to item-level source_title when document doesn't have it.""" + # Create a mock without source_title attribute using spec + mock_doc_obj = MagicMock(spec=["document_id", "text"]) + mock_doc_obj.document_id = "doc789" + mock_doc_obj.text = "Some content" + + mock_similar_docs = [ + { + "document_id": "doc789", + "source_title": "Item Level Title", # Title at item level + "document": mock_doc_obj, + "similarity_score": 0.65, + "metric_scores": {}, + "similarity_reasons": [], + } + ] + mock_search_engine.find_similar_documents.return_value = mock_similar_docs + + mock_protocol.create_response.return_value = {"result": "success"} + + params = { + "target_query": "test", + "comparison_query": "comparison", + } + + await intelligence_handler.handle_find_similar_documents(44, params) + + call_args = mock_protocol.create_response.call_args + result_param = call_args[1]["result"] + structured = result_param["structuredContent"] + similar_docs = structured["similar_documents"] + + assert len(similar_docs) == 1 + # Should fallback to item-level source_title + assert similar_docs[0]["title"] == "Item Level Title" + + @pytest.mark.asyncio + async def test_find_similar_documents_no_text_empty_preview( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test that missing text results in empty content_preview, not error.""" + # Create mock with only document_id, no source_title or text attributes + mock_doc_obj = MagicMock(spec=["document_id"]) + mock_doc_obj.document_id = "doc999" + + mock_similar_docs = [ + { + "document_id": "doc999", + "document": mock_doc_obj, + "similarity_score": 0.50, + "metric_scores": {}, + "similarity_reasons": [], + } + ] + mock_search_engine.find_similar_documents.return_value = mock_similar_docs + + mock_protocol.create_response.return_value = {"result": "success"} + + params = { + "target_query": "test", + "comparison_query": "comparison", + } + + await intelligence_handler.handle_find_similar_documents(45, params) + + call_args = mock_protocol.create_response.call_args + result_param = call_args[1]["result"] + structured = result_param["structuredContent"] + similar_docs = structured["similar_documents"] + + assert len(similar_docs) == 1 + # Content preview should be empty string, not cause error + assert similar_docs[0]["content_preview"] == "" + + @pytest.mark.asyncio + async def test_find_similar_documents_non_string_text( + self, intelligence_handler, mock_search_engine, mock_protocol + ): + """Test handling of non-string text values.""" + mock_doc_obj = MagicMock() + mock_doc_obj.document_id = "doc888" + mock_doc_obj.source_title = "Bad Type Doc" + mock_doc_obj.text = 12345 # Non-string text (bug scenario) + + mock_similar_docs = [ + { + "document_id": "doc888", + "document": mock_doc_obj, + "similarity_score": 0.60, + "metric_scores": {}, + "similarity_reasons": [], + } + ] + mock_search_engine.find_similar_documents.return_value = mock_similar_docs + + mock_protocol.create_response.return_value = {"result": "success"} + + params = { + "target_query": "test", + "comparison_query": "comparison", + } + + await intelligence_handler.handle_find_similar_documents(46, params) + + call_args = mock_protocol.create_response.call_args + result_param = call_args[1]["result"] + structured = result_param["structuredContent"] + similar_docs = structured["similar_documents"] + + assert len(similar_docs) == 1 + # Should handle non-string gracefully with empty preview + assert similar_docs[0]["content_preview"] == "" + @pytest.mark.asyncio async def test_formatter_method_calls( self, intelligence_handler, mock_search_engine, mock_protocol @@ -1374,4 +1626,5 @@ async def test_none_values_in_params( similarity_metrics=None, source_types=None, project_ids=None, + similarity_threshold=0.7, # Default value when not in params ) diff --git a/packages/qdrant-loader-mcp-server/tests/unit/test_logging.py b/packages/qdrant-loader-mcp-server/tests/unit/test_logging.py index 45dd8da19..e1ba89eef 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/test_logging.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/test_logging.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch import pytest +import structlog from qdrant_loader_mcp_server.utils.logging import ( ApplicationFilter, CleanFormatter, @@ -15,6 +16,44 @@ ) +@pytest.fixture(autouse=True) +def reset_logging_state(): + """Reset logging state before and after each test to prevent test pollution.""" + # Store original state + root_logger = logging.getLogger() + original_handlers = root_logger.handlers.copy() + original_level = root_logger.level + + # Reset LoggingConfig state + LoggingConfig._initialized = False + LoggingConfig._current_config = None + + yield + + # Cleanup: remove all handlers and restore original state + for handler in root_logger.handlers[:]: + try: + root_logger.removeHandler(handler) + if isinstance(handler, logging.FileHandler): + handler.close() + except Exception: + pass + + # Restore original handlers + for handler in original_handlers: + if handler not in root_logger.handlers: + root_logger.addHandler(handler) + + root_logger.setLevel(original_level) + + # Reset structlog + structlog.reset_defaults() + + # Reset LoggingConfig state + LoggingConfig._initialized = False + LoggingConfig._current_config = None + + def test_qdrant_version_filter(): """Test QdrantVersionFilter filters version check warnings.""" filter_instance = QdrantVersionFilter() @@ -285,3 +324,74 @@ def test_logging_config_reset_and_reconfigure(): assert LoggingConfig._current_config is not None assert LoggingConfig._current_config[0] == "DEBUG" assert LoggingConfig._current_config[1] == "json" + + +def test_reconfigure_with_level(monkeypatch): + """Test that reconfigure() correctly updates the log level.""" + # Clear env vars to ensure test values are used + monkeypatch.delenv("MCP_LOG_LEVEL", raising=False) + + # Initial setup + LoggingConfig.setup(level="INFO", format="console") + assert LoggingConfig._current_config is not None + assert LoggingConfig._current_config[0] == "INFO" + + # Reconfigure with new level + LoggingConfig.reconfigure(level="DEBUG") + assert LoggingConfig._current_config[0] == "DEBUG" + # Other config values should remain unchanged + assert LoggingConfig._current_config[1] == "console" + + +def test_reconfigure_with_file_and_level(monkeypatch): + """Test that reconfigure() correctly updates both file and level.""" + # Clear env vars to ensure test values are used + monkeypatch.delenv("MCP_LOG_LEVEL", raising=False) + + # Initial setup + LoggingConfig.setup(level="INFO", format="console") + assert LoggingConfig._current_config is not None + + # Reconfigure with both file and level + with patch("logging.FileHandler"): + LoggingConfig.reconfigure(file="/tmp/test.log", level="WARNING") + + assert LoggingConfig._current_config[0] == "WARNING" + assert LoggingConfig._current_config[2] == "/tmp/test.log" + + +def test_reconfigure_level_only_preserves_other_config(monkeypatch): + """Test that reconfigure(level=...) preserves other config values.""" + # Clear env vars to ensure test values are used + monkeypatch.delenv("MCP_LOG_LEVEL", raising=False) + + # Initial setup with specific values + LoggingConfig.setup(level="INFO", format="json", suppress_qdrant_warnings=True) + original_format = LoggingConfig._current_config[1] + original_suppress = LoggingConfig._current_config[3] + + # Reconfigure only level + LoggingConfig.reconfigure(level="ERROR") + + # Level should change + assert LoggingConfig._current_config[0] == "ERROR" + # Other values should be preserved + assert LoggingConfig._current_config[1] == original_format + assert LoggingConfig._current_config[3] == original_suppress + + +def test_reconfigure_without_level_preserves_current_level(monkeypatch): + """Test that reconfigure() without level keeps the current level.""" + # Clear env vars to ensure test values are used + monkeypatch.delenv("MCP_LOG_LEVEL", raising=False) + + # Initial setup + LoggingConfig.setup(level="DEBUG", format="console") + assert LoggingConfig._current_config[0] == "DEBUG" + + # Reconfigure without level (only file) + with patch("logging.FileHandler"): + LoggingConfig.reconfigure(file="/tmp/test.log") + + # Level should remain unchanged + assert LoggingConfig._current_config[0] == "DEBUG" diff --git a/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_async_behavior.py b/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_async_behavior.py index 328ce55e4..e4b444f06 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_async_behavior.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_async_behavior.py @@ -579,8 +579,8 @@ async def slow_operation(delay): parallel_time = time.time() - start_time # Parallel execution should be faster than 3 * 0.1 seconds - # (allowing some overhead) - assert parallel_time < 0.35 # Much less than 3 * 0.1 = 0.3 + # (allowing overhead for CI/slow systems like WSL) + assert parallel_time < 0.5 # Much less than 3 * 0.1 = 0.3 assert len(results) == 3 @pytest.mark.asyncio diff --git a/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_comprehensive.py b/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_comprehensive.py index 550fbbf44..58266b503 100644 --- a/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_comprehensive.py +++ b/packages/qdrant-loader-mcp-server/tests/unit/test_search_handler_comprehensive.py @@ -252,7 +252,7 @@ async def test_handle_search_with_defaults( # Verify defaults were used search_handler.search_engine.search.assert_called_once_with( - query="test", source_types=[], project_ids=[], limit=10 + query="test", source_types=[], project_ids=[], limit=5 ) @pytest.mark.asyncio diff --git a/packages/qdrant-loader/pyproject.toml b/packages/qdrant-loader/pyproject.toml index eaa5d8dbe..e5377c5fc 100644 --- a/packages/qdrant-loader/pyproject.toml +++ b/packages/qdrant-loader/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "qdrant-loader" -version = "0.7.3" +version = "0.7.6" description = "A tool for collecting and vectorizing technical content from multiple sources and storing it in a QDrant vector database." readme = "README.md" requires-python = ">=3.12" @@ -35,7 +35,7 @@ dependencies = [ "structlog>=23.0.0", "httpx>=0.24.0", "openai>=1.0.0", - "qdrant-loader-core[openai]==0.7.3", + "qdrant-loader-core[openai]==0.7.6", "qdrant-client>=1.7.0", "PyYAML>=6.0.0", "beautifulsoup4>=4.12.0", @@ -65,6 +65,7 @@ dependencies = [ "markitdown[all]>=0.1.3", "rich>=13.0.0", "packaging>=21.0", + "prometheus-client>=0.19.0,<1.0.0", ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/packages/qdrant-loader/src/qdrant_loader/cli/commands/config.py b/packages/qdrant-loader/src/qdrant_loader/cli/commands/config.py index 3bc39bb39..b387a6b7d 100644 --- a/packages/qdrant-loader/src/qdrant_loader/cli/commands/config.py +++ b/packages/qdrant-loader/src/qdrant_loader/cli/commands/config.py @@ -42,7 +42,7 @@ def run_show_config( ) if getattr(LoggingConfig, "reconfigure", None): # Core supports reconfigure if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined] - LoggingConfig.reconfigure(file=log_file) # type: ignore[attr-defined] + LoggingConfig.reconfigure(file=log_file, level=log_level) # type: ignore[attr-defined] else: LoggingConfig.setup(level=log_level, format="console", file=log_file) else: diff --git a/packages/qdrant-loader/src/qdrant_loader/cli/commands/config_cmd.py b/packages/qdrant-loader/src/qdrant_loader/cli/commands/config_cmd.py index da2b38361..8c59777ec 100644 --- a/packages/qdrant-loader/src/qdrant_loader/cli/commands/config_cmd.py +++ b/packages/qdrant-loader/src/qdrant_loader/cli/commands/config_cmd.py @@ -22,7 +22,7 @@ def run_config_command( ) if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined] if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined] - LoggingConfig.reconfigure(file=log_file) # type: ignore[attr-defined] + LoggingConfig.reconfigure(file=log_file, level=log_level) # type: ignore[attr-defined] else: LoggingConfig.setup(level=log_level, format="console", file=log_file) else: diff --git a/packages/qdrant-loader/src/qdrant_loader/cli/commands/ingest_cmd.py b/packages/qdrant-loader/src/qdrant_loader/cli/commands/ingest_cmd.py index 79893b3af..7faf0af80 100644 --- a/packages/qdrant-loader/src/qdrant_loader/cli/commands/ingest_cmd.py +++ b/packages/qdrant-loader/src/qdrant_loader/cli/commands/ingest_cmd.py @@ -51,7 +51,7 @@ async def run_ingest_command( ) if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined] if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined] - LoggingConfig.reconfigure(file=log_file) # type: ignore[attr-defined] + LoggingConfig.reconfigure(file=log_file, level=log_level) # type: ignore[attr-defined] else: LoggingConfig.setup(level=log_level, format="console", file=log_file) else: diff --git a/packages/qdrant-loader/src/qdrant_loader/cli/commands/init_cmd.py b/packages/qdrant-loader/src/qdrant_loader/cli/commands/init_cmd.py index cd02001da..365e5242d 100644 --- a/packages/qdrant-loader/src/qdrant_loader/cli/commands/init_cmd.py +++ b/packages/qdrant-loader/src/qdrant_loader/cli/commands/init_cmd.py @@ -42,7 +42,7 @@ async def run_init_command( # Setup logging first (workspace-aware later). Use core reconfigure if available. if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined] if getattr(LoggingConfig, "_initialized", False): # type: ignore[attr-defined] - LoggingConfig.reconfigure(file="qdrant-loader.log") # type: ignore[attr-defined] + LoggingConfig.reconfigure(file="qdrant-loader.log", level=log_level) # type: ignore[attr-defined] else: LoggingConfig.setup( level=log_level, format="console", file="qdrant-loader.log" @@ -88,7 +88,7 @@ async def run_init_command( else "qdrant-loader.log" ) if getattr(LoggingConfig, "reconfigure", None): # type: ignore[attr-defined] - LoggingConfig.reconfigure(file=log_file) # type: ignore[attr-defined] + LoggingConfig.reconfigure(file=log_file, level=log_level) # type: ignore[attr-defined] else: import logging as _py_logging diff --git a/packages/qdrant-loader/src/qdrant_loader/core/qdrant_manager.py b/packages/qdrant-loader/src/qdrant_loader/core/qdrant_manager.py index 280c89d4a..7dc6dbaa7 100644 --- a/packages/qdrant-loader/src/qdrant_loader/core/qdrant_manager.py +++ b/packages/qdrant-loader/src/qdrant_loader/core/qdrant_manager.py @@ -246,12 +246,13 @@ def search( """Search for similar vectors in the collection.""" try: client = self._ensure_client_connected() - search_result = client.search( + # Use query_points API (qdrant-client 1.10+) + query_response = client.query_points( collection_name=self.collection_name, - query_vector=query_vector, + query=query_vector, limit=limit, ) - return search_result + return query_response.points except Exception as e: logger.error("Failed to search collection", error=str(e)) raise @@ -281,13 +282,14 @@ def search_with_project_filter( ] ) - search_result = client.search( + # Use query_points API (qdrant-client 1.10+) + query_response = client.query_points( collection_name=self.collection_name, - query_vector=query_vector, + query=query_vector, query_filter=project_filter, limit=limit, ) - return search_result + return query_response.points except Exception as e: logger.error( "Failed to search collection with project filter", diff --git a/packages/qdrant-loader/tests/unit/connectors/git/test_git_connector.py b/packages/qdrant-loader/tests/unit/connectors/git/test_git_connector.py index 92fa5c169..a4606fa65 100644 --- a/packages/qdrant-loader/tests/unit/connectors/git/test_git_connector.py +++ b/packages/qdrant-loader/tests/unit/connectors/git/test_git_connector.py @@ -136,9 +136,16 @@ async def test_error_handling(self, mock_config): connector = GitConnector(invalid_config) # Test cloning failure - with pytest.raises(RuntimeError): - async with connector: - pass + with patch.object( + connector.git_ops, + "clone", + side_effect=Exception("clone failed"), + ): + with pytest.raises(RuntimeError) as exc: + async with connector: + pass + + assert "Failed to set up Git repository" in str(exc.value) @pytest.mark.asyncio async def test_file_processing(self, mock_config, mock_git_ops): diff --git a/packages/qdrant-loader/tests/unit/connectors/git/test_metadata_extractor.py b/packages/qdrant-loader/tests/unit/connectors/git/test_metadata_extractor.py index 5442271de..d7b546f30 100644 --- a/packages/qdrant-loader/tests/unit/connectors/git/test_metadata_extractor.py +++ b/packages/qdrant-loader/tests/unit/connectors/git/test_metadata_extractor.py @@ -171,7 +171,7 @@ def test_error_handling(self, base_config): assert metadata["file_name"] == "test.md" # File directory should be the relative path from temp_dir assert metadata["file_directory"] == "nonexistent" - + def test_detect_encoding(self, base_config): """Test encoding detection.""" extractor = GitMetadataExtractor(base_config) diff --git a/packages/qdrant-loader/tests/unit/core/file_conversion/test_markitdown_windows_fix.py b/packages/qdrant-loader/tests/unit/core/file_conversion/test_markitdown_windows_fix.py index e3868ee0a..80c350fb4 100644 --- a/packages/qdrant-loader/tests/unit/core/file_conversion/test_markitdown_windows_fix.py +++ b/packages/qdrant-loader/tests/unit/core/file_conversion/test_markitdown_windows_fix.py @@ -59,9 +59,16 @@ def test_markitdown_signal_error_prevention(self): with patch("sys.platform", "win32"): # Mock both validation and MarkItDown to focus on signal error prevention with patch.object(converter, "_validate_file") as mock_validate: - with patch.object( - converter, "_get_markitdown" - ) as mock_get_markitdown: + with ( + patch.object( + converter, "_get_markitdown" + ) as mock_get_markitdown, + patch( + "threading.Thread.start", + autospec=True, + return_value=None, + ), + ): mock_validate.return_value = None # Pass validation mock_markitdown = Mock() diff --git a/packages/qdrant-loader/tests/unit/core/test_qdrant_manager.py b/packages/qdrant-loader/tests/unit/core/test_qdrant_manager.py index aa5e5c0da..1e6fe42e5 100644 --- a/packages/qdrant-loader/tests/unit/core/test_qdrant_manager.py +++ b/packages/qdrant-loader/tests/unit/core/test_qdrant_manager.py @@ -474,7 +474,9 @@ def test_search_success(self, mock_settings, mock_qdrant_client): """Test successful search.""" query_vector = [0.1, 0.2, 0.3] mock_results = [Mock(spec=models.ScoredPoint)] - mock_qdrant_client.search.return_value = mock_results + mock_query_response = Mock() + mock_query_response.points = mock_results + mock_qdrant_client.query_points.return_value = mock_query_response with ( patch("qdrant_loader.core.qdrant_manager.get_global_config"), @@ -487,15 +489,17 @@ def test_search_success(self, mock_settings, mock_qdrant_client): results = manager.search(query_vector, limit=10) assert results == mock_results - mock_qdrant_client.search.assert_called_once_with( - collection_name="test_collection", query_vector=query_vector, limit=10 + mock_qdrant_client.query_points.assert_called_once_with( + collection_name="test_collection", query=query_vector, limit=10 ) def test_search_default_limit(self, mock_settings, mock_qdrant_client): """Test search with default limit.""" query_vector = [0.1, 0.2, 0.3] mock_results = [Mock(spec=models.ScoredPoint)] - mock_qdrant_client.search.return_value = mock_results + mock_query_response = Mock() + mock_query_response.points = mock_results + mock_qdrant_client.query_points.return_value = mock_query_response with ( patch("qdrant_loader.core.qdrant_manager.get_global_config"), @@ -507,14 +511,14 @@ def test_search_default_limit(self, mock_settings, mock_qdrant_client): manager = QdrantManager(mock_settings) manager.search(query_vector) - mock_qdrant_client.search.assert_called_once_with( - collection_name="test_collection", query_vector=query_vector, limit=5 + mock_qdrant_client.query_points.assert_called_once_with( + collection_name="test_collection", query=query_vector, limit=5 ) def test_search_error(self, mock_settings, mock_qdrant_client): """Test search error handling.""" query_vector = [0.1, 0.2, 0.3] - mock_qdrant_client.search.side_effect = Exception("Search failed") + mock_qdrant_client.query_points.side_effect = Exception("Search failed") with ( patch("qdrant_loader.core.qdrant_manager.get_global_config"), diff --git a/pyproject.toml b/pyproject.toml index 7b0ca9d57..7e029517f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "qdrant-loader-workspace" -version = "0.7.3" +version = "0.7.6" description = "A comprehensive toolkit for loading data into Qdrant vector database with MCP server support" readme = "README.md" authors = [ @@ -54,7 +54,8 @@ dev = [ "py-spy", "snakeviz", "memory-profiler", - "prometheus-client", + "prometheus-client>=0.19.0,<1.0.0", + "pytest-xdist>=3.8.0", ] docs = [ "tomli>=2.0.0", diff --git a/release.py b/release.py index a08c1c2a8..56874c0cd 100644 --- a/release.py +++ b/release.py @@ -269,9 +269,17 @@ def get_github_token(dry_run: bool = False) -> str: def extract_repo_info(git_url: str, dry_run: bool = False) -> str: """ - Extract GitHub username and repository name from git remote URL. + Extract the GitHub repository path "owner/repo" from a git remote URL. - Returns the repo info in format "username/repo" + Parameters: + git_url (str): Git remote URL in one of the supported formats (HTTPS, ssh://, or git@). + dry_run (bool): If True, return "unknown/repo" on parse failure instead of exiting. + + Returns: + repo_path (str): The repository path in the form "username/repo". + + Raises: + SystemExit: Exits with status 1 when the URL cannot be parsed and `dry_run` is False. """ logger = logging.getLogger(__name__) logger.debug(f"Extracting repo info from: {git_url}") @@ -312,10 +320,49 @@ def extract_repo_info(git_url: str, dry_run: bool = False) -> str: sys.exit(1) +def extract_changelog_for_version(version: str) -> str: + """Extract changelog content for a specific version from root CHANGELOG.md. + + Args: + version: Version string (e.g., "0.7.4") + + Returns: + Changelog content or empty string if not found + """ + logger = logging.getLogger(__name__) + + changelog_path = "CHANGELOG.md" + if not Path(changelog_path).exists(): + logger.debug(f"CHANGELOG not found at {changelog_path}") + return "" + + with open(changelog_path, encoding="utf-8") as f: + content = f.read() + + # Find version section (format: ## [X.Y.Z] - Date) + import re + version_pattern = rf"## \[{re.escape(version)}\].*?\n(.*?)(?=\n## |\Z)" + match = re.search(version_pattern, content, re.DOTALL) + + if not match: + logger.debug(f"Version {version} not found in {changelog_path}") + return "" + + version_content = match.group(1).strip() + return version_content + + def create_github_release( package_name: str, version: str, token: str, dry_run: bool = False ) -> None: - """Create a GitHub release for a specific package.""" + """Create a GitHub release for a specific package. + + Args: + package_name: Name of the package to release + version: Version string + token: GitHub API token + dry_run: If True, simulate without making changes + """ logger = logging.getLogger(__name__) tag_name = f"{package_name}-v{version}" @@ -326,9 +373,15 @@ def create_github_release( return logger.info(f"Creating GitHub release for {package_name} version {version}") - # Get the latest commits for release notes - stdout, _ = run_command("git log --pretty=format:'%h %s' -n 10") - release_notes = f"## Changes for {package_name} v{version}\n\n```\n{stdout}\n```" + + # Extract changelog directly from root CHANGELOG.md + logger.info("Extracting changelog from root CHANGELOG.md") + release_notes = extract_changelog_for_version(version) + + if not release_notes: + logger.warning("No changelog found, falling back to git log for release notes") + stdout, _ = run_command("git log --pretty=format:'%h %s' -n 10") + release_notes = f"## Changes for {package_name} v{version}\n\n```\n{stdout}\n```" # Create release headers = { @@ -359,11 +412,23 @@ def create_github_release( f"Error creating GitHub release for {package_name}: {response.text}" ) sys.exit(1) + logger.info(f"GitHub release created successfully for {package_name}") def check_main_up_to_date(dry_run: bool = False) -> bool: - """Check if local main branch is up to date with remote main.""" + """ + Verify that the local main branch is synchronized with origin/main. + + Parameters: + dry_run (bool): If True, do not exit the process on mismatch and only simulate checks. + + Returns: + bool: `True` if the local main branch is up to date with origin/main, `False` otherwise. + + Notes: + If the branch is not up to date and `dry_run` is False, the process will exit with status code 1. + """ logger = logging.getLogger(__name__) logger.debug("Checking if main branch is up to date") stdout, _ = run_command("git fetch origin main", dry_run) @@ -379,74 +444,87 @@ def check_main_up_to_date(dry_run: bool = False) -> bool: return True -def check_release_notes_updated(new_version: str, dry_run: bool = False) -> bool: - """Check if RELEASE_NOTES.md has been updated with the new version.""" +def check_changelog_updated(new_version: str, dry_run: bool = False) -> bool: + """ + Verify that the repository root CHANGELOG.md contains a top-level section for the specified new version. + + Checks for a version header matching the pattern `## [X.Y.Z]` (supports `b` beta suffix like `1.2.3b1`) and ignores an `## [Unreleased]` section; logs errors and calls `sys.exit(1)` on failure unless `dry_run` is True. + + Parameters: + new_version (str): The version string to look for (e.g., "1.2.3" or "1.2.3b1"). + dry_run (bool): If True, do not exit the process on failure; return False instead. + + Returns: + bool: `True` if a changelog section for `new_version` is found, `False` otherwise. + """ logger = logging.getLogger(__name__) logger.debug( - f"Checking if RELEASE_NOTES.md has been updated for version {new_version}" + f"Checking if CHANGELOG.md has been updated for version {new_version}" ) - release_notes_path = Path("RELEASE_NOTES.md") - if not release_notes_path.exists(): - logger.error("RELEASE_NOTES.md file not found in the repository root") + changelog_path = Path("CHANGELOG.md") + + if not changelog_path.exists(): + logger.error(f"CHANGELOG.md file not found at {changelog_path}") if not dry_run: sys.exit(1) return False try: - with open(release_notes_path, encoding="utf-8") as f: + with open(changelog_path, encoding="utf-8") as f: content = f.read() - # Look for the version section at the beginning of the file - # Expected format: ## Version X.Y.Z - Date + # Look for the version section + # Expected format: ## [X.Y.Z] - Date import re - # Extract the first version section after the title lines = content.split("\n") - version_pattern = r"^## Version (\d+\.\d+\.\d+(?:b\d+)?)" + version_pattern = r"^## \[(\d+\.\d+\.\d+(?:b\d+)?)\]" + found_version = None for line in lines: - if line.startswith("## Version"): - match = re.match(version_pattern, line) - if match: - found_version = match.group(1) - logger.debug( - f"Found version section in RELEASE_NOTES.md: {found_version}" - ) + if re.match(r'^## \[Unreleased\]', line): + continue # Skip Unreleased section - if found_version == new_version: - logger.debug( - "Release notes are up to date with the new version" - ) - return True - else: - logger.error( - f"RELEASE_NOTES.md has not been updated for version {new_version}" - ) - logger.error( - f"Found version {found_version} but expected {new_version}" - ) - logger.error( - "Please add a release notes section for the new version at the top of RELEASE_NOTES.md" - ) - logger.error( - f"Expected format: ## Version {new_version} - " - ) - if not dry_run: - sys.exit(1) - return False + match = re.match(version_pattern, line) + if match: + found_version = match.group(1) + logger.debug( + f"Found version section in CHANGELOG.md: {found_version}" + ) break - # If we get here, no version section was found - logger.error("No version section found in RELEASE_NOTES.md") - logger.error(f"Please add a release notes section for version {new_version}") - logger.error(f"Expected format: ## Version {new_version} - ") - if not dry_run: - sys.exit(1) - return False + if found_version == new_version: + logger.debug( + "CHANGELOG.md is up to date with the new version" + ) + return True + elif found_version: + logger.error( + f"CHANGELOG.md has not been updated for version {new_version}" + ) + logger.error( + f"Found version {found_version} but expected {new_version}" + ) + logger.error( + f"Please add a changelog section for the new version at the top of {changelog_path}" + ) + logger.error( + f"Expected format: ## [{new_version}] - " + ) + if not dry_run: + sys.exit(1) + return False + else: + logger.error("No version section found in CHANGELOG.md") + logger.error(f"Please add a changelog section for version {new_version}") + logger.error(f"Expected format: ## [{new_version}] - ") + if not dry_run: + sys.exit(1) + return False except Exception as e: - logger.error(f"Error reading RELEASE_NOTES.md: {e}") + logger.error(f"Error reading CHANGELOG.md: {e}") if not dry_run: sys.exit(1) return False @@ -882,10 +960,15 @@ def update_all_internal_dependencies_versions( help="Sync all packages to the same version (uses qdrant-loader as source of truth)", ) def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = False): - """Create a new release with unified versioning for all packages. + """ + Orchestrate a coordinated release across all packages: compute and apply a unified version, run safety checks, update pyproject metadata, commit and push changes, tag, and create GitHub releases. - All packages will always have the same version number. The qdrant-loader - package is used as the source of truth for the current version. + When invoked with sync_versions=True the command only synchronizes all package versions, development-status classifiers, and internal dependency pins to the qdrant-loader package version and then exits. In normal mode it performs repository and CI checks, prompts for a version bump (major/minor/patch/beta/custom), validates CHANGELOG.md, applies the version and classifier updates, pins internal dependencies, commits and pushes changes, creates annotated tags for each releasable package, and creates GitHub releases. Use dry_run=True to simulate all steps without making any persistent changes; use verbose=True to enable more detailed logging. + + Parameters: + dry_run (bool): If True, simulate actions without writing files, running non-whitelisted commands, committing, pushing, or creating releases. + verbose (bool): If True, enable verbose (debug) logging output. + sync_versions (bool): If True, only synchronize all packages to the qdrant-loader version (update versions, classifiers, and internal dependency pins) and exit. """ # Setup logging logger = setup_logging(verbose) @@ -946,7 +1029,7 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = if dry_run: print("๐Ÿ” DRY RUN MODE - No changes will be made\n") - # Run initial safety checks (without release notes check) + # Run initial safety checks (without changelog check) initial_check_results = {} initial_check_results["git_status"] = check_git_status(dry_run) initial_check_results["current_branch"] = check_current_branch(dry_run) @@ -1059,12 +1142,12 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = return sys.exit(1) - # Now check if release notes have been updated for the new version - release_notes_check = check_release_notes_updated(new_version, dry_run) + # Now check if changelog have been updated for the new version + changelog_check = check_changelog_updated(new_version, dry_run) # Combine all check results all_check_results = initial_check_results.copy() - all_check_results["release_notes_updated"] = release_notes_check + all_check_results["changelog_updated"] = changelog_check # Apply the same version to all packages new_versions = {} @@ -1074,15 +1157,15 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = # Display planned change print(f"All packages: {current_version} โ†’ {new_version}") - # Show release notes check result + # Show changelog check result if dry_run: - print("\n๐Ÿ“‹ RELEASE NOTES CHECK") + print("\n๐Ÿ“‹ Changelog CHECK") print("โ”€" * 30) - status = "โœ…" if release_notes_check else "โŒ" - print(f"{status} Release Notes Updated") - if not release_notes_check: + status = "โœ…" if changelog_check else "โŒ" + print(f"{status} Changelog Updated") + if not changelog_check: print( - f" โš ๏ธ RELEASE_NOTES.md needs to be updated for version {new_version}" + f" โš ๏ธ CHANGELOG.md needs to be updated for version {new_version}" ) print() @@ -1142,7 +1225,7 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = print("\n" + "โ”€" * 50) - # Check all results including release notes + # Check all results including changelog all_failed_checks = [] for check_name, passed in all_check_results.items(): if not passed: @@ -1163,7 +1246,7 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = return - # In real mode, exit if any check failed (including release notes) + # In real mode, exit if any check failed (including changelog) if not all(all_check_results.values()): logger.error("One or more safety checks failed. Aborting release.") sys.exit(1) @@ -1202,16 +1285,17 @@ def release(dry_run: bool = False, verbose: bool = False, sync_versions: bool = for package_name in get_packages_for_release(): create_github_release( package_name, new_version, token, dry_run - ) # Use new_version - - print("\n๐ŸŽ‰ RELEASE COMPLETED SUCCESSFULLY!") - print("โ”€" * 40) - print(f"\n๐Ÿ“ฆ Released version: v{new_version}") # Show new_version - print(" All packages released with the same version") - print(f"\n๐Ÿ”„ Updated from: v{current_version}") # Show what we updated from - print(" All packages now have the same new version") - print(" Development Status classifiers updated automatically") - print(" New version committed and pushed to remote repository") + ) + + if not dry_run: + print("\n๐ŸŽ‰ RELEASE COMPLETED SUCCESSFULLY!") + print("โ”€" * 40) + print(f"\n๐Ÿ“ฆ Released version: v{new_version}") + print(" All packages released with the same version") + print(f"\n๐Ÿ”„ Updated from: v{current_version}") + print(" All packages now have the same new version") + print(" Development Status classifiers updated automatically") + print(" New version committed and pushed to remote repository") if __name__ == "__main__": diff --git a/website/builder/core.py b/website/builder/core.py index 1b62c50c4..a557f0d87 100644 --- a/website/builder/core.py +++ b/website/builder/core.py @@ -358,8 +358,8 @@ def build_site( try: if Path("README.md").exists(): self.build_markdown_page("README.md", "docs/README.html") - if Path("RELEASE_NOTES.md").exists(): - self.build_markdown_page("RELEASE_NOTES.md", "docs/RELEASE_NOTES.html") + if Path("CHANGELOG.md").exists(): + self.build_markdown_page("CHANGELOG.md", "docs/CHANGELOG.html") if Path("CONTRIBUTING.md").exists(): self.build_markdown_page("CONTRIBUTING.md", "docs/CONTRIBUTING.html") # License (plain text) rendered via helper diff --git a/website/templates/docs-index.html b/website/templates/docs-index.html index beacdbc50..f7bb9f281 100644 --- a/website/templates/docs-index.html +++ b/website/templates/docs-index.html @@ -53,8 +53,8 @@

CLI
  • - - Release Notes + + Changelog Updates