From f0691ffd6580ded26d71164e0e45dd511346a3a3 Mon Sep 17 00:00:00 2001 From: Shahzad Asghar Date: Thu, 11 Jun 2026 23:29:56 +0300 Subject: [PATCH 1/2] feat(indexing): add markdown document indexing Adds an index-markdown-documents MCP tool that indexes markdown content into Solr, complementing the existing JSON/CSV/XML tools. - New MarkdownDocumentCreator (CommonMark + YAML front matter extension; lightweight, reflection-free, native-image safe): front matter entries become sanitized fields, title resolves from front matter or the first H1, heading texts go into a multi-valued headings field, and the plain-text body into content - Missing ids are derived from a SHA-256 content hash so re-indexing the same markdown stays idempotent (matches the tool hint) - Flow-style YAML lists ([a, b, c]) are expanded to multi-valued fields, matching block-style list behavior - index-data prompt accepts markdown/md formats - Unit tests, MCP integration round-trip test, README/AGENTS docs Refs #69 --- AGENTS.md | 4 +- README.md | 1 + build.gradle.kts | 3 + gradle/libs.versions.toml | 5 + .../mcp/server/indexing/IndexingService.java | 91 +++++- .../IndexingDocumentCreator.java | 34 ++- .../MarkdownDocumentCreator.java | 281 +++++++++++++++++ .../server/McpClientIntegrationTestBase.java | 34 +++ .../IndexingServiceIntegrationTest.java | 4 +- .../server/indexing/IndexingServiceTest.java | 16 + .../server/indexing/MarkdownIndexingTest.java | 287 ++++++++++++++++++ 11 files changed, 749 insertions(+), 11 deletions(-) create mode 100644 src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java create mode 100644 src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java diff --git a/AGENTS.md b/AGENTS.md index 4d9e30dd..8585aa29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -96,7 +96,7 @@ docker run -p 8080:8080 --rm -e PROFILES=http \ Four service classes expose MCP tools via `@McpTool` annotations: - **SearchService** (`search/`) - Full-text search with filtering, faceting, sorting, pagination -- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV, XML formats +- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV, XML, and markdown formats - **CollectionService** (`metadata/`) - List collections, get stats, health checks - **SchemaService** (`schema/`) - Schema introspection and additive modification (add-fields, add-field-types) @@ -104,7 +104,7 @@ Four service classes expose MCP tools via `@McpTool` annotations: `indexing/documentcreator/` uses strategy pattern for format parsing: - `SolrDocumentCreator` - Common interface -- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator` - Format implementations +- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator`, `MarkdownDocumentCreator` - Format implementations - `IndexingDocumentCreator` - Orchestrator that delegates to format-specific creators - `FieldNameSanitizer` - Automatic field name validation for Solr compatibility diff --git a/README.md b/README.md index 20915647..e8169a9f 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,7 @@ For complete setup instructions, see [security-docs/AUTH0_SETUP.md](security-doc | `index-json-documents` | Index documents from a JSON string into a Solr collection | | `index-csv-documents` | Index documents from a CSV string into a Solr collection | | `index-xml-documents` | Index documents from an XML string into a Solr collection | +| `index-markdown-documents` | Index a markdown document into a Solr collection, extracting front matter, title, headings, and body text | ### Collections diff --git a/build.gradle.kts b/build.gradle.kts index 6811ac6f..bdacef7d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -135,6 +135,9 @@ dependencies { implementation(libs.spring.ai.starter.mcp.server.webmvc) implementation(libs.solr.solrj) implementation(libs.commons.csv) + // CommonMark for markdown parsing + implementation(libs.commonmark) + implementation(libs.commonmark.ext.yaml.front.matter) // JSpecify for nullability annotations implementation(libs.jspecify) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 616c2dea..2fac73a2 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -11,6 +11,7 @@ graalvm-native = "0.10.6" spring-ai = "1.1.7" solr = "10.0.0" commons-csv = "1.14.1" +commonmark = "0.28.0" jspecify = "1.0.0" mcp-server-security = "0.0.6" @@ -49,6 +50,10 @@ solr-solrj = { module = "org.apache.solr:solr-solrj", version.ref = "solr" } # Apache Commons commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" } +# CommonMark (markdown parsing) +commonmark = { module = "org.commonmark:commonmark", version.ref = "commonmark" } +commonmark-ext-yaml-front-matter = { module = "org.commonmark:commonmark-ext-yaml-front-matter", version.ref = "commonmark" } + # Null safety jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" } diff --git a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java index cae5ac32..75817541 100644 --- a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java +++ b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java @@ -39,11 +39,11 @@ * Apache Solr collections through Model Context Protocol (MCP) integration. * *

- * This service handles the conversion of JSON, CSV, and XML documents into - * Solr-compatible format and manages the indexing process with robust error - * handling and batch processing capabilities. It employs a schema-less approach - * where Solr automatically detects field types, eliminating the need for - * predefined schema configuration. + * This service handles the conversion of JSON, CSV, XML, and markdown documents + * into Solr-compatible format and manages the indexing process with robust + * error handling and batch processing capabilities. It employs a schema-less + * approach where Solr automatically detects field types, eliminating the need + * for predefined schema configuration. * *

* Core Features: @@ -57,6 +57,8 @@ * with headers *

  • XML Processing: Support for XML documents with element * flattening and attribute handling + *
  • Markdown Processing: Support for markdown documents with + * front matter, title, and heading extraction *
  • Batch Processing: Efficient bulk indexing with * configurable batch sizes *
  • Error Resilience: Individual document fallback when @@ -375,6 +377,79 @@ public String indexXmlDocuments(@McpToolParam(description = "Solr collection to + collection + "'"; } + /** + * Indexes a document from a markdown string into a specified Solr collection. + * + *

    + * This method serves as the primary entry point for markdown document indexing + * operations and is exposed as an MCP tool for AI client interactions. Unlike + * the structured formats (JSON, CSV, XML), markdown is a prose format, so + * searchable structure is extracted from the document content itself. + * + *

    + * Field Extraction: + * + *

    + * + *

    + * MCP Tool Usage: + * + *

    + * AI clients can invoke this method with natural language requests like "index + * this markdown file into my_collection" or "add this README to the search + * index". + * + *

    + * Example Markdown Processing: + * + * 

    {@code
+	 * Input:
+	 * ---
+	 * author: Jane Doe
+	 * ---
+	 * # Getting Started
+	 * Run the installer.
+	 *
+	 * Result: {author:"Jane Doe", title:"Getting Started",
+	 *          headings:["Getting Started"], content:"Getting Started\nRun the installer."}
+	 * }
    + * + * @param collection + * the name of the Solr collection to index documents into + * @param markdown + * markdown string to index, optionally starting with YAML front + * matter + * @throws IOException + * if there are critical errors in Solr communication + * @throws SolrServerException + * if Solr server encounters errors during indexing + * @see IndexingDocumentCreator#createSchemalessDocumentsFromMarkdown(String) + * @see #indexDocuments(String, List) + */ + @PreAuthorize("isAuthenticated()") + @McpTool( + name = "index-markdown-documents", + annotations = @McpTool.McpAnnotations(idempotentHint = true), + description = "Index a document from markdown String into Solr collection, extracting front matter, title, headings, and body text") + public String indexMarkdownDocuments(@McpToolParam(description = "Solr collection to index into") String collection, + @McpToolParam( + description = "Markdown string to index, optionally starting with YAML front matter") String markdown) + throws IOException, SolrServerException { + List schemalessDoc = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + int successCount = indexDocuments(collection, schemalessDoc); + return "Successfully indexed " + successCount + " of " + schemalessDoc.size() + " documents into collection '" + + collection + "'"; + } + /** * Indexes a list of SolrInputDocument objects into a Solr collection using * batch processing. @@ -483,7 +558,9 @@ private static IndexTool resolveIndexTool(String format) { case "json" -> new IndexTool("index-json-documents", "json"); case "csv" -> new IndexTool("index-csv-documents", "csv"); case "xml" -> new IndexTool("index-xml-documents", "xml"); - default -> throw new IllegalArgumentException("format must be one of json/csv/xml, got: " + format); + case "markdown", "md" -> new IndexTool("index-markdown-documents", "markdown"); + default -> + throw new IllegalArgumentException("format must be one of json/csv/xml/markdown, got: " + format); }; } @@ -500,7 +577,7 @@ public String indexDataPrompt( required = true) String collection, @McpArg( name = "format", - description = "Document format: 'json', 'csv', or 'xml'", + description = "Document format: 'json', 'csv', 'xml', or 'markdown'", required = true) String format, @McpArg( name = "sample", diff --git a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java index d89af467..e98321bd 100644 --- a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java +++ b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java @@ -44,6 +44,8 @@ * with headers *
  • XML Processing: Support for XML documents with element * flattening and attribute handling + *
  • Markdown Processing: Support for markdown documents with + * front matter, title, and heading extraction *
  • Field Sanitization: Automatic cleanup of field names for * Solr compatibility * @@ -62,11 +64,14 @@ public class IndexingDocumentCreator { private final JsonDocumentCreator jsonDocumentCreator; + private final MarkdownDocumentCreator markdownDocumentCreator; + public IndexingDocumentCreator(XmlDocumentCreator xmlDocumentCreator, CsvDocumentCreator csvDocumentCreator, - JsonDocumentCreator jsonDocumentCreator) { + JsonDocumentCreator jsonDocumentCreator, MarkdownDocumentCreator markdownDocumentCreator) { this.xmlDocumentCreator = xmlDocumentCreator; this.csvDocumentCreator = csvDocumentCreator; this.jsonDocumentCreator = jsonDocumentCreator; + this.markdownDocumentCreator = markdownDocumentCreator; } /** @@ -134,4 +139,31 @@ public List createSchemalessDocumentsFromXml(String xml) thro return xmlDocumentCreator.create(xml); } + + /** + * Creates a list of schema-less SolrInputDocument objects from a markdown + * string. + * + *

    + * This method delegates markdown processing to the MarkdownDocumentCreator + * utility class. + * + * @param markdown + * markdown string containing document content, optionally starting + * with YAML front matter + * @return list of SolrInputDocument objects ready for indexing + * @throws DocumentProcessingException + * if markdown parsing fails or input validation fails + * @see MarkdownDocumentCreator + */ + public List createSchemalessDocumentsFromMarkdown(String markdown) + throws DocumentProcessingException { + + // Input validation + if (markdown == null || markdown.trim().isEmpty()) { + throw new DocumentProcessingException("Markdown input cannot be null or empty"); + } + + return markdownDocumentCreator.create(markdown); + } } diff --git a/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java new file mode 100644 index 00000000..db560202 --- /dev/null +++ b/src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/MarkdownDocumentCreator.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mcp.server.indexing.documentcreator; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import org.apache.solr.common.SolrInputDocument; +import org.commonmark.Extension; +import org.commonmark.ext.front.matter.YamlFrontMatterExtension; +import org.commonmark.ext.front.matter.YamlFrontMatterVisitor; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.Code; +import org.commonmark.node.CustomBlock; +import org.commonmark.node.Heading; +import org.commonmark.node.Node; +import org.commonmark.node.Text; +import org.commonmark.parser.Parser; +import org.commonmark.renderer.text.TextContentRenderer; +import org.jspecify.annotations.Nullable; +import org.springframework.stereotype.Component; + +/** + * Utility class for processing markdown documents and converting them to + * SolrInputDocument objects. + * + *

    + * Unlike the structured formats (JSON, CSV, XML), markdown is a prose format, + * so this creator extracts searchable structure from the document rather than + * mapping fields one-to-one. Parsing is performed with the CommonMark library, + * which is lightweight and reflection-free (GraalVM native-image safe). + * + *

    + * Field Extraction Rules: + * + *

      + *
    • YAML Front Matter: Each front matter entry becomes a + * document field with a sanitized name. Entries with multiple values become + * multi-valued fields. + *
    • id: Taken from the {@code id} front matter entry when + * present, otherwise derived deterministically from a SHA-256 hash of the input + * so that re-indexing the same markdown overwrites the same document (keeping + * the operation idempotent). + *
    • title: Taken from the {@code title} front matter entry + * when present, otherwise from the first level-1 heading. + *
    • headings: Multi-valued field containing the text of + * every heading, preserving the document outline for searching. + *
    • content: The plain text of the document body (front + * matter excluded), suitable for full-text search. + *
    + * + *

    + * Example Transformation: + * + * 

    {@code
+ * Input markdown:
+ * ---
+ * author: Jane Doe
+ * tags: [search, solr]
+ * ---
+ * # Getting Started
+ * ## Installation
+ * Run the installer.
+ *
+ * Output document:
+ * {author:"Jane Doe", tags:["search","solr"], title:"Getting Started",
+ *  headings:["Getting Started","Installation"], content:"Getting Started\nInstallation\nRun the installer."}
+ * }
    + * + * @see SolrInputDocument + * @see FieldNameSanitizer#sanitizeFieldName(String) + */ +@Component +public class MarkdownDocumentCreator implements SolrDocumentCreator { + + private static final int MAX_INPUT_SIZE_BYTES = 10 * 1024 * 1024; + + /** Solr field holding the document's unique key. */ + public static final String FIELD_ID = "id"; + + /** Solr field holding the document title. */ + public static final String FIELD_TITLE = "title"; + + /** Multi-valued Solr field holding the text of every heading. */ + public static final String FIELD_HEADINGS = "headings"; + + /** Solr field holding the plain text body of the document. */ + public static final String FIELD_CONTENT = "content"; + + private final Parser parser; + + private final TextContentRenderer textContentRenderer; + + public MarkdownDocumentCreator() { + List extensions = List.of(YamlFrontMatterExtension.create()); + this.parser = Parser.builder().extensions(extensions).build(); + this.textContentRenderer = TextContentRenderer.builder().build(); + } + + /** + * Creates a SolrInputDocument from a markdown string. + * + *

    + * The whole input is treated as a single document: front matter entries map to + * fields, the title is resolved from front matter or the first level-1 heading, + * all heading texts are collected into a multi-valued {@code headings} field, + * and the plain text body is stored in {@code content}. + * + * @param markdown + * markdown string, optionally starting with YAML front matter + * @return a single-element list containing the created document, or an empty + * list if the input is blank + * @throws DocumentProcessingException + * if the input exceeds the size limit or parsing fails + */ + @Override + public List create(String markdown) throws DocumentProcessingException { + if (markdown.getBytes(StandardCharsets.UTF_8).length > MAX_INPUT_SIZE_BYTES) { + throw new DocumentProcessingException( + "Input too large: exceeds maximum size of " + MAX_INPUT_SIZE_BYTES + " bytes"); + } + + if (markdown.trim().isEmpty()) { + return List.of(); + } + + Node document; + try { + document = parser.parse(markdown); + } catch (RuntimeException e) { + throw new DocumentProcessingException("Failed to parse markdown document", e); + } + + SolrInputDocument doc = new SolrInputDocument(); + + addFrontMatterFields(document, doc); + + // Solr's default schema requires a unique key. A content-derived id keeps + // re-indexing of the same markdown idempotent (same input, same document) + if (doc.getFieldValue(FIELD_ID) == null) { + doc.addField(FIELD_ID, contentHash(markdown)); + } + + HeadingCollector headingCollector = new HeadingCollector(); + document.accept(headingCollector); + for (String heading : headingCollector.headings) { + doc.addField(FIELD_HEADINGS, heading); + } + + // Front matter title wins; otherwise fall back to the first level-1 heading + if (doc.getFieldValue(FIELD_TITLE) == null && headingCollector.firstTopLevelHeading != null) { + doc.addField(FIELD_TITLE, headingCollector.firstTopLevelHeading); + } + + String content = textContentRenderer.render(document).trim(); + if (!content.isEmpty()) { + doc.addField(FIELD_CONTENT, content); + } + + return List.of(doc); + } + + /** + * Extracts YAML front matter entries into document fields and unlinks the front + * matter block so it is excluded from the rendered body content. + */ + private void addFrontMatterFields(Node document, SolrInputDocument doc) { + YamlFrontMatterVisitor frontMatterVisitor = new YamlFrontMatterVisitor(); + document.accept(frontMatterVisitor); + + frontMatterVisitor.getData().forEach((key, values) -> { + String fieldName = FieldNameSanitizer.sanitizeFieldName(key); + for (String value : flattenFlowSequences(values)) { + if (!value.isEmpty()) { + doc.addField(fieldName, value); + } + } + }); + + // The front matter block is metadata, not body text: remove it so the + // TextContentRenderer output contains only the document body + Node firstChild = document.getFirstChild(); + if (firstChild instanceof CustomBlock) { + firstChild.unlink(); + } + } + + /** + * Expands simple YAML flow sequences into individual values. + * + *

    + * The CommonMark front matter extension parses block-style lists + * ({@code - item}) into multiple values but passes flow-style lists + * ({@code [a, b, c]}) through as a single literal string. Flow style is common + * for tags in real-world markdown (Jekyll, Hugo), so split it here to produce + * the same multi-valued field either way. Values containing commas inside + * quotes are not supported and are kept as-is. + */ + private static List flattenFlowSequences(List values) { + List result = new ArrayList<>(values.size()); + for (String value : values) { + String trimmed = value.trim(); + if (trimmed.length() >= 2 && trimmed.startsWith("[") && trimmed.endsWith("]") && !trimmed.contains("\"") + && !trimmed.contains("'")) { + for (String element : trimmed.substring(1, trimmed.length() - 1).split(",")) { + result.add(element.trim()); + } + } else { + result.add(value); + } + } + return result; + } + + private static String contentHash(String markdown) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(markdown.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is guaranteed to be available on every Java platform + throw new IllegalStateException("SHA-256 MessageDigest not available", e); + } + } + + /** + * AST visitor collecting heading texts and the first level-1 heading for use as + * a title fallback. + */ + private static final class HeadingCollector extends AbstractVisitor { + + private final List headings = new ArrayList<>(); + + @Nullable private String firstTopLevelHeading; + + @Override + public void visit(Heading heading) { + String text = collectText(heading).trim(); + if (!text.isEmpty()) { + headings.add(text); + if (firstTopLevelHeading == null && heading.getLevel() == 1) { + firstTopLevelHeading = text; + } + } + visitChildren(heading); + } + + private static String collectText(Node node) { + StringBuilder builder = new StringBuilder(); + appendText(node, builder); + return builder.toString(); + } + + private static void appendText(Node node, StringBuilder builder) { + if (node instanceof Text text) { + builder.append(text.getLiteral()); + } else if (node instanceof Code code) { + builder.append(code.getLiteral()); + } + for (Node child = node.getFirstChild(); child != null; child = child.getNext()) { + appendText(child, builder); + } + } + } +} diff --git a/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java b/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java index 2f617ec1..56d5e909 100644 --- a/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java +++ b/src/test/java/org/apache/solr/mcp/server/McpClientIntegrationTestBase.java @@ -104,6 +104,7 @@ void listToolsReturnsExpectedTools() { assertTrue(toolNames.contains("create-collection"), "Should have create-collection tool"); assertTrue(toolNames.contains("index-json-documents"), "Should have index-json-documents tool"); + assertTrue(toolNames.contains("index-markdown-documents"), "Should have index-markdown-documents tool"); assertTrue(toolNames.contains("search"), "Should have search tool"); assertTrue(toolNames.contains("list-collections"), "Should have list-collections tool"); assertTrue(toolNames.contains("check-health"), "Should have check-health tool"); @@ -135,6 +136,7 @@ void toolsExposeBehaviorHints() { assertHint(tools, "index-json-documents", false, true, true); assertHint(tools, "index-csv-documents", false, true, true); assertHint(tools, "index-xml-documents", false, true, true); + assertHint(tools, "index-markdown-documents", false, true, true); } private static void assertReadOnly(Map tools, String name) { @@ -398,6 +400,38 @@ void searchWithNewFieldFilters() throws Exception { assertEquals(1, getNumFound(r2), "Multi-valued 'genres' should match on 'crime'"); } + // Self-contained markdown round-trip: independent of the other order-18 test + // (the markdown document carries none of the fields its filter queries touch). + @Test + @Order(18) + void indexMarkdownDocumentAndFindItById() throws Exception { + String markdown = """ + --- + id: md-doc-1 + author: Markdown Author + --- + # Markdown Indexing Guide + + ## Installation + + Index markdown documents through the MCP server. + """; + + CallToolResult indexResult = mcpClient.callTool(new CallToolRequest("index-markdown-documents", + Map.of("collection", COLLECTION, "markdown", markdown))); + + assertNotNull(indexResult); + assertNotError(indexResult); + assertTrue(extractText(indexResult).contains("Successfully indexed 1"), + "Markdown indexing should report one indexed document: " + extractText(indexResult)); + + CallToolResult searchResult = mcpClient + .callTool(new CallToolRequest("search", Map.of("collection", COLLECTION, "query", "id:md-doc-1"))); + Map response = OBJECT_MAPPER.readValue(extractText(searchResult), new TypeReference<>() { + }); + assertEquals(1, getNumFound(response), "Should find the markdown document by its front matter id"); + } + // ===== End-to-end shows workflow (orders 19–27) ===== // Exercises the canonical "set up a new collection with a defined schema, index // real data, then search and introspect" workflow purely through MCP tool diff --git a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java index 933e668a..725848da 100644 --- a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java +++ b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceIntegrationTest.java @@ -27,6 +27,7 @@ import org.apache.solr.mcp.server.indexing.documentcreator.CsvDocumentCreator; import org.apache.solr.mcp.server.indexing.documentcreator.IndexingDocumentCreator; import org.apache.solr.mcp.server.indexing.documentcreator.JsonDocumentCreator; +import org.apache.solr.mcp.server.indexing.documentcreator.MarkdownDocumentCreator; import org.apache.solr.mcp.server.indexing.documentcreator.XmlDocumentCreator; import org.apache.solr.mcp.server.search.SearchResponse; import org.apache.solr.mcp.server.search.SearchService; @@ -72,9 +73,10 @@ void setUp() throws Exception { CsvDocumentCreator csvDocumentCreator = new CsvDocumentCreator(); JsonDocumentCreator jsonDocumentCreator = new JsonDocumentCreator( new com.fasterxml.jackson.databind.ObjectMapper()); + MarkdownDocumentCreator markdownDocumentCreator = new MarkdownDocumentCreator(); indexingDocumentCreator = new IndexingDocumentCreator(xmlDocumentCreator, csvDocumentCreator, - jsonDocumentCreator); + jsonDocumentCreator, markdownDocumentCreator); indexingService = new IndexingService(solrClient, indexingDocumentCreator); searchService = new SearchService(solrClient); diff --git a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java index 88c749ed..b1491d16 100644 --- a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java +++ b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java @@ -354,6 +354,22 @@ void indexDataPrompt_xmlPath_referencesIndexXmlDocuments() { assertTrue(body.contains("index-xml-documents"), "XML path should reference index-xml-documents tool"); } + @Test + void indexDataPrompt_markdownPath_referencesIndexMarkdownDocuments() { + String body = indexingService.indexDataPrompt("library", "markdown", null); + + assertTrue(body.contains("index-markdown-documents"), + "Markdown path should reference index-markdown-documents tool"); + } + + @Test + void indexDataPrompt_mdAliasResolvesToMarkdownTool() { + String body = indexingService.indexDataPrompt("library", "md", null); + + assertTrue(body.contains("index-markdown-documents"), + "'md' alias should reference index-markdown-documents tool"); + } + @Test void indexDataPrompt_unknownFormat_throwsIllegalArgumentException() { IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, diff --git a/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java b/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java new file mode 100644 index 00000000..2ec3a7c5 --- /dev/null +++ b/src/test/java/org/apache/solr/mcp/server/indexing/MarkdownIndexingTest.java @@ -0,0 +1,287 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.mcp.server.indexing; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.mcp.server.indexing.documentcreator.DocumentProcessingException; +import org.apache.solr.mcp.server.indexing.documentcreator.IndexingDocumentCreator; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; + +/** + * Test class for markdown indexing functionality in IndexingService. + * + *

    + * This test verifies that the IndexingService can correctly parse markdown + * content — including YAML front matter, headings, and body text — and convert + * it into SolrInputDocument objects using the schema-less approach. + */ +@SpringBootTest +@TestPropertySource(locations = "classpath:application.properties") +class MarkdownIndexingTest { + + @Autowired + private IndexingDocumentCreator indexingDocumentCreator; + + @Test + void testCreateSchemalessDocumentsFromMarkdownWithFrontMatter() throws Exception { + // Given + String markdown = """ + --- + author: George R.R. Martin + genre: fantasy + published: 1996 + --- + # A Game of Thrones + + ## Synopsis + + The first book in A Song of Ice and Fire. + + ## Reception + + Widely acclaimed. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + + SolrInputDocument doc = documents.getFirst(); + assertThat(doc.getFieldValue("author")).isEqualTo("George R.R. Martin"); + assertThat(doc.getFieldValue("genre")).isEqualTo("fantasy"); + assertThat(doc.getFieldValue("published")).isEqualTo("1996"); + + // Title comes from the first level-1 heading (no front matter title) + assertThat(doc.getFieldValue("title")).isEqualTo("A Game of Thrones"); + + // All heading texts are collected into a multi-valued field + assertThat(doc.getFieldValues("headings")).containsExactly("A Game of Thrones", "Synopsis", "Reception"); + + // Body text is searchable and excludes front matter + String content = (String) doc.getFieldValue("content"); + assertThat(content).contains("The first book in A Song of Ice and Fire."); + assertThat(content).contains("Widely acclaimed."); + assertThat(content).doesNotContain("George R.R. Martin"); + } + + @Test + void testFrontMatterTitleWinsOverFirstHeading() throws Exception { + // Given + String markdown = """ + --- + title: Front Matter Title + --- + # Heading Title + + Some body text. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + assertThat(documents.getFirst().getFieldValue("title")).isEqualTo("Front Matter Title"); + } + + @Test + void testCreateSchemalessDocumentsFromMarkdownWithoutFrontMatter() throws Exception { + // Given + String markdown = """ + # Getting Started + + Install the package and run the server. + + ## Configuration + + Set the `SOLR_URL` environment variable. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + + SolrInputDocument doc = documents.getFirst(); + assertThat(doc.getFieldValue("title")).isEqualTo("Getting Started"); + assertThat(doc.getFieldValues("headings")).containsExactly("Getting Started", "Configuration"); + + String content = (String) doc.getFieldValue("content"); + assertThat(content).contains("Install the package and run the server."); + assertThat(content).contains("SOLR_URL"); + } + + @Test + void testPlainTextMarkdownWithoutHeadings() throws Exception { + // Given + String markdown = "Just a plain paragraph of text without any structure."; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + + SolrInputDocument doc = documents.getFirst(); + assertThat(doc.getFieldValue("title")).isNull(); + assertThat(doc.getFieldValues("headings")).isNull(); + assertThat(doc.getFieldValue("content")).isEqualTo("Just a plain paragraph of text without any structure."); + } + + @Test + void testFrontMatterFieldNamesAreSanitized() throws Exception { + // Given + String markdown = """ + --- + Created-By: Jane Doe + last.updated: 2026-01-01 + --- + # Doc + + Body. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + + SolrInputDocument doc = documents.getFirst(); + assertThat(doc.getFieldValue("created_by")).isEqualTo("Jane Doe"); + assertThat(doc.getFieldValue("last_updated")).isEqualTo("2026-01-01"); + } + + @Test + void testFrontMatterListValuesBecomeMultiValuedFields() throws Exception { + // Given + String markdown = """ + --- + tags: [search, solr, mcp] + --- + # Tagged Document + + Body. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + assertThat(documents.getFirst().getFieldValues("tags")).containsExactly("search", "solr", "mcp"); + } + + @Test + void testFrontMatterBlockListValuesBecomeMultiValuedFields() throws Exception { + // Given + String markdown = """ + --- + tags: + - search + - solr + --- + # Tagged Document + + Body. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + assertThat(documents.getFirst().getFieldValues("tags")).containsExactly("search", "solr"); + } + + @Test + void testFrontMatterIdIsUsedAsDocumentId() throws Exception { + // Given + String markdown = """ + --- + id: doc-42 + --- + # Identified Document + + Body. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + assertThat(documents.getFirst().getFieldValue("id")).isEqualTo("doc-42"); + } + + @Test + void testGeneratedIdIsStableForSameContent() throws Exception { + // Given + String markdown = "# Stable\n\nSame content, same id."; + + // When + List first = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + List second = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + List other = indexingDocumentCreator + .createSchemalessDocumentsFromMarkdown("# Different\n\nOther content."); + + // Then: re-indexing the same markdown must overwrite the same document + Object firstId = first.getFirst().getFieldValue("id"); + assertThat(firstId).isNotNull(); + assertThat(firstId).isEqualTo(second.getFirst().getFieldValue("id")); + assertThat(firstId).isNotEqualTo(other.getFirst().getFieldValue("id")); + } + + @Test + void testEmptyMarkdownThrowsException() { + assertThatThrownBy(() -> indexingDocumentCreator.createSchemalessDocumentsFromMarkdown("")) + .isInstanceOf(DocumentProcessingException.class).hasMessageContaining("cannot be null or empty"); + } + + @Test + void testMarkdownFormattingIsStrippedFromContent() throws Exception { + // Given + String markdown = """ + # Formatted + + Some **bold** and *italic* text with a [link](https://solr.apache.org) and `inline code`. + """; + + // When + List documents = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown); + + // Then + assertThat(documents).hasSize(1); + + String content = (String) documents.getFirst().getFieldValue("content"); + assertThat(content).contains("bold"); + assertThat(content).contains("italic"); + assertThat(content).contains("inline code"); + assertThat(content).doesNotContain("**"); + assertThat(content).doesNotContain("]("); + } +} From e0b250b28046f41d769ecb02f3b15bfe532c07ad Mon Sep 17 00:00:00 2001 From: Shahzad Asghar Date: Thu, 11 Jun 2026 23:42:27 +0300 Subject: [PATCH 2/2] feat(indexing): steer clients away from markdown tool for structured formats Per maintainer feedback on #69, the index-markdown-documents tool description now tells clients not to use it for JSON/CSV/XML input (dedicated tools exist), to convert to markdown only when no dedicated tool covers the source format, and to supply a stable front matter id when indexing converted content. --- .../org/apache/solr/mcp/server/indexing/IndexingService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java index 75817541..cbb4774c 100644 --- a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java +++ b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java @@ -439,7 +439,9 @@ public String indexXmlDocuments(@McpToolParam(description = "Solr collection to @McpTool( name = "index-markdown-documents", annotations = @McpTool.McpAnnotations(idempotentHint = true), - description = "Index a document from markdown String into Solr collection, extracting front matter, title, headings, and body text") + description = "Index a document from markdown String into Solr collection, extracting front matter, title, headings, and body text. " + + "Do NOT use for JSON/CSV/XML input; use index-json-documents, index-csv-documents, or index-xml-documents instead. " + + "Only convert source content to markdown when there is no dedicated tool for the source format, and supply a stable 'id' in the YAML front matter when doing so.") public String indexMarkdownDocuments(@McpToolParam(description = "Solr collection to index into") String collection, @McpToolParam( description = "Markdown string to index, optionally starting with YAML front matter") String markdown)