feat(indexing): add markdown document indexing

AGENTS.md

@@ -96,15 +96,15 @@ docker run -p 8080:8080 --rm -e PROFILES=http \
 Four service classes expose MCP tools via `@McpTool` annotations:
 
 - **SearchService** (`search/`) - Full-text search with filtering, faceting, sorting, pagination
-- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV, XML formats
+- **IndexingService** (`indexing/`) - Document indexing supporting JSON, CSV, XML, and markdown formats
 - **CollectionService** (`metadata/`) - List collections, get stats, health checks
 - **SchemaService** (`schema/`) - Schema introspection and additive modification (add-fields, add-field-types)
 
 ### Document Creators (Strategy Pattern)
 
 `indexing/documentcreator/` uses strategy pattern for format parsing:
 - `SolrDocumentCreator` - Common interface
-- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator` - Format implementations
+- `JsonDocumentCreator`, `CsvDocumentCreator`, `XmlDocumentCreator`, `MarkdownDocumentCreator` - Format implementations
 - `IndexingDocumentCreator` - Orchestrator that delegates to format-specific creators
 - `FieldNameSanitizer` - Automatic field name validation for Solr compatibility
 

README.md

@@ -321,6 +321,7 @@ For complete setup instructions, see [security-docs/AUTH0_SETUP.md](security-doc
 | `index-json-documents` | Index documents from a JSON string into a Solr collection |
 | `index-csv-documents` | Index documents from a CSV string into a Solr collection |
 | `index-xml-documents` | Index documents from an XML string into a Solr collection |
+| `index-markdown-documents` | Index a markdown document into a Solr collection, extracting front matter, title, headings, and body text |
 
 ### Collections
 

build.gradle.kts

@@ -135,6 +135,9 @@ dependencies {
     implementation(libs.spring.ai.starter.mcp.server.webmvc)
     implementation(libs.solr.solrj)
     implementation(libs.commons.csv)
+    // CommonMark for markdown parsing
+    implementation(libs.commonmark)
+    implementation(libs.commonmark.ext.yaml.front.matter)
     // JSpecify for nullability annotations
     implementation(libs.jspecify)
 

gradle/libs.versions.toml

@@ -11,6 +11,7 @@ graalvm-native = "0.10.6"
 spring-ai = "1.1.7"
 solr = "10.0.0"
 commons-csv = "1.14.1"
+commonmark = "0.28.0"
 jspecify = "1.0.0"
 mcp-server-security = "0.0.6"
 
@@ -49,6 +50,10 @@ solr-solrj = { module = "org.apache.solr:solr-solrj", version.ref = "solr" }
 # Apache Commons
 commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons-csv" }
 
+# CommonMark (markdown parsing)
+commonmark = { module = "org.commonmark:commonmark", version.ref = "commonmark" }
+commonmark-ext-yaml-front-matter = { module = "org.commonmark:commonmark-ext-yaml-front-matter", version.ref = "commonmark" }
+
 # Null safety
 jspecify = { module = "org.jspecify:jspecify", version.ref = "jspecify" }
 

src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java

@@ -39,11 +39,11 @@
  * Apache Solr collections through Model Context Protocol (MCP) integration.
  *
  * <p>
- * This service handles the conversion of JSON, CSV, and XML documents into
- * Solr-compatible format and manages the indexing process with robust error
- * handling and batch processing capabilities. It employs a schema-less approach
- * where Solr automatically detects field types, eliminating the need for
- * predefined schema configuration.
+ * This service handles the conversion of JSON, CSV, XML, and markdown documents
+ * into Solr-compatible format and manages the indexing process with robust
+ * error handling and batch processing capabilities. It employs a schema-less
+ * approach where Solr automatically detects field types, eliminating the need
+ * for predefined schema configuration.
  *
  * <p>
  * <strong>Core Features:</strong>
@@ -57,6 +57,8 @@
  * with headers
  * <li><strong>XML Processing</strong>: Support for XML documents with element
  * flattening and attribute handling
+ * <li><strong>Markdown Processing</strong>: Support for markdown documents with
+ * front matter, title, and heading extraction
  * <li><strong>Batch Processing</strong>: Efficient bulk indexing with
  * configurable batch sizes
  * <li><strong>Error Resilience</strong>: Individual document fallback when
@@ -375,6 +377,81 @@ public String indexXmlDocuments(@McpToolParam(description = "Solr collection to
 				+ collection + "'";
 	}
 
+	/**
+	 * Indexes a document from a markdown string into a specified Solr collection.
+	 *
+	 * <p>
+	 * This method serves as the primary entry point for markdown document indexing
+	 * operations and is exposed as an MCP tool for AI client interactions. Unlike
+	 * the structured formats (JSON, CSV, XML), markdown is a prose format, so
+	 * searchable structure is extracted from the document content itself.
+	 *
+	 * <p>
+	 * <strong>Field Extraction:</strong>
+	 *
+	 * <ul>
+	 * <li><strong>YAML Front Matter</strong>: Each entry becomes a document field
+	 * with a sanitized name (multi-valued where applicable)
+	 * <li><strong>title</strong>: From the {@code title} front matter entry, or the
+	 * first level-1 heading
+	 * <li><strong>headings</strong>: Multi-valued field with the text of every
+	 * heading (the document outline)
+	 * <li><strong>content</strong>: Plain text body for full-text search (front
+	 * matter excluded)
+	 * </ul>
+	 *
+	 * <p>
+	 * <strong>MCP Tool Usage:</strong>
+	 *
+	 * <p>
+	 * AI clients can invoke this method with natural language requests like "index
+	 * this markdown file into my_collection" or "add this README to the search
+	 * index".
+	 *
+	 * <p>
+	 * <strong>Example Markdown Processing:</strong>
+	 *
+	 * <pre>{@code
+	 * Input:
+	 * ---
+	 * author: Jane Doe
+	 * ---
+	 * # Getting Started
+	 * Run the installer.
+	 *
+	 * Result: {author:"Jane Doe", title:"Getting Started",
+	 *          headings:["Getting Started"], content:"Getting Started\nRun the installer."}
+	 * }</pre>
+	 *
+	 * @param collection
+	 *            the name of the Solr collection to index documents into
+	 * @param markdown
+	 *            markdown string to index, optionally starting with YAML front
+	 *            matter
+	 * @throws IOException
+	 *             if there are critical errors in Solr communication
+	 * @throws SolrServerException
+	 *             if Solr server encounters errors during indexing
+	 * @see IndexingDocumentCreator#createSchemalessDocumentsFromMarkdown(String)
+	 * @see #indexDocuments(String, List)
+	 */
+	@PreAuthorize("isAuthenticated()")
+	@McpTool(
+			name = "index-markdown-documents",
+			annotations = @McpTool.McpAnnotations(idempotentHint = true),
+			description = "Index a document from markdown String into Solr collection, extracting front matter, title, headings, and body text. "
+					+ "Do NOT use for JSON/CSV/XML input; use index-json-documents, index-csv-documents, or index-xml-documents instead. "
+					+ "Only convert source content to markdown when there is no dedicated tool for the source format, and supply a stable 'id' in the YAML front matter when doing so.")
+	public String indexMarkdownDocuments(@McpToolParam(description = "Solr collection to index into") String collection,
+			@McpToolParam(
+					description = "Markdown string to index, optionally starting with YAML front matter") String markdown)
+			throws IOException, SolrServerException {
+		List<SolrInputDocument> schemalessDoc = indexingDocumentCreator.createSchemalessDocumentsFromMarkdown(markdown);
+		int successCount = indexDocuments(collection, schemalessDoc);
+		return "Successfully indexed " + successCount + " of " + schemalessDoc.size() + " documents into collection '"
+				+ collection + "'";
+	}
+
 	/**
 	 * Indexes a list of SolrInputDocument objects into a Solr collection using
 	 * batch processing.
@@ -483,7 +560,9 @@ private static IndexTool resolveIndexTool(String format) {
 			case "json" -> new IndexTool("index-json-documents", "json");
 			case "csv" -> new IndexTool("index-csv-documents", "csv");
 			case "xml" -> new IndexTool("index-xml-documents", "xml");
-			default -> throw new IllegalArgumentException("format must be one of json/csv/xml, got: " + format);
+			case "markdown", "md" -> new IndexTool("index-markdown-documents", "markdown");
+			default ->
+				throw new IllegalArgumentException("format must be one of json/csv/xml/markdown, got: " + format);
 		};
 	}
 
@@ -500,7 +579,7 @@ public String indexDataPrompt(
 					required = true) String collection,
 			@McpArg(
 					name = "format",
-					description = "Document format: 'json', 'csv', or 'xml'",
+					description = "Document format: 'json', 'csv', 'xml', or 'markdown'",
 					required = true) String format,
 			@McpArg(
 					name = "sample",

src/main/java/org/apache/solr/mcp/server/indexing/documentcreator/IndexingDocumentCreator.java

@@ -44,6 +44,8 @@
  * with headers
  * <li><strong>XML Processing</strong>: Support for XML documents with element
  * flattening and attribute handling
+ * <li><strong>Markdown Processing</strong>: Support for markdown documents with
+ * front matter, title, and heading extraction
  * <li><strong>Field Sanitization</strong>: Automatic cleanup of field names for
  * Solr compatibility
  * </ul>
@@ -62,11 +64,14 @@ public class IndexingDocumentCreator {
 
 	private final JsonDocumentCreator jsonDocumentCreator;
 
+	private final MarkdownDocumentCreator markdownDocumentCreator;
+
 	public IndexingDocumentCreator(XmlDocumentCreator xmlDocumentCreator, CsvDocumentCreator csvDocumentCreator,
-			JsonDocumentCreator jsonDocumentCreator) {
+			JsonDocumentCreator jsonDocumentCreator, MarkdownDocumentCreator markdownDocumentCreator) {
 		this.xmlDocumentCreator = xmlDocumentCreator;
 		this.csvDocumentCreator = csvDocumentCreator;
 		this.jsonDocumentCreator = jsonDocumentCreator;
+		this.markdownDocumentCreator = markdownDocumentCreator;
 	}
 
 	/**
@@ -134,4 +139,31 @@ public List<SolrInputDocument> createSchemalessDocumentsFromXml(String xml) thro
 
 		return xmlDocumentCreator.create(xml);
 	}
+
+	/**
+	 * Creates a list of schema-less SolrInputDocument objects from a markdown
+	 * string.
+	 *
+	 * <p>
+	 * This method delegates markdown processing to the MarkdownDocumentCreator
+	 * utility class.
+	 *
+	 * @param markdown
+	 *            markdown string containing document content, optionally starting
+	 *            with YAML front matter
+	 * @return list of SolrInputDocument objects ready for indexing
+	 * @throws DocumentProcessingException
+	 *             if markdown parsing fails or input validation fails
+	 * @see MarkdownDocumentCreator
+	 */
+	public List<SolrInputDocument> createSchemalessDocumentsFromMarkdown(String markdown)
+			throws DocumentProcessingException {
+
+		// Input validation
+		if (markdown == null || markdown.trim().isEmpty()) {
+			throw new DocumentProcessingException("Markdown input cannot be null or empty");
+		}
+
+		return markdownDocumentCreator.create(markdown);
+	}
 }