From 050452c3c0b672e8c0b64fe18c1dcce197a71e97 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 02:31:56 +0900 Subject: [PATCH 1/2] Add MCP tool contract drift tests (#3199) --- changelog.d/unreleased/3199.internal.md | 17 ++ src/CodeIndex/Mcp/McpToolDefinitions.cs | 5 +- src/CodeIndex/Mcp/McpToolHandlers.cs | 53 ++++-- tests/CodeIndex.Tests/McpToolContractTests.cs | 174 ++++++++++++++++++ 4 files changed, 230 insertions(+), 19 deletions(-) create mode 100644 changelog.d/unreleased/3199.internal.md create mode 100644 tests/CodeIndex.Tests/McpToolContractTests.cs diff --git a/changelog.d/unreleased/3199.internal.md b/changelog.d/unreleased/3199.internal.md new file mode 100644 index 0000000000..57df489441 --- /dev/null +++ b/changelog.d/unreleased/3199.internal.md @@ -0,0 +1,17 @@ +--- +category: internal +issues: + - 3199 +affected: + - src/CodeIndex/Mcp/McpToolDefinitions.cs + - src/CodeIndex/Mcp/McpToolHandlers.cs + - tests/CodeIndex.Tests/McpToolContractTests.cs +--- + +## English + +- **Added MCP tool contract drift tests (#3199)** — MCP tests now compare advertised `tools/list` input properties against the server allowlist and JSON type validator, catching schema, validation, and hidden argument drift before release. + +## 日本語 + +- **MCP ツール契約の drift テストを追加しました (#3199)** — MCP テストが `tools/list` で広告される入力プロパティとサーバーの allowlist / JSON 型バリデータを突き合わせ、schema・validation・隠れ引数のずれをリリース前に検出します。 diff --git a/src/CodeIndex/Mcp/McpToolDefinitions.cs b/src/CodeIndex/Mcp/McpToolDefinitions.cs index a8c32b3aee..c52f12c519 100644 --- a/src/CodeIndex/Mcp/McpToolDefinitions.cs +++ b/src/CodeIndex/Mcp/McpToolDefinitions.cs @@ -348,7 +348,6 @@ private JsonNode HandleToolsList(JsonNode? id) ["path"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Restrict source files to glob-style path patterns. `*` and `?` are wildcards. Accepts a single string or an array; multiple values are OR'd together." }, ["excludePaths"] = new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" }, ["description"] = "Exclude glob-style path patterns. `*` and `?` are wildcards." }, ["excludeTests"] = new JsonObject { ["type"] = "boolean", ["description"] = "Exclude test files", ["default"] = false }, - ["includeGenerated"] = new JsonObject { ["type"] = "boolean", ["description"] = "Include files detected as generated code", ["default"] = false }, ["reverse"] = new JsonObject { ["type"] = "boolean", ["description"] = "Reverse lookup: show files that depend ON the matched path", ["default"] = false }, ["format"] = new JsonObject { ["type"] = "string", ["enum"] = new JsonArray { "edgelist", "json-graph" }, ["description"] = "Structured response format. `edgelist` preserves the existing edges array; `json-graph` returns nodes and edges.", ["default"] = "edgelist" }, ["cycles"] = new JsonObject { ["type"] = "boolean", ["description"] = "Return dependency cycles instead of ordinary edge rows.", ["default"] = false } @@ -373,7 +372,9 @@ private JsonNode HandleToolsList(JsonNode? id) ["properties"] = new JsonObject { ["kind"] = new JsonObject { ["type"] = "string", ["description"] = "Filter by issue kind (replacement_char, bom, null_byte, mixed_line_endings, mixed_line_endings_three_way, cr_only_line_endings, utf16_bom, non_utf8_likely, line_too_long)" }, - ["path"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Filter to paths containing this text. Accepts a single string or an array; multiple values are OR'd together." } + ["path"] = new JsonObject { ["oneOf"] = new JsonArray { new JsonObject { ["type"] = "string" }, new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" } } }, ["description"] = "Filter to paths containing this text. Accepts a single string or an array; multiple values are OR'd together." }, + ["excludePaths"] = new JsonObject { ["type"] = "array", ["items"] = new JsonObject { ["type"] = "string" }, ["description"] = "Exclude any paths containing these texts" }, + ["excludeTests"] = new JsonObject { ["type"] = "boolean", ["description"] = "Exclude likely test files", ["default"] = false } } }, ReadOnlyAnnotations()), diff --git a/src/CodeIndex/Mcp/McpToolHandlers.cs b/src/CodeIndex/Mcp/McpToolHandlers.cs index 828d8c1a87..235ca94f92 100644 --- a/src/CodeIndex/Mcp/McpToolHandlers.cs +++ b/src/CodeIndex/Mcp/McpToolHandlers.cs @@ -28,7 +28,6 @@ public partial class McpServer private static readonly HashSet BoundedEnumLikeScalarArguments = new(StringComparer.Ordinal) { "category", - "direction", "format", "groupBy", "kind", @@ -486,7 +485,7 @@ private static List ReadStringList(JsonNode? args, string propertyName) private static JsonObject? ValidateCommonListArguments(JsonNode? args) { - foreach (var propertyName in new[] { "path", "project", "excludePaths", "names" }) + foreach (var propertyName in new[] { "path", "project", "excludePaths", "names", "sections" }) { if (ValidateStringListArgument(args, propertyName) is JsonObject error) return error; @@ -603,23 +602,31 @@ private static JsonObject AddUnknownArgumentData(JsonObject error, string argume private static bool TryGetExpectedJsonType(string toolName, string argumentName, out string expected) { - if (argumentName is "path" or "project" or "excludePaths" or "names" or "files" or "commits" or "changedBetween") + if (argumentName is "excludePaths" or "names" or "sections" or "files" or "commits" or "changedBetween") { expected = string.Empty; return false; } + if (argumentName == "path") + { + expected = ToolAllowsStringOrArrayPath(toolName) ? "string_or_array" : "string"; + return true; + } + expected = argumentName switch { "limit" or "offset" or "snippetLines" or "maxLineWidth" or "before" or "after" or "focusLine" or "focusColumn" or "focusLength" or "startLine" or "endLine" or - "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or "guardWindow" => "integer", + "maxHops" or "maxDepth" or "depth" or "parallelism" or "maxFileBytes" or + "guardWindow" or "maxOutputBytes" => "integer", "excludeTests" or "includeGenerated" or "rawQuery" or "noDedup" or "exactSubstring" or "exactName" or "exact" or "prefix" or "countOnly" or "includeBody" or "lsp_compatible" or - "regex" or "withPaths" or "rebuild" or "dryRun" or "dry_run" or "force" or "optimize" => "boolean", - "requireBefore" or "requireAfter" or "rejectBefore" or "rejectAfter" => "string_or_array", - "query" or "lang" or "kind" or "format" or "rankBy" or "since" or "path" or "project" or - "solution" or "symbol" or "direction" or "groupBy" or "category" or "language" or + "regex" or "withPaths" or "rebuild" or "dryRun" or "dry_run" or "force" or + "optimize" or "reverse" or "cycles" => "boolean", + "project" or "requireBefore" or "requireAfter" or "rejectBefore" or "rejectAfter" => "string_or_array", + "query" or "lang" or "kind" or "format" or "rankBy" or "since" or "cursor" or + "solution" or "symbol" or "groupBy" or "category" or "language" or "description" or "context" or "toolInvocationContext" or "db" => "string", "queries" or "evidencePaths" or "evidence_paths" => "array", _ => string.Empty, @@ -631,6 +638,14 @@ private static bool TryGetExpectedJsonType(string toolName, string argumentName, return true; } + private static bool ToolAllowsStringOrArrayPath(string toolName) => toolName switch + { + "search" or "definition" or "references" or "callers" or "callees" or "symbols" or + "files" or "find_in_file" or "map" or "analyze_symbol" or "deps" or "impact_analysis" or + "validate" or "unused_symbols" or "symbol_hotspots" => true, + _ => false, + }; + private static bool MatchesExpectedJsonType(JsonNode? node, string expected) => expected switch { "integer" => node is JsonValue value && value.TryGetValue(out _), @@ -674,16 +689,16 @@ private static string DescribeJsonType(JsonNode? node) "references" => new HashSet(StringComparer.Ordinal) { "query", "kind", "lang", "limit", "offset", "maxLineWidth", "lsp_compatible", "path", "excludePaths", "excludeTests", "includeGenerated", "exactName", "exact", "countOnly", "format", "project", "solution" }, "callers" or "callees" => new HashSet(StringComparer.Ordinal) { "query", "kind", "rankBy", "lang", "limit", "offset", "path", "excludePaths", "excludeTests", "includeGenerated", "exactName", "exact", "countOnly", "format", "project", "solution" }, "symbols" => new HashSet(StringComparer.Ordinal) { "query", "names", "kind", "lang", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "since", "exactName", "exact", "project", "solution" }, - "files" => new HashSet(StringComparer.Ordinal) { "query", "lang", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "since" }, + "files" => new HashSet(StringComparer.Ordinal) { "query", "lang", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "since", "project", "solution" }, "find_in_file" => new HashSet(StringComparer.Ordinal) { "query", "path", "limit", "lang", "excludePaths", "excludeTests", "includeGenerated", "before", "after", "snippetLines", "focusLine", "focusColumn", "maxLineWidth", "exact", "regex" }, - "excerpt" => new HashSet(StringComparer.Ordinal) { "path", "startLine", "endLine", "before", "after", "focusLine", "focusColumn", "focusLength", "maxLineWidth" }, - "map" => new HashSet(StringComparer.Ordinal) { "limit", "lang", "path", "excludePaths", "excludeTests", "project", "solution" }, + "excerpt" => new HashSet(StringComparer.Ordinal) { "path", "startLine", "endLine", "before", "after", "focusLine", "focusColumn", "focusLength", "maxLineWidth", "maxOutputBytes" }, + "map" => new HashSet(StringComparer.Ordinal) { "limit", "lang", "path", "excludePaths", "excludeTests", "sections", "depth", "project", "solution" }, "analyze_symbol" => new HashSet(StringComparer.Ordinal) { "query", "lang", "limit", "includeBody", "path", "excludePaths", "excludeTests", "includeGenerated", "exactName", "exact", "maxLineWidth", "project", "solution" }, - "outline" => new HashSet(StringComparer.Ordinal) { "path", "limit", "includeImports", "maxLineWidth" }, + "outline" => new HashSet(StringComparer.Ordinal) { "path" }, "batch_query" => new HashSet(StringComparer.Ordinal) { "queries" }, - "deps" => new HashSet(StringComparer.Ordinal) { "path", "direction", "lang", "limit", "excludePaths", "excludeTests", "project", "solution" }, - "impact_analysis" => new HashSet(StringComparer.Ordinal) { "symbol", "query", "lang", "maxHops", "maxDepth", "depth", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "withPaths", "countOnly", "project", "solution" }, - "validate" => new HashSet(StringComparer.Ordinal) { "path", "lang", "limit", "excludePaths", "excludeTests", "project", "solution" }, + "deps" => new HashSet(StringComparer.Ordinal) { "path", "reverse", "format", "cycles", "lang", "limit", "excludePaths", "excludeTests", "project", "solution" }, + "impact_analysis" => new HashSet(StringComparer.Ordinal) { "query", "lang", "maxHops", "maxDepth", "limit", "path", "excludePaths", "excludeTests", "includeGenerated", "withPaths", "countOnly", "project", "solution" }, + "validate" => new HashSet(StringComparer.Ordinal) { "kind", "path", "excludePaths", "excludeTests", "project", "solution" }, "unused_symbols" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "path", "excludePaths", "excludeTests", "project", "solution" }, "symbol_hotspots" => new HashSet(StringComparer.Ordinal) { "kind", "lang", "limit", "groupBy", "path", "excludePaths", "excludeTests", "project", "solution" }, "index" => new HashSet(StringComparer.Ordinal) { "path", "rebuild", "maxFileBytes" }, @@ -740,8 +755,12 @@ private static string DescribeJsonType(JsonNode? node) if (node is JsonValue scalar && scalar.TryGetValue(out var scalarText)) { - if (propertyName == "names") - return null; + if (propertyName is "excludePaths" or "names" or "sections") + return new JsonObject + { + ["message"] = $"{propertyName} must be an array of strings.", + ["invalid_count"] = 1, + }; if (propertyName == "path" && string.IsNullOrWhiteSpace(scalarText)) return null; if (string.IsNullOrWhiteSpace(scalarText)) diff --git a/tests/CodeIndex.Tests/McpToolContractTests.cs b/tests/CodeIndex.Tests/McpToolContractTests.cs new file mode 100644 index 0000000000..79013ba8d4 --- /dev/null +++ b/tests/CodeIndex.Tests/McpToolContractTests.cs @@ -0,0 +1,174 @@ +using System.Reflection; +using System.Text.Json.Nodes; +using CodeIndex.Mcp; + +namespace CodeIndex.Tests; + +public class McpToolContractTests +{ + private static readonly MethodInfo GetAllowedToolArgumentsMethod = + RequiredPrivateStaticMethod("GetAllowedToolArguments"); + + private static readonly MethodInfo TryGetExpectedJsonTypeMethod = + RequiredPrivateStaticMethod("TryGetExpectedJsonType"); + + private static readonly HashSet<(string Tool, string Argument)> HiddenCompatibilityAliases = + [ + ("backfill_fold", "dryRun"), + ("suggest_improvement", "evidence_paths"), + ]; + + private static readonly HashSet SpecializedListValidatedArguments = new(StringComparer.Ordinal) + { + "excludePaths", + "names", + "sections", + }; + + [Fact] + public void ToolsList_AdvertisedInputPropertiesMatchArgumentAllowlist_Issue3199() + { + var advertisedSchemas = GetAdvertisedToolSchemas(); + var failures = new List(); + + foreach (var (toolName, properties) in advertisedSchemas.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + var advertised = properties.Keys.ToHashSet(StringComparer.Ordinal); + var allowed = GetAllowedToolArguments(toolName); + + var advertisedButRejected = advertised.Except(allowed, StringComparer.Ordinal).Order(StringComparer.Ordinal).ToArray(); + var allowedButHidden = allowed + .Except(advertised, StringComparer.Ordinal) + .Where(argument => !HiddenCompatibilityAliases.Contains((toolName, argument))) + .Order(StringComparer.Ordinal) + .ToArray(); + + if (advertisedButRejected.Length > 0 || allowedButHidden.Length > 0) + { + failures.Add( + $"{toolName}: advertised_but_rejected=[{string.Join(", ", advertisedButRejected)}]; " + + $"allowed_but_hidden=[{string.Join(", ", allowedButHidden)}]"); + } + } + + Assert.True( + failures.Count == 0, + "MCP tools/list schema and argument allowlist drift detected:\n" + string.Join('\n', failures)); + } + + [Fact] + public void ToolsList_AdvertisedInputPropertiesHaveMatchingTypeValidation_Issue3199() + { + var advertisedSchemas = GetAdvertisedToolSchemas(); + var failures = new List(); + + foreach (var (toolName, properties) in advertisedSchemas.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + foreach (var (argumentName, schema) in properties.OrderBy(pair => pair.Key, StringComparer.Ordinal)) + { + var schemaType = ExpectedTypeFromSchema(schema); + if (schemaType == null) + { + failures.Add($"{toolName}.{argumentName}: unsupported schema shape"); + continue; + } + + if (SpecializedListValidatedArguments.Contains(argumentName)) + continue; + + var (hasValidator, validatorType) = TryGetExpectedJsonType(toolName, argumentName); + if (!hasValidator || validatorType != schemaType) + { + failures.Add( + $"{toolName}.{argumentName}: schema={schemaType}; " + + $"validator={(hasValidator ? validatorType : "")}"); + } + } + } + + Assert.True( + failures.Count == 0, + "MCP tools/list schema and argument type validator drift detected:\n" + string.Join('\n', failures)); + } + + private static Dictionary> GetAdvertisedToolSchemas() + { + using var server = new McpServer("unused.db", "test", dbPathExplicit: false, McpToolFilter.AllowAll()); + var request = new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = 1, + ["method"] = "tools/list", + }; + var response = server.HandleMessage(request) + ?? throw new InvalidOperationException("tools/list returned no response."); + + var tools = response["result"]?["tools"]?.AsArray() + ?? throw new InvalidOperationException("tools/list response did not contain result.tools."); + var result = new Dictionary>(StringComparer.Ordinal); + + foreach (var tool in tools) + { + var toolObject = tool?.AsObject() + ?? throw new InvalidOperationException("tools/list returned a non-object tool entry."); + var toolName = toolObject["name"]?.GetValue() + ?? throw new InvalidOperationException("tools/list returned a tool without a name."); + var properties = toolObject["inputSchema"]?["properties"]?.AsObject() + ?? throw new InvalidOperationException($"Tool '{toolName}' did not expose inputSchema.properties."); + + var propertySchemas = new Dictionary(StringComparer.Ordinal); + foreach (var (propertyName, propertySchema) in properties) + { + propertySchemas[propertyName] = propertySchema?.AsObject() + ?? throw new InvalidOperationException($"Tool '{toolName}' property '{propertyName}' did not expose an object schema."); + } + + result.Add(toolName, propertySchemas); + } + + return result; + } + + private static HashSet GetAllowedToolArguments(string toolName) + { + var result = GetAllowedToolArgumentsMethod.Invoke(null, [toolName]); + if (result is not IReadOnlySet allowed) + throw new InvalidOperationException("GetAllowedToolArguments did not return IReadOnlySet."); + return allowed.ToHashSet(StringComparer.Ordinal); + } + + private static (bool HasValidator, string ValidatorType) TryGetExpectedJsonType(string toolName, string argumentName) + { + object?[] args = [toolName, argumentName, string.Empty]; + var hasValidator = (bool)(TryGetExpectedJsonTypeMethod.Invoke(null, args) + ?? throw new InvalidOperationException("TryGetExpectedJsonType returned null.")); + return (hasValidator, (string)args[2]!); + } + + private static string? ExpectedTypeFromSchema(JsonObject schema) + { + if (schema["oneOf"] is JsonArray oneOf) + { + var optionTypes = oneOf + .Select(option => (option as JsonObject)?["type"]?.GetValue()) + .Where(type => type != null) + .Cast() + .ToHashSet(StringComparer.Ordinal); + + return optionTypes.SetEquals(["string", "array"]) ? "string_or_array" : null; + } + + return schema["type"]?.GetValue() switch + { + "array" => "array", + "boolean" => "boolean", + "integer" => "integer", + "string" => "string", + _ => null, + }; + } + + private static MethodInfo RequiredPrivateStaticMethod(string name) + => typeof(McpServer).GetMethod(name, BindingFlags.NonPublic | BindingFlags.Static) + ?? throw new InvalidOperationException($"McpServer.{name} was not found."); +} From 827f3168f06cae69cd750c7488e0b820dbdf45b2 Mon Sep 17 00:00:00 2001 From: Widthdom Date: Fri, 5 Jun 2026 02:39:05 +0900 Subject: [PATCH 2/2] Tighten MCP list contract coverage (#3199) --- tests/CodeIndex.Tests/McpToolContractTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/CodeIndex.Tests/McpToolContractTests.cs b/tests/CodeIndex.Tests/McpToolContractTests.cs index 79013ba8d4..a60d6fa9e5 100644 --- a/tests/CodeIndex.Tests/McpToolContractTests.cs +++ b/tests/CodeIndex.Tests/McpToolContractTests.cs @@ -74,7 +74,16 @@ public void ToolsList_AdvertisedInputPropertiesHaveMatchingTypeValidation_Issue3 } if (SpecializedListValidatedArguments.Contains(argumentName)) + { + if (schemaType != "array") + { + failures.Add( + $"{toolName}.{argumentName}: schema={schemaType}; " + + "specialized_list_validator=array"); + } + continue; + } var (hasValidator, validatorType) = TryGetExpectedJsonType(toolName, argumentName); if (!hasValidator || validatorType != schemaType)