Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -40,5 +40,6 @@ coverage/
*.tmp
*.bak
*.orig
/worktrees/
/*-wtrees/

# Grade reports
.grade-reports/
31 changes: 21 additions & 10 deletions crates/llm-router/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,15 @@ impl OpenAiProvider {
impl LlmProvider for OpenAiProvider {
async fn complete(&self, request: &CompletionRequest) -> Result<CompletionResponse, LlmError> {
let start = std::time::Instant::now();

let body = serde_json::json!({
"model": request.model,
"messages": request.messages,
"temperature": request.temperature.unwrap_or(0.7),
});

let _response = self.client
let _response = self
.client
.post(format!("{}/chat/completions", self.base_url))
.header("Authorization", format!("Bearer {}", self.api_key))
.json(&body)
Expand Down Expand Up @@ -141,19 +142,25 @@ impl LlmRouter {
self.fallback = Some(provider);
}

pub async fn complete(&self, request: &CompletionRequest) -> Result<CompletionResponse, LlmError> {
pub async fn complete(
&self,
request: &CompletionRequest,
) -> Result<CompletionResponse, LlmError> {
// Route based on model prefix
let (prefix, _) = request.model.split_once('/').unwrap_or((&request.model, ""));

let (prefix, _) = request
.model
.split_once('/')
.unwrap_or((&request.model, ""));

if let Some(provider) = self.providers.get(prefix) {
return provider.complete(request).await;
}

// Try fallback
if let Some(fallback) = &self.fallback {
return fallback.complete(request).await;
}

Err(LlmError::InvalidModel(request.model.clone()))
}
}
Expand Down Expand Up @@ -261,8 +268,12 @@ mod tests {
format!("{}", d),
];
// All four messages must be unique.
let unique: std::collections::HashSet<&str> =
messages.iter().map(|s| s.as_str()).collect();
assert_eq!(unique.len(), 4, "duplicate Display for variant: {:?}", messages);
let unique: std::collections::HashSet<&str> = messages.iter().map(|s| s.as_str()).collect();
assert_eq!(
unique.len(),
4,
"duplicate Display for variant: {:?}",
messages
);
}
}
52 changes: 38 additions & 14 deletions crates/mcp-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ use anyhow::Result;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
use thiserror::Error;
use tokio::sync::RwLock;

#[derive(Error, Debug)]
pub enum McpError {
Expand Down Expand Up @@ -69,27 +69,49 @@ impl McpServer {
}
}

pub async fn register_tool(&self, tool: Tool, handler: impl Fn(serde_json::Value) -> Result<serde_json::Value> + Send + Sync + 'static) {
self.tools.write().await.insert(tool.name.clone(), ToolHandler {
tool,
handler: Arc::new(handler),
});
pub async fn register_tool(
&self,
tool: Tool,
handler: impl Fn(serde_json::Value) -> Result<serde_json::Value> + Send + Sync + 'static,
) {
self.tools.write().await.insert(
tool.name.clone(),
ToolHandler {
tool,
handler: Arc::new(handler),
},
);
}

pub async fn register_resource(&self, resource: Resource) {
self.resources.write().await.insert(resource.uri.clone(), resource);
self.resources
.write()
.await
.insert(resource.uri.clone(), resource);
}

pub async fn list_tools(&self) -> Vec<Tool> {
self.tools.read().await.values().map(|h| h.tool.clone()).collect()
self.tools
.read()
.await
.values()
.map(|h| h.tool.clone())
.collect()
}

pub async fn call_tool(&self, name: &str, arguments: serde_json::Value) -> Result<ToolResult, McpError> {
pub async fn call_tool(
&self,
name: &str,
arguments: serde_json::Value,
) -> Result<ToolResult, McpError> {
let tools = self.tools.read().await;
let handler = tools.get(name).ok_or(McpError::ToolNotFound(name.to_string()))?;

let result = (handler.handler)(arguments).map_err(|e| McpError::InvalidRequest(e.to_string()))?;

let handler = tools
.get(name)
.ok_or(McpError::ToolNotFound(name.to_string()))?;

let result =
(handler.handler)(arguments).map_err(|e| McpError::InvalidRequest(e.to_string()))?;
Comment on lines +108 to +113

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: The handler is executed while still borrowing from the tools read lock, so callback execution holds the lock for the full tool runtime and can block writers (register_tool) or deadlock if callbacks re-enter mutation paths. Clone the handler Arc out of the map, drop the lock guard, then invoke the callback. [performance]

Severity Level: Critical 🚨
- ❌ Tool handlers that mutate tools can deadlock call_tool.
- ⚠️ Long-running tools block concurrent tool registration/modification.
Steps of Reproduction ✅
1. In `crates/mcp-server/src/lib.rs:102-121`, `McpServer::call_tool` acquires a read lock
on `self.tools` at line 107 (`let tools = self.tools.read().await;`) and then looks up a
handler at lines 108-110 and invokes it at lines 112-113.

2. In `tests/mcp_server_test.rs:63-88`, `test_call_tool` demonstrates registering a tool
with a closure handler and invoking it via `server.call_tool("echo", ...)`, exercising
this code path.

3. Extend this pattern with a tool handler that captures `server: McpServer` (as done for
other closures in `tests/mcp_server_test.rs:31-35, 46-55, 77-82`) and, inside the handler,
attempts to mutate the tool map (e.g., by calling `server.register_tool(...)` via
`tokio::runtime::Handle::current().block_on`).

4. When `call_tool` executes, it holds the `RwLock` read guard from line 107 for the
entire duration of the handler; the handler then attempts to acquire a write lock via
`register_tool` (lines 72-83 in `lib.rs`), which cannot proceed while the read lock is
held, leading to a deadlock in that task and blocking all writers for as long as tool
callbacks run.

Fix in Cursor | Fix in VSCode Claude

(Use Cmd/Ctrl + Click for best experience)

Prompt for AI Agent 🤖
This is a comment left during a code review.

**Path:** crates/mcp-server/src/lib.rs
**Line:** 108:113
**Comment:**
	*Performance: The handler is executed while still borrowing from the `tools` read lock, so callback execution holds the lock for the full tool runtime and can block writers (`register_tool`) or deadlock if callbacks re-enter mutation paths. Clone the handler `Arc` out of the map, drop the lock guard, then invoke the callback.

Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise.
Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix
👍 | 👎


Ok(ToolResult {
content: vec![ContentItem {
content_type: "text".to_string(),
Expand All @@ -105,7 +127,9 @@ impl McpServer {

pub async fn read_resource(&self, uri: &str) -> Result<String, McpError> {
let resources = self.resources.read().await;
let resource = resources.get(uri).ok_or(McpError::ResourceNotFound(uri.to_string()))?;
let resource = resources
.get(uri)
.ok_or(McpError::ResourceNotFound(uri.to_string()))?;
Ok(format!("Resource: {}", resource.name))
}
}
Expand Down
15 changes: 11 additions & 4 deletions crates/pheno-embedding/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,22 @@ impl OpenAiEmbeddings {
}
}

pub async fn embed(&self, request: &EmbeddingRequest) -> Result<EmbeddingResponse, EmbeddingError> {
let model = request.model.clone().unwrap_or_else(|| "text-embedding-3-small".to_string());

pub async fn embed(
&self,
request: &EmbeddingRequest,
) -> Result<EmbeddingResponse, EmbeddingError> {
let model = request
.model
.clone()
.unwrap_or_else(|| "text-embedding-3-small".to_string());

let body = serde_json::json!({
"input": request.texts,
"model": model,
});

let _response = self.client
let _response = self
.client
.post("https://api.openai.com/v1/embeddings")
.header("Authorization", format!("Bearer {}", self.api_key))
.json(&body)
Expand Down
198 changes: 198 additions & 0 deletions grade.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
#!/usr/bin/env bash
# grade.sh — Fleet-wide project grading engine
# Usage: ./grade.sh [--fast] [--json] [--html]
# --fast : Quick mode (skips heavy checks like fuzz, mutation, perf)
# --json : Output machine-readable JSON
# --html : Output HTML report

set -euo pipefail

FAST=false
JSON=false
HTML=false
REPORT_DIR=".grade-reports"

while [[ $# -gt 0 ]]; do
case "$1" in
--fast) FAST=true; shift ;;
--json) JSON=true; shift ;;
--html) HTML=true; shift ;;
*) echo "Unknown option: $1"; exit 1 ;;
esac
done

mkdir -p "$REPORT_DIR"

# Detect stack
STACK="unknown"
if [[ -f "Cargo.toml" ]]; then STACK="rust"; fi
if [[ -f "package.json" ]]; then STACK="node"; fi
if [[ -f "pyproject.toml" || -f "setup.py" ]]; then STACK="python"; fi
if [[ -f "go.mod" ]]; then STACK="go"; fi
Comment on lines +27 to +31

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: Stack detection overwrites STACK on every matching manifest, so in polyglot repos the final if wins and only one ecosystem is graded. That causes incomplete grading and false pass/fail outcomes because checks from other detected stacks are silently skipped. [logic error]

Severity Level: Critical 🚨
- ❌ Polyglot repos run checks for only one stack.
- ⚠️ Quality gate ignores failing tests in other languages.
- ⚠️ JSON grade report misrepresents overall project health.
Steps of Reproduction ✅
1. Place `grade.sh` at the root of a polyglot repo containing both `Cargo.toml` and
`pyproject.toml` (or `setup.py`) alongside potential `package.json`/`go.mod` manifests,
matching the stack detection logic at `grade.sh:27-31`.

2. From that repo root, run `./grade.sh --json` so that the script executes the stack
detection block at `grade.sh:27-31` and then enters the `case "$STACK" in` dispatch at
`grade.sh:69-125`.

3. Observe that each `if` in `grade.sh:28-31` unconditionally overwrites `STACK`, so the
last matching manifest (e.g. `pyproject.toml` setting `STACK="python"` at line 30)
determines the single stack whose checks are run in the big `case` at `grade.sh:69-120`.

4. Confirm that checks for other present stacks (e.g. Rust `cargo test`, Node `npm test`,
or Go `go test` branches) never execute, so the JSON report written at `grade.sh:143-156`
and the overall grade at `grade.sh:127-139` only reflect the final stack, silently
skipping failures in the other ecosystems.

Fix in Cursor | Fix in VSCode Claude

(Use Cmd/Ctrl + Click for best experience)

Prompt for AI Agent 🤖
This is a comment left during a code review.

**Path:** grade.sh
**Line:** 27:31
**Comment:**
	*Logic Error: Stack detection overwrites `STACK` on every matching manifest, so in polyglot repos the final `if` wins and only one ecosystem is graded. That causes incomplete grading and false pass/fail outcomes because checks from other detected stacks are silently skipped.

Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise.
Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix
👍 | 👎

if [[ -f "Taskfile.yml" || -f "Justfile" ]]; then
: # already has task runner
fi

SCORE=0
MAX=0
CHECKS=()

run_check() {
local name="$1"
local cmd="$2"
local weight="${3:-1}"
local fast_skip="${4:-false}"

if [[ "$FAST" == true && "$fast_skip" == true ]]; then
CHECKS+=("{\"name\":\"$name\",\"status\":\"skipped\",\"score\":0,\"max\":$weight,\"detail\":\"skipped in fast mode\"}")
return 0
fi

MAX=$((MAX + weight))
if eval "$cmd" 2>&1 | tee "$REPORT_DIR/${name}.log" >"$REPORT_DIR/${name}.raw" 2>&1; then
SCORE=$((SCORE + weight))
CHECKS+=("{\"name\":\"$name\",\"status\":\"pass\",\"score\":$weight,\"max\":$weight,\"detail\":\"\"}")
echo " [PASS] $name"
else
local detail="$(head -5 "$REPORT_DIR/${name}.raw" | tr '\n' ' ')"
CHECKS+=("{\"name\":\"$name\",\"status\":\"fail\",\"score\":0,\"max\":$weight,\"detail\":\"$detail\"}")
Comment on lines +57 to +58

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggestion: Command output is interpolated directly into JSON string fields without escaping, so quotes/backslashes/newlines in log text can produce invalid JSON output in both in-memory check entries and grade.json. Escape JSON strings before constructing these objects. [type error]

Severity Level: Major ⚠️
- ⚠️ Machine consumers cannot reliably parse `grade.json`.
- ⚠️ CI dashboards depending on JSON reports may break.
- ⚠️ Debugging failed checks harder without valid structured data.
Steps of Reproduction ✅
1. From the repo root, run `./grade.sh --json` so the Python stack branch at
`grade.sh:96-107` (or any stack) is exercised and JSON output is enabled at
`grade.sh:141-159`.

2. Induce a failing check whose error output contains JSON-breaking characters, e.g. add a
failing assertion in a Python test under `ports/tests/test_model_loader.py:8-37` that
raises `ValueError("bad \"value\"")`, causing pytest (invoked at `grade.sh:98`) to print a
message with embedded double quotes.

3. When the failing check runs, the `run_check` function at `grade.sh:40-61` captures the
first lines of the command's stderr/stdout into `detail` at `grade.sh:57`, without any
JSON escaping, and interpolates it directly into the JSON fragment appended to `CHECKS` at
`grade.sh:58`.

4. After all checks, the JSON report is written at `grade.sh:143-156` using `$(IFS=,; echo
"${CHECKS[*]}")`; attempts to parse `.grade-reports/grade.json` with a standard JSON
parser will fail due to unescaped quotes/backslashes in the `"detail"` field, despite the
file being advertised as machine-readable.

Fix in Cursor | Fix in VSCode Claude

(Use Cmd/Ctrl + Click for best experience)

Prompt for AI Agent 🤖
This is a comment left during a code review.

**Path:** grade.sh
**Line:** 57:58
**Comment:**
	*Type Error: Command output is interpolated directly into JSON string fields without escaping, so quotes/backslashes/newlines in log text can produce invalid JSON output in both in-memory check entries and `grade.json`. Escape JSON strings before constructing these objects.

Validate the correctness of the flagged issue. If correct, How can I resolve this? If you propose a fix, implement it and please make it concise.
Once fix is implemented, also check other comments on the same PR, and ask user if the user wants to fix the rest of the comments as well. if said yes, then fetch all the comments validate the correctness and implement a minimal fix
👍 | 👎

echo " [FAIL] $name"
fi
}

echo "========================================"
echo " GRADE — $(basename "$(pwd)")"
echo " Stack: $STACK"
echo " Mode: $([[ $FAST == true ]] && echo fast || echo full)"
echo "========================================"

case "$STACK" in
rust)
run_check "build" "cargo build --workspace" 2
run_check "test-unit" "cargo test --workspace" 3
run_check "fmt" "cargo fmt -- --check" 2
run_check "clippy" "cargo clippy --workspace --all-targets --all-features -- -D warnings" 2
run_check "deny" "cargo deny check" 1 true
run_check "doc" "cargo doc --workspace --no-deps" 1
run_check "test-snapshot" "cargo test --workspace -- snapshot" 1 true
run_check "test-fuzz" "cargo test --workspace -- fuzz" 1 true
run_check "coverage" "cargo llvm-cov --workspace --fail-under-lines 85" 2 true
run_check "audit" "cargo audit" 1 true
run_check "bench" "cargo bench --workspace" 1 true
;;
node)
run_check "install" "npm ci" 1
run_check "build" "npm run build" 2
run_check "test-unit" "npm test" 3
run_check "lint" "npx eslint . --ext .ts" 2
run_check "fmt" "npx prettier --check '**/*.ts'" 2
run_check "typecheck" "npx tsc --noEmit" 2
run_check "test-e2e" "npm run test:e2e" 2 true
run_check "test-perf" "npm run test:perf" 1 true
run_check "test-mutation" "npx stryker run" 1 true
run_check "coverage" "npx jest --coverage --coverageThreshold='{\"global\":{\"branches\":85,\"functions\":85,\"lines\":85,\"statements\":85}}'" 2 true
run_check "audit" "npm audit --audit-level=moderate" 1
;;
python)
run_check "install" "pip install -e '.[dev]'" 1
run_check "test-unit" "pytest -v" 3
run_check "lint" "ruff check src" 2
run_check "fmt" "ruff format --check src" 2
run_check "typecheck" "mypy src" 2
run_check "test-fuzz" "pytest -v --fuzz" 1 true
run_check "test-mutation" "mutmut run" 1 true
run_check "test-perf" "pytest -v --perf" 1 true
run_check "coverage" "pytest --cov=src --cov-report=term-missing --cov-fail-under=85" 2 true
run_check "security" "bandit -r src" 1
run_check "audit" "pip-audit" 1 true
;;
go)
run_check "build" "go build ./..." 2
run_check "test-unit" "go test ./..." 3
run_check "fmt" "test -z \"\$(gofmt -l .)\"" 2
run_check "vet" "go vet ./..." 2
run_check "lint" "golangci-lint run" 2
run_check "test-race" "go test -race ./..." 2 true
run_check "test-fuzz" "go test -fuzz=. ./..." 1 true
run_check "test-bench" "go test -bench=. ./..." 1 true
run_check "coverage" "go test -coverprofile=coverage.out -covermode=atomic ./... && go tool cover -func=coverage.out | grep total | awk '{print \$3}' | sed 's/%//' | awk '{exit(\$1 < 85 ? 1 : 0)}'" 2 true
run_check "audit" "govulncheck ./..." 1
;;
*)
echo "Unknown stack: $STACK"
exit 1
;;
esac

# Calculate percentage
PCT=$(( SCORE * 100 / MAX ))

# Determine grade
GRADE="F"
if [[ $PCT -ge 95 ]]; then GRADE="A+"; elif [[ $PCT -ge 90 ]]; then GRADE="A"; elif [[ $PCT -ge 85 ]]; then GRADE="B+"; elif [[ $PCT -ge 80 ]]; then GRADE="B"; elif [[ $PCT -ge 70 ]]; then GRADE="C"; elif [[ $PCT -ge 60 ]]; then GRADE="D"; fi

# Output summary
echo ""
echo "========================================"
echo " SCORE: $SCORE / $MAX ($PCT%)"
echo " GRADE: $GRADE"
echo "========================================"

# JSON output
if [[ "$JSON" == true ]]; then
cat > "$REPORT_DIR/grade.json" <<EOF
{
"project": "$(basename "$(pwd)")",
"stack": "$STACK",
"mode": "$([[ $FAST == true ]] && echo fast || echo full)",
"score": $SCORE,
"max": $MAX,
"percentage": $PCT,
"grade": "$GRADE",
"checks": [
$(IFS=,; echo "${CHECKS[*]}")
],
"timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
}
EOF
echo "JSON report: $REPORT_DIR/grade.json"
fi

# HTML output
if [[ "$HTML" == true ]]; then
cat > "$REPORT_DIR/grade.html" <<EOF
<!DOCTYPE html>
<html>
<head><title>Grade Report — $(basename "$(pwd)")</title>
<style>
body{font-family:system-ui,sans-serif;max-width:800px;margin:2rem auto;padding:0 1rem}
h1{border-bottom:2px solid #333}
.score{font-size:3rem;font-weight:bold;color:$([[ $PCT -ge 85 ]] && echo "#2d7" || echo "#d42")}
.grade{font-size:1.5rem}
table{width:100%;border-collapse:collapse;margin:1rem 0}
th,td{padding:0.5rem;text-align:left;border-bottom:1px solid #ddd}
th{background:#f5f5f5}
.pass{color:#2d7}.fail{color:#d42}.skip{color:#888}
</style>
</head>
<body>
<h1>Grade Report — $(basename "$(pwd)")</h1>
<p class="score">$PCT% <span class="grade">($GRADE)</span></p>
<p>Stack: $STACK | Mode: $([[ $FAST == true ]] && echo fast || echo full)</p>
<table>
<tr><th>Check</th><th>Status</th><th>Score</th></tr>
EOF
for check in "${CHECKS[@]}"; do
name=$(echo "$check" | grep -o '"name":"[^"]*"' | cut -d'"' -f4)
status=$(echo "$check" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
score=$(echo "$check" | grep -o '"score":[0-9]*' | cut -d':' -f2)
max=$(echo "$check" | grep -o '"max":[0-9]*' | cut -d':' -f2)
echo "<tr><td>$name</td><td class=\"$status\">$status</td><td>$score/$max</td></tr>" >> "$REPORT_DIR/grade.html"
done
echo "</table><p>Generated: $(date -u)</p></body></html>" >> "$REPORT_DIR/grade.html"
echo "HTML report: $REPORT_DIR/grade.html"
fi

# Exit code
if [[ $PCT -lt 85 ]]; then exit 1; fi
exit 0
Loading
Loading