Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 147 additions & 0 deletions src/core/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,17 @@ pub fn expected_error_kind(message: &str) -> Option<ExpectedErrorKind> {
if is_loopback_unavailable(&lower) {
return Some(ExpectedErrorKind::LoopbackUnavailable);
}
// Check `is_ollama_user_config_rejection` BEFORE the generic network /
// backend-error matchers: the GX "daemon unreachable at localhost" shape
// contains a loopback host but no `Connection refused (os error …)`
// marker, and the XS / MA / KM 400/404 shapes are pure user-config —
// wrong model name, model not pulled, daemon opted-in but not running.
// Route them to the dedicated arm so they share the `ProviderUserState`
// bucket with the composio / OAuth user-state errors instead of falling
// through to capture. See `is_ollama_user_config_rejection`.
if is_ollama_user_config_rejection(&lower) {
return Some(ExpectedErrorKind::ProviderUserState);
}
if is_network_unreachable_message(&lower) {
return Some(ExpectedErrorKind::NetworkUnreachable);
}
Expand Down Expand Up @@ -284,6 +295,77 @@ fn is_loopback_unavailable(lower: &str) -> bool {
|| lower.contains("connection refused (os error 10061)")
}

/// Detect Ollama embed call sites that surface a user-config rejection from
/// the local Ollama daemon — pure user-state errors the UI already surfaces
/// (toast / settings page warning) where Sentry has no remediation path.
///
/// Three canonical wire shapes are covered, all emitted by
/// `openhuman::embeddings::ollama::OllamaEmbedding::embed` and the embed
/// service fallback path:
///
/// - **TAURI-RUST-XS** (~376 events on self-hosted Sentry): user pointed the
/// embedder at a chat / vision model id with a temperature suffix (e.g.
/// `qwen3-vl:4b@0.7`) which Ollama parses as malformed. Wire shape:
/// `ollama embed failed with status 400 Bad Request: {"error":"invalid model name"}`.
/// - **OPENHUMAN-TAURI-MA / -KM** (deferred follow-up from PR #2216): user
/// configured a model id that the local Ollama daemon hasn't pulled yet.
/// Wire shape:
/// `ollama embed failed with status 404 Not Found: {"error":"model \"<id>\" not found, try pulling it first"}`.
/// - **OPENHUMAN-TAURI-GX**: user opted into Ollama embeddings but the
/// daemon isn't running on `localhost:11434`, so the embed service falls
/// back to cloud embeddings for the session. Wire shape:
/// `ollama embeddings opted-in but daemon unreachable at http://localhost:11434; falling back to cloud embeddings for this session`.
///
/// All three are user-config: the user picked the wrong model id, forgot to
/// pull it, or forgot to start the daemon. The remediation is "fix the
/// model id in Settings" / "run `ollama pull <id>`" / "start ollama" —
/// none of which Sentry can do for them.
///
/// The classifier is anchored on the `"ollama embed"` prefix
/// (`"ollama embed failed"` for the 400/404 shapes, `"ollama embeddings opted-in"`
/// for the daemon-unreachable fallback) so unrelated 400/404 errors elsewhere
/// in the codebase that happen to contain `"invalid model name"` or
/// `"not found"` substrings are not silenced.
///
/// Routes to [`ExpectedErrorKind::ProviderUserState`] — the same bucket that
/// holds the composio / gmail / OAuth user-state errors. We deliberately do
/// **not** introduce a dedicated Ollama enum variant: the demotion semantics
/// (drop to `info` log, skip Sentry capture) are identical and adding a new
/// variant for every provider would balloon the enum without changing
/// behavior.
fn is_ollama_user_config_rejection(lower: &str) -> bool {
// XS — 400-status user-config (invalid model name, including the
// temperature-suffix shape `qwen3-vl:4b@0.7` Ollama parses as malformed).
if lower.contains("ollama embed failed") && lower.contains("invalid model name") {
return true;
}

// MA / KM — 404-status pull-required. The wire shape is JSON-escaped
// (`\"<model-id>\" not found`); after lower-casing we still see the
// backslash-quoted form. Anchor on `model \"` + `\" not found` so an
// unrelated 404 that merely contains `"model"` and `"not found"` is not
// swallowed. The `\\"` byte pair in Rust source matches the literal
// `\"` sequence in the wire shape.
if lower.contains("ollama embed failed")
&& lower.contains("model \\\"")
&& lower.contains("\\\" not found")
{
return true;
}

// GX — daemon-unreachable opt-in state. The wire shape is emitted by
// the embed service when the user has opted into Ollama in settings
// but the daemon isn't responding, so the service falls back to cloud
// embeddings for the session. Anchor on the full prefix to keep the
// matcher from colliding with unrelated `"daemon unreachable"`
// messages from other domains (e.g. backend connection-health logs).
if lower.contains("ollama embeddings opted-in but daemon unreachable at") {
return true;
}

false
}

/// Detect transport-level connection failures that fire before any HTTP status
/// is observed — DNS resolution failures, TCP connect refused/reset, TLS
/// handshake failures, or ISP/firewall blocks. The canonical shape is
Expand Down Expand Up @@ -1280,6 +1362,71 @@ mod tests {
);
}

#[test]
fn classifies_ollama_user_config_rejections() {
// TAURI-RUST-XS (~376 events): user pointed embedder at a chat /
// vision model id, sometimes with a temperature suffix like `@0.7`
// that Ollama parses as malformed.
for raw in [
// Canonical XS wire shape from
// `OllamaEmbedding::embed` non-2xx path on a 400 Bad Request.
r#"ollama embed failed with status 400 Bad Request: {"error":"invalid model name"}"#,
// Same shape with a temperature-suffix model id the user pasted
// into Settings → Embeddings → Ollama.
r#"ollama embed failed with status 400 Bad Request: {"error":"invalid model name: qwen3-vl:4b@0.7"}"#,
// OPENHUMAN-TAURI-MA — model not pulled (404 Not Found).
r#"ollama embed failed with status 404 Not Found: {"error":"model \"bge-m3\" not found, try pulling it first"}"#,
// OPENHUMAN-TAURI-KM — same shape, different model id + `:latest` tag.
r#"ollama embed failed with status 404 Not Found: {"error":"model \"nomic-embed-text:latest\" not found, try pulling it first"}"#,
// OPENHUMAN-TAURI-GX — daemon-unreachable opt-in state.
"ollama embeddings opted-in but daemon unreachable at http://localhost:11434; falling back to cloud embeddings for this session",
] {
assert_eq!(
expected_error_kind(raw),
Some(ExpectedErrorKind::ProviderUserState),
"should classify Ollama user-config rejection: {raw}"
);
}
}

#[test]
fn does_not_classify_unrelated_ollama_errors_as_user_config() {
// Unrelated 500 — server-side ollama bug must still reach Sentry.
assert_eq!(
expected_error_kind("ollama embed failed with status 500"),
None
);
// Parse-failure on the response — real bug in either the server
// or our deserializer, must still reach Sentry.
assert_eq!(
expected_error_kind(
"ollama embed response parse failed: invalid type: expected sequence"
),
None
);
// Dimension mismatch — real bug (model dims don't match what we
// recorded), must still reach Sentry.
assert_eq!(
expected_error_kind(
"ollama embed dimension mismatch at index 0: expected 768, got 1024"
),
None
);
// Unrelated `invalid model name` outside Ollama embed call —
// anchor on the `ollama embed` prefix keeps this from being silenced.
assert_eq!(
expected_error_kind("provider config validation failed: invalid model name"),
None
);
// Unrelated `model "…" not found` text without the `ollama embed`
// prefix — anchor keeps this from being silenced even when the
// exact MA/KM wire-shape substring appears in another context.
assert_eq!(
expected_error_kind(r#"provider listing failed: model \"foo\" not found in registry"#),
None
);
}

#[test]
fn classifies_local_ai_capability_unavailable_errors() {
// OPENHUMAN-TAURI-3B: surfaced by `local_ai_download_asset` when a
Expand Down
88 changes: 71 additions & 17 deletions src/openhuman/embeddings/ollama_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -327,32 +327,60 @@ async fn embed_connection_refused() {
);
}

// OPENHUMAN-TAURI-{GP,MA,KM,GX} wire shapes — currently routed through
// `report_error_or_expected` (Sentry classifier ladder) by this PR. The ladder
// matches GP (LocalAiCapabilityUnavailable) today; MA/KM/GX still fall through
// to capture because `observability::expected_error_kind` has no matcher arm
// for "ollama model not found" / "ollama daemon unreachable". Those matcher
// arms are blocked behind PR #2063 + #2188 merging (both touch
// `src/core/observability.rs`) and will land in the follow-up classifier
// batch. Tests below lock the CURRENT state so the follow-up flips them.
// OPENHUMAN-TAURI-{GP,MA,KM,GX} + TAURI-RUST-XS wire shapes — routed through
// `report_error_or_expected` (Sentry classifier ladder) and demoted to
// `ProviderUserState` / `LocalAiCapabilityUnavailable` by the
// `is_ollama_user_config_rejection` matcher in `src/core/observability.rs`.
// GP catches the RAM-tier disable shape; XS/MA/KM/GX cover the four Ollama
// user-config rejection shapes (400 invalid-model-name, 404 model-not-found
// ×2, daemon-unreachable opt-in state).

#[test]
fn ma_wire_shape_current_state_unclassified() {
fn xs_wire_shape_classifies_as_provider_user_state() {
// TAURI-RUST-XS (~376 events on self-hosted Sentry): user pointed
// embedder at a chat / vision model with a temperature suffix
// (`qwen3-vl:4b@0.7`) Ollama parses as malformed.
let msg = r#"ollama embed failed with status 400 Bad Request: {"error":"invalid model name"}"#;
assert_eq!(
crate::core::observability::expected_error_kind(msg),
Some(crate::core::observability::ExpectedErrorKind::ProviderUserState),
"XS — invalid model name 400 is pure user-config, must demote"
);
}

#[test]
fn xs_temperature_suffix_model_classifies_as_provider_user_state() {
// Same XS shape with the temperature-suffix model id embedded in the
// body. Real wire shape from the field report.
let msg = r#"ollama embed failed with status 400 Bad Request: {"error":"invalid model name: qwen3-vl:4b@0.7"}"#;
assert_eq!(
crate::core::observability::expected_error_kind(msg),
Some(crate::core::observability::ExpectedErrorKind::ProviderUserState),
"XS — temperature-suffix model id must also demote"
);
}

#[test]
fn ma_wire_shape_classifies_as_provider_user_state() {
// OPENHUMAN-TAURI-MA: user configured an Ollama model id that the
// local daemon hasn't pulled yet.
let msg = r#"ollama embed failed with status 404 Not Found: {"error":"model \"bge-m3\" not found, try pulling it first"}"#;
assert_eq!(
crate::core::observability::expected_error_kind(msg),
None,
"MA — matcher arm pending follow-up classifier batch (post #2063 + #2188 merge)"
Some(crate::core::observability::ExpectedErrorKind::ProviderUserState),
"MA — model-not-found 404 is pure user-state (pull required), must demote"
);
}

#[test]
fn km_wire_shape_current_state_unclassified() {
fn km_wire_shape_classifies_as_provider_user_state() {
// OPENHUMAN-TAURI-KM: same wire shape as MA with a different model
// id + `:latest` tag.
let msg = r#"ollama embed failed with status 404 Not Found: {"error":"model \"nomic-embed-text:latest\" not found, try pulling it first"}"#;
assert_eq!(
crate::core::observability::expected_error_kind(msg),
None,
"KM — matcher arm pending follow-up classifier batch"
Some(crate::core::observability::ExpectedErrorKind::ProviderUserState),
"KM — pull-required 404 must demote"
);
}

Expand All @@ -363,17 +391,30 @@ fn gp_wire_shape_classifies() {
assert_eq!(
crate::core::observability::expected_error_kind(msg),
Some(crate::core::observability::ExpectedErrorKind::LocalAiCapabilityUnavailable),
"GP — LocalAiCapabilityUnavailable matcher must catch this; closed by this PR"
"GP — LocalAiCapabilityUnavailable matcher must catch this"
);
}

#[test]
fn gx_wire_shape_current_state_unclassified() {
fn gx_wire_shape_classifies_as_provider_user_state() {
// OPENHUMAN-TAURI-GX: user opted into Ollama embeddings in Settings
// but the daemon isn't running on localhost:11434.
let msg = "ollama embeddings opted-in but daemon unreachable at http://localhost:11434; falling back to cloud embeddings for this session";
assert_eq!(
crate::core::observability::expected_error_kind(msg),
Some(crate::core::observability::ExpectedErrorKind::ProviderUserState),
"GX — daemon-unreachable opt-in state is pure user-config (start daemon)"
);
}

#[test]
fn ollama_500_wire_shape_stays_unexpected() {
// Server-side ollama bug — must still reach Sentry.
let msg = "ollama embed failed with status 500 Internal Server Error: model crashed";
assert_eq!(
crate::core::observability::expected_error_kind(msg),
None,
"GX — matcher arm pending follow-up classifier batch"
"real ollama server errors must still reach Sentry"
);
}

Expand All @@ -387,6 +428,19 @@ fn ollama_parse_error_wire_shape_stays_unexpected() {
);
}

#[test]
fn ollama_dimension_mismatch_stays_unexpected() {
// Dimension mismatch — real bug (model dims don't match what we
// recorded for the provider in Settings), must still reach Sentry
// so we can investigate the desync.
let msg = "ollama embed dimension mismatch at index 0: expected 768, got 1024";
assert_eq!(
crate::core::observability::expected_error_kind(msg),
None,
"dimension-mismatch shape must still reach Sentry"
);
}

// ── embed_one (trait default) ───────────────────────────

#[tokio::test]
Expand Down
Loading