From d3c5e1e962fc9f3ce891634ea74931a721952ec0 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 11 Jun 2026 17:30:06 +0100 Subject: [PATCH 01/25] feat(exploit-intelligence): add orchestration module with API endpoints Add a new exploit_intelligence module under modules/fundamental that provides API endpoints for triggering Exploit Intelligence analysis, listing jobs, and retrieving job details. The module includes: - POST /v3/exploit-intelligence/analyze: Creates an analysis job and spawns a background task that submits the SBOM to the EI client service, polls for completion, and ingests returned VEX documents with source: exploit-intelligence labels. - GET /v3/exploit-intelligence/jobs: Paginated list of analysis jobs with filtering by sbom_id and status. - GET /v3/exploit-intelligence/jobs/{id}: Job details including status, finding, report_url, advisory_id, and error details. Configuration is via environment variables: EXPLOIT_INTELLIGENCE_URL, EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS, EXPLOIT_INTELLIGENCE_AUTH_TOKEN. Implements TC-4677 Assisted-by: Claude Code --- Cargo.lock | 2 + common/auth/src/permission.rs | 5 + docs/env-vars.md | 3 + modules/fundamental/Cargo.toml | 1 + modules/fundamental/src/endpoints.rs | 15 +- .../src/exploit_intelligence/endpoints/mod.rs | 389 ++++++++++++++++++ .../exploit_intelligence/endpoints/test.rs | 329 +++++++++++++++ .../src/exploit_intelligence/mod.rs | 4 + .../src/exploit_intelligence/model/mod.rs | 147 +++++++ .../src/exploit_intelligence/service/mod.rs | 228 ++++++++++ modules/fundamental/src/lib.rs | 1 + modules/fundamental/src/test/common.rs | 1 + openapi.yaml | 275 +++++++++++++ server/src/openapi.rs | 1 + server/src/profile/api.rs | 36 ++ 15 files changed, 1436 insertions(+), 1 deletion(-) create mode 100644 modules/fundamental/src/exploit_intelligence/endpoints/mod.rs create mode 100644 modules/fundamental/src/exploit_intelligence/endpoints/test.rs create mode 100644 modules/fundamental/src/exploit_intelligence/mod.rs create mode 100644 modules/fundamental/src/exploit_intelligence/model/mod.rs create mode 100644 modules/fundamental/src/exploit_intelligence/service/mod.rs diff --git a/Cargo.lock b/Cargo.lock index c110546e6..4f4ad7304 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6006,6 +6006,7 @@ dependencies = [ "js-sys", "log", "mime", + "mime_guess", "native-tls", "percent-encoding", "pin-project-lite", @@ -8534,6 +8535,7 @@ dependencies = [ "osv", "packageurl", "regex", + "reqwest 0.13.4", "roxmltree", "rstest", "sanitize-filename", diff --git a/common/auth/src/permission.rs b/common/auth/src/permission.rs index 57e8e1740..f0d8c6375 100644 --- a/common/auth/src/permission.rs +++ b/common/auth/src/permission.rs @@ -117,6 +117,11 @@ permission! { #[strum(serialize = "ai")] Ai, + + #[strum(serialize = "create.exploitIntelligence")] + CreateExploitIntelligence, + #[strum(serialize = "read.exploitIntelligence")] + ReadExploitIntelligence, } } diff --git a/docs/env-vars.md b/docs/env-vars.md index f5dd5489c..a28512bf1 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -62,6 +62,9 @@ | `UI_ISSUER_URL` | Issuer URL used by the UI | `http://localhost:8090/realms/trustify` | | `UI_LOAD_USER` | Whether to load user info | `true` | | `UI_SCOPE` | Scopes to request | `openid` | +| `EXPLOIT_INTELLIGENCE_URL` | Base URL of the Exploit Intelligence client service | | +| `EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS`| Polling interval in seconds for EI analysis completion | `30` | +| `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Authentication token for the Exploit Intelligence service | | ## Data Migration diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 71823b7f2..c35777a38 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -30,6 +30,7 @@ futures-util = { workspace = true } isx = { workspace = true } itertools = { workspace = true } log = { workspace = true } +reqwest = { workspace = true, features = ["json", "multipart"] } sanitize-filename = { workspace = true } sea-orm = { workspace = true } sea-query = { workspace = true } diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 960436618..23ebf0607 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -6,6 +6,8 @@ use trustify_module_ingestor::service::IngestorService; use trustify_module_storage::service::dispatch::DispatchBackend; use utoipa::{IntoParams, ToSchema}; +use crate::exploit_intelligence::service::{ExploitIntelligenceConfig, ExploitIntelligenceService}; + #[derive(Clone, Debug, Eq, PartialEq, Default)] pub struct Config { pub sbom_upload_limit: usize, @@ -13,6 +15,7 @@ pub struct Config { pub max_group_name_length: usize, } +#[allow(clippy::too_many_arguments)] pub fn configure( svc: &mut utoipa_actix_web::service_config::ServiceConfig, config: Config, @@ -21,6 +24,7 @@ pub fn configure( storage: impl Into, analysis: AnalysisService, cache: PaginationCache, + ei_config: Option, ) { let ingestor_service = IngestorService::new(Graph::new(), storage, Some(analysis)); svc.app_data(web::Data::new(ingestor_service)); @@ -45,7 +49,16 @@ pub fn configure( ); crate::vulnerability::endpoints::configure(svc, db_ro.clone(), cache.clone()); crate::weakness::endpoints::configure(svc, db_ro.clone(), cache.clone()); - crate::sbom_group::endpoints::configure(svc, db_rw, db_ro, config.max_group_name_length, cache); + crate::sbom_group::endpoints::configure( + svc, + db_rw.clone(), + db_ro.clone(), + config.max_group_name_length, + cache, + ); + + let ei_service = ExploitIntelligenceService::new(ei_config); + crate::exploit_intelligence::endpoints::configure(svc, db_rw, db_ro, ei_service); } #[derive(Clone, Debug, PartialEq, Eq, Default, ToSchema, serde::Deserialize, IntoParams)] diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs new file mode 100644 index 000000000..5eb0db0b9 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -0,0 +1,389 @@ +#[cfg(test)] +mod test; + +use crate::{ + Error, + exploit_intelligence::{ + model::{ + AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, + ExploitIntelligenceJobSummary, JobStatus, + }, + service::ExploitIntelligenceService, + }, +}; +use actix_web::{HttpResponse, Responder, get, post, web}; +use futures_util::TryStreamExt; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use std::str::FromStr; +use trustify_auth::{CreateExploitIntelligence, ReadExploitIntelligence, authorizer::Require}; +use trustify_common::{ + db, + model::{Paginated, PaginatedResults}, +}; +use trustify_entity::labels::Labels; +use trustify_module_ingestor::service::{Cache, Format, IngestorService}; +use trustify_module_storage::service::StorageBackend; +use uuid::Uuid; + +/// Register exploit intelligence endpoints. +pub fn configure( + config: &mut utoipa_actix_web::service_config::ServiceConfig, + db_rw: db::ReadWrite, + db_ro: db::ReadOnly, + ei_service: ExploitIntelligenceService, +) { + config + .app_data(web::Data::new(db_rw)) + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(analyze) + .service(list_jobs) + .service(get_job); +} + +/// Query parameters for filtering the job list. +#[derive(Debug, Clone, serde::Deserialize, utoipa::IntoParams)] +pub struct JobFilter { + /// Filter by SBOM ID. + pub sbom_id: Option, + /// Filter by job status. + pub status: Option, +} + +/// Trigger an Exploit Intelligence analysis for a CVE against an SBOM. +/// +/// Creates a job record and spawns a background task that submits the SBOM +/// to the EI client service, polls for completion, and ingests any returned +/// VEX document. +#[utoipa::path( + tag = "exploit-intelligence", + operation_id = "analyzeExploitIntelligence", + request_body = AnalyzeRequest, + responses( + (status = 201, description = "Analysis job created", body = AnalyzeResponse), + (status = 404, description = "SBOM not found"), + (status = 503, description = "Exploit Intelligence service not configured"), + ), +)] +#[post("/v3/exploit-intelligence/analyze")] +pub async fn analyze( + ei_service: web::Data, + ingestor: web::Data, + db_rw: web::Data, + db_ro: web::Data, + body: web::Json, + _: Require, +) -> actix_web::Result { + let config = ei_service.config().cloned().ok_or(Error::Unavailable)?; + + // Verify the SBOM exists. + let tx = db_ro.begin().await?; + let sbom = trustify_entity::sbom::Entity::find() + .filter(trustify_entity::sbom::Column::SbomId.eq(body.sbom_id)) + .one(&tx) + .await + .map_err(Error::from)?; + + if sbom.is_none() { + return Ok(HttpResponse::NotFound().finish()); + } + drop(tx); + + // Create the job record. + let job = ei_service + .create_job(body.sbom_id, &body.vulnerability_id, &db_rw) + .await?; + + let job_id = job.id; + let sbom_id = body.sbom_id; + let vulnerability_id = body.vulnerability_id.clone(); + + // Spawn background task to submit and poll. + let ei_svc = ei_service.as_ref().clone(); + let ingestor_svc = ingestor.as_ref().clone(); + let db_rw_clone = db_rw.as_ref().clone(); + + actix_web::rt::spawn(async move { + if let Err(e) = run_analysis( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config, + job_id, + sbom_id, + &vulnerability_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "exploit intelligence analysis failed"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + + Ok(HttpResponse::Created().json(AnalyzeResponse { + job_id, + status: JobStatus::Pending, + })) +} + +/// List Exploit Intelligence analysis jobs. +#[utoipa::path( + tag = "exploit-intelligence", + operation_id = "listExploitIntelligenceJobs", + params( + Paginated, + JobFilter, + ), + responses( + (status = 200, description = "List of analysis jobs", body = PaginatedResults), + ), +)] +#[get("/v3/exploit-intelligence/jobs")] +pub async fn list_jobs( + ei_service: web::Data, + db: web::Data, + web::Query(paginated): web::Query, + web::Query(filter): web::Query, + _: Require, +) -> actix_web::Result { + let tx = db.begin().await?; + let result = ei_service + .fetch_jobs(paginated, filter.sbom_id, filter.status, &tx) + .await?; + Ok(HttpResponse::Ok().json(result)) +} + +/// Get details of a specific Exploit Intelligence analysis job. +#[utoipa::path( + tag = "exploit-intelligence", + operation_id = "getExploitIntelligenceJob", + params( + ("id" = Uuid, Path, description = "Job identifier"), + ), + responses( + (status = 200, description = "Job details", body = ExploitIntelligenceJobDetails), + (status = 404, description = "Job not found"), + ), +)] +#[get("/v3/exploit-intelligence/jobs/{id}")] +pub async fn get_job( + ei_service: web::Data, + db: web::Data, + id: web::Path, + _: Require, +) -> actix_web::Result { + let tx = db.begin().await?; + let job = ei_service.fetch_job(*id, &tx).await?; + + match job { + Some(details) => Ok(HttpResponse::Ok().json(details)), + None => Ok(HttpResponse::NotFound().finish()), + } +} + +/// Background task: submit SBOM to EI client, poll for results, ingest VEX. +async fn run_analysis( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + job_id: Uuid, + sbom_id: Uuid, + vulnerability_id: &str, +) -> Result<(), anyhow::Error> { + // Retrieve the raw SBOM document from storage. + let tx = db_rw.begin().await?; + let sbom_model = trustify_entity::sbom::Entity::find() + .filter(trustify_entity::sbom::Column::SbomId.eq(sbom_id)) + .one(&tx) + .await? + .ok_or_else(|| anyhow::anyhow!("SBOM not found: {sbom_id}"))?; + + let source_doc = + trustify_entity::source_document::Entity::find_by_id(sbom_model.source_document_id) + .one(&tx) + .await? + .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; + + // Build storage key from source document SHA256. + let storage_key = trustify_common::id::Id::from_str(&source_doc.sha256) + .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; + let storage_key: trustify_module_storage::service::StorageKey = storage_key + .try_into() + .map_err(|e: trustify_module_storage::service::StorageKeyError| { + anyhow::anyhow!("storage key error: {e}") + })?; + + let stream = ingestor + .storage() + .retrieve(storage_key) + .await + .map_err(|e| anyhow::anyhow!("storage retrieve error: {e}"))? + .ok_or_else(|| anyhow::anyhow!("SBOM document not found in storage"))?; + + // Collect the stream into bytes. + let sbom_bytes: Vec = stream + .try_fold(Vec::new(), |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await + .map_err(|e| anyhow::anyhow!("failed to read SBOM stream: {e}"))?; + + drop(tx); + + // Submit to EI client. + let client = reqwest::Client::new(); + let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); + + let form = reqwest::multipart::Form::new() + .text("cveId", vulnerability_id.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(sbom_bytes) + .file_name("sbom.json") + .mime_str("application/json")?, + ); + + let mut request = client.post(&upload_url).multipart(form); + + if let Some(token) = &config.auth_token { + request = request.bearer_auth(token); + } + + let response = request.send().await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!("EI client returned {status}: {body}")); + } + + let report_data: serde_json::Value = response.json().await?; + let scan_id = report_data + .pointer("/reportRequestId/reportId") + .and_then(|v| v.as_str()) + .ok_or_else(|| anyhow::anyhow!("missing scanId in EI response"))? + .to_string(); + + // Update job to running with scan ID. + ei_service + .update_job_running(job_id, &scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + // Poll for completion. + let poll_url = format!("{}/api/v1/reports/by-scan-id/{}", config.url, scan_id); + let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); + + loop { + tokio::time::sleep(poll_interval).await; + + let mut poll_request = client.get(&poll_url); + if let Some(token) = &config.auth_token { + poll_request = poll_request.bearer_auth(token); + } + + let poll_response = poll_request.send().await?; + + if !poll_response.status().is_success() { + continue; + } + + let report: serde_json::Value = poll_response.json().await?; + let status = report + .get("status") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + + match status { + "completed" => { + // Extract finding from the analysis result. + use trustify_entity::exploit_intelligence_job::Finding; + let finding = report + .pointer("/report/output/analysis/0/justification/status") + .and_then(|v| v.as_str()) + .map(|s| match s { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + let report_url = format!( + "{}/reports/component/{}/{}", + config.url, vulnerability_id, scan_id + ); + + // Check if VEX document is available. + let vex_json = report.pointer("/report/output/vex"); + let advisory_id = if let Some(vex) = vex_json { + if !vex.is_null() { + // Ingest the VEX document as an advisory with source label. + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest(&vex_bytes, Format::Advisory, labels, None, Cache::Skip, &tx) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + advisory_id = %result.id, + "ingested VEX advisory from exploit intelligence" + ); + + Uuid::from_str(&result.id).ok() + } else { + None + } + } else { + None + }; + + ei_service + .update_job_completed(job_id, finding, Some(report_url), advisory_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + finding = ?finding, + "exploit intelligence analysis completed" + ); + + return Ok(()); + } + "failed" | "expired" => { + let error_msg = report + .pointer("/report/error") + .and_then(|v| v.as_str()) + .unwrap_or("analysis failed") + .to_string(); + + ei_service + .update_job_failed(job_id, &error_msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + return Ok(()); + } + _ => { + // Still in progress (queued, sent, pending), continue polling. + tracing::debug!( + job_id = %job_id, + status = status, + "exploit intelligence analysis still in progress" + ); + } + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs new file mode 100644 index 000000000..82df0936e --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -0,0 +1,329 @@ +use crate::exploit_intelligence::{ + model::{ExploitIntelligenceJobDetails, JobStatus}, + service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, +}; +use actix_web::{ + App, + test::{self, TestRequest}, + web, +}; +use sea_orm::{ActiveModelTrait, Set, TransactionTrait}; +use test_context::test_context; +use test_log::test; +use time::OffsetDateTime; +use trustify_common::{db, model::PaginatedResults}; +use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus}; +use trustify_test_context::TrustifyContext; +use uuid::Uuid; + +/// Helper to create a test service with a fake EI backend URL. +fn test_service() -> ExploitIntelligenceService { + ExploitIntelligenceService::new(Some(ExploitIntelligenceConfig { + url: "http://localhost:9999".to_string(), + poll_interval_secs: 1, + auth_token: None, + })) +} + +/// Helper to insert a job record directly for testing query endpoints. +async fn insert_test_job( + db: &trustify_common::db::Database, + id: Uuid, + sbom_id: Option, + vulnerability_id: &str, + status: ExploitIntelligenceJobStatus, + finding: Option, + error_message: Option<&str>, +) -> exploit_intelligence_job::Model { + let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job::ActiveModel { + id: Set(id), + scan_id: Set(format!("scan-{id}")), + sbom_id: Set(sbom_id), + vulnerability_id: Set(vulnerability_id.to_string()), + status: Set(status), + source_url: Set(None), + report_url: Set(None), + advisory_id: Set(None), + finding: Set(finding), + error_message: Set(error_message.map(|s| s.to_string())), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await.expect("begin tx"); + let result = model.insert(&tx).await.expect("insert job"); + tx.commit().await.expect("commit tx"); + result +} + +/// Verifies that the GET job details endpoint returns the correct state +/// for a completed job. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a completed job record in the database + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-1234", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::NotVulnerable), + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When requesting the job details + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response contains the completed job details + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Completed); + assert_eq!(body.vulnerability_id, "CVE-2024-1234"); + assert_eq!(body.finding.as_deref(), Some("not_vulnerable")); + + Ok(()) +} + +/// Verifies that the GET job details endpoint returns the correct state +/// for a pending job. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_pending(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a pending job record in the database + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-5678", + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When requesting the job details + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response shows the pending status + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Pending); + assert_eq!(body.finding, None); + + Ok(()) +} + +/// Verifies that the GET job details endpoint returns the correct state +/// for a failed job. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_failed(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a failed job record in the database + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-9999", + ExploitIntelligenceJobStatus::Failed, + None, + Some("analysis timed out"), + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When requesting the job details + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response shows the failed status with error details + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Failed); + assert_eq!(body.error_message.as_deref(), Some("analysis timed out")); + + Ok(()) +} + +/// Verifies that requesting a non-existent job returns 404. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When requesting a non-existent job + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{}", Uuid::now_v7())) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then a 404 is returned + assert_eq!(resp.status(), 404); + + Ok(()) +} + +/// Verifies that the list endpoint supports pagination. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn list_jobs_paginated(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given three job records in the database + for i in 0..3 { + insert_test_job( + &ctx.db, + Uuid::now_v7(), + None, + &format!("CVE-2024-{i:04}"), + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + } + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::list_jobs), + ) + .await; + + // When requesting with limit=2 + let req = TestRequest::get() + .uri("/v3/exploit-intelligence/jobs?limit=2") + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then only 2 items are returned but total reflects all 3 + assert_eq!(resp.status(), 200); + let body: PaginatedResults = test::read_body_json(resp).await; + assert_eq!(body.items.len(), 2); + assert_eq!(body.total, Some(3)); + + Ok(()) +} + +/// Verifies that the list endpoint supports filtering by sbom_id. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given jobs for two different SBOMs + let target_sbom_id = Uuid::now_v7(); + let other_sbom_id = Uuid::now_v7(); + + insert_test_job( + &ctx.db, + Uuid::now_v7(), + Some(target_sbom_id), + "CVE-2024-0001", + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + + insert_test_job( + &ctx.db, + Uuid::now_v7(), + Some(other_sbom_id), + "CVE-2024-0002", + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::list_jobs), + ) + .await; + + // When filtering by the target SBOM ID + let req = TestRequest::get() + .uri(&format!( + "/v3/exploit-intelligence/jobs?sbom_id={target_sbom_id}" + )) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then only the matching job is returned + assert_eq!(resp.status(), 200); + let body: PaginatedResults = test::read_body_json(resp).await; + assert_eq!(body.items.len(), 1); + assert_eq!(body.total, Some(1)); + + Ok(()) +} diff --git a/modules/fundamental/src/exploit_intelligence/mod.rs b/modules/fundamental/src/exploit_intelligence/mod.rs new file mode 100644 index 000000000..9560fd556 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod endpoints; + +pub mod model; +pub mod service; diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs new file mode 100644 index 000000000..8c71f80ba --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -0,0 +1,147 @@ +use serde::{Deserialize, Serialize}; +use time::OffsetDateTime; +use trustify_entity::exploit_intelligence_job::ExploitIntelligenceJobStatus; +use utoipa::ToSchema; +use uuid::Uuid; + +/// Request body for triggering an Exploit Intelligence analysis. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct AnalyzeRequest { + /// The SBOM identifier to analyze. + pub sbom_id: Uuid, + /// The vulnerability identifier (e.g., "CVE-2024-9680"). + pub vulnerability_id: String, +} + +/// Response returned when an analysis job is successfully created. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct AnalyzeResponse { + /// The unique job identifier. + pub job_id: Uuid, + /// Current lifecycle status of the job. + pub status: JobStatus, +} + +/// Summary of an Exploit Intelligence analysis job. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ExploitIntelligenceJobSummary { + /// The unique job identifier. + pub id: Uuid, + /// FK to the SBOM being analysed. + pub sbom_id: Option, + /// CVE identifier. + pub vulnerability_id: String, + /// Current lifecycle status of the job. + pub status: JobStatus, + /// Timestamp when the job was created. + #[schema(value_type = String)] + pub created: OffsetDateTime, + /// Timestamp when the job was last updated. + #[schema(value_type = String)] + pub updated: OffsetDateTime, +} + +/// Detailed view of an Exploit Intelligence analysis job. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ExploitIntelligenceJobDetails { + /// The unique job identifier. + pub id: Uuid, + /// FK to the SBOM being analysed. + pub sbom_id: Option, + /// CVE identifier. + pub vulnerability_id: String, + /// Current lifecycle status of the job. + pub status: JobStatus, + /// Analysis result from Exploit Intelligence. + pub finding: Option, + /// Link to the EI human-readable justification report. + pub report_url: Option, + /// FK to advisory — set when a VEX document is ingested from the result. + pub advisory_id: Option, + /// Error details for failed analyses. + pub error_message: Option, + /// Timestamp when the job was created. + #[schema(value_type = String)] + pub created: OffsetDateTime, + /// Timestamp when the job was last updated. + #[schema(value_type = String)] + pub updated: OffsetDateTime, +} + +/// Lifecycle status of an Exploit Intelligence analysis job (API representation). +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum JobStatus { + /// Job has been created but not yet started. + Pending, + /// Analysis is in progress. + Running, + /// Analysis completed successfully. + Completed, + /// Analysis failed with an error. + Failed, +} + +impl From for JobStatus { + fn from(status: ExploitIntelligenceJobStatus) -> Self { + match status { + ExploitIntelligenceJobStatus::Pending => Self::Pending, + ExploitIntelligenceJobStatus::Running => Self::Running, + ExploitIntelligenceJobStatus::Completed => Self::Completed, + ExploitIntelligenceJobStatus::Failed => Self::Failed, + } + } +} + +impl From for ExploitIntelligenceJobStatus { + fn from(status: JobStatus) -> Self { + match status { + JobStatus::Pending => Self::Pending, + JobStatus::Running => Self::Running, + JobStatus::Completed => Self::Completed, + JobStatus::Failed => Self::Failed, + } + } +} + +impl ExploitIntelligenceJobSummary { + /// Create a summary from an entity model. + pub fn from_entity(model: &trustify_entity::exploit_intelligence_job::Model) -> Self { + Self { + id: model.id, + sbom_id: model.sbom_id, + vulnerability_id: model.vulnerability_id.clone(), + status: model.status.into(), + created: model.created, + updated: model.updated, + } + } +} + +impl ExploitIntelligenceJobDetails { + /// Create a detailed view from an entity model. + pub fn from_entity(model: &trustify_entity::exploit_intelligence_job::Model) -> Self { + Self { + id: model.id, + sbom_id: model.sbom_id, + vulnerability_id: model.vulnerability_id.clone(), + status: model.status.into(), + finding: model.finding.map(|f| match f { + trustify_entity::exploit_intelligence_job::Finding::Vulnerable => { + "vulnerable".to_string() + } + trustify_entity::exploit_intelligence_job::Finding::NotVulnerable => { + "not_vulnerable".to_string() + } + trustify_entity::exploit_intelligence_job::Finding::Uncertain => { + "uncertain".to_string() + } + }), + report_url: model.report_url.clone(), + advisory_id: model.advisory_id, + error_message: model.error_message.clone(), + created: model.created, + updated: model.updated, + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs new file mode 100644 index 000000000..8ba3ec60e --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -0,0 +1,228 @@ +use crate::{ + Error, + exploit_intelligence::model::{ + ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, + }, +}; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, + QueryOrder, QuerySelect, Set, TransactionTrait, +}; +use time::OffsetDateTime; +use tracing::{Instrument, info_span, instrument}; +use trustify_common::{ + db, + model::{Paginated, PaginatedResults}, +}; +use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus}; +use uuid::Uuid; + +/// Configuration for the Exploit Intelligence client connection. +#[derive(Clone, Debug)] +pub struct ExploitIntelligenceConfig { + /// Base URL of the Exploit Intelligence client service. + pub url: String, + /// Polling interval in seconds for checking analysis completion. + pub poll_interval_secs: u64, + /// Optional authentication token for the EI service. + pub auth_token: Option, +} + +/// Service for orchestrating Exploit Intelligence analysis jobs. +#[derive(Clone)] +pub struct ExploitIntelligenceService { + config: Option, +} + +impl ExploitIntelligenceService { + /// Create a new service instance with optional configuration. + /// + /// When `config` is `None`, the service will return `Error::Unavailable` + /// for all analysis operations. + pub fn new(config: Option) -> Self { + Self { config } + } + + /// Create an analysis job record in the database. + /// + /// Returns the created job details. The caller is responsible for + /// spawning the background task that submits the request and polls for + /// results. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn create_job( + &self, + sbom_id: Uuid, + vulnerability_id: &str, + db: &db::ReadWrite, + ) -> Result { + let _config = self.config.as_ref().ok_or(Error::Unavailable)?; + + let now = OffsetDateTime::now_utc(); + let job_id = Uuid::now_v7(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + scan_id: Set(String::new()), + sbom_id: Set(Some(sbom_id)), + vulnerability_id: Set(vulnerability_id.to_string()), + status: Set(ExploitIntelligenceJobStatus::Pending), + source_url: Set(None), + report_url: Set(None), + advisory_id: Set(None), + finding: Set(None), + error_message: Set(None), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await?; + let result = model.insert(&tx).await.map_err(Error::from)?; + tx.commit().await?; + + Ok(ExploitIntelligenceJobDetails::from_entity(&result)) + } + + /// Fetch details of a single job by its ID. + #[instrument(skip(self, connection), err(level = tracing::Level::INFO))] + pub async fn fetch_job( + &self, + id: Uuid, + connection: &C, + ) -> Result, Error> { + let result = exploit_intelligence_job::Entity::find_by_id(id) + .one(connection) + .instrument(info_span!("fetch exploit intelligence job")) + .await + .map_err(Error::from)?; + + Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m))) + } + + /// Fetch a paginated list of jobs, optionally filtered by SBOM ID and status. + #[instrument(skip(self, connection), err(level = tracing::Level::INFO))] + pub async fn fetch_jobs( + &self, + paginated: Paginated, + sbom_id: Option, + status: Option, + connection: &C, + ) -> Result, Error> { + let mut query = exploit_intelligence_job::Entity::find(); + + if let Some(sbom_id) = sbom_id { + query = query.filter(exploit_intelligence_job::Column::SbomId.eq(sbom_id)); + } + + if let Some(status) = status { + let db_status: ExploitIntelligenceJobStatus = status.into(); + query = query.filter(exploit_intelligence_job::Column::Status.eq(db_status)); + } + + let total = query + .clone() + .count(connection) + .instrument(info_span!("count exploit intelligence jobs")) + .await + .map_err(Error::from)?; + + let results = query + .order_by_desc(exploit_intelligence_job::Column::Created) + .offset(paginated.offset) + .limit(paginated.limit) + .all(connection) + .instrument(info_span!("fetch exploit intelligence jobs")) + .await + .map_err(Error::from)?; + + let items = results + .iter() + .map(ExploitIntelligenceJobSummary::from_entity) + .collect(); + + Ok(PaginatedResults { + items, + total: Some(total), + }) + } + + /// Update a job's status to running and store the scan ID from the EI service. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_job_running( + &self, + job_id: Uuid, + scan_id: &str, + db: &db::ReadWrite, + ) -> Result<(), Error> { + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + scan_id: Set(scan_id.to_string()), + status: Set(ExploitIntelligenceJobStatus::Running), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Update a job to completed with the analysis finding and optional advisory reference. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_job_completed( + &self, + job_id: Uuid, + finding: exploit_intelligence_job::Finding, + report_url: Option, + advisory_id: Option, + db: &db::ReadWrite, + ) -> Result<(), Error> { + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + status: Set(ExploitIntelligenceJobStatus::Completed), + finding: Set(Some(finding)), + report_url: Set(report_url), + advisory_id: Set(advisory_id), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Update a job to failed with an error message. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_job_failed( + &self, + job_id: Uuid, + error_message: &str, + db: &db::ReadWrite, + ) -> Result<(), Error> { + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + status: Set(ExploitIntelligenceJobStatus::Failed), + error_message: Set(Some(error_message.to_string())), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Returns the EI configuration, if available. + pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { + self.config.as_ref() + } +} diff --git a/modules/fundamental/src/lib.rs b/modules/fundamental/src/lib.rs index c8b3c183e..e6bd84cc3 100644 --- a/modules/fundamental/src/lib.rs +++ b/modules/fundamental/src/lib.rs @@ -4,6 +4,7 @@ pub mod advisory; pub mod common; pub mod endpoints; pub mod error; +pub mod exploit_intelligence; pub mod license; pub mod organization; pub mod product; diff --git a/modules/fundamental/src/test/common.rs b/modules/fundamental/src/test/common.rs index 9b56aae55..7ac67044a 100644 --- a/modules/fundamental/src/test/common.rs +++ b/modules/fundamental/src/test/common.rs @@ -27,6 +27,7 @@ pub async fn caller_with( ctx.storage.clone(), analysis.clone(), cache, + None, ); trustify_module_analysis::endpoints::configure(svc, db_ro, analysis); }) diff --git a/openapi.yaml b/openapi.yaml index 653e46cfe..a1dc8ca16 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1033,6 +1033,115 @@ paths: description: Uploaded the dataset '400': description: The file could not be parsed as an dataset + /api/v3/exploit-intelligence/analyze: + post: + tags: + - exploit-intelligence + summary: Trigger an Exploit Intelligence analysis for a CVE against an SBOM. + description: |- + Creates a job record and spawns a background task that submits the SBOM + to the EI client service, polls for completion, and ingests any returned + VEX document. + operationId: analyzeExploitIntelligence + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AnalyzeRequest' + required: true + responses: + '201': + description: Analysis job created + content: + application/json: + schema: + $ref: '#/components/schemas/AnalyzeResponse' + '404': + description: SBOM not found + '503': + description: Exploit Intelligence service not configured + /api/v3/exploit-intelligence/jobs: + get: + tags: + - exploit-intelligence + summary: List Exploit Intelligence analysis jobs. + operationId: listExploitIntelligenceJobs + parameters: + - name: offset + in: query + description: |- + The first item to return, skipping all that come before it. + + NOTE: The order of items is defined by the API being called. + required: false + schema: + type: integer + format: int64 + minimum: 0 + - name: limit + in: query + description: |- + The maximum number of entries to return. + + Zero means: return no items (the total count is still computed if requested). + required: false + schema: + type: integer + format: int64 + minimum: 0 + - name: total + in: query + description: Whether to compute and return the total count of matching items. + required: false + schema: + type: boolean + - name: sbom_id + in: query + description: Filter by SBOM ID. + required: false + schema: + type: + - string + - 'null' + format: uuid + - name: status + in: query + description: Filter by job status. + required: false + schema: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/JobStatus' + responses: + '200': + description: List of analysis jobs + content: + application/json: + schema: + $ref: '#/components/schemas/PaginatedResults_ExploitIntelligenceJobSummary' + /api/v3/exploit-intelligence/jobs/{id}: + get: + tags: + - exploit-intelligence + summary: Get details of a specific Exploit Intelligence analysis job. + operationId: getExploitIntelligenceJob + parameters: + - name: id + in: path + description: Job identifier + required: true + schema: + type: string + format: uuid + responses: + '200': + description: Job details + content: + application/json: + schema: + $ref: '#/components/schemas/ExploitIntelligenceJobDetails' + '404': + description: Job not found /api/v3/group/sbom: get: tags: @@ -4340,6 +4449,34 @@ components: cache: $ref: '#/components/schemas/CacheStatusDetails' description: Details about the cache + AnalyzeRequest: + type: object + description: Request body for triggering an Exploit Intelligence analysis. + required: + - sbom_id + - vulnerability_id + properties: + sbom_id: + type: string + format: uuid + description: The SBOM identifier to analyze. + vulnerability_id: + type: string + description: The vulnerability identifier (e.g., "CVE-2024-9680"). + AnalyzeResponse: + type: object + description: Response returned when an analysis job is successfully created. + required: + - job_id + - status + properties: + job_id: + type: string + format: uuid + description: The unique job identifier. + status: + $ref: '#/components/schemas/JobStatus' + description: Current lifecycle status of the job. BasePurlDetails: allOf: - $ref: '#/components/schemas/BasePurlHead' @@ -4636,6 +4773,91 @@ components: message: type: string description: A human-readable error message + ExploitIntelligenceJobDetails: + type: object + description: Detailed view of an Exploit Intelligence analysis job. + required: + - id + - vulnerability_id + - status + - created + - updated + properties: + advisory_id: + type: + - string + - 'null' + format: uuid + description: FK to advisory — set when a VEX document is ingested from the result. + created: + type: string + description: Timestamp when the job was created. + error_message: + type: + - string + - 'null' + description: Error details for failed analyses. + finding: + type: + - string + - 'null' + description: 'Analysis result: "vulnerable", "not_vulnerable", or "uncertain".' + id: + type: string + format: uuid + description: The unique job identifier. + report_url: + type: + - string + - 'null' + description: Link to the EI human-readable justification report. + sbom_id: + type: + - string + - 'null' + format: uuid + description: FK to the SBOM being analysed. + status: + $ref: '#/components/schemas/JobStatus' + description: Current lifecycle status of the job. + updated: + type: string + description: Timestamp when the job was last updated. + vulnerability_id: + type: string + description: CVE identifier. + ExploitIntelligenceJobSummary: + type: object + description: Summary of an Exploit Intelligence analysis job. + required: + - id + - vulnerability_id + - status + - created + - updated + properties: + created: + type: string + description: Timestamp when the job was created. + id: + type: string + format: uuid + description: The unique job identifier. + sbom_id: + type: + - string + - 'null' + format: uuid + description: FK to the SBOM being analysed. + status: + $ref: '#/components/schemas/JobStatus' + description: Current lifecycle status of the job. + updated: + type: string + description: Timestamp when the job was last updated. + vulnerability_id: + type: string + description: CVE identifier. ExternalReferenceQuery: type: object properties: @@ -4911,6 +5133,14 @@ components: items: type: string description: Warnings that occurred during the import process + JobStatus: + type: string + description: Lifecycle status of an Exploit Intelligence analysis job (API representation). + enum: + - pending + - running + - completed + - failed Labels: type: object additionalProperties: @@ -5132,6 +5362,51 @@ components: - 'null' format: int64 minimum: 0 + PaginatedResults_ExploitIntelligenceJobSummary: + type: object + required: + - items + properties: + items: + type: array + items: + type: object + description: Summary of an Exploit Intelligence analysis job. + required: + - id + - vulnerability_id + - status + - created + - updated + properties: + created: + type: string + description: Timestamp when the job was created. + id: + type: string + format: uuid + description: The unique job identifier. + sbom_id: + type: + - string + - 'null' + format: uuid + description: FK to the SBOM being analysed. + status: + $ref: '#/components/schemas/JobStatus' + description: Current lifecycle status of the job. + updated: + type: string + description: Timestamp when the job was last updated. + vulnerability_id: + type: string + description: CVE identifier. + total: + type: + - integer + - 'null' + format: int64 + minimum: 0 PaginatedResults_GroupDetails: type: object required: diff --git a/server/src/openapi.rs b/server/src/openapi.rs index 4e8bf4348..43b292182 100644 --- a/server/src/openapi.rs +++ b/server/src/openapi.rs @@ -26,6 +26,7 @@ pub async fn create_openapi() -> anyhow::Result { auth: None, analysis, read_only: false, + ei_config: None, }, ); }) diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 7758159c9..6136bf113 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -92,6 +92,22 @@ pub struct Run { )] pub scan_limit: BinaryByteSize, + /// Base URL of the Exploit Intelligence client service. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_URL")] + pub exploit_intelligence_url: Option, + + /// Polling interval in seconds for EI analysis completion (default: 30). + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS", + default_value_t = 30 + )] + pub exploit_intelligence_poll_interval_secs: u64, + + /// Authentication token for the Exploit Intelligence service. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_AUTH_TOKEN")] + pub exploit_intelligence_auth_token: Option, + // flattened commands must go last // /// Analysis configuration @@ -194,6 +210,9 @@ struct InitData { config: ModuleConfig, analysis: AnalysisService, read_only: bool, + ei_config: Option< + trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig, + >, } /// Groups all module configurations. @@ -298,6 +317,14 @@ impl InitData { }, }; + let ei_config = run.exploit_intelligence_url.map(|url| { + trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { + url, + poll_interval_secs: run.exploit_intelligence_poll_interval_secs, + auth_token: run.exploit_intelligence_auth_token, + } + }); + Ok(InitData { analysis: AnalysisService::new(run.analysis, db_ro.clone()), authenticator, @@ -315,6 +342,7 @@ impl InitData { embedded_oidc, ui, read_only: run.read_only, + ei_config, }) } @@ -341,6 +369,7 @@ impl InitData { auth: self.authenticator.clone(), analysis: self.analysis.clone(), read_only: self.read_only, + ei_config: self.ei_config.clone(), }, ); }) @@ -391,6 +420,9 @@ pub(crate) struct Config { pub(crate) analysis: AnalysisService, pub(crate) auth: Option>, pub(crate) read_only: bool, + pub(crate) ei_config: Option< + trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig, + >, } pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfig, config: Config) { @@ -408,6 +440,7 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi auth, analysis, read_only, + ei_config, } = config; let graph = Graph::new(); @@ -441,6 +474,7 @@ pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfi storage, analysis.clone(), cache, + ei_config.clone(), ); trustify_module_analysis::endpoints::configure(svc, db_ro.clone(), analysis); trustify_module_user::endpoints::configure(svc); @@ -521,6 +555,7 @@ mod test { auth: None, analysis, read_only: false, + ei_config: None, }, ); }) @@ -594,6 +629,7 @@ mod test { auth: None, analysis, read_only, + ei_config: None, }, ); }) From 67197b29d870e570e512b167a46f6c76fd7fda9a Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 12 Jun 2026 13:45:02 +0100 Subject: [PATCH 02/25] fix(exploit-intelligence): add polling timeout, job deduplication, and integration tests - Add max_poll_duration_secs field to ExploitIntelligenceConfig (default 1800s) with EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS env var - Add polling timeout logic in run_analysis: calculates max iterations from config and breaks with Failed status when exceeded - Add find_active_job method for deduplication: queries for Pending/Running jobs matching the same sbom_id + vulnerability_id - In analyze endpoint, check for active jobs before creating duplicates - Add three integration tests: deduplication, timeout failure, finding-only flow - Document new env var in docs/env-vars.md Co-Authored-By: Claude Opus 4.6 --- docs/env-vars.md | 1 + .../src/exploit_intelligence/endpoints/mod.rs | 28 +++ .../exploit_intelligence/endpoints/test.rs | 176 +++++++++++++++++- .../src/exploit_intelligence/service/mod.rs | 33 +++- server/src/profile/api.rs | 9 + 5 files changed, 242 insertions(+), 5 deletions(-) diff --git a/docs/env-vars.md b/docs/env-vars.md index a28512bf1..10715a17f 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -64,6 +64,7 @@ | `UI_SCOPE` | Scopes to request | `openid` | | `EXPLOIT_INTELLIGENCE_URL` | Base URL of the Exploit Intelligence client service | | | `EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS`| Polling interval in seconds for EI analysis completion | `30` | +| `EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS`| Maximum duration in seconds before EI polling is considered timed out | `1800` | | `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Authentication token for the Exploit Intelligence service | | ## Data Migration diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 5eb0db0b9..dbf331d47 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -87,6 +87,19 @@ pub async fn analyze( if sbom.is_none() { return Ok(HttpResponse::NotFound().finish()); } + + // Check for an existing active job (deduplication). + if let Some(active_job) = ei_service + .find_active_job(body.sbom_id, &body.vulnerability_id, &tx) + .await? + { + drop(tx); + return Ok(HttpResponse::Created().json(AnalyzeResponse { + job_id: active_job.id, + status: active_job.status, + })); + } + drop(tx); // Create the job record. @@ -277,10 +290,25 @@ async fn run_analysis( // Poll for completion. let poll_url = format!("{}/api/v1/reports/by-scan-id/{}", config.url, scan_id); let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); + let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); + let mut poll_count: u64 = 0; loop { tokio::time::sleep(poll_interval).await; + poll_count += 1; + if poll_count > max_polls { + let msg = format!( + "analysis timed out after {}s", + config.max_poll_duration_secs + ); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(anyhow::anyhow!("{msg}")); + } + let mut poll_request = client.get(&poll_url); if let Some(token) = &config.auth_token { poll_request = poll_request.bearer_auth(token); diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 82df0936e..e78825aec 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -1,5 +1,5 @@ use crate::exploit_intelligence::{ - model::{ExploitIntelligenceJobDetails, JobStatus}, + model::{AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, JobStatus}, service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, }; use actix_web::{ @@ -13,7 +13,7 @@ use test_log::test; use time::OffsetDateTime; use trustify_common::{db, model::PaginatedResults}; use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus}; -use trustify_test_context::TrustifyContext; +use trustify_test_context::{TrustifyContext, app::TestApp}; use uuid::Uuid; /// Helper to create a test service with a fake EI backend URL. @@ -21,6 +21,7 @@ fn test_service() -> ExploitIntelligenceService { ExploitIntelligenceService::new(Some(ExploitIntelligenceConfig { url: "http://localhost:9999".to_string(), poll_interval_secs: 1, + max_poll_duration_secs: 1800, auth_token: None, })) } @@ -38,7 +39,7 @@ async fn insert_test_job( let now = OffsetDateTime::now_utc(); let model = exploit_intelligence_job::ActiveModel { id: Set(id), - scan_id: Set(format!("scan-{id}")), + scan_id: Set(Some(format!("scan-{id}"))), sbom_id: Set(sbom_id), vulnerability_id: Set(vulnerability_id.to_string()), status: Set(status), @@ -327,3 +328,172 @@ async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<() Ok(()) } + +/// Verifies job deduplication: submitting an analyze request for an SBOM+CVE +/// that already has an active (Pending or Running) job returns the existing +/// job instead of creating a duplicate. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a real SBOM in the database + let ingest_result = ctx + .ingest_document("cyclonedx/application.cdx.json") + .await?; + let sbom_id = Uuid::parse_str( + ingest_result + .id + .strip_prefix("urn:uuid:") + .expect("sbom id should have urn:uuid: prefix"), + )?; + + // And an existing pending job for that SBOM and CVE + let existing_job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + existing_job_id, + Some(sbom_id), + "CVE-2024-DEDUP", + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + + let ei_service = test_service(); + let db_rw = db::ReadWrite::new(ctx.db.clone()); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_rw)) + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .app_data(web::Data::new(ctx.ingestor.clone())) + .service(super::analyze), + ) + .await; + + // When submitting an analyze request for the same SBOM + CVE + let req = TestRequest::post() + .uri("/v3/exploit-intelligence/analyze") + .set_json(AnalyzeRequest { + sbom_id, + vulnerability_id: "CVE-2024-DEDUP".to_string(), + }) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response returns the existing job (not a new one) + assert_eq!(resp.status(), 201); + let body: AnalyzeResponse = test::read_body_json(resp).await; + assert_eq!(body.job_id, existing_job_id); + assert_eq!(body.status, JobStatus::Pending); + + Ok(()) +} + +/// Verifies that a job updated with a timeout error message is correctly +/// reflected when fetched via the GET endpoint. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a running job record in the database + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-TIMEOUT", + ExploitIntelligenceJobStatus::Running, + None, + None, + ) + .await; + + // When the service marks it as failed with a timeout error + let ei_service = test_service(); + let db_rw = db::ReadWrite::new(ctx.db.clone()); + ei_service + .update_job_failed(job_id, "analysis timed out after 1800s", &db_rw) + .await?; + + // Then fetching the job via GET shows the failed status and timeout message + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Failed); + assert_eq!( + body.error_message.as_deref(), + Some("analysis timed out after 1800s") + ); + assert_eq!(body.finding, None); + + Ok(()) +} + +/// Verifies that a completed job with a finding but no advisory link is +/// returned correctly -- simulating a "vulnerable" result where no VEX +/// document was generated by the EI service. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a completed job with a "vulnerable" finding but no advisory + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-VULN", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::Vulnerable), + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When fetching the job details + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response shows the finding without an advisory link + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Completed); + assert_eq!(body.finding.as_deref(), Some("vulnerable")); + assert_eq!(body.advisory_id, None); + assert_eq!(body.error_message, None); + + Ok(()) +} diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 8ba3ec60e..22c66490f 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -24,6 +24,8 @@ pub struct ExploitIntelligenceConfig { pub url: String, /// Polling interval in seconds for checking analysis completion. pub poll_interval_secs: u64, + /// Maximum duration in seconds before a polling loop is considered timed out. + pub max_poll_duration_secs: u64, /// Optional authentication token for the EI service. pub auth_token: Option, } @@ -62,7 +64,7 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), - scan_id: Set(String::new()), + scan_id: Set(None), sbom_id: Set(Some(sbom_id)), vulnerability_id: Set(vulnerability_id.to_string()), status: Set(ExploitIntelligenceJobStatus::Pending), @@ -158,7 +160,7 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), - scan_id: Set(scan_id.to_string()), + scan_id: Set(Some(scan_id.to_string())), status: Set(ExploitIntelligenceJobStatus::Running), updated: Set(now), ..Default::default() @@ -221,6 +223,33 @@ impl ExploitIntelligenceService { Ok(()) } + /// Find an existing job in Pending or Running state for the given SBOM and vulnerability. + /// + /// Used for deduplication: if an active job already exists for the same + /// (sbom_id, vulnerability_id) pair, the caller can return it instead of + /// creating a duplicate. + #[instrument(skip(self, connection), err(level = tracing::Level::INFO))] + pub async fn find_active_job( + &self, + sbom_id: Uuid, + vulnerability_id: &str, + connection: &C, + ) -> Result, Error> { + let result = exploit_intelligence_job::Entity::find() + .filter(exploit_intelligence_job::Column::SbomId.eq(sbom_id)) + .filter(exploit_intelligence_job::Column::VulnerabilityId.eq(vulnerability_id)) + .filter(exploit_intelligence_job::Column::Status.is_in([ + ExploitIntelligenceJobStatus::Pending, + ExploitIntelligenceJobStatus::Running, + ])) + .one(connection) + .instrument(info_span!("find active exploit intelligence job")) + .await + .map_err(Error::from)?; + + Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m))) + } + /// Returns the EI configuration, if available. pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { self.config.as_ref() diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 6136bf113..25085051b 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -104,6 +104,14 @@ pub struct Run { )] pub exploit_intelligence_poll_interval_secs: u64, + /// Maximum duration in seconds before EI polling is considered timed out (default: 1800 = 30 minutes). + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS", + default_value_t = 1800 + )] + pub exploit_intelligence_max_poll_duration_secs: u64, + /// Authentication token for the Exploit Intelligence service. #[arg(long, env = "EXPLOIT_INTELLIGENCE_AUTH_TOKEN")] pub exploit_intelligence_auth_token: Option, @@ -321,6 +329,7 @@ impl InitData { trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { url, poll_interval_secs: run.exploit_intelligence_poll_interval_secs, + max_poll_duration_secs: run.exploit_intelligence_max_poll_duration_secs, auth_token: run.exploit_intelligence_auth_token, } }); From 409e0edb5b2b94b4820545bb322ea2bf3424336e Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:35:20 +0100 Subject: [PATCH 03/25] fix(exploit-intelligence): expose finding as typed enum in API response The entity layer already has a proper Finding enum (Vulnerable, NotVulnerable, Uncertain), but the API DTOs were converting it to Option. This exposes a JobFinding enum in the API model, matching the pattern used by JobStatus. Co-Authored-By: Claude Opus 4.6 --- .../src/exploit_intelligence/model/mod.rs | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index 8c71f80ba..1e2f206f4 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; use time::OffsetDateTime; -use trustify_entity::exploit_intelligence_job::ExploitIntelligenceJobStatus; +use trustify_entity::exploit_intelligence_job::{ExploitIntelligenceJobStatus, Finding}; use utoipa::ToSchema; use uuid::Uuid; @@ -33,6 +33,10 @@ pub struct ExploitIntelligenceJobSummary { pub vulnerability_id: String, /// Current lifecycle status of the job. pub status: JobStatus, + /// Analysis result from Exploit Intelligence. + pub finding: Option, + /// Link to the EI human-readable justification report. + pub report_url: Option, /// Timestamp when the job was created. #[schema(value_type = String)] pub created: OffsetDateTime, @@ -53,7 +57,7 @@ pub struct ExploitIntelligenceJobDetails { /// Current lifecycle status of the job. pub status: JobStatus, /// Analysis result from Exploit Intelligence. - pub finding: Option, + pub finding: Option, /// Link to the EI human-readable justification report. pub report_url: Option, /// FK to advisory — set when a VEX document is ingested from the result. @@ -93,6 +97,28 @@ impl From for JobStatus { } } +/// Analysis finding from Exploit Intelligence (API representation). +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum JobFinding { + /// The analysed component is vulnerable to the CVE. + Vulnerable, + /// The analysed component is not vulnerable to the CVE. + NotVulnerable, + /// The analysis was inconclusive. + Uncertain, +} + +impl From for JobFinding { + fn from(finding: Finding) -> Self { + match finding { + Finding::Vulnerable => Self::Vulnerable, + Finding::NotVulnerable => Self::NotVulnerable, + Finding::Uncertain => Self::Uncertain, + } + } +} + impl From for ExploitIntelligenceJobStatus { fn from(status: JobStatus) -> Self { match status { @@ -112,6 +138,8 @@ impl ExploitIntelligenceJobSummary { sbom_id: model.sbom_id, vulnerability_id: model.vulnerability_id.clone(), status: model.status.into(), + finding: model.finding.map(JobFinding::from), + report_url: model.report_url.clone(), created: model.created, updated: model.updated, } @@ -126,17 +154,7 @@ impl ExploitIntelligenceJobDetails { sbom_id: model.sbom_id, vulnerability_id: model.vulnerability_id.clone(), status: model.status.into(), - finding: model.finding.map(|f| match f { - trustify_entity::exploit_intelligence_job::Finding::Vulnerable => { - "vulnerable".to_string() - } - trustify_entity::exploit_intelligence_job::Finding::NotVulnerable => { - "not_vulnerable".to_string() - } - trustify_entity::exploit_intelligence_job::Finding::Uncertain => { - "uncertain".to_string() - } - }), + finding: model.finding.map(JobFinding::from), report_url: model.report_url.clone(), advisory_id: model.advisory_id, error_message: model.error_message.clone(), From 256cd62bc1354076a4af0c36b41adce382117122 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:37:08 +0100 Subject: [PATCH 04/25] fix(exploit-intelligence): replace fragile JSON pointer parsing with typed structs The EI response was parsed via serde_json::Value with hardcoded JSON Pointer paths, making the code silently break on any upstream schema change. Introduces typed Rust structs (UploadResponse, PollResponse, etc.) so contract mismatches surface as deserialization errors. Co-Authored-By: Claude Opus 4.6 --- .../src/exploit_intelligence/ei_client.rs | 44 +++++++++++++++++++ .../src/exploit_intelligence/endpoints/mod.rs | 44 ++++++++++--------- .../src/exploit_intelligence/mod.rs | 1 + 3 files changed, 69 insertions(+), 20 deletions(-) create mode 100644 modules/fundamental/src/exploit_intelligence/ei_client.rs diff --git a/modules/fundamental/src/exploit_intelligence/ei_client.rs b/modules/fundamental/src/exploit_intelligence/ei_client.rs new file mode 100644 index 000000000..7ca1f1e06 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/ei_client.rs @@ -0,0 +1,44 @@ +use serde::Deserialize; + +/// Response from POST /api/v1/products/upload-cyclonedx +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct UploadResponse { + pub report_request_id: ReportRequestId, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReportRequestId { + pub report_id: String, +} + +/// Response from GET /api/v1/reports/by-scan-id/{scan_id} +#[derive(Debug, Deserialize)] +pub struct PollResponse { + pub status: String, + pub report: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Report { + pub output: Option, + pub error: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ReportOutput { + pub analysis: Option>, + /// VEX kept as `serde_json::Value` since it is forwarded directly to the ingestor. + pub vex: Option, +} + +#[derive(Debug, Deserialize)] +pub struct AnalysisEntry { + pub justification: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Justification { + pub status: Option, +} diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index dbf331d47..c1212a68f 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -274,12 +274,8 @@ async fn run_analysis( return Err(anyhow::anyhow!("EI client returned {status}: {body}")); } - let report_data: serde_json::Value = response.json().await?; - let scan_id = report_data - .pointer("/reportRequestId/reportId") - .and_then(|v| v.as_str()) - .ok_or_else(|| anyhow::anyhow!("missing scanId in EI response"))? - .to_string(); + let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; + let scan_id = upload.report_request_id.report_id; // Update job to running with scan ID. ei_service @@ -320,19 +316,21 @@ async fn run_analysis( continue; } - let report: serde_json::Value = poll_response.json().await?; - let status = report - .get("status") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); + let poll: crate::exploit_intelligence::ei_client::PollResponse = + poll_response.json().await?; - match status { + match poll.status.as_str() { "completed" => { // Extract finding from the analysis result. use trustify_entity::exploit_intelligence_job::Finding; - let finding = report - .pointer("/report/output/analysis/0/justification/status") - .and_then(|v| v.as_str()) + let finding = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.analysis.as_ref()) + .and_then(|a| a.first()) + .and_then(|e| e.justification.as_ref()) + .and_then(|j| j.status.as_deref()) .map(|s| match s { "false" => Finding::NotVulnerable, "true" => Finding::Vulnerable, @@ -346,7 +344,12 @@ async fn run_analysis( ); // Check if VEX document is available. - let vex_json = report.pointer("/report/output/vex"); + let vex_json = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + let advisory_id = if let Some(vex) = vex_json { if !vex.is_null() { // Ingest the VEX document as an advisory with source label. @@ -391,9 +394,10 @@ async fn run_analysis( return Ok(()); } "failed" | "expired" => { - let error_msg = report - .pointer("/report/error") - .and_then(|v| v.as_str()) + let error_msg = poll + .report + .as_ref() + .and_then(|r| r.error.as_deref()) .unwrap_or("analysis failed") .to_string(); @@ -408,7 +412,7 @@ async fn run_analysis( // Still in progress (queued, sent, pending), continue polling. tracing::debug!( job_id = %job_id, - status = status, + status = poll.status, "exploit intelligence analysis still in progress" ); } diff --git a/modules/fundamental/src/exploit_intelligence/mod.rs b/modules/fundamental/src/exploit_intelligence/mod.rs index 9560fd556..e4185bd74 100644 --- a/modules/fundamental/src/exploit_intelligence/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/mod.rs @@ -1,3 +1,4 @@ +pub mod ei_client; pub(crate) mod endpoints; pub mod model; From 295ed3cd9e420a7ccecf4767585cc83b9fbf3982 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:38:49 +0100 Subject: [PATCH 05/25] fix(exploit-intelligence): configure HTTP client with timeouts and connection pooling reqwest::Client was created bare with no timeouts or pooling config, and a new instance was constructed per background task. Now a shared client is built once with connect_timeout(10s), request timeout(60s), pool_max_idle_per_host(4), and pool_idle_timeout(90s). Co-Authored-By: Claude Opus 4.6 --- .../src/exploit_intelligence/endpoints/mod.rs | 7 +++++- .../src/exploit_intelligence/service/mod.rs | 24 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index c1212a68f..5ff7f8404 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -75,6 +75,10 @@ pub async fn analyze( _: Require, ) -> actix_web::Result { let config = ei_service.config().cloned().ok_or(Error::Unavailable)?; + let client = ei_service + .http_client() + .cloned() + .ok_or(Error::Unavailable)?; // Verify the SBOM exists. let tx = db_ro.begin().await?; @@ -122,6 +126,7 @@ pub async fn analyze( &ingestor_svc, &db_rw_clone, &config, + &client, job_id, sbom_id, &vulnerability_id, @@ -202,6 +207,7 @@ async fn run_analysis( ingestor: &IngestorService, db_rw: &db::ReadWrite, config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, job_id: Uuid, sbom_id: Uuid, vulnerability_id: &str, @@ -248,7 +254,6 @@ async fn run_analysis( drop(tx); // Submit to EI client. - let client = reqwest::Client::new(); let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); let form = reqwest::multipart::Form::new() diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 22c66490f..c3a689448 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -8,6 +8,7 @@ use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, }; +use std::time::Duration; use time::OffsetDateTime; use tracing::{Instrument, info_span, instrument}; use trustify_common::{ @@ -34,15 +35,34 @@ pub struct ExploitIntelligenceConfig { #[derive(Clone)] pub struct ExploitIntelligenceService { config: Option, + http_client: Option, } impl ExploitIntelligenceService { /// Create a new service instance with optional configuration. /// /// When `config` is `None`, the service will return `Error::Unavailable` - /// for all analysis operations. + /// for all analysis operations. When configuration is present, a shared + /// HTTP client is built with sensible timeouts and connection pooling. pub fn new(config: Option) -> Self { - Self { config } + let http_client = config.as_ref().map(|_| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) + .pool_max_idle_per_host(4) + .pool_idle_timeout(Duration::from_secs(90)) + .build() + .expect("failed to build HTTP client") + }); + Self { + config, + http_client, + } + } + + /// Returns the shared HTTP client, if configuration is present. + pub fn http_client(&self) -> Option<&reqwest::Client> { + self.http_client.as_ref() } /// Create an analysis job record in the database. From 16c38e10987865a510bc5482980463847340352f Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:40:51 +0100 Subject: [PATCH 06/25] fix(exploit-intelligence): add OIDC client credentials auth with token refresh Replaces the static bearer token with a TokenProvider abstraction supporting both static tokens (backward compatible via EXPLOIT_INTELLIGENCE_AUTH_TOKEN) and OIDC client credentials flow with automatic token refresh before expiry. New env vars: EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL, EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID, EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET. Co-Authored-By: Claude Opus 4.6 --- modules/fundamental/Cargo.toml | 2 +- .../src/exploit_intelligence/auth.rs | 133 ++++++++++++++++++ .../src/exploit_intelligence/endpoints/mod.rs | 12 +- .../exploit_intelligence/endpoints/test.rs | 2 +- .../src/exploit_intelligence/mod.rs | 1 + .../src/exploit_intelligence/service/mod.rs | 4 +- server/src/profile/api.rs | 33 ++++- 7 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 modules/fundamental/src/exploit_intelligence/auth.rs diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index c35777a38..8aa8f451f 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -30,7 +30,7 @@ futures-util = { workspace = true } isx = { workspace = true } itertools = { workspace = true } log = { workspace = true } -reqwest = { workspace = true, features = ["json", "multipart"] } +reqwest = { workspace = true, features = ["form", "json", "multipart"] } sanitize-filename = { workspace = true } sea-orm = { workspace = true } sea-query = { workspace = true } diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs new file mode 100644 index 000000000..328454268 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -0,0 +1,133 @@ +use std::sync::Arc; +use std::time::Duration; +use time::OffsetDateTime; +use tokio::sync::RwLock; + +/// Provides bearer tokens for EI service authentication. +/// Supports static tokens and OIDC client credentials flow. +#[derive(Clone)] +pub enum TokenProvider { + /// A fixed bearer token (from env var / CLI arg). + Static(String), + /// OIDC client credentials grant with automatic refresh. + Oidc(Arc), +} + +pub struct OidcState { + client: reqwest::Client, + token_url: String, + client_id: String, + client_secret: String, + /// Cached token + expiry + cached: RwLock>, +} + +struct CachedToken { + access_token: String, + expires_at: OffsetDateTime, +} + +#[derive(serde::Deserialize)] +struct TokenResponse { + access_token: String, + expires_in: Option, + // token_type, scope, etc. ignored +} + +impl std::fmt::Debug for TokenProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Static(_) => f.debug_tuple("Static").field(&"[redacted]").finish(), + Self::Oidc(_) => f.debug_tuple("Oidc").field(&"[...]").finish(), + } + } +} + +impl TokenProvider { + /// Create a static token provider. + pub fn static_token(token: String) -> Self { + Self::Static(token) + } + + /// Create an OIDC client credentials provider. + /// + /// Builds an internal HTTP client with sensible timeouts for token + /// exchange requests. + pub fn oidc(token_url: String, client_id: String, client_secret: String) -> Self { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(30)) + .build() + .expect("failed to build OIDC HTTP client"); + Self::Oidc(Arc::new(OidcState { + client, + token_url, + client_id, + client_secret, + cached: RwLock::new(None), + })) + } + + /// Get a valid bearer token string. For OIDC, refreshes automatically if expired. + pub async fn get_token(&self) -> Result { + match self { + Self::Static(token) => Ok(token.clone()), + Self::Oidc(state) => state.get_or_refresh().await, + } + } +} + +impl OidcState { + async fn get_or_refresh(&self) -> Result { + // Check cached token under read lock + { + let cached = self.cached.read().await; + if let Some(ref tok) = *cached { + // Refresh 30s before expiry + if tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) { + return Ok(tok.access_token.clone()); + } + } + } + + // Need refresh -- acquire write lock + let mut cached = self.cached.write().await; + // Double-check after acquiring write lock (another task may have refreshed) + if let Some(ref tok) = *cached { + if tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) { + return Ok(tok.access_token.clone()); + } + } + + let response = self + .client + .post(&self.token_url) + .form(&[ + ("grant_type", "client_credentials"), + ("client_id", &self.client_id), + ("client_secret", &self.client_secret), + ]) + .send() + .await?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "OIDC token request failed ({status}): {body}" + )); + } + + let token_resp: TokenResponse = response.json().await?; + let expires_in = token_resp.expires_in.unwrap_or(300); // default 5 min + let expires_at = OffsetDateTime::now_utc() + Duration::from_secs(expires_in); + + let access_token = token_resp.access_token.clone(); + *cached = Some(CachedToken { + access_token: token_resp.access_token, + expires_at, + }); + + Ok(access_token) + } +} diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 5ff7f8404..b52297e93 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -267,8 +267,10 @@ async fn run_analysis( let mut request = client.post(&upload_url).multipart(form); - if let Some(token) = &config.auth_token { - request = request.bearer_auth(token); + if let Some(ref provider) = config.token_provider { + let token = provider.get_token().await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + request = request.bearer_auth(&token); } let response = request.send().await?; @@ -311,8 +313,10 @@ async fn run_analysis( } let mut poll_request = client.get(&poll_url); - if let Some(token) = &config.auth_token { - poll_request = poll_request.bearer_auth(token); + if let Some(ref provider) = config.token_provider { + let token = provider.get_token().await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + poll_request = poll_request.bearer_auth(&token); } let poll_response = poll_request.send().await?; diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index e78825aec..49978b11b 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -22,7 +22,7 @@ fn test_service() -> ExploitIntelligenceService { url: "http://localhost:9999".to_string(), poll_interval_secs: 1, max_poll_duration_secs: 1800, - auth_token: None, + token_provider: None, })) } diff --git a/modules/fundamental/src/exploit_intelligence/mod.rs b/modules/fundamental/src/exploit_intelligence/mod.rs index e4185bd74..f568f9fa1 100644 --- a/modules/fundamental/src/exploit_intelligence/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/mod.rs @@ -1,3 +1,4 @@ +pub mod auth; pub mod ei_client; pub(crate) mod endpoints; diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index c3a689448..e8ab71be4 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -27,8 +27,8 @@ pub struct ExploitIntelligenceConfig { pub poll_interval_secs: u64, /// Maximum duration in seconds before a polling loop is considered timed out. pub max_poll_duration_secs: u64, - /// Optional authentication token for the EI service. - pub auth_token: Option, + /// Optional token provider for the EI service (static or OIDC). + pub token_provider: Option, } /// Service for orchestrating Exploit Intelligence analysis jobs. diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 25085051b..b1e19e347 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -112,10 +112,22 @@ pub struct Run { )] pub exploit_intelligence_max_poll_duration_secs: u64, - /// Authentication token for the Exploit Intelligence service. + /// Authentication token for the Exploit Intelligence service (static token, for backward compatibility). #[arg(long, env = "EXPLOIT_INTELLIGENCE_AUTH_TOKEN")] pub exploit_intelligence_auth_token: Option, + /// OIDC token endpoint URL for EI service authentication (client credentials flow). + #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL")] + pub exploit_intelligence_oidc_token_url: Option, + + /// OIDC client ID for EI service authentication. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID")] + pub exploit_intelligence_oidc_client_id: Option, + + /// OIDC client secret for EI service authentication. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET")] + pub exploit_intelligence_oidc_client_secret: Option, + // flattened commands must go last // /// Analysis configuration @@ -326,11 +338,28 @@ impl InitData { }; let ei_config = run.exploit_intelligence_url.map(|url| { + use trustify_module_fundamental::exploit_intelligence::auth::TokenProvider; + + // Build token provider: prefer OIDC if all three vars are set, + // fall back to static token, or None. + let token_provider = match ( + run.exploit_intelligence_oidc_token_url, + run.exploit_intelligence_oidc_client_id, + run.exploit_intelligence_oidc_client_secret, + ) { + (Some(token_url), Some(client_id), Some(client_secret)) => { + Some(TokenProvider::oidc(token_url, client_id, client_secret)) + } + _ => run + .exploit_intelligence_auth_token + .map(TokenProvider::static_token), + }; + trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { url, poll_interval_secs: run.exploit_intelligence_poll_interval_secs, max_poll_duration_secs: run.exploit_intelligence_max_poll_duration_secs, - auth_token: run.exploit_intelligence_auth_token, + token_provider, } }); From 1a070e834d6ca23eec0d90c55f4532c6c027ecb5 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:43:54 +0100 Subject: [PATCH 07/25] fix(exploit-intelligence): add retry logic with exponential backoff Upload requests retry up to 3 times with exponential backoff (1s, 2s, 4s) on transient HTTP errors (429, 502, 503, 504) or network failures. Poll loop classifies errors as transient (retryable) or permanent (immediate failure). Up to 5 consecutive transient poll failures are tolerated before the job is marked failed. Co-Authored-By: Claude Opus 4.6 --- .../src/exploit_intelligence/endpoints/mod.rs | 193 +++++++++++++++--- 1 file changed, 168 insertions(+), 25 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index b52297e93..498fad308 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -13,6 +13,7 @@ use crate::{ }; use actix_web::{HttpResponse, Responder, get, post, web}; use futures_util::TryStreamExt; +use reqwest::StatusCode; use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use std::str::FromStr; use trustify_auth::{CreateExploitIntelligence, ReadExploitIntelligence, authorizer::Require}; @@ -25,6 +26,46 @@ use trustify_module_ingestor::service::{Cache, Format, IngestorService}; use trustify_module_storage::service::StorageBackend; use uuid::Uuid; +// --------------------------------------------------------------------------- +// Retry helpers for HTTP requests to the EI service +// --------------------------------------------------------------------------- + +/// Retry categories for HTTP responses. +#[allow(dead_code)] +enum RetryDecision { + /// Success -- return the response. + Success, + /// Transient failure -- retry after backoff. + Transient(String), + /// Permanent failure -- abort immediately. + Permanent(String), +} + +/// Classify an HTTP status code as success, transient, or permanent. +fn classify_status(status: StatusCode) -> RetryDecision { + if status.is_success() { + return RetryDecision::Success; + } + + match status.as_u16() { + // Transient -- worth retrying. + 408 | 429 | 502 | 503 | 504 => { + RetryDecision::Transient(format!("transient HTTP error: {status}")) + } + // Permanent -- no point retrying. + _ => RetryDecision::Permanent(format!("HTTP error: {status}")), + } +} + +/// Returns `true` when a `reqwest::Error` looks transient (connection or +/// timeout problem on the network layer). +fn is_transient_network_error(err: &reqwest::Error) -> bool { + err.is_connect() || err.is_timeout() +} + +/// Maximum number of consecutive transient poll failures before we give up. +const MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES: u32 = 5; + /// Register exploit intelligence endpoints. pub fn configure( config: &mut utoipa_actix_web::service_config::ServiceConfig, @@ -253,34 +294,86 @@ async fn run_analysis( drop(tx); - // Submit to EI client. + // Submit to EI client with retry on transient failures. let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); - let form = reqwest::multipart::Form::new() - .text("cveId", vulnerability_id.to_string()) - .part( - "file", - reqwest::multipart::Part::bytes(sbom_bytes) - .file_name("sbom.json") - .mime_str("application/json")?, - ); - - let mut request = client.post(&upload_url).multipart(form); - - if let Some(ref provider) = config.token_provider { - let token = provider.get_token().await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - request = request.bearer_auth(&token); - } + const UPLOAD_MAX_ATTEMPTS: u32 = 3; + let mut upload_response: Option = None; + + for attempt in 0..UPLOAD_MAX_ATTEMPTS { + // Build a fresh multipart form for each attempt (forms are consumed by send). + let form = reqwest::multipart::Form::new() + .text("cveId", vulnerability_id.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(sbom_bytes.clone()) + .file_name("sbom.json") + .mime_str("application/json")?, + ); - let response = request.send().await?; + let mut request = client.post(&upload_url).multipart(form); - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("EI client returned {status}: {body}")); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + request = request.bearer_auth(&token); + } + + match request.send().await { + Ok(resp) => { + let status = resp.status(); + match classify_status(status) { + RetryDecision::Success => { + upload_response = Some(resp); + break; + } + RetryDecision::Transient(reason) => { + if attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + reason = %reason, + "upload to EI client failed with transient error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" + )); + } + } + RetryDecision::Permanent(_) => { + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!("EI client returned {status}: {body}")); + } + } + } + Err(err) => { + if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + error = %err, + "upload to EI client failed with network error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + return Err(anyhow::anyhow!("EI upload network error: {err}")); + } + } + } } + let response = upload_response + .ok_or_else(|| anyhow::anyhow!("EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; + let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; let scan_id = upload.report_request_id.report_id; @@ -295,6 +388,7 @@ async fn run_analysis( let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); let mut poll_count: u64 = 0; + let mut consecutive_transient_failures: u32 = 0; loop { tokio::time::sleep(poll_interval).await; @@ -314,17 +408,66 @@ async fn run_analysis( let mut poll_request = client.get(&poll_url); if let Some(ref provider) = config.token_provider { - let token = provider.get_token().await + let token = provider + .get_token() + .await .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; poll_request = poll_request.bearer_auth(&token); } - let poll_response = poll_request.send().await?; + let poll_response = match poll_request.send().await { + Ok(resp) => resp, + Err(err) => { + // Network-level error during polling. + tracing::warn!( + job_id = %job_id, + error = %err, + "network error while polling EI service" + ); + consecutive_transient_failures += 1; + if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { + return Err(anyhow::anyhow!( + "EI service unreachable: {consecutive_transient_failures} consecutive poll failures" + )); + } + continue; + } + }; if !poll_response.status().is_success() { - continue; + let status = poll_response.status(); + match classify_status(status) { + RetryDecision::Success => { + // Not reachable (is_success() was false), but satisfy match. + unreachable!(); + } + RetryDecision::Transient(reason) => { + consecutive_transient_failures += 1; + tracing::debug!( + job_id = %job_id, + reason = %reason, + consecutive_failures = consecutive_transient_failures, + "transient error while polling EI service, will retry" + ); + if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { + return Err(anyhow::anyhow!( + "EI service unhealthy: {consecutive_transient_failures} consecutive transient failures ({status})" + )); + } + continue; + } + RetryDecision::Permanent(_) => { + let body = poll_response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI poll returned permanent error {status}: {body}" + )); + } + } } + // Successful response -- reset consecutive failure counter. + consecutive_transient_failures = 0; + let poll: crate::exploit_intelligence::ei_client::PollResponse = poll_response.json().await?; From 2cc075b02ced8f4f8c569ebb5c0079e2eebd2798 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 22 Jun 2026 13:48:51 +0100 Subject: [PATCH 08/25] fix(exploit-intelligence): recover orphaned jobs after server restart After a server restart, EI jobs that were Pending or Running become orphaned because their background tasks no longer exist. - Pending jobs (no scan_id) are marked Failed with a descriptive message telling the user to re-trigger the analysis. - Running jobs (scan_id present) get their poll loops re-spawned so the EI backend result is picked up transparently. Recovery runs inside the EI endpoints `configure()` where the actix runtime is available, using `actix_web::rt::spawn` (not `tokio::spawn`) because IngestorService contains non-Send types. The poll loop is extracted into a standalone `poll_for_result()` function shared by `run_analysis` (normal path) and `recover_and_resume` (restart recovery path). Co-Authored-By: Claude Opus 4.6 --- modules/fundamental/src/endpoints.rs | 10 +- .../src/exploit_intelligence/endpoints/mod.rs | 124 +++++++++++++++++- .../src/exploit_intelligence/service/mod.rs | 100 +++++++++++++- server/src/profile/api.rs | 5 + 4 files changed, 227 insertions(+), 12 deletions(-) diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 23ebf0607..5d7526260 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -27,7 +27,7 @@ pub fn configure( ei_config: Option, ) { let ingestor_service = IngestorService::new(Graph::new(), storage, Some(analysis)); - svc.app_data(web::Data::new(ingestor_service)); + svc.app_data(web::Data::new(ingestor_service.clone())); crate::advisory::endpoints::configure( svc, @@ -58,7 +58,13 @@ pub fn configure( ); let ei_service = ExploitIntelligenceService::new(ei_config); - crate::exploit_intelligence::endpoints::configure(svc, db_rw, db_ro, ei_service); + crate::exploit_intelligence::endpoints::configure( + svc, + db_rw, + db_ro, + ei_service, + ingestor_service, + ); } #[derive(Clone, Debug, PartialEq, Eq, Default, ToSchema, serde::Deserialize, IntoParams)] diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 498fad308..1090c926f 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -66,13 +66,24 @@ fn is_transient_network_error(err: &reqwest::Error) -> bool { /// Maximum number of consecutive transient poll failures before we give up. const MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES: u32 = 5; -/// Register exploit intelligence endpoints. +/// Register exploit intelligence endpoints and spawn recovery for orphaned jobs. pub fn configure( config: &mut utoipa_actix_web::service_config::ServiceConfig, db_rw: db::ReadWrite, db_ro: db::ReadOnly, ei_service: ExploitIntelligenceService, + ingestor: IngestorService, ) { + // Spawn recovery for orphaned jobs from a previous server lifetime. + if ei_service.config().is_some() { + let ei_svc = ei_service.clone(); + let ingestor_svc = ingestor.clone(); + let db_rw_clone = db_rw.clone(); + actix_web::rt::spawn(async move { + recover_and_resume(&ei_svc, &ingestor_svc, &db_rw_clone).await; + }); + } + config .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) @@ -82,6 +93,82 @@ pub fn configure( .service(get_job); } +/// Recover orphaned EI jobs after a server restart. +/// +/// Pending jobs (never submitted) are marked as Failed. Running jobs +/// (scan_id present) get their poll loops re-spawned so the EI backend +/// result is picked up transparently. +async fn recover_and_resume( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, +) { + let result = match ei_service.recover_orphaned_jobs(db_rw).await { + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "failed to recover orphaned exploit intelligence jobs"); + return; + } + }; + + let config = match ei_service.config().cloned() { + Some(c) => c, + None => return, + }; + + let client = match ei_service.http_client().cloned() { + Some(c) => c, + None => return, + }; + + for job in result.running_jobs { + // Running jobs are guaranteed to have a scan_id. + let scan_id = match job.scan_id { + Some(ref s) => s.clone(), + None => { + tracing::error!(job_id = %job.id, "running job has no scan_id, marking failed"); + let _ = ei_service + .update_job_failed( + job.id, + "Running job missing scan_id during recovery", + db_rw, + ) + .await; + continue; + } + }; + + let job_id = job.id; + let vulnerability_id = job.vulnerability_id.clone(); + let ei_svc = ei_service.clone(); + let ingestor_svc = ingestor.clone(); + let db_rw_clone = db_rw.clone(); + let config_clone = config.clone(); + let client_clone = client.clone(); + + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); + if let Err(e) = poll_for_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &scan_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } +} + /// Query parameters for filtering the job list. #[derive(Debug, Clone, serde::Deserialize, utoipa::IntoParams)] pub struct JobFilter { @@ -268,7 +355,9 @@ async fn run_analysis( .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; // Build storage key from source document SHA256. - let storage_key = trustify_common::id::Id::from_str(&source_doc.sha256) + // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:…". + let sha256_with_prefix = format!("sha256:{}", source_doc.sha256); + let storage_key = trustify_common::id::Id::from_str(&sha256_with_prefix) .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; let storage_key: trustify_module_storage::service::StorageKey = storage_key .try_into() @@ -383,7 +472,36 @@ async fn run_analysis( .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - // Poll for completion. + // Hand off to the poll loop. + poll_for_result( + ei_service, + ingestor, + db_rw, + config, + client, + job_id, + vulnerability_id, + &scan_id, + ) + .await +} + +/// Poll the EI service for an in-flight analysis and ingest the result. +/// +/// This is the second half of the analysis lifecycle: it assumes the SBOM has +/// already been uploaded and a `scan_id` obtained. It is called both from +/// `run_analysis` (normal path) and from the startup recovery path for +/// orphaned Running jobs. +pub(crate) async fn poll_for_result( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, + job_id: Uuid, + vulnerability_id: &str, + scan_id: &str, +) -> Result<(), anyhow::Error> { let poll_url = format!("{}/api/v1/reports/by-scan-id/{}", config.url, scan_id); let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index e8ab71be4..aa66a684a 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -5,8 +5,8 @@ use crate::{ }, }; use sea_orm::{ - ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, PaginatorTrait, QueryFilter, - QueryOrder, QuerySelect, Set, TransactionTrait, + ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, IntoActiveModel, PaginatorTrait, + QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, }; use std::time::Duration; use time::OffsetDateTime; @@ -27,14 +27,26 @@ pub struct ExploitIntelligenceConfig { pub poll_interval_secs: u64, /// Maximum duration in seconds before a polling loop is considered timed out. pub max_poll_duration_secs: u64, - /// Optional token provider for the EI service (static or OIDC). + /// Optional token provider for EI service authentication. + /// Supports static tokens and OIDC client credentials flow. pub token_provider: Option, } +/// Result of the startup recovery pass over orphaned EI jobs. +pub struct RecoveryResult { + /// Number of Pending jobs that were marked as Failed. + pub pending_failed: u64, + /// Running jobs whose poll loops should be resumed by the caller. + pub running_jobs: Vec, +} + /// Service for orchestrating Exploit Intelligence analysis jobs. #[derive(Clone)] pub struct ExploitIntelligenceService { config: Option, + /// Shared HTTP client for EI background tasks. Only present when `config` + /// is `Some` (i.e. EI is enabled). Built once and reused across all + /// analysis invocations to benefit from connection pooling. http_client: Option, } @@ -42,8 +54,7 @@ impl ExploitIntelligenceService { /// Create a new service instance with optional configuration. /// /// When `config` is `None`, the service will return `Error::Unavailable` - /// for all analysis operations. When configuration is present, a shared - /// HTTP client is built with sensible timeouts and connection pooling. + /// for all analysis operations and no HTTP client is created. pub fn new(config: Option) -> Self { let http_client = config.as_ref().map(|_| { reqwest::Client::builder() @@ -52,7 +63,7 @@ impl ExploitIntelligenceService { .pool_max_idle_per_host(4) .pool_idle_timeout(Duration::from_secs(90)) .build() - .expect("failed to build HTTP client") + .expect("failed to build reqwest client for exploit intelligence") }); Self { config, @@ -60,7 +71,7 @@ impl ExploitIntelligenceService { } } - /// Returns the shared HTTP client, if configuration is present. + /// Returns a reference to the shared HTTP client, if EI is enabled. pub fn http_client(&self) -> Option<&reqwest::Client> { self.http_client.as_ref() } @@ -274,4 +285,79 @@ impl ExploitIntelligenceService { pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { self.config.as_ref() } + + /// Recover orphaned jobs after a server restart. + /// + /// **Pending** jobs (no scan_id yet) are marked as Failed because the SBOM + /// was never submitted to the EI service. + /// + /// **Running** jobs (scan_id is present) are returned to the caller so + /// their poll loops can be resumed — the EI service may still be working + /// on them. + pub async fn recover_orphaned_jobs( + &self, + db: &db::ReadWrite, + ) -> Result { + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + // Find all Pending or Running jobs. + let orphaned = exploit_intelligence_job::Entity::find() + .filter(exploit_intelligence_job::Column::Status.is_in([ + ExploitIntelligenceJobStatus::Pending, + ExploitIntelligenceJobStatus::Running, + ])) + .all(&tx) + .await + .map_err(Error::from)?; + + let mut pending_failed: u64 = 0; + let mut running_jobs = Vec::new(); + + for job in orphaned { + match job.status { + ExploitIntelligenceJobStatus::Pending => { + let mut active = job.into_active_model(); + active.status = Set(ExploitIntelligenceJobStatus::Failed); + active.error_message = Set(Some( + "Server restarted before analysis was submitted. \ + Please re-trigger the analysis." + .to_string(), + )); + active.updated = Set(now); + active.update(&tx).await.map_err(Error::from)?; + pending_failed += 1; + } + ExploitIntelligenceJobStatus::Running => { + // Keep them in Running state — the caller will probe the + // EI backend and either resume polling or fail them. + running_jobs.push(job); + } + _ => { + // Should not happen given the filter, but be safe. + } + } + } + + tx.commit().await?; + + if pending_failed > 0 { + tracing::warn!( + pending_failed, + "marked orphaned pending exploit intelligence jobs as failed after restart" + ); + } + + if !running_jobs.is_empty() { + tracing::info!( + count = running_jobs.len(), + "found orphaned running exploit intelligence jobs to resume" + ); + } + + Ok(RecoveryResult { + pending_failed, + running_jobs, + }) + } } diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index b1e19e347..f9f6aa76c 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -386,6 +386,11 @@ impl InitData { #[allow(unused_mut)] async fn run(mut self) -> anyhow::Result<()> { + // Recovery for orphaned exploit intelligence jobs is handled inside + // the EI endpoints `configure()`, which spawns a background task. + // This avoids needing the IngestorService here (it is only available + // inside the module configuration). + let ui = Arc::new(UiResources::new(&self.ui)?); let http = { From 280ea9376dece618ca1cb1cf83e2c2cc5a992cc5 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 25 Jun 2026 14:38:23 +0100 Subject: [PATCH 09/25] feat(exploit-intelligence): add multi-component SPDX product flow --- docs/env-vars.md | 5 +- .../src/exploit_intelligence/ei_client.rs | 66 ++ .../src/exploit_intelligence/endpoints/mod.rs | 759 +++++++++++++++--- .../src/exploit_intelligence/model/mod.rs | 136 ++++ .../src/exploit_intelligence/service/mod.rs | 324 +++++++- 5 files changed, 1166 insertions(+), 124 deletions(-) diff --git a/docs/env-vars.md b/docs/env-vars.md index 10715a17f..b19efb583 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -65,7 +65,10 @@ | `EXPLOIT_INTELLIGENCE_URL` | Base URL of the Exploit Intelligence client service | | | `EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS`| Polling interval in seconds for EI analysis completion | `30` | | `EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS`| Maximum duration in seconds before EI polling is considered timed out | `1800` | -| `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Authentication token for the Exploit Intelligence service | | +| `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Static authentication token for the Exploit Intelligence service (used when OIDC is not configured) | | +| `EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL` | OIDC token endpoint URL for EI service authentication (client credentials flow) | | +| `EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID` | OIDC client ID for EI service authentication | | +| `EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET`| OIDC client secret for EI service authentication | | ## Data Migration diff --git a/modules/fundamental/src/exploit_intelligence/ei_client.rs b/modules/fundamental/src/exploit_intelligence/ei_client.rs index 7ca1f1e06..cb57306b0 100644 --- a/modules/fundamental/src/exploit_intelligence/ei_client.rs +++ b/modules/fundamental/src/exploit_intelligence/ei_client.rs @@ -42,3 +42,69 @@ pub struct AnalysisEntry { pub struct Justification { pub status: Option, } + +// ── SPDX multi-component product flow ──────────────────────────────── + +/// Response from POST /api/v1/products/upload-spdx +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SpdxUploadResponse { + pub product_id: String, +} + +/// Response from GET /api/v1/products/{productId} +#[derive(Debug, Deserialize)] +pub struct ProductStatusResponse { + pub data: ProductData, + pub summary: ProductSummary, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProductData { + pub product_id: String, + pub name: Option, + pub version: Option, + pub components: Vec, + #[serde(default)] + pub submission_failures: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProductComponent { + pub component_id: Option, + pub name: String, + pub version: Option, + pub purl: Option, + pub scan_id: Option, + pub state: Option, +} + +#[derive(Debug, Deserialize)] +pub struct SubmissionFailure { + pub name: Option, + pub version: Option, + pub purl: Option, + pub error: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProductSummary { + pub product_state: String, + #[serde(default)] + pub status_counts: std::collections::HashMap, + #[serde(default)] + pub justification_status_counts: std::collections::HashMap, +} + +/// A single component's report within a product. +/// Same shape as the by-scan-id response but embedded in an array from the product reports endpoint. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComponentReportEntry { + pub scan_id: Option, + pub status: String, + pub report: Option, +} diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 1090c926f..5a6266650 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -122,22 +122,6 @@ async fn recover_and_resume( }; for job in result.running_jobs { - // Running jobs are guaranteed to have a scan_id. - let scan_id = match job.scan_id { - Some(ref s) => s.clone(), - None => { - tracing::error!(job_id = %job.id, "running job has no scan_id, marking failed"); - let _ = ei_service - .update_job_failed( - job.id, - "Running job missing scan_id during recovery", - db_rw, - ) - .await; - continue; - } - }; - let job_id = job.id; let vulnerability_id = job.vulnerability_id.clone(); let ei_svc = ei_service.clone(); @@ -146,26 +130,62 @@ async fn recover_and_resume( let config_clone = config.clone(); let client_clone = client.clone(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); - if let Err(e) = poll_for_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &vulnerability_id, - &scan_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; - } - }); + if let Some(ref product_id) = job.product_id { + // Multi-component SPDX flow — resume product-level polling. + let product_id = product_id.clone(); + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, product_id = %product_id, "resuming orphaned SPDX product polling"); + if let Err(e) = poll_for_product_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &product_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } else if let Some(ref scan_id) = job.scan_id { + // Single-component CycloneDX flow — resume scan-level polling. + let scan_id = scan_id.clone(); + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); + if let Err(e) = poll_for_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &scan_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } else { + tracing::error!(job_id = %job_id, "running job has no scan_id or product_id, marking failed"); + let _ = ei_service + .update_job_failed( + job_id, + "Running job missing scan_id and product_id during recovery", + db_rw, + ) + .await; + } } } @@ -383,107 +403,227 @@ async fn run_analysis( drop(tx); - // Submit to EI client with retry on transient failures. - let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); - - const UPLOAD_MAX_ATTEMPTS: u32 = 3; - let mut upload_response: Option = None; - - for attempt in 0..UPLOAD_MAX_ATTEMPTS { - // Build a fresh multipart form for each attempt (forms are consumed by send). - let form = reqwest::multipart::Form::new() - .text("cveId", vulnerability_id.to_string()) - .part( - "file", - reqwest::multipart::Part::bytes(sbom_bytes.clone()) - .file_name("sbom.json") - .mime_str("application/json")?, - ); + // Detect SBOM format from labels. + let sbom_format = sbom_model + .labels + .0 + .get("type") + .map(String::as_str) + .unwrap_or("cyclonedx"); + + let is_spdx = sbom_format == "spdx"; + + if is_spdx { + // SPDX multi-component flow: upload to /api/v1/products/upload-spdx + let upload_url = format!("{}/api/v1/products/upload-spdx", config.url); + + const UPLOAD_MAX_ATTEMPTS: u32 = 3; + let mut upload_response: Option = None; + + for attempt in 0..UPLOAD_MAX_ATTEMPTS { + let form = reqwest::multipart::Form::new() + .text("cveId", vulnerability_id.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(sbom_bytes.clone()) + .file_name("sbom.json") + .mime_str("application/json")?, + ); - let mut request = client.post(&upload_url).multipart(form); + let mut request = client.post(&upload_url).multipart(form); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - request = request.bearer_auth(&token); - } + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + request = request.bearer_auth(&token); + } - match request.send().await { - Ok(resp) => { - let status = resp.status(); - match classify_status(status) { - RetryDecision::Success => { - upload_response = Some(resp); - break; - } - RetryDecision::Transient(reason) => { - if attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - reason = %reason, - "upload to EI client failed with transient error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { + match request.send().await { + Ok(resp) => { + let status = resp.status(); + match classify_status(status) { + RetryDecision::Success => { + upload_response = Some(resp); + break; + } + RetryDecision::Transient(reason) => { + if attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + reason = %reason, + "SPDX upload to EI client failed with transient error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI SPDX upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" + )); + } + } + RetryDecision::Permanent(_) => { let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" - )); + return Err(anyhow::anyhow!("EI client returned {status}: {body}")); } } - RetryDecision::Permanent(_) => { - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("EI client returned {status}: {body}")); + } + Err(err) => { + if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + error = %err, + "SPDX upload to EI client failed with network error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + return Err(anyhow::anyhow!("EI upload network error: {err}")); } } } - Err(err) => { - if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - error = %err, - "upload to EI client failed with network error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { - return Err(anyhow::anyhow!("EI upload network error: {err}")); + } + + let response = upload_response + .ok_or_else(|| anyhow::anyhow!("EI SPDX upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; + + let spdx_upload: crate::exploit_intelligence::ei_client::SpdxUploadResponse = + response.json().await?; + let product_id = spdx_upload.product_id; + + // Update job to running with the product ID. + ei_service + .update_job_running_with_product(job_id, &product_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product uploaded to EI, starting product-level polling" + ); + + // Hand off to the product-level poll loop. + poll_for_product_result( + ei_service, + ingestor, + db_rw, + config, + client, + job_id, + vulnerability_id, + &product_id, + ) + .await + } else { + // CycloneDX single-component flow (existing behavior). + let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); + + const UPLOAD_MAX_ATTEMPTS: u32 = 3; + let mut upload_response: Option = None; + + for attempt in 0..UPLOAD_MAX_ATTEMPTS { + // Build a fresh multipart form for each attempt (forms are consumed by send). + let form = reqwest::multipart::Form::new() + .text("cveId", vulnerability_id.to_string()) + .part( + "file", + reqwest::multipart::Part::bytes(sbom_bytes.clone()) + .file_name("sbom.json") + .mime_str("application/json")?, + ); + + let mut request = client.post(&upload_url).multipart(form); + + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + request = request.bearer_auth(&token); + } + + match request.send().await { + Ok(resp) => { + let status = resp.status(); + match classify_status(status) { + RetryDecision::Success => { + upload_response = Some(resp); + break; + } + RetryDecision::Transient(reason) => { + if attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + reason = %reason, + "upload to EI client failed with transient error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" + )); + } + } + RetryDecision::Permanent(_) => { + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!("EI client returned {status}: {body}")); + } + } + } + Err(err) => { + if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { + let backoff = std::time::Duration::from_secs(1 << attempt); + tracing::warn!( + job_id = %job_id, + attempt = attempt + 1, + backoff_secs = backoff.as_secs(), + error = %err, + "upload to EI client failed with network error, retrying" + ); + tokio::time::sleep(backoff).await; + } else { + return Err(anyhow::anyhow!("EI upload network error: {err}")); + } } } } - } - let response = upload_response - .ok_or_else(|| anyhow::anyhow!("EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; + let response = upload_response + .ok_or_else(|| anyhow::anyhow!("EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; - let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; - let scan_id = upload.report_request_id.report_id; + let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; + let scan_id = upload.report_request_id.report_id; - // Update job to running with scan ID. - ei_service - .update_job_running(job_id, &scan_id, db_rw) + // Update job to running with scan ID. + ei_service + .update_job_running(job_id, &scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + // Hand off to the poll loop. + poll_for_result( + ei_service, + ingestor, + db_rw, + config, + client, + job_id, + vulnerability_id, + &scan_id, + ) .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - - // Hand off to the poll loop. - poll_for_result( - ei_service, - ingestor, - db_rw, - config, - client, - job_id, - vulnerability_id, - &scan_id, - ) - .await + } } /// Poll the EI service for an in-flight analysis and ingest the result. @@ -689,3 +829,380 @@ pub(crate) async fn poll_for_result( } } } + +/// Poll the EI service for a multi-component product and process per-component results. +/// +/// This is the SPDX counterpart of `poll_for_result`. It polls the product-level +/// endpoint until all components are complete, then fetches each component's +/// report individually. +pub(crate) async fn poll_for_product_result( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, + job_id: Uuid, + vulnerability_id: &str, + product_id: &str, +) -> Result<(), anyhow::Error> { + let product_url = format!("{}/api/v1/products/{}", config.url, product_id); + let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); + let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); + let mut poll_count: u64 = 0; + let mut consecutive_transient_failures: u32 = 0; + let mut components_initialized = false; + + // Maps EI scan_id -> our internal component_id (populated on first successful poll). + let mut scan_to_component: std::collections::HashMap = + std::collections::HashMap::new(); + + loop { + tokio::time::sleep(poll_interval).await; + + poll_count += 1; + if poll_count > max_polls { + let msg = format!( + "SPDX product analysis timed out after {}s", + config.max_poll_duration_secs + ); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(anyhow::anyhow!("{msg}")); + } + + // Poll the product endpoint. + let mut poll_request = client.get(&product_url); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + poll_request = poll_request.bearer_auth(&token); + } + + let poll_response = match poll_request.send().await { + Ok(resp) => resp, + Err(err) => { + tracing::warn!( + job_id = %job_id, + error = %err, + "network error while polling EI product" + ); + consecutive_transient_failures += 1; + if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { + return Err(anyhow::anyhow!( + "EI service unreachable: {consecutive_transient_failures} consecutive product poll failures" + )); + } + continue; + } + }; + + if !poll_response.status().is_success() { + let status = poll_response.status(); + match classify_status(status) { + RetryDecision::Success => unreachable!(), + RetryDecision::Transient(reason) => { + consecutive_transient_failures += 1; + tracing::debug!( + job_id = %job_id, + reason = %reason, + consecutive_failures = consecutive_transient_failures, + "transient error while polling EI product, will retry" + ); + if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { + return Err(anyhow::anyhow!( + "EI service unhealthy: {consecutive_transient_failures} consecutive transient failures ({status})" + )); + } + continue; + } + RetryDecision::Permanent(_) => { + let body = poll_response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI product poll returned permanent error {status}: {body}" + )); + } + } + } + + consecutive_transient_failures = 0; + + let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = + poll_response.json().await?; + + // On first successful poll, initialize component records from the product response. + if !components_initialized { + for component in &product.data.components { + let purl = component.purl.as_deref().unwrap_or("unknown"); + let name = &component.name; + + let component_id = ei_service + .create_component(job_id, purl, name, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + + if let Some(ref scan_id) = component.scan_id { + scan_to_component.insert(scan_id.clone(), component_id); + + // If the component already has a scan_id, mark it as running. + ei_service + .update_component_running(component_id, scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + } + } + + // Record submission failures as excluded components. + for failure in &product.data.submission_failures { + let purl = failure.purl.as_deref().unwrap_or("unknown"); + let name = failure.name.as_deref().unwrap_or("unknown"); + let error = failure.error.as_deref().unwrap_or("excluded by EI"); + + ei_service + .create_excluded_component(job_id, purl, name, error, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; + } + + components_initialized = true; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + components = product.data.components.len(), + excluded = product.data.submission_failures.len(), + "initialized component records for SPDX product" + ); + } + + // Check product-level state. + match product.summary.product_state.as_str() { + "completed" => { + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed, fetching per-component reports" + ); + + // Fetch each component's report and process results. + let mut any_vulnerable = false; + let mut all_not_vulnerable = true; + + for component in &product.data.components { + let scan_id = match component.scan_id.as_ref() { + Some(id) => id, + None => continue, + }; + + let component_id = match scan_to_component.get(scan_id) { + Some(id) => *id, + None => continue, + }; + + // Fetch the per-component report. + let report_url_str = format!( + "{}/api/v1/reports/by-scan-id/{}", + config.url, scan_id + ); + + let mut report_request = client.get(&report_url_str); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + report_request = report_request.bearer_auth(&token); + } + + let report_response = match report_request.send().await { + Ok(resp) if resp.status().is_success() => resp, + Ok(resp) => { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + tracing::warn!( + job_id = %job_id, + scan_id = %scan_id, + "failed to fetch component report: {status}: {body}" + ); + ei_service + .update_component_failed( + component_id, + &format!("failed to fetch report: {status}"), + db_rw, + ) + .await + .map_err(|e| { + anyhow::anyhow!("failed to update component: {e}") + })?; + all_not_vulnerable = false; + continue; + } + Err(err) => { + tracing::warn!( + job_id = %job_id, + scan_id = %scan_id, + error = %err, + "network error fetching component report" + ); + ei_service + .update_component_failed( + component_id, + &format!("network error: {err}"), + db_rw, + ) + .await + .map_err(|e| { + anyhow::anyhow!("failed to update component: {e}") + })?; + all_not_vulnerable = false; + continue; + } + }; + + let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = + report_response.json().await?; + + // Extract finding. + use trustify_entity::exploit_intelligence_job::Finding; + let finding = poll_resp + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.analysis.as_ref()) + .and_then(|a| a.first()) + .and_then(|e| e.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + match finding { + Finding::Vulnerable => { + any_vulnerable = true; + all_not_vulnerable = false; + } + Finding::Uncertain => { + all_not_vulnerable = false; + } + Finding::NotVulnerable => {} + } + + let component_report_url = format!( + "{}/reports/component/{}/{}", + config.url, vulnerability_id, scan_id + ); + + // Ingest VEX document if present. + let vex_json = poll_resp + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + let advisory_id = if let Some(vex) = vex_json { + if !vex.is_null() { + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest( + &vex_bytes, + Format::Advisory, + labels, + None, + Cache::Skip, + &tx, + ) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + component_purl = component.purl.as_deref().unwrap_or("?"), + advisory_id = %result.id, + "ingested VEX advisory for SPDX component" + ); + + Uuid::from_str(&result.id).ok() + } else { + None + } + } else { + None + }; + + ei_service + .update_component_completed( + component_id, + finding, + Some(component_report_url), + advisory_id, + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + } + + // Compute aggregate finding for the parent job. + let aggregate_finding = if any_vulnerable { + trustify_entity::exploit_intelligence_job::Finding::Vulnerable + } else if all_not_vulnerable { + trustify_entity::exploit_intelligence_job::Finding::NotVulnerable + } else { + trustify_entity::exploit_intelligence_job::Finding::Uncertain + }; + + let product_report_url = + format!("{}/reports/product/{}", config.url, product_id); + + ei_service + .update_job_completed( + job_id, + aggregate_finding, + Some(product_report_url), + None, // No single advisory for the parent job + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + finding = ?aggregate_finding, + "SPDX product analysis completed" + ); + + return Ok(()); + } + "failed" => { + let msg = format!("EI product analysis failed for product {product_id}"); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Ok(()); + } + _ => { + // Still in progress (queued, in_progress), continue polling. + tracing::debug!( + job_id = %job_id, + product_id = %product_id, + state = product.summary.product_state, + "SPDX product analysis still in progress" + ); + } + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index 1e2f206f4..cddd55eb9 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -37,6 +37,20 @@ pub struct ExploitIntelligenceJobSummary { pub finding: Option, /// Link to the EI human-readable justification report. pub report_url: Option, + /// EI product ID for multi-component SPDX flow. + pub product_id: Option, + /// Total components in the product. + pub total_components: Option, + /// Number of completed components (computed at query time, not stored). + pub completed_components: Option, + /// Number of failed/excluded components (computed at query time, not stored). + pub failed_components: Option, + /// Number of completed components with Vulnerable finding (computed at query time). + pub vulnerable_components: Option, + /// Number of completed components with NotVulnerable finding (computed at query time). + pub not_vulnerable_components: Option, + /// Number of completed components with Uncertain finding (computed at query time). + pub uncertain_components: Option, /// Timestamp when the job was created. #[schema(value_type = String)] pub created: OffsetDateTime, @@ -64,6 +78,22 @@ pub struct ExploitIntelligenceJobDetails { pub advisory_id: Option, /// Error details for failed analyses. pub error_message: Option, + /// EI product ID for multi-component SPDX flow. + pub product_id: Option, + /// Total components in the product. + pub total_components: Option, + /// Number of completed components (computed at query time, not stored). + pub completed_components: Option, + /// Number of failed/excluded components (computed at query time, not stored). + pub failed_components: Option, + /// Number of completed components with Vulnerable finding (computed at query time). + pub vulnerable_components: Option, + /// Number of completed components with NotVulnerable finding (computed at query time). + pub not_vulnerable_components: Option, + /// Number of completed components with Uncertain finding (computed at query time). + pub uncertain_components: Option, + /// Per-component analysis results (populated for multi-component jobs). + pub components: Vec, /// Timestamp when the job was created. #[schema(value_type = String)] pub created: OffsetDateTime, @@ -72,6 +102,23 @@ pub struct ExploitIntelligenceJobDetails { pub updated: OffsetDateTime, } +/// Per-component analysis result within a multi-component product job. +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ComponentResult { + pub id: Uuid, + pub component_purl: String, + pub component_name: String, + pub status: JobStatus, + pub finding: Option, + pub report_url: Option, + pub advisory_id: Option, + pub error_message: Option, + #[schema(value_type = String)] + pub created: OffsetDateTime, + #[schema(value_type = String)] + pub updated: OffsetDateTime, +} + /// Lifecycle status of an Exploit Intelligence analysis job (API representation). #[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq, ToSchema)] #[serde(rename_all = "snake_case")] @@ -130,6 +177,24 @@ impl From for ExploitIntelligenceJobStatus { } } +impl ComponentResult { + /// Create a component result from an entity model. + pub fn from_entity(model: &trustify_entity::exploit_intelligence_job_component::Model) -> Self { + Self { + id: model.id, + component_purl: model.component_purl.clone(), + component_name: model.component_name.clone(), + status: model.status.into(), + finding: model.finding.map(JobFinding::from), + report_url: model.report_url.clone(), + advisory_id: model.advisory_id, + error_message: model.error_message.clone(), + created: model.created, + updated: model.updated, + } + } +} + impl ExploitIntelligenceJobSummary { /// Create a summary from an entity model. pub fn from_entity(model: &trustify_entity::exploit_intelligence_job::Model) -> Self { @@ -140,10 +205,35 @@ impl ExploitIntelligenceJobSummary { status: model.status.into(), finding: model.finding.map(JobFinding::from), report_url: model.report_url.clone(), + product_id: model.product_id.clone(), + total_components: model.total_components, + completed_components: None, + failed_components: None, + vulnerable_components: None, + not_vulnerable_components: None, + uncertain_components: None, created: model.created, updated: model.updated, } } + + /// Create a summary from an entity model with pre-computed component counts. + pub fn from_entity_with_counts( + model: &trustify_entity::exploit_intelligence_job::Model, + completed_components: Option, + failed_components: Option, + vulnerable_components: Option, + not_vulnerable_components: Option, + uncertain_components: Option, + ) -> Self { + let mut summary = Self::from_entity(model); + summary.completed_components = completed_components; + summary.failed_components = failed_components; + summary.vulnerable_components = vulnerable_components; + summary.not_vulnerable_components = not_vulnerable_components; + summary.uncertain_components = uncertain_components; + summary + } } impl ExploitIntelligenceJobDetails { @@ -158,8 +248,54 @@ impl ExploitIntelligenceJobDetails { report_url: model.report_url.clone(), advisory_id: model.advisory_id, error_message: model.error_message.clone(), + product_id: model.product_id.clone(), + total_components: model.total_components, + completed_components: None, + failed_components: None, + vulnerable_components: None, + not_vulnerable_components: None, + uncertain_components: None, + components: Vec::new(), created: model.created, updated: model.updated, } } + + /// Create a detailed view from an entity model with component results. + pub fn from_entity_with_components( + model: &trustify_entity::exploit_intelligence_job::Model, + component_models: &[trustify_entity::exploit_intelligence_job_component::Model], + ) -> Self { + let mut details = Self::from_entity(model); + details.components = component_models + .iter() + .map(ComponentResult::from_entity) + .collect(); + let completed = component_models + .iter() + .filter(|c| c.status == ExploitIntelligenceJobStatus::Completed) + .count() as i32; + let failed = component_models + .iter() + .filter(|c| c.status == ExploitIntelligenceJobStatus::Failed) + .count() as i32; + let vulnerable = component_models + .iter() + .filter(|c| c.finding == Some(Finding::Vulnerable)) + .count() as i32; + let not_vulnerable = component_models + .iter() + .filter(|c| c.finding == Some(Finding::NotVulnerable)) + .count() as i32; + let uncertain = component_models + .iter() + .filter(|c| c.finding == Some(Finding::Uncertain)) + .count() as i32; + details.completed_components = Some(completed); + details.failed_components = Some(failed); + details.vulnerable_components = Some(vulnerable); + details.not_vulnerable_components = Some(not_vulnerable); + details.uncertain_components = Some(uncertain); + details + } } diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index aa66a684a..4dc797622 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -7,7 +7,9 @@ use crate::{ use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, IntoActiveModel, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, + sea_query::Expr, }; +use std::collections::HashMap; use std::time::Duration; use time::OffsetDateTime; use tracing::{Instrument, info_span, instrument}; @@ -16,6 +18,7 @@ use trustify_common::{ model::{Paginated, PaginatedResults}, }; use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus}; +use trustify_entity::exploit_intelligence_job_component; use uuid::Uuid; /// Configuration for the Exploit Intelligence client connection. @@ -104,6 +107,8 @@ impl ExploitIntelligenceService { advisory_id: Set(None), finding: Set(None), error_message: Set(None), + product_id: Set(None), + total_components: Set(None), created: Set(now), updated: Set(now), }; @@ -116,6 +121,9 @@ impl ExploitIntelligenceService { } /// Fetch details of a single job by its ID. + /// + /// For multi-component jobs (those with a `product_id`), component records + /// are loaded automatically and component counts are computed. #[instrument(skip(self, connection), err(level = tracing::Level::INFO))] pub async fn fetch_job( &self, @@ -128,7 +136,17 @@ impl ExploitIntelligenceService { .await .map_err(Error::from)?; - Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m))) + match result { + Some(model) => { + if model.product_id.is_some() { + let components = self.fetch_components(model.id, connection).await?; + Ok(Some(ExploitIntelligenceJobDetails::from_entity_with_components(&model, &components))) + } else { + Ok(Some(ExploitIntelligenceJobDetails::from_entity(&model))) + } + } + None => Ok(None), + } } /// Fetch a paginated list of jobs, optionally filtered by SBOM ID and status. @@ -167,9 +185,69 @@ impl ExploitIntelligenceService { .await .map_err(Error::from)?; + // Compute component counts per job via aggregate query, grouped by status and finding. + // Keyed by job_id → (completed, failed, total, vulnerable, not_vulnerable, uncertain). + let job_ids: Vec = results.iter().map(|j| j.id).collect(); + let mut counts_map: HashMap = HashMap::new(); + + if !job_ids.is_empty() { + let rows: Vec<(Uuid, i32, Option, i64)> = + exploit_intelligence_job_component::Entity::find() + .select_only() + .column(exploit_intelligence_job_component::Column::JobId) + .column(exploit_intelligence_job_component::Column::Status) + .column(exploit_intelligence_job_component::Column::Finding) + .expr(Expr::col(exploit_intelligence_job_component::Column::Id).count()) + .filter(exploit_intelligence_job_component::Column::JobId.is_in(job_ids)) + .group_by(exploit_intelligence_job_component::Column::JobId) + .group_by(exploit_intelligence_job_component::Column::Status) + .group_by(exploit_intelligence_job_component::Column::Finding) + .into_tuple() + .all(connection) + .instrument(info_span!("count exploit intelligence job components")) + .await + .map_err(Error::from)?; + + for (job_id, status, finding, count) in rows { + let entry = counts_map.entry(job_id).or_insert((0, 0, 0, 0, 0, 0)); + let count = count as i32; + entry.2 += count; // total + if status == ExploitIntelligenceJobStatus::Completed as i32 { + entry.0 += count; // completed + match finding { + Some(0) => entry.3 += count, // vulnerable + Some(1) => entry.4 += count, // not_vulnerable + Some(2) => entry.5 += count, // uncertain + _ => {} + } + } else if status == ExploitIntelligenceJobStatus::Failed as i32 { + entry.1 += count; // failed + } + } + } + let items = results .iter() - .map(ExploitIntelligenceJobSummary::from_entity) + .map(|model| { + if let Some(&(completed, failed, total, vulnerable, not_vulnerable, uncertain)) = + counts_map.get(&model.id) + { + let mut summary = ExploitIntelligenceJobSummary::from_entity_with_counts( + model, + Some(completed), + Some(failed), + Some(vulnerable), + Some(not_vulnerable), + Some(uncertain), + ); + if summary.total_components.is_none() { + summary.total_components = Some(total); + } + summary + } else { + ExploitIntelligenceJobSummary::from_entity(model) + } + }) .collect(); Ok(PaginatedResults { @@ -281,6 +359,248 @@ impl ExploitIntelligenceService { Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m))) } + /// Create an analysis job with product_id and total_components set (multi-component SPDX flow). + /// + /// Returns the created job details. The caller is responsible for creating + /// component records and spawning background tasks. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn create_job_with_product( + &self, + sbom_id: Uuid, + vulnerability_id: &str, + product_id: &str, + total_components: i32, + db: &db::ReadWrite, + ) -> Result { + let _config = self.config.as_ref().ok_or(Error::Unavailable)?; + let now = OffsetDateTime::now_utc(); + let job_id = Uuid::now_v7(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + scan_id: Set(None), + sbom_id: Set(Some(sbom_id)), + vulnerability_id: Set(vulnerability_id.to_string()), + status: Set(ExploitIntelligenceJobStatus::Pending), + source_url: Set(None), + report_url: Set(None), + advisory_id: Set(None), + finding: Set(None), + error_message: Set(None), + product_id: Set(Some(product_id.to_string())), + total_components: Set(Some(total_components)), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await?; + let result = model.insert(&tx).await.map_err(Error::from)?; + tx.commit().await?; + + Ok(ExploitIntelligenceJobDetails::from_entity(&result)) + } + + /// Create a component record for a multi-component product job. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn create_component( + &self, + job_id: Uuid, + component_purl: &str, + component_name: &str, + db: &db::ReadWrite, + ) -> Result { + use trustify_entity::exploit_intelligence_job_component; + + let now = OffsetDateTime::now_utc(); + let component_id = Uuid::now_v7(); + + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + job_id: Set(job_id), + component_purl: Set(component_purl.to_string()), + component_name: Set(component_name.to_string()), + scan_id: Set(None), + status: Set(ExploitIntelligenceJobStatus::Pending), + finding: Set(None), + report_url: Set(None), + advisory_id: Set(None), + error_message: Set(None), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await?; + model.insert(&tx).await.map_err(Error::from)?; + tx.commit().await?; + + Ok(component_id) + } + + /// Create a component record marked as excluded (Failed with an error message). + /// + /// Used for components that cannot be analysed (e.g., unsupported ecosystem). + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn create_excluded_component( + &self, + job_id: Uuid, + component_purl: &str, + component_name: &str, + error_message: &str, + db: &db::ReadWrite, + ) -> Result { + use trustify_entity::exploit_intelligence_job_component; + + let now = OffsetDateTime::now_utc(); + let component_id = Uuid::now_v7(); + + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + job_id: Set(job_id), + component_purl: Set(component_purl.to_string()), + component_name: Set(component_name.to_string()), + scan_id: Set(None), + status: Set(ExploitIntelligenceJobStatus::Failed), + finding: Set(None), + report_url: Set(None), + advisory_id: Set(None), + error_message: Set(Some(error_message.to_string())), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await?; + model.insert(&tx).await.map_err(Error::from)?; + tx.commit().await?; + + Ok(component_id) + } + + /// Update a component's status to running and store the scan ID from the EI service. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_component_running( + &self, + component_id: Uuid, + scan_id: &str, + db: &db::ReadWrite, + ) -> Result<(), Error> { + use trustify_entity::exploit_intelligence_job_component; + + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + scan_id: Set(Some(scan_id.to_string())), + status: Set(ExploitIntelligenceJobStatus::Running), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Update a component to completed with the analysis finding. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_component_completed( + &self, + component_id: Uuid, + finding: exploit_intelligence_job::Finding, + report_url: Option, + advisory_id: Option, + db: &db::ReadWrite, + ) -> Result<(), Error> { + use trustify_entity::exploit_intelligence_job_component; + + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + status: Set(ExploitIntelligenceJobStatus::Completed), + finding: Set(Some(finding)), + report_url: Set(report_url), + advisory_id: Set(advisory_id), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Update a component to failed with an error message. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_component_failed( + &self, + component_id: Uuid, + error_message: &str, + db: &db::ReadWrite, + ) -> Result<(), Error> { + use trustify_entity::exploit_intelligence_job_component; + + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + status: Set(ExploitIntelligenceJobStatus::Failed), + error_message: Set(Some(error_message.to_string())), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Update a job's status to running and store the product ID from the EI service. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn update_job_running_with_product( + &self, + job_id: Uuid, + product_id: &str, + db: &db::ReadWrite, + ) -> Result<(), Error> { + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + product_id: Set(Some(product_id.to_string())), + status: Set(ExploitIntelligenceJobStatus::Running), + updated: Set(now), + ..Default::default() + }; + + model.update(&tx).await.map_err(Error::from)?; + tx.commit().await?; + Ok(()) + } + + /// Fetch all component records for a given job, ordered by creation time. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn fetch_components( + &self, + job_id: Uuid, + connection: &C, + ) -> Result, Error> { + use trustify_entity::exploit_intelligence_job_component; + + let results = exploit_intelligence_job_component::Entity::find() + .filter(exploit_intelligence_job_component::Column::JobId.eq(job_id)) + .order_by_asc(exploit_intelligence_job_component::Column::Created) + .all(connection) + .instrument(info_span!("fetch exploit intelligence job components")) + .await + .map_err(Error::from)?; + + Ok(results) + } + /// Returns the EI configuration, if available. pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { self.config.as_ref() From 4cbbe0dc85656ebf99f48df73b912b002ec37604 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 25 Jun 2026 15:51:18 +0100 Subject: [PATCH 10/25] feat(exploit-intelligence): add separate UI URL config for report deep-links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report_url was incorrectly built from the API base URL (config.url), which points to the agent-morpheus-client backend. The EI web UI may be hosted at a different URL. Add EXPLOIT_INTELLIGENCE_UI_URL as a separate optional config — when set, completed jobs include clickable deep-links to the EI web UI; when unset, report_url is null. Also fix pre-existing test compilation issues (missing entity fields, wrong assertion method on JobFinding enum). Co-Authored-By: Claude Opus 4.6 --- .../src/exploit_intelligence/endpoints/mod.rs | 25 +++++++++---------- .../exploit_intelligence/endpoints/test.rs | 9 ++++--- .../src/exploit_intelligence/service/mod.rs | 3 +++ server/src/profile/api.rs | 5 ++++ 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 5a6266650..0b8a3a38f 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -748,10 +748,9 @@ pub(crate) async fn poll_for_result( }) .unwrap_or(Finding::Uncertain); - let report_url = format!( - "{}/reports/component/{}/{}", - config.url, vulnerability_id, scan_id - ); + let report_url = config.ui_url.as_ref().map(|ui_url| { + format!("{}/reports/component/{}/{}", ui_url, vulnerability_id, scan_id) + }); // Check if VEX document is available. let vex_json = poll @@ -791,7 +790,7 @@ pub(crate) async fn poll_for_result( }; ei_service - .update_job_completed(job_id, finding, Some(report_url), advisory_id, db_rw) + .update_job_completed(job_id, finding, report_url, advisory_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; @@ -1093,10 +1092,9 @@ pub(crate) async fn poll_for_product_result( Finding::NotVulnerable => {} } - let component_report_url = format!( - "{}/reports/component/{}/{}", - config.url, vulnerability_id, scan_id - ); + let component_report_url = config.ui_url.as_ref().map(|ui_url| { + format!("{}/reports/component/{}/{}", ui_url, vulnerability_id, scan_id) + }); // Ingest VEX document if present. let vex_json = poll_resp @@ -1146,7 +1144,7 @@ pub(crate) async fn poll_for_product_result( .update_component_completed( component_id, finding, - Some(component_report_url), + component_report_url, advisory_id, db_rw, ) @@ -1163,14 +1161,15 @@ pub(crate) async fn poll_for_product_result( trustify_entity::exploit_intelligence_job::Finding::Uncertain }; - let product_report_url = - format!("{}/reports/product/{}", config.url, product_id); + let product_report_url = config.ui_url.as_ref().map(|ui_url| { + format!("{}/reports/product/{}", ui_url, product_id) + }); ei_service .update_job_completed( job_id, aggregate_finding, - Some(product_report_url), + product_report_url, None, // No single advisory for the parent job db_rw, ) diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 49978b11b..0a21fd424 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -1,5 +1,5 @@ use crate::exploit_intelligence::{ - model::{AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, JobStatus}, + model::{AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, JobFinding, JobStatus}, service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, }; use actix_web::{ @@ -20,6 +20,7 @@ use uuid::Uuid; fn test_service() -> ExploitIntelligenceService { ExploitIntelligenceService::new(Some(ExploitIntelligenceConfig { url: "http://localhost:9999".to_string(), + ui_url: Some("http://localhost:9999".to_string()), poll_interval_secs: 1, max_poll_duration_secs: 1800, token_provider: None, @@ -48,6 +49,8 @@ async fn insert_test_job( advisory_id: Set(None), finding: Set(finding), error_message: Set(error_message.map(|s| s.to_string())), + product_id: Set(None), + total_components: Set(None), created: Set(now), updated: Set(now), }; @@ -100,7 +103,7 @@ async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { assert_eq!(body.id, job_id); assert_eq!(body.status, JobStatus::Completed); assert_eq!(body.vulnerability_id, "CVE-2024-1234"); - assert_eq!(body.finding.as_deref(), Some("not_vulnerable")); + assert_eq!(body.finding, Some(JobFinding::NotVulnerable)); Ok(()) } @@ -491,7 +494,7 @@ async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Resu let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; assert_eq!(body.id, job_id); assert_eq!(body.status, JobStatus::Completed); - assert_eq!(body.finding.as_deref(), Some("vulnerable")); + assert_eq!(body.finding, Some(JobFinding::Vulnerable)); assert_eq!(body.advisory_id, None); assert_eq!(body.error_message, None); diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 4dc797622..451f5cea8 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -26,6 +26,9 @@ use uuid::Uuid; pub struct ExploitIntelligenceConfig { /// Base URL of the Exploit Intelligence client service. pub url: String, + /// Base URL of the Exploit Intelligence web UI for deep-linking to reports. + /// When set, completed jobs include a clickable report URL. + pub ui_url: Option, /// Polling interval in seconds for checking analysis completion. pub poll_interval_secs: u64, /// Maximum duration in seconds before a polling loop is considered timed out. diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index f9f6aa76c..2c2363970 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -96,6 +96,10 @@ pub struct Run { #[arg(long, env = "EXPLOIT_INTELLIGENCE_URL")] pub exploit_intelligence_url: Option, + /// Base URL of the Exploit Intelligence web UI for deep-linking to reports. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_UI_URL")] + pub exploit_intelligence_ui_url: Option, + /// Polling interval in seconds for EI analysis completion (default: 30). #[arg( long, @@ -357,6 +361,7 @@ impl InitData { trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { url, + ui_url: run.exploit_intelligence_ui_url, poll_interval_secs: run.exploit_intelligence_poll_interval_secs, max_poll_duration_secs: run.exploit_intelligence_max_poll_duration_secs, token_provider, From 22ee0bf2d93a411f8608d5a94087cccccf0e0788 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 26 Jun 2026 13:59:18 +0100 Subject: [PATCH 11/25] fix(exploit-intelligence): address review findings across module - Regenerate OpenAPI spec with missing JobFinding, ComponentResult schemas and fields - Fix TOCTOU race in analyze endpoint by using single RW transaction for dedup check-and-insert - Add test authorizer to all read endpoint tests (get_job, list_jobs) - Fix advisory_id parsing by stripping urn:uuid: prefix before Uuid::from_str - Return 200 OK for deduplicated jobs instead of 201 Created - Enforce minimum 5s poll interval via clap validation and defense-in-depth .max(5) - Replace magic integer literals with named Finding enum constants in aggregate query - Make total count conditional on paginated.total to skip unnecessary count queries - Make component initialization idempotent for orphaned jobs by checking existing records - Use read-only transaction for SBOM retrieval in run_analysis - URL-encode path segments in all EI client API and report URL construction - Add tests: happy-path analyze, running status, status filter, 404 SBOM not found, 503 EI unconfigured - Document EXPLOIT_INTELLIGENCE_UI_URL environment variable - Warn when OIDC args are partially configured instead of silently falling back - Increase default OIDC expires_in from 300s to 3600s - Add err.is_request() to transient network error detection for DNS failures - Log warning when SBOM format label is missing and defaulting to CycloneDX - Add doc comments to all ComponentResult fields - Introduce EiRuntime wrapper struct for config/client coupling Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/env-vars.md | 1 + modules/fundamental/Cargo.toml | 1 + .../src/exploit_intelligence/auth.rs | 2 +- .../src/exploit_intelligence/endpoints/mod.rs | 145 +++++++--- .../exploit_intelligence/endpoints/test.rs | 259 +++++++++++++++++- .../src/exploit_intelligence/model/mod.rs | 10 + .../src/exploit_intelligence/service/mod.rs | 84 +++--- openapi.yaml | 219 ++++++++++++++- server/src/profile/api.rs | 24 +- 9 files changed, 659 insertions(+), 86 deletions(-) diff --git a/docs/env-vars.md b/docs/env-vars.md index b19efb583..497aaef28 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -63,6 +63,7 @@ | `UI_LOAD_USER` | Whether to load user info | `true` | | `UI_SCOPE` | Scopes to request | `openid` | | `EXPLOIT_INTELLIGENCE_URL` | Base URL of the Exploit Intelligence client service | | +| `EXPLOIT_INTELLIGENCE_UI_URL` | Base URL of the EI web UI for deep-linking to reports | | | `EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS`| Polling interval in seconds for EI analysis completion | `30` | | `EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS`| Maximum duration in seconds before EI polling is considered timed out | `1800` | | `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Static authentication token for the Exploit Intelligence service (used when OIDC is not configured) | | diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 8aa8f451f..9c6296073 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -46,6 +46,7 @@ time = { workspace = true } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-futures = { workspace = true, features = ["futures-03"] } +urlencoding = { workspace = true } utoipa = { workspace = true, features = ["actix_extras", "uuid", "time"] } utoipa-actix-web = { workspace = true } uuid = { workspace = true } diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs index 328454268..74daaed03 100644 --- a/modules/fundamental/src/exploit_intelligence/auth.rs +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -119,7 +119,7 @@ impl OidcState { } let token_resp: TokenResponse = response.json().await?; - let expires_in = token_resp.expires_in.unwrap_or(300); // default 5 min + let expires_in = token_resp.expires_in.unwrap_or(3600); // default 1 hour let expires_at = OffsetDateTime::now_utc() + Duration::from_secs(expires_in); let access_token = token_resp.access_token.clone(); diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 0b8a3a38f..cbad40762 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -14,7 +14,7 @@ use crate::{ use actix_web::{HttpResponse, Responder, get, post, web}; use futures_util::TryStreamExt; use reqwest::StatusCode; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use std::str::FromStr; use trustify_auth::{CreateExploitIntelligence, ReadExploitIntelligence, authorizer::Require}; use trustify_common::{ @@ -57,10 +57,10 @@ fn classify_status(status: StatusCode) -> RetryDecision { } } -/// Returns `true` when a `reqwest::Error` looks transient (connection or -/// timeout problem on the network layer). +/// Returns `true` when a `reqwest::Error` looks transient (connection, +/// timeout, or request-building problem on the network layer). fn is_transient_network_error(err: &reqwest::Error) -> bool { - err.is_connect() || err.is_timeout() + err.is_connect() || err.is_timeout() || err.is_request() } /// Maximum number of consecutive transient poll failures before we give up. @@ -111,15 +111,12 @@ async fn recover_and_resume( } }; - let config = match ei_service.config().cloned() { - Some(c) => c, - None => return, - }; - - let client = match ei_service.http_client().cloned() { - Some(c) => c, + let runtime = match ei_service.runtime().cloned() { + Some(rt) => rt, None => return, }; + let config = runtime.config; + let client = runtime.client; for job in result.running_jobs { let job_id = job.id; @@ -208,6 +205,7 @@ pub struct JobFilter { operation_id = "analyzeExploitIntelligence", request_body = AnalyzeRequest, responses( + (status = 200, description = "Active analysis job already exists", body = AnalyzeResponse), (status = 201, description = "Analysis job created", body = AnalyzeResponse), (status = 404, description = "SBOM not found"), (status = 503, description = "Exploit Intelligence service not configured"), @@ -218,18 +216,19 @@ pub async fn analyze( ei_service: web::Data, ingestor: web::Data, db_rw: web::Data, - db_ro: web::Data, body: web::Json, _: Require, ) -> actix_web::Result { - let config = ei_service.config().cloned().ok_or(Error::Unavailable)?; - let client = ei_service - .http_client() - .cloned() - .ok_or(Error::Unavailable)?; + let runtime = ei_service.runtime().cloned().ok_or(Error::Unavailable)?; + let config = runtime.config; + let client = runtime.client; + + // Use a single read-write transaction for the existence check, active-job + // deduplication, and job creation to prevent TOCTOU race conditions where + // two concurrent requests could both pass the check and create duplicate jobs. + let tx = db_rw.begin().await.map_err(Error::from)?; // Verify the SBOM exists. - let tx = db_ro.begin().await?; let sbom = trustify_entity::sbom::Entity::find() .filter(trustify_entity::sbom::Column::SbomId.eq(body.sbom_id)) .one(&tx) @@ -245,20 +244,20 @@ pub async fn analyze( .find_active_job(body.sbom_id, &body.vulnerability_id, &tx) .await? { - drop(tx); - return Ok(HttpResponse::Created().json(AnalyzeResponse { + // No need to commit — no mutations were made. + return Ok(HttpResponse::Ok().json(AnalyzeResponse { job_id: active_job.id, status: active_job.status, })); } - drop(tx); - - // Create the job record. + // Create the job record within the same transaction. let job = ei_service - .create_job(body.sbom_id, &body.vulnerability_id, &db_rw) + .create_job(body.sbom_id, &body.vulnerability_id, &tx) .await?; + tx.commit().await.map_err(Error::from)?; + let job_id = job.id; let sbom_id = body.sbom_id; let vulnerability_id = body.vulnerability_id.clone(); @@ -360,8 +359,10 @@ async fn run_analysis( sbom_id: Uuid, vulnerability_id: &str, ) -> Result<(), anyhow::Error> { - // Retrieve the raw SBOM document from storage. - let tx = db_rw.begin().await?; + // Retrieve the raw SBOM document from storage (read-only – no mutations needed). + let tx = db_rw + .begin_with_config(None, Some(AccessMode::ReadOnly)) + .await?; let sbom_model = trustify_entity::sbom::Entity::find() .filter(trustify_entity::sbom::Column::SbomId.eq(sbom_id)) .one(&tx) @@ -404,12 +405,16 @@ async fn run_analysis( drop(tx); // Detect SBOM format from labels. - let sbom_format = sbom_model - .labels - .0 - .get("type") - .map(String::as_str) - .unwrap_or("cyclonedx"); + let sbom_format = match sbom_model.labels.0.get("type") { + Some(t) => t.as_str(), + None => { + tracing::warn!( + sbom_id = %sbom_id, + "SBOM has no 'type' label, defaulting to CycloneDX format" + ); + "cyclonedx" + } + }; let is_spdx = sbom_format == "spdx"; @@ -642,9 +647,14 @@ pub(crate) async fn poll_for_result( vulnerability_id: &str, scan_id: &str, ) -> Result<(), anyhow::Error> { - let poll_url = format!("{}/api/v1/reports/by-scan-id/{}", config.url, scan_id); - let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); - let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); + let poll_url = format!( + "{}/api/v1/reports/by-scan-id/{}", + config.url, + urlencoding::encode(scan_id) + ); + let poll_secs = config.poll_interval_secs.max(5); + let poll_interval = std::time::Duration::from_secs(poll_secs); + let max_polls = config.max_poll_duration_secs / poll_secs; let mut poll_count: u64 = 0; let mut consecutive_transient_failures: u32 = 0; @@ -749,7 +759,12 @@ pub(crate) async fn poll_for_result( .unwrap_or(Finding::Uncertain); let report_url = config.ui_url.as_ref().map(|ui_url| { - format!("{}/reports/component/{}/{}", ui_url, vulnerability_id, scan_id) + format!( + "{}/reports/component/{}/{}", + ui_url, + urlencoding::encode(vulnerability_id), + urlencoding::encode(scan_id) + ) }); // Check if VEX document is available. @@ -781,7 +796,10 @@ pub(crate) async fn poll_for_result( "ingested VEX advisory from exploit intelligence" ); - Uuid::from_str(&result.id).ok() + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) } else { None } @@ -844,9 +862,14 @@ pub(crate) async fn poll_for_product_result( vulnerability_id: &str, product_id: &str, ) -> Result<(), anyhow::Error> { - let product_url = format!("{}/api/v1/products/{}", config.url, product_id); - let poll_interval = std::time::Duration::from_secs(config.poll_interval_secs); - let max_polls = config.max_poll_duration_secs / config.poll_interval_secs.max(1); + let product_url = format!( + "{}/api/v1/products/{}", + config.url, + urlencoding::encode(product_id) + ); + let poll_secs = config.poll_interval_secs.max(5); + let poll_interval = std::time::Duration::from_secs(poll_secs); + let max_polls = config.max_poll_duration_secs / poll_secs; let mut poll_count: u64 = 0; let mut consecutive_transient_failures: u32 = 0; let mut components_initialized = false; @@ -855,6 +878,27 @@ pub(crate) async fn poll_for_product_result( let mut scan_to_component: std::collections::HashMap = std::collections::HashMap::new(); + // Check whether component records already exist for this job (e.g. after a + // crash/restart that orphaned a running job). If they do, skip re-initialization + // to avoid duplicate component rows. + let existing_components = ei_service + .fetch_components(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to fetch existing components: {e}"))?; + if !existing_components.is_empty() { + for c in &existing_components { + if let Some(ref scan_id) = c.scan_id { + scan_to_component.insert(scan_id.clone(), c.id); + } + } + components_initialized = true; + tracing::info!( + job_id = %job_id, + existing_components = existing_components.len(), + "resumed with existing component records, skipping re-initialization" + ); + } + loop { tokio::time::sleep(poll_interval).await; @@ -1004,7 +1048,8 @@ pub(crate) async fn poll_for_product_result( // Fetch the per-component report. let report_url_str = format!( "{}/api/v1/reports/by-scan-id/{}", - config.url, scan_id + config.url, + urlencoding::encode(scan_id) ); let mut report_request = client.get(&report_url_str); @@ -1093,7 +1138,12 @@ pub(crate) async fn poll_for_product_result( } let component_report_url = config.ui_url.as_ref().map(|ui_url| { - format!("{}/reports/component/{}/{}", ui_url, vulnerability_id, scan_id) + format!( + "{}/reports/component/{}/{}", + ui_url, + urlencoding::encode(vulnerability_id), + urlencoding::encode(scan_id) + ) }); // Ingest VEX document if present. @@ -1132,7 +1182,10 @@ pub(crate) async fn poll_for_product_result( "ingested VEX advisory for SPDX component" ); - Uuid::from_str(&result.id).ok() + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) } else { None } @@ -1162,7 +1215,11 @@ pub(crate) async fn poll_for_product_result( }; let product_report_url = config.ui_url.as_ref().map(|ui_url| { - format!("{}/reports/product/{}", ui_url, product_id) + format!( + "{}/reports/product/{}", + ui_url, + urlencoding::encode(product_id) + ) }); ei_service diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 0a21fd424..10402a511 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -1,5 +1,8 @@ use crate::exploit_intelligence::{ - model::{AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, JobFinding, JobStatus}, + model::{ + AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, + ExploitIntelligenceJobSummary, JobFinding, JobStatus, + }, service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, }; use actix_web::{ @@ -84,6 +87,7 @@ async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -131,6 +135,7 @@ async fn get_job_pending(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -177,6 +182,7 @@ async fn get_job_failed(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -209,6 +215,7 @@ async fn get_job_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -251,6 +258,7 @@ async fn list_jobs_paginated(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::list_jobs), @@ -308,6 +316,7 @@ async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<() let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::list_jobs), @@ -427,6 +436,7 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -476,6 +486,7 @@ async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Resu let app = test::init_service( App::new() + .add_test_authorizer() .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) .service(super::get_job), @@ -500,3 +511,249 @@ async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Resu Ok(()) } + +/// Verifies that POSTing an analyze request for a valid SBOM creates a new job +/// and returns 201 with the job_id and a "pending" status. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn analyze_creates_new_job(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a real SBOM in the database + let ingest_result = ctx + .ingest_document("cyclonedx/application.cdx.json") + .await?; + let sbom_id = Uuid::parse_str( + ingest_result + .id + .strip_prefix("urn:uuid:") + .expect("sbom id should have urn:uuid: prefix"), + )?; + + let ei_service = test_service(); + let db_rw = db::ReadWrite::new(ctx.db.clone()); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_rw)) + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .app_data(web::Data::new(ctx.ingestor.clone())) + .service(super::analyze), + ) + .await; + + // When submitting an analyze request (no pre-existing job) + let req = TestRequest::post() + .uri("/v3/exploit-intelligence/analyze") + .set_json(AnalyzeRequest { + sbom_id, + vulnerability_id: "CVE-2024-NEW".to_string(), + }) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then a new job is created and returned with 201 status + assert_eq!(resp.status(), 201); + let body: AnalyzeResponse = test::read_body_json(resp).await; + assert_eq!(body.status, JobStatus::Pending); + // The job_id should be a valid UUID (non-nil) + assert_ne!(body.job_id, Uuid::nil()); + + Ok(()) +} + +/// Verifies that the GET job details endpoint returns the correct state +/// for a running job. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn get_job_running(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given a running job record in the database + let job_id = Uuid::now_v7(); + insert_test_job( + &ctx.db, + job_id, + None, + "CVE-2024-RUN", + ExploitIntelligenceJobStatus::Running, + None, + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::get_job), + ) + .await; + + // When requesting the job details + let req = TestRequest::get() + .uri(&format!("/v3/exploit-intelligence/jobs/{job_id}")) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then the response shows the running status + assert_eq!(resp.status(), 200); + let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; + assert_eq!(body.id, job_id); + assert_eq!(body.status, JobStatus::Running); + assert_eq!(body.vulnerability_id, "CVE-2024-RUN"); + assert_eq!(body.finding, None); + + Ok(()) +} + +/// Verifies that the list endpoint supports filtering by job status, +/// returning only jobs that match the requested status. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given jobs with different statuses + insert_test_job( + &ctx.db, + Uuid::now_v7(), + None, + "CVE-2024-S001", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::NotVulnerable), + None, + ) + .await; + + insert_test_job( + &ctx.db, + Uuid::now_v7(), + None, + "CVE-2024-S002", + ExploitIntelligenceJobStatus::Pending, + None, + None, + ) + .await; + + insert_test_job( + &ctx.db, + Uuid::now_v7(), + None, + "CVE-2024-S003", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::Vulnerable), + None, + ) + .await; + + let ei_service = test_service(); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .service(super::list_jobs), + ) + .await; + + // When filtering by completed status + let req = TestRequest::get() + .uri("/v3/exploit-intelligence/jobs?status=completed") + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then only the two completed jobs are returned + assert_eq!(resp.status(), 200); + let body: PaginatedResults = test::read_body_json(resp).await; + assert_eq!(body.items.len(), 2); + assert_eq!(body.total, Some(2)); + for item in &body.items { + assert_eq!(item.status, JobStatus::Completed); + } + + Ok(()) +} + +/// Verifies that POSTing an analyze request with a non-existent SBOM ID +/// returns 404. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn analyze_sbom_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given an app configured with EI support + let ei_service = test_service(); + let db_rw = db::ReadWrite::new(ctx.db.clone()); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_rw)) + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .app_data(web::Data::new(ctx.ingestor.clone())) + .service(super::analyze), + ) + .await; + + // When submitting an analyze request with a non-existent SBOM ID + let req = TestRequest::post() + .uri("/v3/exploit-intelligence/analyze") + .set_json(AnalyzeRequest { + sbom_id: Uuid::now_v7(), + vulnerability_id: "CVE-2024-GHOST".to_string(), + }) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then a 404 is returned + assert_eq!(resp.status(), 404); + + Ok(()) +} + +/// Verifies that POSTing an analyze request when Exploit Intelligence is not +/// configured returns 503 Service Unavailable. +#[test_context(TrustifyContext)] +#[test(actix_web::test)] +async fn analyze_ei_not_configured(ctx: &TrustifyContext) -> anyhow::Result<()> { + // Given an app with no EI configuration + let ei_service = ExploitIntelligenceService::new(None); + let db_rw = db::ReadWrite::new(ctx.db.clone()); + let db_ro = db::ReadOnly::new(ctx.db.clone()); + + let app = test::init_service( + App::new() + .add_test_authorizer() + .app_data(web::Data::new(db_rw)) + .app_data(web::Data::new(db_ro)) + .app_data(web::Data::new(ei_service)) + .app_data(web::Data::new(ctx.ingestor.clone())) + .service(super::analyze), + ) + .await; + + // When submitting an analyze request + let req = TestRequest::post() + .uri("/v3/exploit-intelligence/analyze") + .set_json(AnalyzeRequest { + sbom_id: Uuid::now_v7(), + vulnerability_id: "CVE-2024-NOCONFIG".to_string(), + }) + .to_request(); + + let resp = test::call_service(&app, req).await; + + // Then a 503 Service Unavailable is returned + assert_eq!(resp.status(), 503); + + Ok(()) +} diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index cddd55eb9..154b8dffa 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -105,16 +105,26 @@ pub struct ExploitIntelligenceJobDetails { /// Per-component analysis result within a multi-component product job. #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ComponentResult { + /// The unique component record identifier. pub id: Uuid, + /// Package URL (purl) of the component being analysed. pub component_purl: String, + /// Human-readable name of the component. pub component_name: String, + /// Current lifecycle status of this component's analysis. pub status: JobStatus, + /// Analysis result from Exploit Intelligence, if completed. pub finding: Option, + /// Link to the EI human-readable justification report for this component. pub report_url: Option, + /// FK to advisory — set when a VEX document is ingested from the result. pub advisory_id: Option, + /// Error details for failed component analyses. pub error_message: Option, + /// Timestamp when the component record was created. #[schema(value_type = String)] pub created: OffsetDateTime, + /// Timestamp when the component record was last updated. #[schema(value_type = String)] pub updated: OffsetDateTime, } diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 451f5cea8..80bb96f41 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -17,7 +17,7 @@ use trustify_common::{ db, model::{Paginated, PaginatedResults}, }; -use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus}; +use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStatus, Finding}; use trustify_entity::exploit_intelligence_job_component; use uuid::Uuid; @@ -30,6 +30,7 @@ pub struct ExploitIntelligenceConfig { /// When set, completed jobs include a clickable report URL. pub ui_url: Option, /// Polling interval in seconds for checking analysis completion. + /// Minimum enforced: 5 seconds (values below are clamped at the polling site). pub poll_interval_secs: u64, /// Maximum duration in seconds before a polling loop is considered timed out. pub max_poll_duration_secs: u64, @@ -46,14 +47,23 @@ pub struct RecoveryResult { pub running_jobs: Vec, } +/// Holds the EI configuration together with a shared HTTP client. +/// +/// These two values are always present or absent together: when EI is +/// enabled both exist; when it is disabled neither does. +#[derive(Clone, Debug)] +pub struct EiRuntime { + /// Configuration for the Exploit Intelligence client connection. + pub config: ExploitIntelligenceConfig, + /// Shared HTTP client for EI background tasks. Built once and reused + /// across all analysis invocations to benefit from connection pooling. + pub client: reqwest::Client, +} + /// Service for orchestrating Exploit Intelligence analysis jobs. #[derive(Clone)] pub struct ExploitIntelligenceService { - config: Option, - /// Shared HTTP client for EI background tasks. Only present when `config` - /// is `Some` (i.e. EI is enabled). Built once and reused across all - /// analysis invocations to benefit from connection pooling. - http_client: Option, + runtime: Option, } impl ExploitIntelligenceService { @@ -62,24 +72,27 @@ impl ExploitIntelligenceService { /// When `config` is `None`, the service will return `Error::Unavailable` /// for all analysis operations and no HTTP client is created. pub fn new(config: Option) -> Self { - let http_client = config.as_ref().map(|_| { - reqwest::Client::builder() + let runtime = config.map(|config| { + let client = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(60)) .pool_max_idle_per_host(4) .pool_idle_timeout(Duration::from_secs(90)) .build() - .expect("failed to build reqwest client for exploit intelligence") + .expect("failed to build reqwest client for exploit intelligence"); + EiRuntime { config, client } }); - Self { - config, - http_client, - } + Self { runtime } } /// Returns a reference to the shared HTTP client, if EI is enabled. pub fn http_client(&self) -> Option<&reqwest::Client> { - self.http_client.as_ref() + self.runtime.as_ref().map(|rt| &rt.client) + } + + /// Returns the runtime (config + client) bundle, if EI is enabled. + pub fn runtime(&self) -> Option<&EiRuntime> { + self.runtime.as_ref() } /// Create an analysis job record in the database. @@ -87,14 +100,17 @@ impl ExploitIntelligenceService { /// Returns the created job details. The caller is responsible for /// spawning the background task that submits the request and polls for /// results. + /// + /// The caller must provide a connection (typically a transaction) and is + /// responsible for committing it. #[instrument(skip_all, err(level = tracing::Level::INFO))] - pub async fn create_job( + pub async fn create_job( &self, sbom_id: Uuid, vulnerability_id: &str, - db: &db::ReadWrite, + connection: &C, ) -> Result { - let _config = self.config.as_ref().ok_or(Error::Unavailable)?; + let _config = self.runtime.as_ref().ok_or(Error::Unavailable)?; let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); @@ -116,9 +132,7 @@ impl ExploitIntelligenceService { updated: Set(now), }; - let tx = db.begin().await?; - let result = model.insert(&tx).await.map_err(Error::from)?; - tx.commit().await?; + let result = model.insert(connection).await.map_err(Error::from)?; Ok(ExploitIntelligenceJobDetails::from_entity(&result)) } @@ -172,12 +186,18 @@ impl ExploitIntelligenceService { query = query.filter(exploit_intelligence_job::Column::Status.eq(db_status)); } - let total = query - .clone() - .count(connection) - .instrument(info_span!("count exploit intelligence jobs")) - .await - .map_err(Error::from)?; + let total = if paginated.total { + Some( + query + .clone() + .count(connection) + .instrument(info_span!("count exploit intelligence jobs")) + .await + .map_err(Error::from)?, + ) + } else { + None + }; let results = query .order_by_desc(exploit_intelligence_job::Column::Created) @@ -218,9 +238,9 @@ impl ExploitIntelligenceService { if status == ExploitIntelligenceJobStatus::Completed as i32 { entry.0 += count; // completed match finding { - Some(0) => entry.3 += count, // vulnerable - Some(1) => entry.4 += count, // not_vulnerable - Some(2) => entry.5 += count, // uncertain + Some(v) if v == Finding::Vulnerable as i32 => entry.3 += count, + Some(v) if v == Finding::NotVulnerable as i32 => entry.4 += count, + Some(v) if v == Finding::Uncertain as i32 => entry.5 += count, _ => {} } } else if status == ExploitIntelligenceJobStatus::Failed as i32 { @@ -255,7 +275,7 @@ impl ExploitIntelligenceService { Ok(PaginatedResults { items, - total: Some(total), + total, }) } @@ -375,7 +395,7 @@ impl ExploitIntelligenceService { total_components: i32, db: &db::ReadWrite, ) -> Result { - let _config = self.config.as_ref().ok_or(Error::Unavailable)?; + let _config = self.runtime.as_ref().ok_or(Error::Unavailable)?; let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); @@ -606,7 +626,7 @@ impl ExploitIntelligenceService { /// Returns the EI configuration, if available. pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { - self.config.as_ref() + self.runtime.as_ref().map(|rt| &rt.config) } /// Recover orphaned jobs after a server restart. diff --git a/openapi.yaml b/openapi.yaml index a1dc8ca16..6918acd6c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1050,6 +1050,12 @@ paths: $ref: '#/components/schemas/AnalyzeRequest' required: true responses: + '200': + description: Active analysis job already exists + content: + application/json: + schema: + $ref: '#/components/schemas/AnalyzeResponse' '201': description: Analysis job created content: @@ -4697,6 +4703,57 @@ components: period: type: string description: The period the importer should be run. + ComponentResult: + type: object + description: Per-component analysis result within a multi-component product job. + required: + - id + - component_purl + - component_name + - status + - created + - updated + properties: + advisory_id: + type: + - string + - 'null' + format: uuid + description: FK to advisory — set when a VEX document is ingested from the result. + component_name: + type: string + description: Human-readable name of the component. + component_purl: + type: string + description: Package URL (purl) of the component being analysed. + created: + type: string + description: Timestamp when the component record was created. + error_message: + type: + - string + - 'null' + description: Error details for failed component analyses. + finding: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/JobFinding' + description: Analysis result from Exploit Intelligence, if completed. + id: + type: string + format: uuid + description: The unique component record identifier. + report_url: + type: + - string + - 'null' + description: Link to the EI human-readable justification report for this component. + status: + $ref: '#/components/schemas/JobStatus' + description: Current lifecycle status of this component's analysis. + updated: + type: string + description: Timestamp when the component record was last updated. Cpe: type: string format: uri @@ -4780,6 +4837,7 @@ components: - id - vulnerability_id - status + - components - created - updated properties: @@ -4789,6 +4847,17 @@ components: - 'null' format: uuid description: FK to advisory — set when a VEX document is ingested from the result. + completed_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components (computed at query time, not stored). + components: + type: array + items: + $ref: '#/components/schemas/ComponentResult' + description: Per-component analysis results (populated for multi-component jobs). created: type: string description: Timestamp when the job was created. @@ -4797,15 +4866,32 @@ components: - string - 'null' description: Error details for failed analyses. - finding: + failed_components: type: - - string + - integer - 'null' - description: 'Analysis result: "vulnerable", "not_vulnerable", or "uncertain".' + format: int32 + description: Number of failed/excluded components (computed at query time, not stored). + finding: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/JobFinding' + description: Analysis result from Exploit Intelligence. id: type: string format: uuid description: The unique job identifier. + not_vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with NotVulnerable finding (computed at query time). + product_id: + type: + - string + - 'null' + description: EI product ID for multi-component SPDX flow. report_url: type: - string @@ -4820,12 +4906,30 @@ components: status: $ref: '#/components/schemas/JobStatus' description: Current lifecycle status of the job. + total_components: + type: + - integer + - 'null' + format: int32 + description: Total components in the product. + uncertain_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Uncertain finding (computed at query time). updated: type: string description: Timestamp when the job was last updated. vulnerability_id: type: string description: CVE identifier. + vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Vulnerable finding (computed at query time). ExploitIntelligenceJobSummary: type: object description: Summary of an Exploit Intelligence analysis job. @@ -4836,13 +4940,46 @@ components: - created - updated properties: + completed_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components (computed at query time, not stored). created: type: string description: Timestamp when the job was created. + failed_components: + type: + - integer + - 'null' + format: int32 + description: Number of failed/excluded components (computed at query time, not stored). + finding: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/JobFinding' + description: Analysis result from Exploit Intelligence. id: type: string format: uuid description: The unique job identifier. + not_vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with NotVulnerable finding (computed at query time). + product_id: + type: + - string + - 'null' + description: EI product ID for multi-component SPDX flow. + report_url: + type: + - string + - 'null' + description: Link to the EI human-readable justification report. sbom_id: type: - string @@ -4852,12 +4989,30 @@ components: status: $ref: '#/components/schemas/JobStatus' description: Current lifecycle status of the job. + total_components: + type: + - integer + - 'null' + format: int32 + description: Total components in the product. + uncertain_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Uncertain finding (computed at query time). updated: type: string description: Timestamp when the job was last updated. vulnerability_id: type: string description: CVE identifier. + vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Vulnerable finding (computed at query time). ExternalReferenceQuery: type: object properties: @@ -5133,6 +5288,13 @@ components: items: type: string description: Warnings that occurred during the import process + JobFinding: + type: string + description: Analysis finding from Exploit Intelligence (API representation). + enum: + - vulnerable + - not_vulnerable + - uncertain JobStatus: type: string description: Lifecycle status of an Exploit Intelligence analysis job (API representation). @@ -5379,13 +5541,46 @@ components: - created - updated properties: + completed_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components (computed at query time, not stored). created: type: string description: Timestamp when the job was created. + failed_components: + type: + - integer + - 'null' + format: int32 + description: Number of failed/excluded components (computed at query time, not stored). + finding: + oneOf: + - type: 'null' + - $ref: '#/components/schemas/JobFinding' + description: Analysis result from Exploit Intelligence. id: type: string format: uuid description: The unique job identifier. + not_vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with NotVulnerable finding (computed at query time). + product_id: + type: + - string + - 'null' + description: EI product ID for multi-component SPDX flow. + report_url: + type: + - string + - 'null' + description: Link to the EI human-readable justification report. sbom_id: type: - string @@ -5395,12 +5590,30 @@ components: status: $ref: '#/components/schemas/JobStatus' description: Current lifecycle status of the job. + total_components: + type: + - integer + - 'null' + format: int32 + description: Total components in the product. + uncertain_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Uncertain finding (computed at query time). updated: type: string description: Timestamp when the job was last updated. vulnerability_id: type: string description: CVE identifier. + vulnerable_components: + type: + - integer + - 'null' + format: int32 + description: Number of completed components with Vulnerable finding (computed at query time). total: type: - integer diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 2c2363970..29e310d88 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -100,11 +100,12 @@ pub struct Run { #[arg(long, env = "EXPLOIT_INTELLIGENCE_UI_URL")] pub exploit_intelligence_ui_url: Option, - /// Polling interval in seconds for EI analysis completion (default: 30). + /// Polling interval in seconds for EI analysis completion (default: 30, minimum: 5). #[arg( long, env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS", - default_value_t = 30 + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(5..) )] pub exploit_intelligence_poll_interval_secs: u64, @@ -346,17 +347,30 @@ impl InitData { // Build token provider: prefer OIDC if all three vars are set, // fall back to static token, or None. - let token_provider = match ( + let oidc_args = ( run.exploit_intelligence_oidc_token_url, run.exploit_intelligence_oidc_client_id, run.exploit_intelligence_oidc_client_secret, - ) { + ); + + let token_provider = match oidc_args { (Some(token_url), Some(client_id), Some(client_secret)) => { Some(TokenProvider::oidc(token_url, client_id, client_secret)) } - _ => run + (None, None, None) => run .exploit_intelligence_auth_token .map(TokenProvider::static_token), + _ => { + log::warn!( + "Incomplete EI OIDC configuration: all three of \ + EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL, \ + EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID, and \ + EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET must be set together. \ + Falling back to static token or no auth." + ); + run.exploit_intelligence_auth_token + .map(TokenProvider::static_token) + } }; trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { From 2981c8cedcc178960d69094cb6fe6c945c7a6b56 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Tue, 30 Jun 2026 15:59:43 +0100 Subject: [PATCH 12/25] fix(exploit-intelligence): rename component_purl to component_ref and fix SubmissionFailure deserialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename component_purl → component_ref in the API model and service layer to reflect that the field stores both purls and container image references. Fix SubmissionFailure.purl → .image to match what morpheus-client actually sends, resolving silent deserialization failures that stored "unknown" for failed component references. Co-Authored-By: Claude Opus 4.6 --- modules/fundamental/src/exploit_intelligence/ei_client.rs | 2 +- .../fundamental/src/exploit_intelligence/endpoints/mod.rs | 4 ++-- modules/fundamental/src/exploit_intelligence/model/mod.rs | 6 +++--- .../fundamental/src/exploit_intelligence/service/mod.rs | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/ei_client.rs b/modules/fundamental/src/exploit_intelligence/ei_client.rs index cb57306b0..8f0e4a0d7 100644 --- a/modules/fundamental/src/exploit_intelligence/ei_client.rs +++ b/modules/fundamental/src/exploit_intelligence/ei_client.rs @@ -85,7 +85,7 @@ pub struct ProductComponent { pub struct SubmissionFailure { pub name: Option, pub version: Option, - pub purl: Option, + pub image: Option, pub error: Option, } diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index cbad40762..c49f8c40c 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -1000,12 +1000,12 @@ pub(crate) async fn poll_for_product_result( // Record submission failures as excluded components. for failure in &product.data.submission_failures { - let purl = failure.purl.as_deref().unwrap_or("unknown"); + let image = failure.image.as_deref().unwrap_or("unknown"); let name = failure.name.as_deref().unwrap_or("unknown"); let error = failure.error.as_deref().unwrap_or("excluded by EI"); ei_service - .create_excluded_component(job_id, purl, name, error, db_rw) + .create_excluded_component(job_id, image, name, error, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; } diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index 154b8dffa..930cabb51 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -107,8 +107,8 @@ pub struct ExploitIntelligenceJobDetails { pub struct ComponentResult { /// The unique component record identifier. pub id: Uuid, - /// Package URL (purl) of the component being analysed. - pub component_purl: String, + /// Identifier of the component (purl or container image reference). + pub component_ref: String, /// Human-readable name of the component. pub component_name: String, /// Current lifecycle status of this component's analysis. @@ -192,7 +192,7 @@ impl ComponentResult { pub fn from_entity(model: &trustify_entity::exploit_intelligence_job_component::Model) -> Self { Self { id: model.id, - component_purl: model.component_purl.clone(), + component_ref: model.component_ref.clone(), component_name: model.component_name.clone(), status: model.status.into(), finding: model.finding.map(JobFinding::from), diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 80bb96f41..b9b34b59c 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -428,7 +428,7 @@ impl ExploitIntelligenceService { pub async fn create_component( &self, job_id: Uuid, - component_purl: &str, + component_ref: &str, component_name: &str, db: &db::ReadWrite, ) -> Result { @@ -440,7 +440,7 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job_component::ActiveModel { id: Set(component_id), job_id: Set(job_id), - component_purl: Set(component_purl.to_string()), + component_ref: Set(component_ref.to_string()), component_name: Set(component_name.to_string()), scan_id: Set(None), status: Set(ExploitIntelligenceJobStatus::Pending), @@ -466,7 +466,7 @@ impl ExploitIntelligenceService { pub async fn create_excluded_component( &self, job_id: Uuid, - component_purl: &str, + component_ref: &str, component_name: &str, error_message: &str, db: &db::ReadWrite, @@ -479,7 +479,7 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job_component::ActiveModel { id: Set(component_id), job_id: Set(job_id), - component_purl: Set(component_purl.to_string()), + component_ref: Set(component_ref.to_string()), component_name: Set(component_name.to_string()), scan_id: Set(None), status: Set(ExploitIntelligenceJobStatus::Failed), From 63fbdea87451085096c685d7f1c8463876c219d4 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Tue, 7 Jul 2026 16:59:21 +0100 Subject: [PATCH 13/25] fix(exploit-intelligence): case-insensitive finding status comparison The EI service returns finding status as uppercase ("TRUE"/"FALSE") but the mapping matched lowercase only, causing all findings to fall through to "uncertain". Use to_ascii_lowercase() before matching. Co-Authored-By: Claude Opus 4.6 (1M context) --- modules/fundamental/src/exploit_intelligence/endpoints/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index c49f8c40c..822535bb8 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -751,7 +751,7 @@ pub(crate) async fn poll_for_result( .and_then(|a| a.first()) .and_then(|e| e.justification.as_ref()) .and_then(|j| j.status.as_deref()) - .map(|s| match s { + .map(|s| match s.to_ascii_lowercase().as_str() { "false" => Finding::NotVulnerable, "true" => Finding::Vulnerable, _ => Finding::Uncertain, @@ -1119,7 +1119,7 @@ pub(crate) async fn poll_for_product_result( .and_then(|a| a.first()) .and_then(|e| e.justification.as_ref()) .and_then(|j| j.status.as_deref()) - .map(|s| match s { + .map(|s| match s.to_ascii_lowercase().as_str() { "false" => Finding::NotVulnerable, "true" => Finding::Vulnerable, _ => Finding::Uncertain, From 0004082f1780eb690aa158f85470c5cb6a6301bc Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Tue, 7 Jul 2026 17:02:42 +0100 Subject: [PATCH 14/25] feat(exploit-intelligence): adapt orchestration to unified entity schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the EI orchestration layer for the unified schema where all per-analysis results (scan_id, finding, advisory_id) live on the component table — even CycloneDX single-component jobs now create exactly one component row. Key changes: - CycloneDX flow creates a component record before upload, stores scan_id on the component via update_component_running - Delete update_job_running (scan_id no longer on job) - Simplify update_job_completed to status-only (no finding/advisory) - Recovery loads components from DB instead of reading job.scan_id - SPDX flow uses GET /reports?productId= to discover component scan IDs and findings, since the product endpoint returns empty components - poll_for_result takes component_id and updates component on completion - report_url computed at API response time from ui_url + identifiers - Aggregate finding computed from component data in model layer - Remove component_name (dropped from entity) - Remove advisory_id from job details (lives on components) - Add ProductReportEntry/ReportVuln structs for reports-list endpoint - Fix ProductData deserialization (alias "id", default components) Depends on TC-4675 entity/migration changes (will not compile alone). Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/exploit_intelligence/ei_client.rs | 27 +- .../src/exploit_intelligence/endpoints/mod.rs | 482 +++++++++--------- .../exploit_intelligence/endpoints/test.rs | 105 ++-- .../src/exploit_intelligence/model/mod.rs | 141 ++++- .../src/exploit_intelligence/service/mod.rs | 149 +++--- 5 files changed, 513 insertions(+), 391 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/ei_client.rs b/modules/fundamental/src/exploit_intelligence/ei_client.rs index 8f0e4a0d7..8c89a5868 100644 --- a/modules/fundamental/src/exploit_intelligence/ei_client.rs +++ b/modules/fundamental/src/exploit_intelligence/ei_client.rs @@ -62,9 +62,11 @@ pub struct ProductStatusResponse { #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ProductData { + #[serde(alias = "id")] pub product_id: String, pub name: Option, pub version: Option, + #[serde(default)] pub components: Vec, #[serde(default)] pub submission_failures: Vec, @@ -99,12 +101,25 @@ pub struct ProductSummary { pub justification_status_counts: std::collections::HashMap, } -/// A single component's report within a product. -/// Same shape as the by-scan-id response but embedded in an array from the product reports endpoint. +/// Entry from GET /api/v1/reports?productId={id}. +/// +/// Each entry represents a single component's analysis report within a product. +/// The `vulns` array contains the per-CVE justification from the EI engine. +/// Note: this endpoint does NOT include the full report output (no VEX data). +/// For VEX ingestion, use `GET /api/v1/reports/by-scan-id/{scanId}` per component. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct ComponentReportEntry { - pub scan_id: Option, - pub status: String, - pub report: Option, +pub struct ProductReportEntry { + pub scan_id: String, + pub state: String, + pub image_name: String, + #[serde(default)] + pub vulns: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ReportVuln { + pub vuln_id: Option, + pub justification: Option, } diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 822535bb8..a1403ca77 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -150,38 +150,56 @@ async fn recover_and_resume( .await; } }); - } else if let Some(ref scan_id) = job.scan_id { - // Single-component CycloneDX flow — resume scan-level polling. - let scan_id = scan_id.clone(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); - if let Err(e) = poll_for_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &vulnerability_id, - &scan_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + } else { + // Single-component CycloneDX flow — look up scan_id from the component. + let components = match ei_service.fetch_components(job_id, db_rw).await { + Ok(c) => c, + Err(e) => { + tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); + let _ = ei_service + .update_job_failed(job_id, &format!("recovery error: {e}"), db_rw) .await; + continue; } - }); - } else { - tracing::error!(job_id = %job_id, "running job has no scan_id or product_id, marking failed"); - let _ = ei_service - .update_job_failed( - job_id, - "Running job missing scan_id and product_id during recovery", - db_rw, - ) - .await; + }; + + let first_component = components.first(); + let scan_id = first_component.and_then(|c| c.scan_id.as_deref()); + let component_id = first_component.map(|c| c.id); + + if let (Some(scan_id), Some(component_id)) = (scan_id, component_id) { + let scan_id = scan_id.to_string(); + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); + if let Err(e) = poll_for_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &scan_id, + component_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } else { + tracing::error!(job_id = %job_id, "running job has no component with scan_id, marking failed"); + let _ = ei_service + .update_job_failed( + job_id, + "Running job missing component scan_id during recovery", + db_rw, + ) + .await; + } } } } @@ -527,7 +545,14 @@ async fn run_analysis( ) .await } else { - // CycloneDX single-component flow (existing behavior). + // CycloneDX single-component flow. + // Create the component record before uploading so it is tracked. + let component_ref = sbom_model.document_id.clone().unwrap_or_else(|| sbom_model.node_id.clone()); + let component_id = ei_service + .create_component(job_id, &component_ref, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); const UPLOAD_MAX_ATTEMPTS: u32 = 3; @@ -610,11 +635,11 @@ async fn run_analysis( let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; let scan_id = upload.report_request_id.report_id; - // Update job to running with scan ID. + // Update component to running with scan ID. ei_service - .update_job_running(job_id, &scan_id, db_rw) + .update_component_running(component_id, &scan_id, db_rw) .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; // Hand off to the poll loop. poll_for_result( @@ -626,17 +651,21 @@ async fn run_analysis( job_id, vulnerability_id, &scan_id, + component_id, ) .await } } -/// Poll the EI service for an in-flight analysis and ingest the result. +/// Poll the EI service for an in-flight single-component analysis and ingest the result. /// /// This is the second half of the analysis lifecycle: it assumes the SBOM has /// already been uploaded and a `scan_id` obtained. It is called both from /// `run_analysis` (normal path) and from the startup recovery path for /// orphaned Running jobs. +/// +/// Updates both the component record (with finding/advisory) and the parent +/// job record (status only) upon completion. pub(crate) async fn poll_for_result( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, @@ -644,8 +673,9 @@ pub(crate) async fn poll_for_result( config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, client: &reqwest::Client, job_id: Uuid, - vulnerability_id: &str, + _vulnerability_id: &str, scan_id: &str, + component_id: Uuid, ) -> Result<(), anyhow::Error> { let poll_url = format!( "{}/api/v1/reports/by-scan-id/{}", @@ -758,15 +788,6 @@ pub(crate) async fn poll_for_result( }) .unwrap_or(Finding::Uncertain); - let report_url = config.ui_url.as_ref().map(|ui_url| { - format!( - "{}/reports/component/{}/{}", - ui_url, - urlencoding::encode(vulnerability_id), - urlencoding::encode(scan_id) - ) - }); - // Check if VEX document is available. let vex_json = poll .report @@ -807,8 +828,15 @@ pub(crate) async fn poll_for_result( None }; + // Update the component with the per-analysis results. + ei_service + .update_component_completed(component_id, finding, advisory_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + + // Mark the parent job as completed (finding lives on the component). ei_service - .update_job_completed(job_id, finding, report_url, advisory_id, db_rw) + .update_job_completed(job_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; @@ -859,7 +887,7 @@ pub(crate) async fn poll_for_product_result( config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, client: &reqwest::Client, job_id: Uuid, - vulnerability_id: &str, + _vulnerability_id: &str, product_id: &str, ) -> Result<(), anyhow::Error> { let product_url = format!( @@ -976,228 +1004,202 @@ pub(crate) async fn poll_for_product_result( let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = poll_response.json().await?; - // On first successful poll, initialize component records from the product response. - if !components_initialized { - for component in &product.data.components { - let purl = component.purl.as_deref().unwrap_or("unknown"); - let name = &component.name; - - let component_id = ei_service - .create_component(job_id, purl, name, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + // Check product-level state. + match product.summary.product_state.as_str() { + "completed" => { + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed, fetching per-component reports" + ); - if let Some(ref scan_id) = component.scan_id { - scan_to_component.insert(scan_id.clone(), component_id); + // The EI product endpoint does not return per-component + // data (its `components` array is always empty). Instead, + // fetch the reports list filtered by product ID to discover + // each component's scan ID and justification. + let reports_url = format!( + "{}/api/v1/reports?productId={}", + config.url, + urlencoding::encode(product_id) + ); - // If the component already has a scan_id, mark it as running. - ei_service - .update_component_running(component_id, scan_id, db_rw) + let mut reports_request = client.get(&reports_url); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + reports_request = reports_request.bearer_auth(&token); } - } - - // Record submission failures as excluded components. - for failure in &product.data.submission_failures { - let image = failure.image.as_deref().unwrap_or("unknown"); - let name = failure.name.as_deref().unwrap_or("unknown"); - let error = failure.error.as_deref().unwrap_or("excluded by EI"); - ei_service - .create_excluded_component(job_id, image, name, error, db_rw) + let reports_response = reports_request + .send() .await - .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; - } + .map_err(|e| anyhow::anyhow!("failed to fetch product reports: {e}"))?; - components_initialized = true; + if !reports_response.status().is_success() { + let status = reports_response.status(); + let body = reports_response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI product reports request failed ({status}): {body}" + )); + } - tracing::info!( - job_id = %job_id, - product_id = %product_id, - components = product.data.components.len(), - excluded = product.data.submission_failures.len(), - "initialized component records for SPDX product" - ); - } + let component_reports: Vec = + reports_response.json().await?; + + // Record submission failures as excluded components. + if !components_initialized { + for failure in &product.data.submission_failures { + let image = failure.image.as_deref().unwrap_or("unknown"); + let error = failure.error.as_deref().unwrap_or("excluded by EI"); + + ei_service + .create_excluded_component(job_id, image, error, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; + } + } - // Check product-level state. - match product.summary.product_state.as_str() { - "completed" => { tracing::info!( job_id = %job_id, product_id = %product_id, - "SPDX product analysis completed, fetching per-component reports" + reports = component_reports.len(), + excluded = product.data.submission_failures.len(), + "processing per-component reports for SPDX product" ); - // Fetch each component's report and process results. - let mut any_vulnerable = false; - let mut all_not_vulnerable = true; + use trustify_entity::exploit_intelligence_job::Finding; - for component in &product.data.components { - let scan_id = match component.scan_id.as_ref() { - Some(id) => id, - None => continue, - }; + for entry in &component_reports { + let scan_id = &entry.scan_id; + let image_name = &entry.image_name; - let component_id = match scan_to_component.get(scan_id) { - Some(id) => *id, - None => continue, + // Create a component record for this report. + let component_id = if !components_initialized { + let id = ei_service + .create_component(job_id, image_name, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + ei_service + .update_component_running(id, scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + id + } else { + match scan_to_component.get(scan_id) { + Some(id) => *id, + None => continue, + } }; - // Fetch the per-component report. - let report_url_str = format!( + // Handle non-completed reports (expired, failed on EI side). + if entry.state != "completed" { + ei_service + .update_component_failed( + component_id, + &format!("EI report state: {}", entry.state), + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + continue; + } + + // Extract finding from the justification in the reports list. + let finding = entry + .vulns + .first() + .and_then(|v| v.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s.to_ascii_lowercase().as_str() { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + // Fetch the full by-scan-id report for VEX ingestion. + let full_report_url = format!( "{}/api/v1/reports/by-scan-id/{}", config.url, urlencoding::encode(scan_id) ); - let mut report_request = client.get(&report_url_str); + let mut full_request = client.get(&full_report_url); if let Some(ref provider) = config.token_provider { let token = provider .get_token() .await .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - report_request = report_request.bearer_auth(&token); + full_request = full_request.bearer_auth(&token); } - let report_response = match report_request.send().await { - Ok(resp) if resp.status().is_success() => resp, - Ok(resp) => { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - tracing::warn!( - job_id = %job_id, - scan_id = %scan_id, - "failed to fetch component report: {status}: {body}" - ); - ei_service - .update_component_failed( - component_id, - &format!("failed to fetch report: {status}"), - db_rw, - ) - .await - .map_err(|e| { - anyhow::anyhow!("failed to update component: {e}") - })?; - all_not_vulnerable = false; - continue; + let advisory_id = match full_request.send().await { + Ok(resp) if resp.status().is_success() => { + let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = + resp.json().await?; + + let vex_json = poll_resp + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + if let Some(vex) = vex_json { + if !vex.is_null() { + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest( + &vex_bytes, + Format::Advisory, + labels, + None, + Cache::Skip, + &tx, + ) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + component = image_name, + advisory_id = %result.id, + "ingested VEX advisory for SPDX component" + ); + + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) + } else { + None + } + } else { + None + } } - Err(err) => { + _ => { tracing::warn!( job_id = %job_id, scan_id = %scan_id, - error = %err, - "network error fetching component report" + "failed to fetch full report for VEX ingestion, skipping" ); - ei_service - .update_component_failed( - component_id, - &format!("network error: {err}"), - db_rw, - ) - .await - .map_err(|e| { - anyhow::anyhow!("failed to update component: {e}") - })?; - all_not_vulnerable = false; - continue; - } - }; - - let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = - report_response.json().await?; - - // Extract finding. - use trustify_entity::exploit_intelligence_job::Finding; - let finding = poll_resp - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.analysis.as_ref()) - .and_then(|a| a.first()) - .and_then(|e| e.justification.as_ref()) - .and_then(|j| j.status.as_deref()) - .map(|s| match s.to_ascii_lowercase().as_str() { - "false" => Finding::NotVulnerable, - "true" => Finding::Vulnerable, - _ => Finding::Uncertain, - }) - .unwrap_or(Finding::Uncertain); - - match finding { - Finding::Vulnerable => { - any_vulnerable = true; - all_not_vulnerable = false; - } - Finding::Uncertain => { - all_not_vulnerable = false; - } - Finding::NotVulnerable => {} - } - - let component_report_url = config.ui_url.as_ref().map(|ui_url| { - format!( - "{}/reports/component/{}/{}", - ui_url, - urlencoding::encode(vulnerability_id), - urlencoding::encode(scan_id) - ) - }); - - // Ingest VEX document if present. - let vex_json = poll_resp - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.vex.as_ref()); - - let advisory_id = if let Some(vex) = vex_json { - if !vex.is_null() { - let vex_bytes = serde_json::to_vec(vex)?; - let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); - - let tx = db_rw.begin().await?; - let result = ingestor - .ingest( - &vex_bytes, - Format::Advisory, - labels, - None, - Cache::Skip, - &tx, - ) - .await - .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; - tx.commit().await?; - - tracing::info!( - job_id = %job_id, - component_purl = component.purl.as_deref().unwrap_or("?"), - advisory_id = %result.id, - "ingested VEX advisory for SPDX component" - ); - - result - .id - .strip_prefix("urn:uuid:") - .and_then(|id| Uuid::from_str(id).ok()) - } else { None } - } else { - None }; ei_service .update_component_completed( component_id, finding, - component_report_url, advisory_id, db_rw, ) @@ -1205,38 +1207,16 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; } - // Compute aggregate finding for the parent job. - let aggregate_finding = if any_vulnerable { - trustify_entity::exploit_intelligence_job::Finding::Vulnerable - } else if all_not_vulnerable { - trustify_entity::exploit_intelligence_job::Finding::NotVulnerable - } else { - trustify_entity::exploit_intelligence_job::Finding::Uncertain - }; - - let product_report_url = config.ui_url.as_ref().map(|ui_url| { - format!( - "{}/reports/product/{}", - ui_url, - urlencoding::encode(product_id) - ) - }); - + // Mark the parent job as completed. The aggregate finding is + // computed from component records at query time. ei_service - .update_job_completed( - job_id, - aggregate_finding, - product_report_url, - None, // No single advisory for the parent job - db_rw, - ) + .update_job_completed(job_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; tracing::info!( job_id = %job_id, product_id = %product_id, - finding = ?aggregate_finding, "SPDX product analysis completed" ); diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 10402a511..8b7c1bf59 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -34,23 +34,17 @@ fn test_service() -> ExploitIntelligenceService { async fn insert_test_job( db: &trustify_common::db::Database, id: Uuid, - sbom_id: Option, + sbom_id: Uuid, vulnerability_id: &str, status: ExploitIntelligenceJobStatus, - finding: Option, error_message: Option<&str>, ) -> exploit_intelligence_job::Model { let now = OffsetDateTime::now_utc(); let model = exploit_intelligence_job::ActiveModel { id: Set(id), - scan_id: Set(Some(format!("scan-{id}"))), sbom_id: Set(sbom_id), vulnerability_id: Set(vulnerability_id.to_string()), status: Set(status), - source_url: Set(None), - report_url: Set(None), - advisory_id: Set(None), - finding: Set(finding), error_message: Set(error_message.map(|s| s.to_string())), product_id: Set(None), total_components: Set(None), @@ -64,23 +58,60 @@ async fn insert_test_job( result } +/// Helper to insert a component record for a job, with optional finding. +async fn insert_test_component( + db: &trustify_common::db::Database, + job_id: Uuid, + component_ref: &str, + status: ExploitIntelligenceJobStatus, + finding: Option, +) -> trustify_entity::exploit_intelligence_job_component::Model { + use trustify_entity::exploit_intelligence_job_component; + + let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(Uuid::now_v7()), + job_id: Set(job_id), + component_ref: Set(component_ref.to_string()), + scan_id: Set(Some(format!("scan-{job_id}"))), + status: Set(status), + finding: Set(finding), + advisory_id: Set(None), + error_message: Set(None), + created: Set(now), + updated: Set(now), + }; + + let tx = db.begin().await.expect("begin tx"); + let result = model.insert(&tx).await.expect("insert component"); + tx.commit().await.expect("commit tx"); + result +} + /// Verifies that the GET job details endpoint returns the correct state /// for a completed job. #[test_context(TrustifyContext)] #[test(actix_web::test)] async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { - // Given a completed job record in the database + // Given a completed job with a component record in the database let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-1234", ExploitIntelligenceJobStatus::Completed, - Some(exploit_intelligence_job::Finding::NotVulnerable), None, ) .await; + insert_test_component( + &ctx.db, + job_id, + "test-component", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::NotVulnerable), + ) + .await; let ei_service = test_service(); let db_ro = db::ReadOnly::new(ctx.db.clone()); @@ -122,11 +153,10 @@ async fn get_job_pending(ctx: &TrustifyContext) -> anyhow::Result<()> { insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-5678", ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; @@ -169,10 +199,9 @@ async fn get_job_failed(ctx: &TrustifyContext) -> anyhow::Result<()> { insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-9999", ExploitIntelligenceJobStatus::Failed, - None, Some("analysis timed out"), ) .await; @@ -244,11 +273,10 @@ async fn list_jobs_paginated(ctx: &TrustifyContext) -> anyhow::Result<()> { insert_test_job( &ctx.db, Uuid::now_v7(), - None, + Uuid::nil(), &format!("CVE-2024-{i:04}"), ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; } @@ -292,22 +320,20 @@ async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<() insert_test_job( &ctx.db, Uuid::now_v7(), - Some(target_sbom_id), + target_sbom_id, "CVE-2024-0001", ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; insert_test_job( &ctx.db, Uuid::now_v7(), - Some(other_sbom_id), + other_sbom_id, "CVE-2024-0002", ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; @@ -363,11 +389,10 @@ async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Resu insert_test_job( &ctx.db, existing_job_id, - Some(sbom_id), + sbom_id, "CVE-2024-DEDUP", ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; @@ -416,11 +441,10 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-TIMEOUT", ExploitIntelligenceJobStatus::Running, None, - None, ) .await; @@ -462,24 +486,30 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { Ok(()) } -/// Verifies that a completed job with a finding but no advisory link is -/// returned correctly -- simulating a "vulnerable" result where no VEX -/// document was generated by the EI service. +/// Verifies that a completed job with a vulnerable component finding is +/// returned correctly -- the aggregate finding is computed from components. #[test_context(TrustifyContext)] #[test(actix_web::test)] async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Result<()> { - // Given a completed job with a "vulnerable" finding but no advisory + // Given a completed job with a vulnerable component let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-VULN", ExploitIntelligenceJobStatus::Completed, - Some(exploit_intelligence_job::Finding::Vulnerable), None, ) .await; + insert_test_component( + &ctx.db, + job_id, + "vulnerable-component", + ExploitIntelligenceJobStatus::Completed, + Some(exploit_intelligence_job::Finding::Vulnerable), + ) + .await; let ei_service = test_service(); let db_ro = db::ReadOnly::new(ctx.db.clone()); @@ -500,13 +530,12 @@ async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Resu let resp = test::call_service(&app, req).await; - // Then the response shows the finding without an advisory link + // Then the response shows the aggregate finding computed from the component assert_eq!(resp.status(), 200); let body: ExploitIntelligenceJobDetails = test::read_body_json(resp).await; assert_eq!(body.id, job_id); assert_eq!(body.status, JobStatus::Completed); assert_eq!(body.finding, Some(JobFinding::Vulnerable)); - assert_eq!(body.advisory_id, None); assert_eq!(body.error_message, None); Ok(()) @@ -574,11 +603,10 @@ async fn get_job_running(ctx: &TrustifyContext) -> anyhow::Result<()> { insert_test_job( &ctx.db, job_id, - None, + Uuid::nil(), "CVE-2024-RUN", ExploitIntelligenceJobStatus::Running, None, - None, ) .await; @@ -621,10 +649,9 @@ async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> insert_test_job( &ctx.db, Uuid::now_v7(), - None, + Uuid::nil(), "CVE-2024-S001", ExploitIntelligenceJobStatus::Completed, - Some(exploit_intelligence_job::Finding::NotVulnerable), None, ) .await; @@ -632,21 +659,19 @@ async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> insert_test_job( &ctx.db, Uuid::now_v7(), - None, + Uuid::nil(), "CVE-2024-S002", ExploitIntelligenceJobStatus::Pending, None, - None, ) .await; insert_test_job( &ctx.db, Uuid::now_v7(), - None, + Uuid::nil(), "CVE-2024-S003", ExploitIntelligenceJobStatus::Completed, - Some(exploit_intelligence_job::Finding::Vulnerable), None, ) .await; diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index 930cabb51..74f320532 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -28,7 +28,7 @@ pub struct ExploitIntelligenceJobSummary { /// The unique job identifier. pub id: Uuid, /// FK to the SBOM being analysed. - pub sbom_id: Option, + pub sbom_id: Uuid, /// CVE identifier. pub vulnerability_id: String, /// Current lifecycle status of the job. @@ -65,17 +65,15 @@ pub struct ExploitIntelligenceJobDetails { /// The unique job identifier. pub id: Uuid, /// FK to the SBOM being analysed. - pub sbom_id: Option, + pub sbom_id: Uuid, /// CVE identifier. pub vulnerability_id: String, /// Current lifecycle status of the job. pub status: JobStatus, - /// Analysis result from Exploit Intelligence. + /// Analysis result computed from component findings. pub finding: Option, /// Link to the EI human-readable justification report. pub report_url: Option, - /// FK to advisory — set when a VEX document is ingested from the result. - pub advisory_id: Option, /// Error details for failed analyses. pub error_message: Option, /// EI product ID for multi-component SPDX flow. @@ -102,15 +100,13 @@ pub struct ExploitIntelligenceJobDetails { pub updated: OffsetDateTime, } -/// Per-component analysis result within a multi-component product job. +/// Per-component analysis result within an analysis job. #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ComponentResult { /// The unique component record identifier. pub id: Uuid, /// Identifier of the component (purl or container image reference). pub component_ref: String, - /// Human-readable name of the component. - pub component_name: String, /// Current lifecycle status of this component's analysis. pub status: JobStatus, /// Analysis result from Exploit Intelligence, if completed. @@ -189,14 +185,30 @@ impl From for ExploitIntelligenceJobStatus { impl ComponentResult { /// Create a component result from an entity model. - pub fn from_entity(model: &trustify_entity::exploit_intelligence_job_component::Model) -> Self { + /// + /// `ui_url` and `vulnerability_id` are used to compute the report URL + /// for completed components with a scan ID. + pub fn from_entity( + model: &trustify_entity::exploit_intelligence_job_component::Model, + ui_url: Option<&str>, + vulnerability_id: &str, + ) -> Self { + let report_url = ui_url.and_then(|base| { + model.scan_id.as_ref().map(|scan_id| { + format!( + "{}/reports/component/{}/{}", + base, + urlencoding::encode(vulnerability_id), + urlencoding::encode(scan_id), + ) + }) + }); Self { id: model.id, component_ref: model.component_ref.clone(), - component_name: model.component_name.clone(), status: model.status.into(), finding: model.finding.map(JobFinding::from), - report_url: model.report_url.clone(), + report_url, advisory_id: model.advisory_id, error_message: model.error_message.clone(), created: model.created, @@ -205,16 +217,57 @@ impl ComponentResult { } } +/// Compute the EI report URL from a job's identifiers and the UI base URL. +/// +/// For multi-component jobs (with `product_id`), returns the product-level URL. +/// For single-component jobs, returns `None` — the component's own report URL +/// (in [`ComponentResult`]) provides the link instead. +/// Compute the EI report URL from a job's identifiers and the UI base URL. +/// +/// For multi-component (SPDX) jobs with a `product_id`, builds a product URL. +/// For single-component (CycloneDX) jobs, uses the component's `scan_id` +/// passed via `first_scan_id` to build a component URL. +fn compute_job_report_url( + model: &trustify_entity::exploit_intelligence_job::Model, + ui_url: Option<&str>, + first_scan_id: Option<&str>, +) -> Option { + let base = ui_url?; + if let Some(ref product_id) = model.product_id { + Some(format!( + "{}/reports/product/{}/{}", + base, + urlencoding::encode(product_id), + urlencoding::encode(&model.vulnerability_id), + )) + } else { + first_scan_id.map(|scan_id| { + format!( + "{}/reports/component/{}/{}", + base, + urlencoding::encode(&model.vulnerability_id), + urlencoding::encode(scan_id), + ) + }) + } +} + impl ExploitIntelligenceJobSummary { /// Create a summary from an entity model. - pub fn from_entity(model: &trustify_entity::exploit_intelligence_job::Model) -> Self { + /// + /// `ui_url` is used to compute the report URL at response time. + /// Finding is not available without component data and will be `None`. + pub fn from_entity( + model: &trustify_entity::exploit_intelligence_job::Model, + ui_url: Option<&str>, + ) -> Self { Self { id: model.id, sbom_id: model.sbom_id, vulnerability_id: model.vulnerability_id.clone(), status: model.status.into(), - finding: model.finding.map(JobFinding::from), - report_url: model.report_url.clone(), + finding: None, + report_url: compute_job_report_url(model, ui_url, None), product_id: model.product_id.clone(), total_components: model.total_components, completed_components: None, @@ -228,35 +281,61 @@ impl ExploitIntelligenceJobSummary { } /// Create a summary from an entity model with pre-computed component counts. + /// + /// The aggregate finding is computed from component count data: + /// - If any components are vulnerable, the finding is `Vulnerable`. + /// - If all completed components are not vulnerable, the finding is `NotVulnerable`. + /// - Otherwise, the finding is `Uncertain`. pub fn from_entity_with_counts( model: &trustify_entity::exploit_intelligence_job::Model, + ui_url: Option<&str>, completed_components: Option, failed_components: Option, vulnerable_components: Option, not_vulnerable_components: Option, uncertain_components: Option, ) -> Self { - let mut summary = Self::from_entity(model); + let mut summary = Self::from_entity(model, ui_url); summary.completed_components = completed_components; summary.failed_components = failed_components; summary.vulnerable_components = vulnerable_components; summary.not_vulnerable_components = not_vulnerable_components; summary.uncertain_components = uncertain_components; + + // Compute aggregate finding from component counts. + let vuln = vulnerable_components.unwrap_or(0); + let not_vuln = not_vulnerable_components.unwrap_or(0); + let completed = completed_components.unwrap_or(0); + + if vuln > 0 { + summary.finding = Some(JobFinding::Vulnerable); + } else if completed > 0 && not_vuln == completed { + summary.finding = Some(JobFinding::NotVulnerable); + } else if completed > 0 { + summary.finding = Some(JobFinding::Uncertain); + } + // else: no completed components yet, finding stays None + summary } } impl ExploitIntelligenceJobDetails { /// Create a detailed view from an entity model. - pub fn from_entity(model: &trustify_entity::exploit_intelligence_job::Model) -> Self { + /// + /// `ui_url` is used to compute the report URL at response time. + /// Finding is not available without component data and will be `None`. + pub fn from_entity( + model: &trustify_entity::exploit_intelligence_job::Model, + ui_url: Option<&str>, + ) -> Self { Self { id: model.id, sbom_id: model.sbom_id, vulnerability_id: model.vulnerability_id.clone(), status: model.status.into(), - finding: model.finding.map(JobFinding::from), - report_url: model.report_url.clone(), - advisory_id: model.advisory_id, + finding: None, + report_url: compute_job_report_url(model, ui_url, None), error_message: model.error_message.clone(), product_id: model.product_id.clone(), total_components: model.total_components, @@ -272,14 +351,24 @@ impl ExploitIntelligenceJobDetails { } /// Create a detailed view from an entity model with component results. + /// + /// The aggregate finding is computed from the component models. pub fn from_entity_with_components( model: &trustify_entity::exploit_intelligence_job::Model, component_models: &[trustify_entity::exploit_intelligence_job_component::Model], + ui_url: Option<&str>, ) -> Self { - let mut details = Self::from_entity(model); + let first_scan_id = component_models + .first() + .and_then(|c| c.scan_id.as_deref()); + let mut details = Self { + report_url: compute_job_report_url(model, ui_url, first_scan_id), + ..Self::from_entity(model, ui_url) + }; + let vuln_id = &model.vulnerability_id; details.components = component_models .iter() - .map(ComponentResult::from_entity) + .map(|c| ComponentResult::from_entity(c, ui_url, vuln_id)) .collect(); let completed = component_models .iter() @@ -306,6 +395,16 @@ impl ExploitIntelligenceJobDetails { details.vulnerable_components = Some(vulnerable); details.not_vulnerable_components = Some(not_vulnerable); details.uncertain_components = Some(uncertain); + + // Compute aggregate finding from components. + if vulnerable > 0 { + details.finding = Some(JobFinding::Vulnerable); + } else if completed > 0 && not_vulnerable == completed { + details.finding = Some(JobFinding::NotVulnerable); + } else if completed > 0 { + details.finding = Some(JobFinding::Uncertain); + } + details } } diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index b9b34b59c..a823a71b4 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -95,6 +95,11 @@ impl ExploitIntelligenceService { self.runtime.as_ref() } + /// Returns the EI web UI base URL, if configured. + pub fn ui_url(&self) -> Option<&str> { + self.runtime.as_ref().and_then(|r| r.config.ui_url.as_deref()) + } + /// Create an analysis job record in the database. /// /// Returns the created job details. The caller is responsible for @@ -117,14 +122,9 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), - scan_id: Set(None), - sbom_id: Set(Some(sbom_id)), + sbom_id: Set(sbom_id), vulnerability_id: Set(vulnerability_id.to_string()), status: Set(ExploitIntelligenceJobStatus::Pending), - source_url: Set(None), - report_url: Set(None), - advisory_id: Set(None), - finding: Set(None), error_message: Set(None), product_id: Set(None), total_components: Set(None), @@ -134,13 +134,14 @@ impl ExploitIntelligenceService { let result = model.insert(connection).await.map_err(Error::from)?; - Ok(ExploitIntelligenceJobDetails::from_entity(&result)) + Ok(ExploitIntelligenceJobDetails::from_entity(&result, self.ui_url())) } /// Fetch details of a single job by its ID. /// - /// For multi-component jobs (those with a `product_id`), component records - /// are loaded automatically and component counts are computed. + /// Component records are always loaded so that the aggregate finding and + /// component counts can be computed. Every job (including single-component + /// CycloneDX jobs) now has at least one component row. #[instrument(skip(self, connection), err(level = tracing::Level::INFO))] pub async fn fetch_job( &self, @@ -155,11 +156,11 @@ impl ExploitIntelligenceService { match result { Some(model) => { - if model.product_id.is_some() { - let components = self.fetch_components(model.id, connection).await?; - Ok(Some(ExploitIntelligenceJobDetails::from_entity_with_components(&model, &components))) + let components = self.fetch_components(model.id, connection).await?; + if components.is_empty() { + Ok(Some(ExploitIntelligenceJobDetails::from_entity(&model, self.ui_url()))) } else { - Ok(Some(ExploitIntelligenceJobDetails::from_entity(&model))) + Ok(Some(ExploitIntelligenceJobDetails::from_entity_with_components(&model, &components, self.ui_url()))) } } None => Ok(None), @@ -214,7 +215,7 @@ impl ExploitIntelligenceService { let mut counts_map: HashMap = HashMap::new(); if !job_ids.is_empty() { - let rows: Vec<(Uuid, i32, Option, i64)> = + let rows: Vec<(Uuid, ExploitIntelligenceJobStatus, Option, i64)> = exploit_intelligence_job_component::Entity::find() .select_only() .column(exploit_intelligence_job_component::Column::JobId) @@ -235,41 +236,86 @@ impl ExploitIntelligenceService { let entry = counts_map.entry(job_id).or_insert((0, 0, 0, 0, 0, 0)); let count = count as i32; entry.2 += count; // total - if status == ExploitIntelligenceJobStatus::Completed as i32 { + if status == ExploitIntelligenceJobStatus::Completed { entry.0 += count; // completed match finding { - Some(v) if v == Finding::Vulnerable as i32 => entry.3 += count, - Some(v) if v == Finding::NotVulnerable as i32 => entry.4 += count, - Some(v) if v == Finding::Uncertain as i32 => entry.5 += count, + Some(Finding::Vulnerable) => entry.3 += count, + Some(Finding::NotVulnerable) => entry.4 += count, + Some(Finding::Uncertain) => entry.5 += count, _ => {} } - } else if status == ExploitIntelligenceJobStatus::Failed as i32 { + } else if status == ExploitIntelligenceJobStatus::Failed { entry.1 += count; // failed } } } + // Fetch the first scan_id per job for report URL computation. + // Single-component jobs need this since scan_id lives on the component. + let job_ids_for_scan: Vec = results + .iter() + .filter(|j| j.product_id.is_none()) + .map(|j| j.id) + .collect(); + let mut scan_id_map: HashMap = HashMap::new(); + if !job_ids_for_scan.is_empty() { + let scan_rows: Vec<(Uuid, Option)> = + exploit_intelligence_job_component::Entity::find() + .select_only() + .column(exploit_intelligence_job_component::Column::JobId) + .column(exploit_intelligence_job_component::Column::ScanId) + .filter(exploit_intelligence_job_component::Column::JobId.is_in(job_ids_for_scan)) + .filter(exploit_intelligence_job_component::Column::ScanId.is_not_null()) + .into_tuple() + .all(connection) + .instrument(info_span!("fetch component scan_ids")) + .await + .map_err(Error::from)?; + for (job_id, scan_id) in scan_rows { + if let Some(sid) = scan_id { + scan_id_map.entry(job_id).or_insert(sid); + } + } + } + let items = results .iter() .map(|model| { - if let Some(&(completed, failed, total, vulnerable, not_vulnerable, uncertain)) = + let mut summary = if let Some(&(completed, failed, total, vulnerable, not_vulnerable, uncertain)) = counts_map.get(&model.id) { - let mut summary = ExploitIntelligenceJobSummary::from_entity_with_counts( + let mut s = ExploitIntelligenceJobSummary::from_entity_with_counts( model, + self.ui_url(), Some(completed), Some(failed), Some(vulnerable), Some(not_vulnerable), Some(uncertain), ); - if summary.total_components.is_none() { - summary.total_components = Some(total); + if s.total_components.is_none() { + s.total_components = Some(total); } - summary + s } else { - ExploitIntelligenceJobSummary::from_entity(model) + ExploitIntelligenceJobSummary::from_entity(model, self.ui_url()) + }; + + // Set report_url from component scan_id for single-component jobs. + if model.product_id.is_none() { + if let Some(scan_id) = scan_id_map.get(&model.id) { + if let Some(base) = self.ui_url() { + summary.report_url = Some(format!( + "{}/reports/component/{}/{}", + base, + urlencoding::encode(&model.vulnerability_id), + urlencoding::encode(scan_id), + )); + } + } } + + summary }) .collect(); @@ -279,38 +325,11 @@ impl ExploitIntelligenceService { }) } - /// Update a job's status to running and store the scan ID from the EI service. - #[instrument(skip_all, err(level = tracing::Level::INFO))] - pub async fn update_job_running( - &self, - job_id: Uuid, - scan_id: &str, - db: &db::ReadWrite, - ) -> Result<(), Error> { - let tx = db.begin().await?; - let now = OffsetDateTime::now_utc(); - - let model = exploit_intelligence_job::ActiveModel { - id: Set(job_id), - scan_id: Set(Some(scan_id.to_string())), - status: Set(ExploitIntelligenceJobStatus::Running), - updated: Set(now), - ..Default::default() - }; - - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; - Ok(()) - } - - /// Update a job to completed with the analysis finding and optional advisory reference. + /// Update a job to completed. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_completed( &self, job_id: Uuid, - finding: exploit_intelligence_job::Finding, - report_url: Option, - advisory_id: Option, db: &db::ReadWrite, ) -> Result<(), Error> { let tx = db.begin().await?; @@ -319,9 +338,6 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), status: Set(ExploitIntelligenceJobStatus::Completed), - finding: Set(Some(finding)), - report_url: Set(report_url), - advisory_id: Set(advisory_id), updated: Set(now), ..Default::default() }; @@ -379,7 +395,7 @@ impl ExploitIntelligenceService { .await .map_err(Error::from)?; - Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m))) + Ok(result.map(|m| ExploitIntelligenceJobDetails::from_entity(&m, self.ui_url()))) } /// Create an analysis job with product_id and total_components set (multi-component SPDX flow). @@ -401,14 +417,9 @@ impl ExploitIntelligenceService { let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), - scan_id: Set(None), - sbom_id: Set(Some(sbom_id)), + sbom_id: Set(sbom_id), vulnerability_id: Set(vulnerability_id.to_string()), status: Set(ExploitIntelligenceJobStatus::Pending), - source_url: Set(None), - report_url: Set(None), - advisory_id: Set(None), - finding: Set(None), error_message: Set(None), product_id: Set(Some(product_id.to_string())), total_components: Set(Some(total_components)), @@ -420,16 +431,15 @@ impl ExploitIntelligenceService { let result = model.insert(&tx).await.map_err(Error::from)?; tx.commit().await?; - Ok(ExploitIntelligenceJobDetails::from_entity(&result)) + Ok(ExploitIntelligenceJobDetails::from_entity(&result, self.ui_url())) } - /// Create a component record for a multi-component product job. + /// Create a component record for an analysis job. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn create_component( &self, job_id: Uuid, component_ref: &str, - component_name: &str, db: &db::ReadWrite, ) -> Result { use trustify_entity::exploit_intelligence_job_component; @@ -441,11 +451,9 @@ impl ExploitIntelligenceService { id: Set(component_id), job_id: Set(job_id), component_ref: Set(component_ref.to_string()), - component_name: Set(component_name.to_string()), scan_id: Set(None), status: Set(ExploitIntelligenceJobStatus::Pending), finding: Set(None), - report_url: Set(None), advisory_id: Set(None), error_message: Set(None), created: Set(now), @@ -467,7 +475,6 @@ impl ExploitIntelligenceService { &self, job_id: Uuid, component_ref: &str, - component_name: &str, error_message: &str, db: &db::ReadWrite, ) -> Result { @@ -480,11 +487,9 @@ impl ExploitIntelligenceService { id: Set(component_id), job_id: Set(job_id), component_ref: Set(component_ref.to_string()), - component_name: Set(component_name.to_string()), scan_id: Set(None), status: Set(ExploitIntelligenceJobStatus::Failed), finding: Set(None), - report_url: Set(None), advisory_id: Set(None), error_message: Set(Some(error_message.to_string())), created: Set(now), @@ -530,7 +535,6 @@ impl ExploitIntelligenceService { &self, component_id: Uuid, finding: exploit_intelligence_job::Finding, - report_url: Option, advisory_id: Option, db: &db::ReadWrite, ) -> Result<(), Error> { @@ -543,7 +547,6 @@ impl ExploitIntelligenceService { id: Set(component_id), status: Set(ExploitIntelligenceJobStatus::Completed), finding: Set(Some(finding)), - report_url: Set(report_url), advisory_id: Set(advisory_id), updated: Set(now), ..Default::default() From d45f56c8f48eeb0f80af00a8619c3e59277abe83 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 9 Jul 2026 15:40:57 +0100 Subject: [PATCH 15/25] refactor(exploit-intelligence): use backon crate for retry and polling Replace custom common::retry module with the backon crate, which is already a transitive dependency via walker-common. Upload retries use backon's ExponentialBuilder directly, and polling uses ConstantBuilder with a Cell-based consecutive failure counter in the .when() predicate. Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 1 + Cargo.toml | 1 + modules/fundamental/Cargo.toml | 1 + .../src/exploit_intelligence/endpoints/mod.rs | 1175 ++++++++--------- 4 files changed, 559 insertions(+), 619 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f4ad7304..4182c5061 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8511,6 +8511,7 @@ dependencies = [ "actix-web", "anyhow", "async-trait", + "backon", "bytes", "bytesize", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 47c750262..3491c796f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ aws-config = { version = "1.8.14", features = ["behavior-version-latest"] } aws-sdk-s3 = { version = "1.124.0", default-features = false, features = ["behavior-version-latest", "default-https-client", "http-1x", "rt-tokio", "sigv4a"] } aws-smithy-http-client = { version = "1.2.0", features = ["rustls-aws-lc"] } aws-smithy-types = { version = "1.4.5" } +backon = "1.6" base16ct = "1" base64 = "0.22" build-info = "0.0.44" diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 9c6296073..51f726715 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -21,6 +21,7 @@ trustify-query-derive = { workspace = true } actix-http = { workspace = true } actix-web = { workspace = true } +backon = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } cpe = { workspace = true } diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index a1403ca77..59c41aeaa 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -13,7 +13,6 @@ use crate::{ }; use actix_web::{HttpResponse, Responder, get, post, web}; use futures_util::TryStreamExt; -use reqwest::StatusCode; use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use std::str::FromStr; use trustify_auth::{CreateExploitIntelligence, ReadExploitIntelligence, authorizer::Require}; @@ -21,50 +20,72 @@ use trustify_common::{ db, model::{Paginated, PaginatedResults}, }; +use backon::{ExponentialBuilder, ConstantBuilder, Retryable}; +use std::cell::Cell; use trustify_entity::labels::Labels; use trustify_module_ingestor::service::{Cache, Format, IngestorService}; use trustify_module_storage::service::StorageBackend; use uuid::Uuid; -// --------------------------------------------------------------------------- -// Retry helpers for HTTP requests to the EI service -// --------------------------------------------------------------------------- - -/// Retry categories for HTTP responses. -#[allow(dead_code)] -enum RetryDecision { - /// Success -- return the response. - Success, - /// Transient failure -- retry after backoff. +/// Error classification for EI HTTP requests. +enum EiRequestError { + /// Retryable failure (network timeout, 5xx, etc.) Transient(String), - /// Permanent failure -- abort immediately. - Permanent(String), + /// Fatal failure (auth error, 4xx, parse error) + Permanent(anyhow::Error), } -/// Classify an HTTP status code as success, transient, or permanent. -fn classify_status(status: StatusCode) -> RetryDecision { - if status.is_success() { - return RetryDecision::Success; +impl EiRequestError { + fn is_transient(&self) -> bool { + matches!(self, Self::Transient(_)) } - match status.as_u16() { - // Transient -- worth retrying. - 408 | 429 | 502 | 503 | 504 => { - RetryDecision::Transient(format!("transient HTTP error: {status}")) + fn into_anyhow(self) -> anyhow::Error { + match self { + Self::Transient(msg) => anyhow::anyhow!(msg), + Self::Permanent(err) => err, } - // Permanent -- no point retrying. - _ => RetryDecision::Permanent(format!("HTTP error: {status}")), } } -/// Returns `true` when a `reqwest::Error` looks transient (connection, -/// timeout, or request-building problem on the network layer). -fn is_transient_network_error(err: &reqwest::Error) -> bool { - err.is_connect() || err.is_timeout() || err.is_request() +impl std::fmt::Display for EiRequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Transient(msg) => write!(f, "transient: {msg}"), + Self::Permanent(err) => write!(f, "permanent: {err}"), + } + } } -/// Maximum number of consecutive transient poll failures before we give up. -const MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES: u32 = 5; +/// Classify an HTTP response or reqwest error into an EiRequestError. +fn classify_response_error( + result: Result, +) -> Result { + match result { + Ok(resp) => { + let status = resp.status(); + if status.is_success() { + Ok(resp) + } else { + match status.as_u16() { + 408 | 429 | 502 | 503 | 504 => Err(EiRequestError::Transient(format!( + "transient HTTP status {status}" + ))), + _ => Err(EiRequestError::Permanent(anyhow::anyhow!("HTTP {status}"))), + } + } + } + Err(err) => { + if err.is_connect() || err.is_timeout() || err.is_request() { + Err(EiRequestError::Transient(format!("network error: {err}"))) + } else { + Err(EiRequestError::Permanent(anyhow::anyhow!( + "network error: {err}" + ))) + } + } + } +} /// Register exploit intelligence endpoints and spawn recovery for orphaned jobs. pub fn configure( @@ -440,81 +461,44 @@ async fn run_analysis( // SPDX multi-component flow: upload to /api/v1/products/upload-spdx let upload_url = format!("{}/api/v1/products/upload-spdx", config.url); - const UPLOAD_MAX_ATTEMPTS: u32 = 3; - let mut upload_response: Option = None; - - for attempt in 0..UPLOAD_MAX_ATTEMPTS { - let form = reqwest::multipart::Form::new() - .text("cveId", vulnerability_id.to_string()) - .part( - "file", - reqwest::multipart::Part::bytes(sbom_bytes.clone()) - .file_name("sbom.json") - .mime_str("application/json")?, - ); + let response = (|| { + let url = upload_url.clone(); + let sbom = sbom_bytes.clone(); + let vuln_id = vulnerability_id.to_string(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let form = reqwest::multipart::Form::new() + .text("cveId", vuln_id) + .part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); - let mut request = client.post(&upload_url).multipart(form); + let mut request = cli.post(&url).multipart(form); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - request = request.bearer_auth(&token); - } - - match request.send().await { - Ok(resp) => { - let status = resp.status(); - match classify_status(status) { - RetryDecision::Success => { - upload_response = Some(resp); - break; - } - RetryDecision::Transient(reason) => { - if attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - reason = %reason, - "SPDX upload to EI client failed with transient error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI SPDX upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" - )); - } - } - RetryDecision::Permanent(_) => { - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("EI client returned {status}: {body}")); - } - } - } - Err(err) => { - if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - error = %err, - "SPDX upload to EI client failed with network error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { - return Err(anyhow::anyhow!("EI upload network error: {err}")); - } + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + request = request.bearer_auth(&token); } - } - } - let response = upload_response - .ok_or_else(|| anyhow::anyhow!("EI SPDX upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; + classify_response_error(request.send().await) + } + }) + .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .when(|e| e.is_transient()) + .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) + .await + .map_err(EiRequestError::into_anyhow)?; let spdx_upload: crate::exploit_intelligence::ei_client::SpdxUploadResponse = response.json().await?; @@ -555,82 +539,44 @@ async fn run_analysis( let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); - const UPLOAD_MAX_ATTEMPTS: u32 = 3; - let mut upload_response: Option = None; - - for attempt in 0..UPLOAD_MAX_ATTEMPTS { - // Build a fresh multipart form for each attempt (forms are consumed by send). - let form = reqwest::multipart::Form::new() - .text("cveId", vulnerability_id.to_string()) - .part( - "file", - reqwest::multipart::Part::bytes(sbom_bytes.clone()) - .file_name("sbom.json") - .mime_str("application/json")?, - ); + let response = (|| { + let url = upload_url.clone(); + let sbom = sbom_bytes.clone(); + let vuln_id = vulnerability_id.to_string(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let form = reqwest::multipart::Form::new() + .text("cveId", vuln_id) + .part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); - let mut request = client.post(&upload_url).multipart(form); + let mut request = cli.post(&url).multipart(form); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - request = request.bearer_auth(&token); - } - - match request.send().await { - Ok(resp) => { - let status = resp.status(); - match classify_status(status) { - RetryDecision::Success => { - upload_response = Some(resp); - break; - } - RetryDecision::Transient(reason) => { - if attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - reason = %reason, - "upload to EI client failed with transient error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts: {status}: {body}" - )); - } - } - RetryDecision::Permanent(_) => { - let body = resp.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!("EI client returned {status}: {body}")); - } - } - } - Err(err) => { - if is_transient_network_error(&err) && attempt + 1 < UPLOAD_MAX_ATTEMPTS { - let backoff = std::time::Duration::from_secs(1 << attempt); - tracing::warn!( - job_id = %job_id, - attempt = attempt + 1, - backoff_secs = backoff.as_secs(), - error = %err, - "upload to EI client failed with network error, retrying" - ); - tokio::time::sleep(backoff).await; - } else { - return Err(anyhow::anyhow!("EI upload network error: {err}")); - } + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + request = request.bearer_auth(&token); } - } - } - let response = upload_response - .ok_or_else(|| anyhow::anyhow!("EI upload failed after {UPLOAD_MAX_ATTEMPTS} attempts"))?; + classify_response_error(request.send().await) + } + }) + .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .when(|e| e.is_transient()) + .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) + .await + .map_err(EiRequestError::into_anyhow)?; let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; let scan_id = upload.report_request_id.report_id; @@ -683,195 +629,189 @@ pub(crate) async fn poll_for_result( urlencoding::encode(scan_id) ); let poll_secs = config.poll_interval_secs.max(5); - let poll_interval = std::time::Duration::from_secs(poll_secs); - let max_polls = config.max_poll_duration_secs / poll_secs; - let mut poll_count: u64 = 0; - let mut consecutive_transient_failures: u32 = 0; - - loop { - tokio::time::sleep(poll_interval).await; - - poll_count += 1; - if poll_count > max_polls { - let msg = format!( - "analysis timed out after {}s", - config.max_poll_duration_secs - ); + let consecutive_failures = Cell::new(0u32); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(config.max_poll_duration_secs), + (|| { + let url = poll_url.clone(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let mut poll_request = cli.get(&url); + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + poll_request = poll_request.bearer_auth(&token); + } + + let resp = classify_response_error(poll_request.send().await)?; + + let poll: crate::exploit_intelligence::ei_client::PollResponse = + resp.json().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to parse EI poll response: {e}" + )) + })?; + + match poll.status.as_str() { + "completed" | "failed" | "expired" => Ok(poll), + _ => { + tracing::debug!( + status = poll.status, + "exploit intelligence analysis still in progress" + ); + Err(EiRequestError::Transient("pending".to_string())) + } + } + } + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(poll_secs)) + .without_max_times(), + ) + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < 5 + } + EiRequestError::Permanent(_) => false, + }) + .notify(|e, dur| { + if !matches!(e, EiRequestError::Transient(r) if r == "pending") { + tracing::warn!(?dur, %e, "transient poll failure"); + } + }), + ) + .await; + + // Convert timeout/retry result to our expected shape. + let poll = match result { + Ok(Ok(poll)) => poll, + Ok(Err(e)) => { + let err = e.into_anyhow(); + let msg = err.to_string(); ei_service .update_job_failed(job_id, &msg, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Err(anyhow::anyhow!("{msg}")); + return Err(err); } - - let mut poll_request = client.get(&poll_url); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() + Err(_elapsed) => { + let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); + ei_service + .update_job_failed(job_id, &msg, db_rw) .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - poll_request = poll_request.bearer_auth(&token); + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(anyhow::anyhow!("{msg}")); } + }; - let poll_response = match poll_request.send().await { - Ok(resp) => resp, - Err(err) => { - // Network-level error during polling. - tracing::warn!( - job_id = %job_id, - error = %err, - "network error while polling EI service" - ); - consecutive_transient_failures += 1; - if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { - return Err(anyhow::anyhow!( - "EI service unreachable: {consecutive_transient_failures} consecutive poll failures" - )); - } - continue; - } - }; - - if !poll_response.status().is_success() { - let status = poll_response.status(); - match classify_status(status) { - RetryDecision::Success => { - // Not reachable (is_success() was false), but satisfy match. - unreachable!(); - } - RetryDecision::Transient(reason) => { - consecutive_transient_failures += 1; - tracing::debug!( + // Process the terminal poll response. + match poll.status.as_str() { + "completed" => { + // Extract finding from the analysis result. + use trustify_entity::exploit_intelligence_job::Finding; + let finding = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.analysis.as_ref()) + .and_then(|a| a.first()) + .and_then(|e| e.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s.to_ascii_lowercase().as_str() { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + // Check if VEX document is available. + let vex_json = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + let advisory_id = if let Some(vex) = vex_json { + if !vex.is_null() { + // Ingest the VEX document as an advisory with source label. + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest(&vex_bytes, Format::Advisory, labels, None, Cache::Skip, &tx) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( job_id = %job_id, - reason = %reason, - consecutive_failures = consecutive_transient_failures, - "transient error while polling EI service, will retry" + advisory_id = %result.id, + "ingested VEX advisory from exploit intelligence" ); - if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { - return Err(anyhow::anyhow!( - "EI service unhealthy: {consecutive_transient_failures} consecutive transient failures ({status})" - )); - } - continue; - } - RetryDecision::Permanent(_) => { - let body = poll_response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI poll returned permanent error {status}: {body}" - )); - } - } - } - - // Successful response -- reset consecutive failure counter. - consecutive_transient_failures = 0; - - let poll: crate::exploit_intelligence::ei_client::PollResponse = - poll_response.json().await?; - - match poll.status.as_str() { - "completed" => { - // Extract finding from the analysis result. - use trustify_entity::exploit_intelligence_job::Finding; - let finding = poll - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.analysis.as_ref()) - .and_then(|a| a.first()) - .and_then(|e| e.justification.as_ref()) - .and_then(|j| j.status.as_deref()) - .map(|s| match s.to_ascii_lowercase().as_str() { - "false" => Finding::NotVulnerable, - "true" => Finding::Vulnerable, - _ => Finding::Uncertain, - }) - .unwrap_or(Finding::Uncertain); - // Check if VEX document is available. - let vex_json = poll - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.vex.as_ref()); - - let advisory_id = if let Some(vex) = vex_json { - if !vex.is_null() { - // Ingest the VEX document as an advisory with source label. - let vex_bytes = serde_json::to_vec(vex)?; - let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); - - let tx = db_rw.begin().await?; - let result = ingestor - .ingest(&vex_bytes, Format::Advisory, labels, None, Cache::Skip, &tx) - .await - .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; - tx.commit().await?; - - tracing::info!( - job_id = %job_id, - advisory_id = %result.id, - "ingested VEX advisory from exploit intelligence" - ); - - result - .id - .strip_prefix("urn:uuid:") - .and_then(|id| Uuid::from_str(id).ok()) - } else { - None - } + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) } else { None - }; + } + } else { + None + }; - // Update the component with the per-analysis results. - ei_service - .update_component_completed(component_id, finding, advisory_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + // Update the component with the per-analysis results. + ei_service + .update_component_completed(component_id, finding, advisory_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - // Mark the parent job as completed (finding lives on the component). - ei_service - .update_job_completed(job_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + // Mark the parent job as completed (finding lives on the component). + ei_service + .update_job_completed(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - tracing::info!( - job_id = %job_id, - finding = ?finding, - "exploit intelligence analysis completed" - ); + tracing::info!( + job_id = %job_id, + finding = ?finding, + "exploit intelligence analysis completed" + ); - return Ok(()); - } - "failed" | "expired" => { - let error_msg = poll - .report - .as_ref() - .and_then(|r| r.error.as_deref()) - .unwrap_or("analysis failed") - .to_string(); + Ok(()) + } + "failed" | "expired" => { + let error_msg = poll + .report + .as_ref() + .and_then(|r| r.error.as_deref()) + .unwrap_or("analysis failed") + .to_string(); - ei_service - .update_job_failed(job_id, &error_msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + ei_service + .update_job_failed(job_id, &error_msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Ok(()); - } - _ => { - // Still in progress (queued, sent, pending), continue polling. - tracing::debug!( - job_id = %job_id, - status = poll.status, - "exploit intelligence analysis still in progress" - ); - } + Ok(()) } + _ => unreachable!("backon retry only returns terminal states"), } } @@ -896,10 +836,8 @@ pub(crate) async fn poll_for_product_result( urlencoding::encode(product_id) ); let poll_secs = config.poll_interval_secs.max(5); - let poll_interval = std::time::Duration::from_secs(poll_secs); - let max_polls = config.max_poll_duration_secs / poll_secs; - let mut poll_count: u64 = 0; - let mut consecutive_transient_failures: u32 = 0; + let consecutive_failures = Cell::new(0u32); + let mut components_initialized = false; // Maps EI scan_id -> our internal component_id (populated on first successful poll). @@ -927,318 +865,317 @@ pub(crate) async fn poll_for_product_result( ); } - loop { - tokio::time::sleep(poll_interval).await; + let result = tokio::time::timeout( + std::time::Duration::from_secs(config.max_poll_duration_secs), + (|| { + let url = product_url.clone(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let mut poll_request = cli.get(&url); + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + poll_request = poll_request.bearer_auth(&token); + } - poll_count += 1; - if poll_count > max_polls { - let msg = format!( - "SPDX product analysis timed out after {}s", - config.max_poll_duration_secs - ); + let resp = classify_response_error(poll_request.send().await)?; + + let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = + resp.json().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to parse EI product response: {e}" + )) + })?; + + match product.summary.product_state.as_str() { + "completed" | "failed" => Ok(product), + _ => { + tracing::debug!( + state = product.summary.product_state, + "SPDX product analysis still in progress" + ); + Err(EiRequestError::Transient("pending".to_string())) + } + } + } + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(poll_secs)) + .without_max_times(), + ) + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < 5 + } + EiRequestError::Permanent(_) => false, + }) + .notify(|e, dur| { + if !matches!(e, EiRequestError::Transient(r) if r == "pending") { + tracing::warn!(?dur, %e, "transient poll failure"); + } + }), + ) + .await; + + // Convert timeout/retry result to our expected shape. + let product = match result { + Ok(Ok(product)) => product, + Ok(Err(e)) => { + let err = e.into_anyhow(); + let msg = err.to_string(); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(err); + } + Err(_elapsed) => { + let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); ei_service .update_job_failed(job_id, &msg, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; return Err(anyhow::anyhow!("{msg}")); } + }; + + // Process the terminal product response. + match product.summary.product_state.as_str() { + "completed" => { + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed, fetching per-component reports" + ); + + // The EI product endpoint does not return per-component + // data (its `components` array is always empty). Instead, + // fetch the reports list filtered by product ID to discover + // each component's scan ID and justification. + let reports_url = format!( + "{}/api/v1/reports?productId={}", + config.url, + urlencoding::encode(product_id) + ); + + let mut reports_request = client.get(&reports_url); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + reports_request = reports_request.bearer_auth(&token); + } - // Poll the product endpoint. - let mut poll_request = client.get(&product_url); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() + let reports_response = reports_request + .send() .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - poll_request = poll_request.bearer_auth(&token); - } + .map_err(|e| anyhow::anyhow!("failed to fetch product reports: {e}"))?; + + if !reports_response.status().is_success() { + let status = reports_response.status(); + let body = reports_response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI product reports request failed ({status}): {body}" + )); + } - let poll_response = match poll_request.send().await { - Ok(resp) => resp, - Err(err) => { - tracing::warn!( - job_id = %job_id, - error = %err, - "network error while polling EI product" - ); - consecutive_transient_failures += 1; - if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { - return Err(anyhow::anyhow!( - "EI service unreachable: {consecutive_transient_failures} consecutive product poll failures" - )); + let component_reports: Vec = + reports_response.json().await?; + + // Record submission failures as excluded components. + if !components_initialized { + for failure in &product.data.submission_failures { + let image = failure.image.as_deref().unwrap_or("unknown"); + let error = failure.error.as_deref().unwrap_or("excluded by EI"); + + ei_service + .create_excluded_component(job_id, image, error, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; } - continue; } - }; - - if !poll_response.status().is_success() { - let status = poll_response.status(); - match classify_status(status) { - RetryDecision::Success => unreachable!(), - RetryDecision::Transient(reason) => { - consecutive_transient_failures += 1; - tracing::debug!( - job_id = %job_id, - reason = %reason, - consecutive_failures = consecutive_transient_failures, - "transient error while polling EI product, will retry" - ); - if consecutive_transient_failures > MAX_CONSECUTIVE_TRANSIENT_POLL_FAILURES { - return Err(anyhow::anyhow!( - "EI service unhealthy: {consecutive_transient_failures} consecutive transient failures ({status})" - )); + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + reports = component_reports.len(), + excluded = product.data.submission_failures.len(), + "processing per-component reports for SPDX product" + ); + + use trustify_entity::exploit_intelligence_job::Finding; + + for entry in &component_reports { + let scan_id = &entry.scan_id; + let image_name = &entry.image_name; + + // Create a component record for this report. + let component_id = if !components_initialized { + let id = ei_service + .create_component(job_id, image_name, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + ei_service + .update_component_running(id, scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + id + } else { + match scan_to_component.get(scan_id) { + Some(id) => *id, + None => continue, } + }; + + // Handle non-completed reports (expired, failed on EI side). + if entry.state != "completed" { + ei_service + .update_component_failed( + component_id, + &format!("EI report state: {}", entry.state), + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; continue; } - RetryDecision::Permanent(_) => { - let body = poll_response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI product poll returned permanent error {status}: {body}" - )); - } - } - } - - consecutive_transient_failures = 0; - let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = - poll_response.json().await?; - - // Check product-level state. - match product.summary.product_state.as_str() { - "completed" => { - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product analysis completed, fetching per-component reports" - ); + // Extract finding from the justification in the reports list. + let finding = entry + .vulns + .first() + .and_then(|v| v.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s.to_ascii_lowercase().as_str() { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); - // The EI product endpoint does not return per-component - // data (its `components` array is always empty). Instead, - // fetch the reports list filtered by product ID to discover - // each component's scan ID and justification. - let reports_url = format!( - "{}/api/v1/reports?productId={}", + // Fetch the full by-scan-id report for VEX ingestion. + let full_report_url = format!( + "{}/api/v1/reports/by-scan-id/{}", config.url, - urlencoding::encode(product_id) + urlencoding::encode(scan_id) ); - let mut reports_request = client.get(&reports_url); + let mut full_request = client.get(&full_report_url); if let Some(ref provider) = config.token_provider { let token = provider .get_token() .await .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - reports_request = reports_request.bearer_auth(&token); - } - - let reports_response = reports_request - .send() - .await - .map_err(|e| anyhow::anyhow!("failed to fetch product reports: {e}"))?; - - if !reports_response.status().is_success() { - let status = reports_response.status(); - let body = reports_response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI product reports request failed ({status}): {body}" - )); - } - - let component_reports: Vec = - reports_response.json().await?; - - // Record submission failures as excluded components. - if !components_initialized { - for failure in &product.data.submission_failures { - let image = failure.image.as_deref().unwrap_or("unknown"); - let error = failure.error.as_deref().unwrap_or("excluded by EI"); - - ei_service - .create_excluded_component(job_id, image, error, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; - } + full_request = full_request.bearer_auth(&token); } - tracing::info!( - job_id = %job_id, - product_id = %product_id, - reports = component_reports.len(), - excluded = product.data.submission_failures.len(), - "processing per-component reports for SPDX product" - ); - - use trustify_entity::exploit_intelligence_job::Finding; - - for entry in &component_reports { - let scan_id = &entry.scan_id; - let image_name = &entry.image_name; - - // Create a component record for this report. - let component_id = if !components_initialized { - let id = ei_service - .create_component(job_id, image_name, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; - ei_service - .update_component_running(id, scan_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - id - } else { - match scan_to_component.get(scan_id) { - Some(id) => *id, - None => continue, - } - }; - - // Handle non-completed reports (expired, failed on EI side). - if entry.state != "completed" { - ei_service - .update_component_failed( - component_id, - &format!("EI report state: {}", entry.state), - db_rw, - ) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - continue; - } - - // Extract finding from the justification in the reports list. - let finding = entry - .vulns - .first() - .and_then(|v| v.justification.as_ref()) - .and_then(|j| j.status.as_deref()) - .map(|s| match s.to_ascii_lowercase().as_str() { - "false" => Finding::NotVulnerable, - "true" => Finding::Vulnerable, - _ => Finding::Uncertain, - }) - .unwrap_or(Finding::Uncertain); - - // Fetch the full by-scan-id report for VEX ingestion. - let full_report_url = format!( - "{}/api/v1/reports/by-scan-id/{}", - config.url, - urlencoding::encode(scan_id) - ); - - let mut full_request = client.get(&full_report_url); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - full_request = full_request.bearer_auth(&token); - } + let advisory_id = match full_request.send().await { + Ok(resp) if resp.status().is_success() => { + let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = + resp.json().await?; + + let vex_json = poll_resp + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + if let Some(vex) = vex_json { + if !vex.is_null() { + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest( + &vex_bytes, + Format::Advisory, + labels, + None, + Cache::Skip, + &tx, + ) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + component = image_name, + advisory_id = %result.id, + "ingested VEX advisory for SPDX component" + ); - let advisory_id = match full_request.send().await { - Ok(resp) if resp.status().is_success() => { - let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = - resp.json().await?; - - let vex_json = poll_resp - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.vex.as_ref()); - - if let Some(vex) = vex_json { - if !vex.is_null() { - let vex_bytes = serde_json::to_vec(vex)?; - let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); - - let tx = db_rw.begin().await?; - let result = ingestor - .ingest( - &vex_bytes, - Format::Advisory, - labels, - None, - Cache::Skip, - &tx, - ) - .await - .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; - tx.commit().await?; - - tracing::info!( - job_id = %job_id, - component = image_name, - advisory_id = %result.id, - "ingested VEX advisory for SPDX component" - ); - - result - .id - .strip_prefix("urn:uuid:") - .and_then(|id| Uuid::from_str(id).ok()) - } else { - None - } + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) } else { None } - } - _ => { - tracing::warn!( - job_id = %job_id, - scan_id = %scan_id, - "failed to fetch full report for VEX ingestion, skipping" - ); + } else { None } - }; - - ei_service - .update_component_completed( - component_id, - finding, - advisory_id, - db_rw, - ) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - } + } + _ => { + tracing::warn!( + job_id = %job_id, + scan_id = %scan_id, + "failed to fetch full report for VEX ingestion, skipping" + ); + None + } + }; - // Mark the parent job as completed. The aggregate finding is - // computed from component records at query time. ei_service - .update_job_completed(job_id, db_rw) + .update_component_completed( + component_id, + finding, + advisory_id, + db_rw, + ) .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + } - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product analysis completed" - ); + // Mark the parent job as completed. The aggregate finding is + // computed from component records at query time. + ei_service + .update_job_completed(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Ok(()); - } - "failed" => { - let msg = format!("EI product analysis failed for product {product_id}"); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Ok(()); - } - _ => { - // Still in progress (queued, in_progress), continue polling. - tracing::debug!( - job_id = %job_id, - product_id = %product_id, - state = product.summary.product_state, - "SPDX product analysis still in progress" - ); - } + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed" + ); + + Ok(()) + } + "failed" => { + let msg = format!("EI product analysis failed for product {product_id}"); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + Ok(()) } + _ => unreachable!("backon retry only returns terminal states"), } } From 880435a1df3856728fa3608b0d1b1a914b7e3740 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 13 Jul 2026 11:45:09 +0100 Subject: [PATCH 16/25] refactor(exploit-intelligence): move business logic from endpoints to runner module Move background processing functions (run_analysis, poll_for_result, poll_for_product_result, recover_and_resume) out of the endpoints layer into a new runner/ module. Endpoints now contain only HTTP handlers. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/exploit_intelligence/endpoints/mod.rs | 975 +----------------- .../src/exploit_intelligence/mod.rs | 1 + .../exploit_intelligence/runner/analysis.rs | 230 +++++ .../src/exploit_intelligence/runner/mod.rs | 63 ++ .../exploit_intelligence/runner/polling.rs | 589 +++++++++++ .../exploit_intelligence/runner/recovery.rs | 116 +++ 6 files changed, 1002 insertions(+), 972 deletions(-) create mode 100644 modules/fundamental/src/exploit_intelligence/runner/analysis.rs create mode 100644 modules/fundamental/src/exploit_intelligence/runner/mod.rs create mode 100644 modules/fundamental/src/exploit_intelligence/runner/polling.rs create mode 100644 modules/fundamental/src/exploit_intelligence/runner/recovery.rs diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 59c41aeaa..7022627a4 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -8,85 +8,20 @@ use crate::{ AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, }, + runner::{analysis::run_analysis, recovery::recover_and_resume}, service::ExploitIntelligenceService, }, }; use actix_web::{HttpResponse, Responder, get, post, web}; -use futures_util::TryStreamExt; -use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; -use std::str::FromStr; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use trustify_auth::{CreateExploitIntelligence, ReadExploitIntelligence, authorizer::Require}; use trustify_common::{ db, model::{Paginated, PaginatedResults}, }; -use backon::{ExponentialBuilder, ConstantBuilder, Retryable}; -use std::cell::Cell; -use trustify_entity::labels::Labels; -use trustify_module_ingestor::service::{Cache, Format, IngestorService}; -use trustify_module_storage::service::StorageBackend; +use trustify_module_ingestor::service::IngestorService; use uuid::Uuid; -/// Error classification for EI HTTP requests. -enum EiRequestError { - /// Retryable failure (network timeout, 5xx, etc.) - Transient(String), - /// Fatal failure (auth error, 4xx, parse error) - Permanent(anyhow::Error), -} - -impl EiRequestError { - fn is_transient(&self) -> bool { - matches!(self, Self::Transient(_)) - } - - fn into_anyhow(self) -> anyhow::Error { - match self { - Self::Transient(msg) => anyhow::anyhow!(msg), - Self::Permanent(err) => err, - } - } -} - -impl std::fmt::Display for EiRequestError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Transient(msg) => write!(f, "transient: {msg}"), - Self::Permanent(err) => write!(f, "permanent: {err}"), - } - } -} - -/// Classify an HTTP response or reqwest error into an EiRequestError. -fn classify_response_error( - result: Result, -) -> Result { - match result { - Ok(resp) => { - let status = resp.status(); - if status.is_success() { - Ok(resp) - } else { - match status.as_u16() { - 408 | 429 | 502 | 503 | 504 => Err(EiRequestError::Transient(format!( - "transient HTTP status {status}" - ))), - _ => Err(EiRequestError::Permanent(anyhow::anyhow!("HTTP {status}"))), - } - } - } - Err(err) => { - if err.is_connect() || err.is_timeout() || err.is_request() { - Err(EiRequestError::Transient(format!("network error: {err}"))) - } else { - Err(EiRequestError::Permanent(anyhow::anyhow!( - "network error: {err}" - ))) - } - } - } -} - /// Register exploit intelligence endpoints and spawn recovery for orphaned jobs. pub fn configure( config: &mut utoipa_actix_web::service_config::ServiceConfig, @@ -114,117 +49,6 @@ pub fn configure( .service(get_job); } -/// Recover orphaned EI jobs after a server restart. -/// -/// Pending jobs (never submitted) are marked as Failed. Running jobs -/// (scan_id present) get their poll loops re-spawned so the EI backend -/// result is picked up transparently. -async fn recover_and_resume( - ei_service: &ExploitIntelligenceService, - ingestor: &IngestorService, - db_rw: &db::ReadWrite, -) { - let result = match ei_service.recover_orphaned_jobs(db_rw).await { - Ok(r) => r, - Err(e) => { - tracing::error!(error = %e, "failed to recover orphaned exploit intelligence jobs"); - return; - } - }; - - let runtime = match ei_service.runtime().cloned() { - Some(rt) => rt, - None => return, - }; - let config = runtime.config; - let client = runtime.client; - - for job in result.running_jobs { - let job_id = job.id; - let vulnerability_id = job.vulnerability_id.clone(); - let ei_svc = ei_service.clone(); - let ingestor_svc = ingestor.clone(); - let db_rw_clone = db_rw.clone(); - let config_clone = config.clone(); - let client_clone = client.clone(); - - if let Some(ref product_id) = job.product_id { - // Multi-component SPDX flow — resume product-level polling. - let product_id = product_id.clone(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, product_id = %product_id, "resuming orphaned SPDX product polling"); - if let Err(e) = poll_for_product_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &vulnerability_id, - &product_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; - } - }); - } else { - // Single-component CycloneDX flow — look up scan_id from the component. - let components = match ei_service.fetch_components(job_id, db_rw).await { - Ok(c) => c, - Err(e) => { - tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); - let _ = ei_service - .update_job_failed(job_id, &format!("recovery error: {e}"), db_rw) - .await; - continue; - } - }; - - let first_component = components.first(); - let scan_id = first_component.and_then(|c| c.scan_id.as_deref()); - let component_id = first_component.map(|c| c.id); - - if let (Some(scan_id), Some(component_id)) = (scan_id, component_id) { - let scan_id = scan_id.to_string(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); - if let Err(e) = poll_for_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &vulnerability_id, - &scan_id, - component_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; - } - }); - } else { - tracing::error!(job_id = %job_id, "running job has no component with scan_id, marking failed"); - let _ = ei_service - .update_job_failed( - job_id, - "Running job missing component scan_id during recovery", - db_rw, - ) - .await; - } - } - } -} - /// Query parameters for filtering the job list. #[derive(Debug, Clone, serde::Deserialize, utoipa::IntoParams)] pub struct JobFilter { @@ -386,796 +210,3 @@ pub async fn get_job( None => Ok(HttpResponse::NotFound().finish()), } } - -/// Background task: submit SBOM to EI client, poll for results, ingest VEX. -async fn run_analysis( - ei_service: &ExploitIntelligenceService, - ingestor: &IngestorService, - db_rw: &db::ReadWrite, - config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, - client: &reqwest::Client, - job_id: Uuid, - sbom_id: Uuid, - vulnerability_id: &str, -) -> Result<(), anyhow::Error> { - // Retrieve the raw SBOM document from storage (read-only – no mutations needed). - let tx = db_rw - .begin_with_config(None, Some(AccessMode::ReadOnly)) - .await?; - let sbom_model = trustify_entity::sbom::Entity::find() - .filter(trustify_entity::sbom::Column::SbomId.eq(sbom_id)) - .one(&tx) - .await? - .ok_or_else(|| anyhow::anyhow!("SBOM not found: {sbom_id}"))?; - - let source_doc = - trustify_entity::source_document::Entity::find_by_id(sbom_model.source_document_id) - .one(&tx) - .await? - .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; - - // Build storage key from source document SHA256. - // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:…". - let sha256_with_prefix = format!("sha256:{}", source_doc.sha256); - let storage_key = trustify_common::id::Id::from_str(&sha256_with_prefix) - .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; - let storage_key: trustify_module_storage::service::StorageKey = storage_key - .try_into() - .map_err(|e: trustify_module_storage::service::StorageKeyError| { - anyhow::anyhow!("storage key error: {e}") - })?; - - let stream = ingestor - .storage() - .retrieve(storage_key) - .await - .map_err(|e| anyhow::anyhow!("storage retrieve error: {e}"))? - .ok_or_else(|| anyhow::anyhow!("SBOM document not found in storage"))?; - - // Collect the stream into bytes. - let sbom_bytes: Vec = stream - .try_fold(Vec::new(), |mut acc, chunk| async move { - acc.extend_from_slice(&chunk); - Ok(acc) - }) - .await - .map_err(|e| anyhow::anyhow!("failed to read SBOM stream: {e}"))?; - - drop(tx); - - // Detect SBOM format from labels. - let sbom_format = match sbom_model.labels.0.get("type") { - Some(t) => t.as_str(), - None => { - tracing::warn!( - sbom_id = %sbom_id, - "SBOM has no 'type' label, defaulting to CycloneDX format" - ); - "cyclonedx" - } - }; - - let is_spdx = sbom_format == "spdx"; - - if is_spdx { - // SPDX multi-component flow: upload to /api/v1/products/upload-spdx - let upload_url = format!("{}/api/v1/products/upload-spdx", config.url); - - let response = (|| { - let url = upload_url.clone(); - let sbom = sbom_bytes.clone(); - let vuln_id = vulnerability_id.to_string(); - let cli = client.clone(); - let tp = config.token_provider.clone(); - async move { - let form = reqwest::multipart::Form::new() - .text("cveId", vuln_id) - .part( - "file", - reqwest::multipart::Part::bytes(sbom) - .file_name("sbom.json") - .mime_str("application/json") - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) - })?, - ); - - let mut request = cli.post(&url).multipart(form); - - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; - request = request.bearer_auth(&token); - } - - classify_response_error(request.send().await) - } - }) - .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) - .when(|e| e.is_transient()) - .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) - .await - .map_err(EiRequestError::into_anyhow)?; - - let spdx_upload: crate::exploit_intelligence::ei_client::SpdxUploadResponse = - response.json().await?; - let product_id = spdx_upload.product_id; - - // Update job to running with the product ID. - ei_service - .update_job_running_with_product(job_id, &product_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product uploaded to EI, starting product-level polling" - ); - - // Hand off to the product-level poll loop. - poll_for_product_result( - ei_service, - ingestor, - db_rw, - config, - client, - job_id, - vulnerability_id, - &product_id, - ) - .await - } else { - // CycloneDX single-component flow. - // Create the component record before uploading so it is tracked. - let component_ref = sbom_model.document_id.clone().unwrap_or_else(|| sbom_model.node_id.clone()); - let component_id = ei_service - .create_component(job_id, &component_ref, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; - - let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); - - let response = (|| { - let url = upload_url.clone(); - let sbom = sbom_bytes.clone(); - let vuln_id = vulnerability_id.to_string(); - let cli = client.clone(); - let tp = config.token_provider.clone(); - async move { - let form = reqwest::multipart::Form::new() - .text("cveId", vuln_id) - .part( - "file", - reqwest::multipart::Part::bytes(sbom) - .file_name("sbom.json") - .mime_str("application/json") - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) - })?, - ); - - let mut request = cli.post(&url).multipart(form); - - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; - request = request.bearer_auth(&token); - } - - classify_response_error(request.send().await) - } - }) - .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) - .when(|e| e.is_transient()) - .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) - .await - .map_err(EiRequestError::into_anyhow)?; - - let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; - let scan_id = upload.report_request_id.report_id; - - // Update component to running with scan ID. - ei_service - .update_component_running(component_id, &scan_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - - // Hand off to the poll loop. - poll_for_result( - ei_service, - ingestor, - db_rw, - config, - client, - job_id, - vulnerability_id, - &scan_id, - component_id, - ) - .await - } -} - -/// Poll the EI service for an in-flight single-component analysis and ingest the result. -/// -/// This is the second half of the analysis lifecycle: it assumes the SBOM has -/// already been uploaded and a `scan_id` obtained. It is called both from -/// `run_analysis` (normal path) and from the startup recovery path for -/// orphaned Running jobs. -/// -/// Updates both the component record (with finding/advisory) and the parent -/// job record (status only) upon completion. -pub(crate) async fn poll_for_result( - ei_service: &ExploitIntelligenceService, - ingestor: &IngestorService, - db_rw: &db::ReadWrite, - config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, - client: &reqwest::Client, - job_id: Uuid, - _vulnerability_id: &str, - scan_id: &str, - component_id: Uuid, -) -> Result<(), anyhow::Error> { - let poll_url = format!( - "{}/api/v1/reports/by-scan-id/{}", - config.url, - urlencoding::encode(scan_id) - ); - let poll_secs = config.poll_interval_secs.max(5); - let consecutive_failures = Cell::new(0u32); - - let result = tokio::time::timeout( - std::time::Duration::from_secs(config.max_poll_duration_secs), - (|| { - let url = poll_url.clone(); - let cli = client.clone(); - let tp = config.token_provider.clone(); - async move { - let mut poll_request = cli.get(&url); - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; - poll_request = poll_request.bearer_auth(&token); - } - - let resp = classify_response_error(poll_request.send().await)?; - - let poll: crate::exploit_intelligence::ei_client::PollResponse = - resp.json().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to parse EI poll response: {e}" - )) - })?; - - match poll.status.as_str() { - "completed" | "failed" | "expired" => Ok(poll), - _ => { - tracing::debug!( - status = poll.status, - "exploit intelligence analysis still in progress" - ); - Err(EiRequestError::Transient("pending".to_string())) - } - } - } - }) - .retry( - ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(poll_secs)) - .without_max_times(), - ) - .when(|e| match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true - } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < 5 - } - EiRequestError::Permanent(_) => false, - }) - .notify(|e, dur| { - if !matches!(e, EiRequestError::Transient(r) if r == "pending") { - tracing::warn!(?dur, %e, "transient poll failure"); - } - }), - ) - .await; - - // Convert timeout/retry result to our expected shape. - let poll = match result { - Ok(Ok(poll)) => poll, - Ok(Err(e)) => { - let err = e.into_anyhow(); - let msg = err.to_string(); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Err(err); - } - Err(_elapsed) => { - let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Err(anyhow::anyhow!("{msg}")); - } - }; - - // Process the terminal poll response. - match poll.status.as_str() { - "completed" => { - // Extract finding from the analysis result. - use trustify_entity::exploit_intelligence_job::Finding; - let finding = poll - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.analysis.as_ref()) - .and_then(|a| a.first()) - .and_then(|e| e.justification.as_ref()) - .and_then(|j| j.status.as_deref()) - .map(|s| match s.to_ascii_lowercase().as_str() { - "false" => Finding::NotVulnerable, - "true" => Finding::Vulnerable, - _ => Finding::Uncertain, - }) - .unwrap_or(Finding::Uncertain); - - // Check if VEX document is available. - let vex_json = poll - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.vex.as_ref()); - - let advisory_id = if let Some(vex) = vex_json { - if !vex.is_null() { - // Ingest the VEX document as an advisory with source label. - let vex_bytes = serde_json::to_vec(vex)?; - let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); - - let tx = db_rw.begin().await?; - let result = ingestor - .ingest(&vex_bytes, Format::Advisory, labels, None, Cache::Skip, &tx) - .await - .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; - tx.commit().await?; - - tracing::info!( - job_id = %job_id, - advisory_id = %result.id, - "ingested VEX advisory from exploit intelligence" - ); - - result - .id - .strip_prefix("urn:uuid:") - .and_then(|id| Uuid::from_str(id).ok()) - } else { - None - } - } else { - None - }; - - // Update the component with the per-analysis results. - ei_service - .update_component_completed(component_id, finding, advisory_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - - // Mark the parent job as completed (finding lives on the component). - ei_service - .update_job_completed(job_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - - tracing::info!( - job_id = %job_id, - finding = ?finding, - "exploit intelligence analysis completed" - ); - - Ok(()) - } - "failed" | "expired" => { - let error_msg = poll - .report - .as_ref() - .and_then(|r| r.error.as_deref()) - .unwrap_or("analysis failed") - .to_string(); - - ei_service - .update_job_failed(job_id, &error_msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - - Ok(()) - } - _ => unreachable!("backon retry only returns terminal states"), - } -} - -/// Poll the EI service for a multi-component product and process per-component results. -/// -/// This is the SPDX counterpart of `poll_for_result`. It polls the product-level -/// endpoint until all components are complete, then fetches each component's -/// report individually. -pub(crate) async fn poll_for_product_result( - ei_service: &ExploitIntelligenceService, - ingestor: &IngestorService, - db_rw: &db::ReadWrite, - config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, - client: &reqwest::Client, - job_id: Uuid, - _vulnerability_id: &str, - product_id: &str, -) -> Result<(), anyhow::Error> { - let product_url = format!( - "{}/api/v1/products/{}", - config.url, - urlencoding::encode(product_id) - ); - let poll_secs = config.poll_interval_secs.max(5); - let consecutive_failures = Cell::new(0u32); - - let mut components_initialized = false; - - // Maps EI scan_id -> our internal component_id (populated on first successful poll). - let mut scan_to_component: std::collections::HashMap = - std::collections::HashMap::new(); - - // Check whether component records already exist for this job (e.g. after a - // crash/restart that orphaned a running job). If they do, skip re-initialization - // to avoid duplicate component rows. - let existing_components = ei_service - .fetch_components(job_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to fetch existing components: {e}"))?; - if !existing_components.is_empty() { - for c in &existing_components { - if let Some(ref scan_id) = c.scan_id { - scan_to_component.insert(scan_id.clone(), c.id); - } - } - components_initialized = true; - tracing::info!( - job_id = %job_id, - existing_components = existing_components.len(), - "resumed with existing component records, skipping re-initialization" - ); - } - - let result = tokio::time::timeout( - std::time::Duration::from_secs(config.max_poll_duration_secs), - (|| { - let url = product_url.clone(); - let cli = client.clone(); - let tp = config.token_provider.clone(); - async move { - let mut poll_request = cli.get(&url); - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; - poll_request = poll_request.bearer_auth(&token); - } - - let resp = classify_response_error(poll_request.send().await)?; - - let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = - resp.json().await.map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to parse EI product response: {e}" - )) - })?; - - match product.summary.product_state.as_str() { - "completed" | "failed" => Ok(product), - _ => { - tracing::debug!( - state = product.summary.product_state, - "SPDX product analysis still in progress" - ); - Err(EiRequestError::Transient("pending".to_string())) - } - } - } - }) - .retry( - ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(poll_secs)) - .without_max_times(), - ) - .when(|e| match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true - } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < 5 - } - EiRequestError::Permanent(_) => false, - }) - .notify(|e, dur| { - if !matches!(e, EiRequestError::Transient(r) if r == "pending") { - tracing::warn!(?dur, %e, "transient poll failure"); - } - }), - ) - .await; - - // Convert timeout/retry result to our expected shape. - let product = match result { - Ok(Ok(product)) => product, - Ok(Err(e)) => { - let err = e.into_anyhow(); - let msg = err.to_string(); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Err(err); - } - Err(_elapsed) => { - let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - return Err(anyhow::anyhow!("{msg}")); - } - }; - - // Process the terminal product response. - match product.summary.product_state.as_str() { - "completed" => { - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product analysis completed, fetching per-component reports" - ); - - // The EI product endpoint does not return per-component - // data (its `components` array is always empty). Instead, - // fetch the reports list filtered by product ID to discover - // each component's scan ID and justification. - let reports_url = format!( - "{}/api/v1/reports?productId={}", - config.url, - urlencoding::encode(product_id) - ); - - let mut reports_request = client.get(&reports_url); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - reports_request = reports_request.bearer_auth(&token); - } - - let reports_response = reports_request - .send() - .await - .map_err(|e| anyhow::anyhow!("failed to fetch product reports: {e}"))?; - - if !reports_response.status().is_success() { - let status = reports_response.status(); - let body = reports_response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "EI product reports request failed ({status}): {body}" - )); - } - - let component_reports: Vec = - reports_response.json().await?; - - // Record submission failures as excluded components. - if !components_initialized { - for failure in &product.data.submission_failures { - let image = failure.image.as_deref().unwrap_or("unknown"); - let error = failure.error.as_deref().unwrap_or("excluded by EI"); - - ei_service - .create_excluded_component(job_id, image, error, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; - } - } - - tracing::info!( - job_id = %job_id, - product_id = %product_id, - reports = component_reports.len(), - excluded = product.data.submission_failures.len(), - "processing per-component reports for SPDX product" - ); - - use trustify_entity::exploit_intelligence_job::Finding; - - for entry in &component_reports { - let scan_id = &entry.scan_id; - let image_name = &entry.image_name; - - // Create a component record for this report. - let component_id = if !components_initialized { - let id = ei_service - .create_component(job_id, image_name, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; - ei_service - .update_component_running(id, scan_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - id - } else { - match scan_to_component.get(scan_id) { - Some(id) => *id, - None => continue, - } - }; - - // Handle non-completed reports (expired, failed on EI side). - if entry.state != "completed" { - ei_service - .update_component_failed( - component_id, - &format!("EI report state: {}", entry.state), - db_rw, - ) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - continue; - } - - // Extract finding from the justification in the reports list. - let finding = entry - .vulns - .first() - .and_then(|v| v.justification.as_ref()) - .and_then(|j| j.status.as_deref()) - .map(|s| match s.to_ascii_lowercase().as_str() { - "false" => Finding::NotVulnerable, - "true" => Finding::Vulnerable, - _ => Finding::Uncertain, - }) - .unwrap_or(Finding::Uncertain); - - // Fetch the full by-scan-id report for VEX ingestion. - let full_report_url = format!( - "{}/api/v1/reports/by-scan-id/{}", - config.url, - urlencoding::encode(scan_id) - ); - - let mut full_request = client.get(&full_report_url); - if let Some(ref provider) = config.token_provider { - let token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - full_request = full_request.bearer_auth(&token); - } - - let advisory_id = match full_request.send().await { - Ok(resp) if resp.status().is_success() => { - let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = - resp.json().await?; - - let vex_json = poll_resp - .report - .as_ref() - .and_then(|r| r.output.as_ref()) - .and_then(|o| o.vex.as_ref()); - - if let Some(vex) = vex_json { - if !vex.is_null() { - let vex_bytes = serde_json::to_vec(vex)?; - let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); - - let tx = db_rw.begin().await?; - let result = ingestor - .ingest( - &vex_bytes, - Format::Advisory, - labels, - None, - Cache::Skip, - &tx, - ) - .await - .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; - tx.commit().await?; - - tracing::info!( - job_id = %job_id, - component = image_name, - advisory_id = %result.id, - "ingested VEX advisory for SPDX component" - ); - - result - .id - .strip_prefix("urn:uuid:") - .and_then(|id| Uuid::from_str(id).ok()) - } else { - None - } - } else { - None - } - } - _ => { - tracing::warn!( - job_id = %job_id, - scan_id = %scan_id, - "failed to fetch full report for VEX ingestion, skipping" - ); - None - } - }; - - ei_service - .update_component_completed( - component_id, - finding, - advisory_id, - db_rw, - ) - .await - .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - } - - // Mark the parent job as completed. The aggregate finding is - // computed from component records at query time. - ei_service - .update_job_completed(job_id, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product analysis completed" - ); - - Ok(()) - } - "failed" => { - let msg = format!("EI product analysis failed for product {product_id}"); - ei_service - .update_job_failed(job_id, &msg, db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - Ok(()) - } - _ => unreachable!("backon retry only returns terminal states"), - } -} diff --git a/modules/fundamental/src/exploit_intelligence/mod.rs b/modules/fundamental/src/exploit_intelligence/mod.rs index f568f9fa1..930cf84be 100644 --- a/modules/fundamental/src/exploit_intelligence/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/mod.rs @@ -3,4 +3,5 @@ pub mod ei_client; pub(crate) mod endpoints; pub mod model; +pub(crate) mod runner; pub mod service; diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs new file mode 100644 index 000000000..4358d4ab9 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -0,0 +1,230 @@ +use crate::exploit_intelligence::{ + runner::{EiRequestError, classify_response_error}, + service::ExploitIntelligenceService, +}; +use backon::{ExponentialBuilder, Retryable}; +use futures_util::TryStreamExt; +use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use std::str::FromStr; +use trustify_common::db; +use trustify_module_ingestor::service::IngestorService; +use trustify_module_storage::service::StorageBackend; +use uuid::Uuid; + +use super::polling::{poll_for_product_result, poll_for_result}; + +/// Background task: submit SBOM to EI client, poll for results, ingest VEX. +pub(crate) async fn run_analysis( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, + job_id: Uuid, + sbom_id: Uuid, + vulnerability_id: &str, +) -> Result<(), anyhow::Error> { + // Retrieve the raw SBOM document from storage (read-only – no mutations needed). + let tx = db_rw + .begin_with_config(None, Some(AccessMode::ReadOnly)) + .await?; + let sbom_model = trustify_entity::sbom::Entity::find() + .filter(trustify_entity::sbom::Column::SbomId.eq(sbom_id)) + .one(&tx) + .await? + .ok_or_else(|| anyhow::anyhow!("SBOM not found: {sbom_id}"))?; + + let source_doc = + trustify_entity::source_document::Entity::find_by_id(sbom_model.source_document_id) + .one(&tx) + .await? + .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; + + // Build storage key from source document SHA256. + // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:…". + let sha256_with_prefix = format!("sha256:{}", source_doc.sha256); + let storage_key = trustify_common::id::Id::from_str(&sha256_with_prefix) + .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; + let storage_key: trustify_module_storage::service::StorageKey = storage_key + .try_into() + .map_err(|e: trustify_module_storage::service::StorageKeyError| { + anyhow::anyhow!("storage key error: {e}") + })?; + + let stream = ingestor + .storage() + .retrieve(storage_key) + .await + .map_err(|e| anyhow::anyhow!("storage retrieve error: {e}"))? + .ok_or_else(|| anyhow::anyhow!("SBOM document not found in storage"))?; + + // Collect the stream into bytes. + let sbom_bytes: Vec = stream + .try_fold(Vec::new(), |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await + .map_err(|e| anyhow::anyhow!("failed to read SBOM stream: {e}"))?; + + drop(tx); + + // Detect SBOM format from labels. + let sbom_format = match sbom_model.labels.0.get("type") { + Some(t) => t.as_str(), + None => { + tracing::warn!( + sbom_id = %sbom_id, + "SBOM has no 'type' label, defaulting to CycloneDX format" + ); + "cyclonedx" + } + }; + + let is_spdx = sbom_format == "spdx"; + + if is_spdx { + // SPDX multi-component flow: upload to /api/v1/products/upload-spdx + let upload_url = format!("{}/api/v1/products/upload-spdx", config.url); + + let response = (|| { + let url = upload_url.clone(); + let sbom = sbom_bytes.clone(); + let vuln_id = vulnerability_id.to_string(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let form = reqwest::multipart::Form::new() + .text("cveId", vuln_id) + .part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); + + let mut request = cli.post(&url).multipart(form); + + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + request = request.bearer_auth(&token); + } + + classify_response_error(request.send().await) + } + }) + .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .when(|e| e.is_transient()) + .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) + .await + .map_err(EiRequestError::into_anyhow)?; + + let spdx_upload: crate::exploit_intelligence::ei_client::SpdxUploadResponse = + response.json().await?; + let product_id = spdx_upload.product_id; + + // Update job to running with the product ID. + ei_service + .update_job_running_with_product(job_id, &product_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product uploaded to EI, starting product-level polling" + ); + + // Hand off to the product-level poll loop. + poll_for_product_result( + ei_service, + ingestor, + db_rw, + config, + client, + job_id, + vulnerability_id, + &product_id, + ) + .await + } else { + // CycloneDX single-component flow. + // Create the component record before uploading so it is tracked. + let component_ref = sbom_model.document_id.clone().unwrap_or_else(|| sbom_model.node_id.clone()); + let component_id = ei_service + .create_component(job_id, &component_ref, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + + let upload_url = format!("{}/api/v1/products/upload-cyclonedx", config.url); + + let response = (|| { + let url = upload_url.clone(); + let sbom = sbom_bytes.clone(); + let vuln_id = vulnerability_id.to_string(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let form = reqwest::multipart::Form::new() + .text("cveId", vuln_id) + .part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); + + let mut request = cli.post(&url).multipart(form); + + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + request = request.bearer_auth(&token); + } + + classify_response_error(request.send().await) + } + }) + .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .when(|e| e.is_transient()) + .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) + .await + .map_err(EiRequestError::into_anyhow)?; + + let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; + let scan_id = upload.report_request_id.report_id; + + // Update component to running with scan ID. + ei_service + .update_component_running(component_id, &scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + + // Hand off to the poll loop. + poll_for_result( + ei_service, + ingestor, + db_rw, + config, + client, + job_id, + vulnerability_id, + &scan_id, + component_id, + ) + .await + } +} diff --git a/modules/fundamental/src/exploit_intelligence/runner/mod.rs b/modules/fundamental/src/exploit_intelligence/runner/mod.rs new file mode 100644 index 000000000..191c932ce --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/mod.rs @@ -0,0 +1,63 @@ +pub(crate) mod analysis; +pub(crate) mod polling; +pub(crate) mod recovery; + +/// Error classification for EI HTTP requests. +pub(crate) enum EiRequestError { + /// Retryable failure (network timeout, 5xx, etc.) + Transient(String), + /// Fatal failure (auth error, 4xx, parse error) + Permanent(anyhow::Error), +} + +impl EiRequestError { + pub(crate) fn is_transient(&self) -> bool { + matches!(self, Self::Transient(_)) + } + + pub(crate) fn into_anyhow(self) -> anyhow::Error { + match self { + Self::Transient(msg) => anyhow::anyhow!(msg), + Self::Permanent(err) => err, + } + } +} + +impl std::fmt::Display for EiRequestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Transient(msg) => write!(f, "transient: {msg}"), + Self::Permanent(err) => write!(f, "permanent: {err}"), + } + } +} + +/// Classify an HTTP response or reqwest error into an EiRequestError. +pub(crate) fn classify_response_error( + result: Result, +) -> Result { + match result { + Ok(resp) => { + let status = resp.status(); + if status.is_success() { + Ok(resp) + } else { + match status.as_u16() { + 408 | 429 | 502 | 503 | 504 => Err(EiRequestError::Transient(format!( + "transient HTTP status {status}" + ))), + _ => Err(EiRequestError::Permanent(anyhow::anyhow!("HTTP {status}"))), + } + } + } + Err(err) => { + if err.is_connect() || err.is_timeout() || err.is_request() { + Err(EiRequestError::Transient(format!("network error: {err}"))) + } else { + Err(EiRequestError::Permanent(anyhow::anyhow!( + "network error: {err}" + ))) + } + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs new file mode 100644 index 000000000..7fe4ed3cf --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -0,0 +1,589 @@ +use crate::exploit_intelligence::{ + runner::{EiRequestError, classify_response_error}, + service::ExploitIntelligenceService, +}; +use backon::{ConstantBuilder, Retryable}; +use sea_orm::TransactionTrait; +use std::cell::Cell; +use std::str::FromStr; +use trustify_common::db; +use trustify_entity::labels::Labels; +use trustify_module_ingestor::service::{Cache, Format, IngestorService}; +use uuid::Uuid; + +/// Poll the EI service for an in-flight single-component analysis and ingest the result. +/// +/// This is the second half of the analysis lifecycle: it assumes the SBOM has +/// already been uploaded and a `scan_id` obtained. It is called both from +/// `run_analysis` (normal path) and from the startup recovery path for +/// orphaned Running jobs. +/// +/// Updates both the component record (with finding/advisory) and the parent +/// job record (status only) upon completion. +pub(crate) async fn poll_for_result( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, + job_id: Uuid, + _vulnerability_id: &str, + scan_id: &str, + component_id: Uuid, +) -> Result<(), anyhow::Error> { + let poll_url = format!( + "{}/api/v1/reports/by-scan-id/{}", + config.url, + urlencoding::encode(scan_id) + ); + let poll_secs = config.poll_interval_secs.max(5); + let consecutive_failures = Cell::new(0u32); + + let result = tokio::time::timeout( + std::time::Duration::from_secs(config.max_poll_duration_secs), + (|| { + let url = poll_url.clone(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let mut poll_request = cli.get(&url); + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + poll_request = poll_request.bearer_auth(&token); + } + + let resp = classify_response_error(poll_request.send().await)?; + + let poll: crate::exploit_intelligence::ei_client::PollResponse = + resp.json().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to parse EI poll response: {e}" + )) + })?; + + match poll.status.as_str() { + "completed" | "failed" | "expired" => Ok(poll), + _ => { + tracing::debug!( + status = poll.status, + "exploit intelligence analysis still in progress" + ); + Err(EiRequestError::Transient("pending".to_string())) + } + } + } + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(poll_secs)) + .without_max_times(), + ) + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < 5 + } + EiRequestError::Permanent(_) => false, + }) + .notify(|e, dur| { + if !matches!(e, EiRequestError::Transient(r) if r == "pending") { + tracing::warn!(?dur, %e, "transient poll failure"); + } + }), + ) + .await; + + // Convert timeout/retry result to our expected shape. + let poll = match result { + Ok(Ok(poll)) => poll, + Ok(Err(e)) => { + let err = e.into_anyhow(); + let msg = err.to_string(); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(err); + } + Err(_elapsed) => { + let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(anyhow::anyhow!("{msg}")); + } + }; + + // Process the terminal poll response. + match poll.status.as_str() { + "completed" => { + // Extract finding from the analysis result. + use trustify_entity::exploit_intelligence_job::Finding; + let finding = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.analysis.as_ref()) + .and_then(|a| a.first()) + .and_then(|e| e.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s.to_ascii_lowercase().as_str() { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + // Check if VEX document is available. + let vex_json = poll + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + let advisory_id = if let Some(vex) = vex_json { + if !vex.is_null() { + // Ingest the VEX document as an advisory with source label. + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest(&vex_bytes, Format::Advisory, labels, None, Cache::Skip, &tx) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + advisory_id = %result.id, + "ingested VEX advisory from exploit intelligence" + ); + + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) + } else { + None + } + } else { + None + }; + + // Update the component with the per-analysis results. + ei_service + .update_component_completed(component_id, finding, advisory_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + + // Mark the parent job as completed (finding lives on the component). + ei_service + .update_job_completed(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + finding = ?finding, + "exploit intelligence analysis completed" + ); + + Ok(()) + } + "failed" | "expired" => { + let error_msg = poll + .report + .as_ref() + .and_then(|r| r.error.as_deref()) + .unwrap_or("analysis failed") + .to_string(); + + ei_service + .update_job_failed(job_id, &error_msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + Ok(()) + } + _ => unreachable!("backon retry only returns terminal states"), + } +} + +/// Poll the EI service for a multi-component product and process per-component results. +/// +/// This is the SPDX counterpart of `poll_for_result`. It polls the product-level +/// endpoint until all components are complete, then fetches each component's +/// report individually. +pub(crate) async fn poll_for_product_result( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, + config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + client: &reqwest::Client, + job_id: Uuid, + _vulnerability_id: &str, + product_id: &str, +) -> Result<(), anyhow::Error> { + let product_url = format!( + "{}/api/v1/products/{}", + config.url, + urlencoding::encode(product_id) + ); + let poll_secs = config.poll_interval_secs.max(5); + let consecutive_failures = Cell::new(0u32); + + let mut components_initialized = false; + + // Maps EI scan_id -> our internal component_id (populated on first successful poll). + let mut scan_to_component: std::collections::HashMap = + std::collections::HashMap::new(); + + // Check whether component records already exist for this job (e.g. after a + // crash/restart that orphaned a running job). If they do, skip re-initialization + // to avoid duplicate component rows. + let existing_components = ei_service + .fetch_components(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to fetch existing components: {e}"))?; + if !existing_components.is_empty() { + for c in &existing_components { + if let Some(ref scan_id) = c.scan_id { + scan_to_component.insert(scan_id.clone(), c.id); + } + } + components_initialized = true; + tracing::info!( + job_id = %job_id, + existing_components = existing_components.len(), + "resumed with existing component records, skipping re-initialization" + ); + } + + let result = tokio::time::timeout( + std::time::Duration::from_secs(config.max_poll_duration_secs), + (|| { + let url = product_url.clone(); + let cli = client.clone(); + let tp = config.token_provider.clone(); + async move { + let mut poll_request = cli.get(&url); + if let Some(ref provider) = tp { + let token = provider.get_token().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to obtain auth token: {e}" + )) + })?; + poll_request = poll_request.bearer_auth(&token); + } + + let resp = classify_response_error(poll_request.send().await)?; + + let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = + resp.json().await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!( + "failed to parse EI product response: {e}" + )) + })?; + + match product.summary.product_state.as_str() { + "completed" | "failed" => Ok(product), + _ => { + tracing::debug!( + state = product.summary.product_state, + "SPDX product analysis still in progress" + ); + Err(EiRequestError::Transient("pending".to_string())) + } + } + } + }) + .retry( + ConstantBuilder::default() + .with_delay(std::time::Duration::from_secs(poll_secs)) + .without_max_times(), + ) + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < 5 + } + EiRequestError::Permanent(_) => false, + }) + .notify(|e, dur| { + if !matches!(e, EiRequestError::Transient(r) if r == "pending") { + tracing::warn!(?dur, %e, "transient poll failure"); + } + }), + ) + .await; + + // Convert timeout/retry result to our expected shape. + let product = match result { + Ok(Ok(product)) => product, + Ok(Err(e)) => { + let err = e.into_anyhow(); + let msg = err.to_string(); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(err); + } + Err(_elapsed) => { + let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + return Err(anyhow::anyhow!("{msg}")); + } + }; + + // Process the terminal product response. + match product.summary.product_state.as_str() { + "completed" => { + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed, fetching per-component reports" + ); + + // The EI product endpoint does not return per-component + // data (its `components` array is always empty). Instead, + // fetch the reports list filtered by product ID to discover + // each component's scan ID and justification. + let reports_url = format!( + "{}/api/v1/reports?productId={}", + config.url, + urlencoding::encode(product_id) + ); + + let mut reports_request = client.get(&reports_url); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + reports_request = reports_request.bearer_auth(&token); + } + + let reports_response = reports_request + .send() + .await + .map_err(|e| anyhow::anyhow!("failed to fetch product reports: {e}"))?; + + if !reports_response.status().is_success() { + let status = reports_response.status(); + let body = reports_response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "EI product reports request failed ({status}): {body}" + )); + } + + let component_reports: Vec = + reports_response.json().await?; + + // Record submission failures as excluded components. + if !components_initialized { + for failure in &product.data.submission_failures { + let image = failure.image.as_deref().unwrap_or("unknown"); + let error = failure.error.as_deref().unwrap_or("excluded by EI"); + + ei_service + .create_excluded_component(job_id, image, error, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create excluded component: {e}"))?; + } + } + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + reports = component_reports.len(), + excluded = product.data.submission_failures.len(), + "processing per-component reports for SPDX product" + ); + + use trustify_entity::exploit_intelligence_job::Finding; + + for entry in &component_reports { + let scan_id = &entry.scan_id; + let image_name = &entry.image_name; + + // Create a component record for this report. + let component_id = if !components_initialized { + let id = ei_service + .create_component(job_id, image_name, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + ei_service + .update_component_running(id, scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + id + } else { + match scan_to_component.get(scan_id) { + Some(id) => *id, + None => continue, + } + }; + + // Handle non-completed reports (expired, failed on EI side). + if entry.state != "completed" { + ei_service + .update_component_failed( + component_id, + &format!("EI report state: {}", entry.state), + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + continue; + } + + // Extract finding from the justification in the reports list. + let finding = entry + .vulns + .first() + .and_then(|v| v.justification.as_ref()) + .and_then(|j| j.status.as_deref()) + .map(|s| match s.to_ascii_lowercase().as_str() { + "false" => Finding::NotVulnerable, + "true" => Finding::Vulnerable, + _ => Finding::Uncertain, + }) + .unwrap_or(Finding::Uncertain); + + // Fetch the full by-scan-id report for VEX ingestion. + let full_report_url = format!( + "{}/api/v1/reports/by-scan-id/{}", + config.url, + urlencoding::encode(scan_id) + ); + + let mut full_request = client.get(&full_report_url); + if let Some(ref provider) = config.token_provider { + let token = provider + .get_token() + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; + full_request = full_request.bearer_auth(&token); + } + + let advisory_id = match full_request.send().await { + Ok(resp) if resp.status().is_success() => { + let poll_resp: crate::exploit_intelligence::ei_client::PollResponse = + resp.json().await?; + + let vex_json = poll_resp + .report + .as_ref() + .and_then(|r| r.output.as_ref()) + .and_then(|o| o.vex.as_ref()); + + if let Some(vex) = vex_json { + if !vex.is_null() { + let vex_bytes = serde_json::to_vec(vex)?; + let mut labels = Labels::default(); + labels + .0 + .insert("source".to_string(), "exploit-intelligence".to_string()); + + let tx = db_rw.begin().await?; + let result = ingestor + .ingest( + &vex_bytes, + Format::Advisory, + labels, + None, + Cache::Skip, + &tx, + ) + .await + .map_err(|e| anyhow::anyhow!("VEX ingestion failed: {e}"))?; + tx.commit().await?; + + tracing::info!( + job_id = %job_id, + component = image_name, + advisory_id = %result.id, + "ingested VEX advisory for SPDX component" + ); + + result + .id + .strip_prefix("urn:uuid:") + .and_then(|id| Uuid::from_str(id).ok()) + } else { + None + } + } else { + None + } + } + _ => { + tracing::warn!( + job_id = %job_id, + scan_id = %scan_id, + "failed to fetch full report for VEX ingestion, skipping" + ); + None + } + }; + + ei_service + .update_component_completed( + component_id, + finding, + advisory_id, + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + } + + // Mark the parent job as completed. The aggregate finding is + // computed from component records at query time. + ei_service + .update_job_completed(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis completed" + ); + + Ok(()) + } + "failed" => { + let msg = format!("EI product analysis failed for product {product_id}"); + ei_service + .update_job_failed(job_id, &msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + Ok(()) + } + _ => unreachable!("backon retry only returns terminal states"), + } +} diff --git a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs new file mode 100644 index 000000000..c6adf31c3 --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs @@ -0,0 +1,116 @@ +use crate::exploit_intelligence::service::ExploitIntelligenceService; +use trustify_common::db; +use trustify_module_ingestor::service::IngestorService; + +use super::polling::{poll_for_product_result, poll_for_result}; + +/// Recover orphaned EI jobs after a server restart. +/// +/// Pending jobs (never submitted) are marked as Failed. Running jobs +/// (scan_id present) get their poll loops re-spawned so the EI backend +/// result is picked up transparently. +pub(crate) async fn recover_and_resume( + ei_service: &ExploitIntelligenceService, + ingestor: &IngestorService, + db_rw: &db::ReadWrite, +) { + let result = match ei_service.recover_orphaned_jobs(db_rw).await { + Ok(r) => r, + Err(e) => { + tracing::error!(error = %e, "failed to recover orphaned exploit intelligence jobs"); + return; + } + }; + + let runtime = match ei_service.runtime().cloned() { + Some(rt) => rt, + None => return, + }; + let config = runtime.config; + let client = runtime.client; + + for job in result.running_jobs { + let job_id = job.id; + let vulnerability_id = job.vulnerability_id.clone(); + let ei_svc = ei_service.clone(); + let ingestor_svc = ingestor.clone(); + let db_rw_clone = db_rw.clone(); + let config_clone = config.clone(); + let client_clone = client.clone(); + + if let Some(ref product_id) = job.product_id { + // Multi-component SPDX flow — resume product-level polling. + let product_id = product_id.clone(); + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, product_id = %product_id, "resuming orphaned SPDX product polling"); + if let Err(e) = poll_for_product_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &product_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } else { + // Single-component CycloneDX flow — look up scan_id from the component. + let components = match ei_service.fetch_components(job_id, db_rw).await { + Ok(c) => c, + Err(e) => { + tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); + let _ = ei_service + .update_job_failed(job_id, &format!("recovery error: {e}"), db_rw) + .await; + continue; + } + }; + + let first_component = components.first(); + let scan_id = first_component.and_then(|c| c.scan_id.as_deref()); + let component_id = first_component.map(|c| c.id); + + if let (Some(scan_id), Some(component_id)) = (scan_id, component_id) { + let scan_id = scan_id.to_string(); + actix_web::rt::spawn(async move { + tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); + if let Err(e) = poll_for_result( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &config_clone, + &client_clone, + job_id, + &vulnerability_id, + &scan_id, + component_id, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } else { + tracing::error!(job_id = %job_id, "running job has no component with scan_id, marking failed"); + let _ = ei_service + .update_job_failed( + job_id, + "Running job missing component scan_id during recovery", + db_rw, + ) + .await; + } + } + } +} From e308729c59ead812ff2ecfcfdf4ca66bf5018f6c Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Mon, 13 Jul 2026 14:40:29 +0100 Subject: [PATCH 17/25] refactor(exploit-intelligence): make retry/poll values configurable with humantime Switch poll interval and max duration env vars from raw seconds to humantime format, and make upload retry count, retry delay, and max consecutive poll failures configurable via new env vars with reasonable defaults. Also fix clippy expect_used violation in service constructor. Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 1 + docs/env-vars.md | 7 +++- modules/fundamental/src/endpoints.rs | 3 +- .../exploit_intelligence/endpoints/test.rs | 10 +++-- .../exploit_intelligence/runner/analysis.rs | 4 +- .../exploit_intelligence/runner/polling.rs | 23 ++++++----- .../src/exploit_intelligence/service/mod.rs | 40 +++++++++++-------- server/Cargo.toml | 1 + server/src/profile/api.rs | 40 +++++++++++-------- 9 files changed, 78 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4182c5061..f05f8af59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8794,6 +8794,7 @@ dependencies = [ "clap", "futures", "garage-door", + "humantime", "log", "rand 0.10.2", "rstest", diff --git a/docs/env-vars.md b/docs/env-vars.md index 497aaef28..378783fef 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -64,8 +64,11 @@ | `UI_SCOPE` | Scopes to request | `openid` | | `EXPLOIT_INTELLIGENCE_URL` | Base URL of the Exploit Intelligence client service | | | `EXPLOIT_INTELLIGENCE_UI_URL` | Base URL of the EI web UI for deep-linking to reports | | -| `EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS`| Polling interval in seconds for EI analysis completion | `30` | -| `EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS`| Maximum duration in seconds before EI polling is considered timed out | `1800` | +| `EXPLOIT_INTELLIGENCE_POLL_INTERVAL`| Polling interval for EI analysis completion (humantime) | `30s` | +| `EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION`| Maximum duration before EI polling is considered timed out (humantime) | `30m` | +| `EXPLOIT_INTELLIGENCE_UPLOAD_MAX_RETRIES`| Maximum number of upload retry attempts | `3` | +| `EXPLOIT_INTELLIGENCE_UPLOAD_RETRY_DELAY`| Initial delay between upload retries (humantime, exponential backoff) | `1s` | +| `EXPLOIT_INTELLIGENCE_MAX_CONSECUTIVE_POLL_FAILURES`| Maximum consecutive poll failures before giving up | `5` | | `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Static authentication token for the Exploit Intelligence service (used when OIDC is not configured) | | | `EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL` | OIDC token endpoint URL for EI service authentication (client credentials flow) | | | `EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID` | OIDC client ID for EI service authentication | | diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 5d7526260..1f47f1004 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -57,7 +57,8 @@ pub fn configure( cache, ); - let ei_service = ExploitIntelligenceService::new(ei_config); + let ei_service = ExploitIntelligenceService::new(ei_config) + .unwrap_or_else(|e| panic!("failed to create ExploitIntelligenceService: {e}")); crate::exploit_intelligence::endpoints::configure( svc, db_rw, diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 8b7c1bf59..7eb0d85ee 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -24,10 +24,14 @@ fn test_service() -> ExploitIntelligenceService { ExploitIntelligenceService::new(Some(ExploitIntelligenceConfig { url: "http://localhost:9999".to_string(), ui_url: Some("http://localhost:9999".to_string()), - poll_interval_secs: 1, - max_poll_duration_secs: 1800, + poll_interval: std::time::Duration::from_secs(1), + max_poll_duration: std::time::Duration::from_secs(1800), + upload_max_retries: 3, + upload_retry_delay: std::time::Duration::from_secs(1), + max_consecutive_poll_failures: 5, token_provider: None, })) + .unwrap() } /// Helper to insert a job record directly for testing query endpoints. @@ -751,7 +755,7 @@ async fn analyze_sbom_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn analyze_ei_not_configured(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given an app with no EI configuration - let ei_service = ExploitIntelligenceService::new(None); + let ei_service = ExploitIntelligenceService::new(None).unwrap(); let db_rw = db::ReadWrite::new(ctx.db.clone()); let db_ro = db::ReadOnly::new(ctx.db.clone()); diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index 4358d4ab9..ae6acc957 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -120,7 +120,7 @@ pub(crate) async fn run_analysis( classify_response_error(request.send().await) } }) - .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .retry(ExponentialBuilder::default().with_min_delay(config.upload_retry_delay).with_max_times(config.upload_max_retries)) .when(|e| e.is_transient()) .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) .await @@ -198,7 +198,7 @@ pub(crate) async fn run_analysis( classify_response_error(request.send().await) } }) - .retry(ExponentialBuilder::default().with_min_delay(std::time::Duration::from_secs(1)).with_max_times(3)) + .retry(ExponentialBuilder::default().with_min_delay(config.upload_retry_delay).with_max_times(config.upload_max_retries)) .when(|e| e.is_transient()) .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) .await diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index 7fe4ed3cf..156362503 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -6,6 +6,7 @@ use backon::{ConstantBuilder, Retryable}; use sea_orm::TransactionTrait; use std::cell::Cell; use std::str::FromStr; +use std::time::Duration; use trustify_common::db; use trustify_entity::labels::Labels; use trustify_module_ingestor::service::{Cache, Format, IngestorService}; @@ -36,11 +37,12 @@ pub(crate) async fn poll_for_result( config.url, urlencoding::encode(scan_id) ); - let poll_secs = config.poll_interval_secs.max(5); + let poll_interval = config.poll_interval.max(Duration::from_secs(5)); + let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); let result = tokio::time::timeout( - std::time::Duration::from_secs(config.max_poll_duration_secs), + config.max_poll_duration, (|| { let url = poll_url.clone(); let cli = client.clone(); @@ -79,7 +81,7 @@ pub(crate) async fn poll_for_result( }) .retry( ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(poll_secs)) + .with_delay(poll_interval) .without_max_times(), ) .when(|e| match e { @@ -90,7 +92,7 @@ pub(crate) async fn poll_for_result( EiRequestError::Transient(_) => { let n = consecutive_failures.get() + 1; consecutive_failures.set(n); - n < 5 + n < max_consecutive } EiRequestError::Permanent(_) => false, }) @@ -115,7 +117,7 @@ pub(crate) async fn poll_for_result( return Err(err); } Err(_elapsed) => { - let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); + let msg = format!("polling timed out after {:?}", config.max_poll_duration); ei_service .update_job_failed(job_id, &msg, db_rw) .await @@ -243,7 +245,8 @@ pub(crate) async fn poll_for_product_result( config.url, urlencoding::encode(product_id) ); - let poll_secs = config.poll_interval_secs.max(5); + let poll_interval = config.poll_interval.max(Duration::from_secs(5)); + let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); let mut components_initialized = false; @@ -274,7 +277,7 @@ pub(crate) async fn poll_for_product_result( } let result = tokio::time::timeout( - std::time::Duration::from_secs(config.max_poll_duration_secs), + config.max_poll_duration, (|| { let url = product_url.clone(); let cli = client.clone(); @@ -313,7 +316,7 @@ pub(crate) async fn poll_for_product_result( }) .retry( ConstantBuilder::default() - .with_delay(std::time::Duration::from_secs(poll_secs)) + .with_delay(poll_interval) .without_max_times(), ) .when(|e| match e { @@ -324,7 +327,7 @@ pub(crate) async fn poll_for_product_result( EiRequestError::Transient(_) => { let n = consecutive_failures.get() + 1; consecutive_failures.set(n); - n < 5 + n < max_consecutive } EiRequestError::Permanent(_) => false, }) @@ -349,7 +352,7 @@ pub(crate) async fn poll_for_product_result( return Err(err); } Err(_elapsed) => { - let msg = format!("polling timed out after {}s", config.max_poll_duration_secs); + let msg = format!("polling timed out after {:?}", config.max_poll_duration); ei_service .update_job_failed(job_id, &msg, db_rw) .await diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index a823a71b4..087803c3a 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -29,11 +29,17 @@ pub struct ExploitIntelligenceConfig { /// Base URL of the Exploit Intelligence web UI for deep-linking to reports. /// When set, completed jobs include a clickable report URL. pub ui_url: Option, - /// Polling interval in seconds for checking analysis completion. + /// Polling interval for checking analysis completion. /// Minimum enforced: 5 seconds (values below are clamped at the polling site). - pub poll_interval_secs: u64, - /// Maximum duration in seconds before a polling loop is considered timed out. - pub max_poll_duration_secs: u64, + pub poll_interval: Duration, + /// Maximum duration before a polling loop is considered timed out. + pub max_poll_duration: Duration, + /// Maximum number of upload retry attempts (exponential backoff). + pub upload_max_retries: usize, + /// Initial delay between upload retries (exponential backoff). + pub upload_retry_delay: Duration, + /// Maximum consecutive transient poll failures before giving up. + pub max_consecutive_poll_failures: u32, /// Optional token provider for EI service authentication. /// Supports static tokens and OIDC client credentials flow. pub token_provider: Option, @@ -71,18 +77,20 @@ impl ExploitIntelligenceService { /// /// When `config` is `None`, the service will return `Error::Unavailable` /// for all analysis operations and no HTTP client is created. - pub fn new(config: Option) -> Self { - let runtime = config.map(|config| { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(60)) - .pool_max_idle_per_host(4) - .pool_idle_timeout(Duration::from_secs(90)) - .build() - .expect("failed to build reqwest client for exploit intelligence"); - EiRuntime { config, client } - }); - Self { runtime } + pub fn new(config: Option) -> Result { + let runtime = config + .map(|config| -> Result { + let client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(60)) + .pool_max_idle_per_host(4) + .pool_idle_timeout(Duration::from_secs(90)) + .build() + .map_err(|e| anyhow::anyhow!("failed to build reqwest client: {e}"))?; + Ok(EiRuntime { config, client }) + }) + .transpose()?; + Ok(Self { runtime }) } /// Returns a reference to the shared HTTP client, if EI is enabled. diff --git a/server/Cargo.toml b/server/Cargo.toml index 3db494c9d..47eb5388b 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -26,6 +26,7 @@ bytesize = { workspace = true } clap = { workspace = true, features = ["derive"] } futures = { workspace = true } garage-door = { workspace = true, optional = true } +humantime = { workspace = true } log = { workspace = true } rand = { workspace = true } serde = { workspace = true } diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 29e310d88..21f2df485 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -100,22 +100,25 @@ pub struct Run { #[arg(long, env = "EXPLOIT_INTELLIGENCE_UI_URL")] pub exploit_intelligence_ui_url: Option, - /// Polling interval in seconds for EI analysis completion (default: 30, minimum: 5). - #[arg( - long, - env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL_SECS", - default_value_t = 30, - value_parser = clap::value_parser!(u64).range(5..) - )] - pub exploit_intelligence_poll_interval_secs: u64, + /// Polling interval for EI analysis completion (humantime, e.g. "30s"). + #[arg(long, env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL", default_value = "30s")] + pub exploit_intelligence_poll_interval: humantime::Duration, - /// Maximum duration in seconds before EI polling is considered timed out (default: 1800 = 30 minutes). - #[arg( - long, - env = "EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION_SECS", - default_value_t = 1800 - )] - pub exploit_intelligence_max_poll_duration_secs: u64, + /// Maximum duration before EI polling is considered timed out (humantime, e.g. "30m"). + #[arg(long, env = "EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION", default_value = "30m")] + pub exploit_intelligence_max_poll_duration: humantime::Duration, + + /// Maximum number of upload retry attempts for EI analysis. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_UPLOAD_MAX_RETRIES", default_value_t = 3)] + pub exploit_intelligence_upload_max_retries: usize, + + /// Initial delay between upload retries (humantime, exponential backoff). + #[arg(long, env = "EXPLOIT_INTELLIGENCE_UPLOAD_RETRY_DELAY", default_value = "1s")] + pub exploit_intelligence_upload_retry_delay: humantime::Duration, + + /// Maximum consecutive poll failures before giving up. + #[arg(long, env = "EXPLOIT_INTELLIGENCE_MAX_CONSECUTIVE_POLL_FAILURES", default_value_t = 5)] + pub exploit_intelligence_max_consecutive_poll_failures: u32, /// Authentication token for the Exploit Intelligence service (static token, for backward compatibility). #[arg(long, env = "EXPLOIT_INTELLIGENCE_AUTH_TOKEN")] @@ -376,8 +379,11 @@ impl InitData { trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { url, ui_url: run.exploit_intelligence_ui_url, - poll_interval_secs: run.exploit_intelligence_poll_interval_secs, - max_poll_duration_secs: run.exploit_intelligence_max_poll_duration_secs, + poll_interval: run.exploit_intelligence_poll_interval.into(), + max_poll_duration: run.exploit_intelligence_max_poll_duration.into(), + upload_max_retries: run.exploit_intelligence_upload_max_retries, + upload_retry_delay: run.exploit_intelligence_upload_retry_delay.into(), + max_consecutive_poll_failures: run.exploit_intelligence_max_consecutive_poll_failures, token_provider, } }); From 49065f560448395c2c31fb46735a7ba3b3d5494d Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Tue, 14 Jul 2026 13:54:32 +0100 Subject: [PATCH 18/25] feat(exploit-intelligence): add heartbeat locking for multi-instance safety Add optimistic-locking heartbeat mechanism to prevent multiple trustify instances from polling the same EI job concurrently. Recovery uses SELECT ... FOR UPDATE SKIP LOCKED to atomically claim orphaned jobs. Key design decisions: - CAS guards (WHERE heartbeat = ) on all terminal status updates so only the owning instance can transition job state - CancellationToken for cooperative work cancellation on heartbeat loss - Partial unique index on (sbom_id, vulnerability_id) for active jobs to prevent duplicate job creation races - SharedHeartbeat (Arc) threaded through poll functions for ownership verification at every DB write Co-Authored-By: Claude Opus 4.6 (1M context) --- entity/src/exploit_intelligence_job.rs | 4 + ...0002230_create_exploit_intelligence_job.rs | 2 + modules/fundamental/Cargo.toml | 2 +- .../src/exploit_intelligence/auth.rs | 8 +- .../src/exploit_intelligence/endpoints/mod.rs | 39 +- .../exploit_intelligence/endpoints/test.rs | 4 +- .../exploit_intelligence/runner/analysis.rs | 27 +- .../exploit_intelligence/runner/heartbeat.rs | 193 ++++++++++ .../src/exploit_intelligence/runner/mod.rs | 1 + .../exploit_intelligence/runner/polling.rs | 194 +++++++--- .../exploit_intelligence/runner/recovery.rs | 67 ++-- .../src/exploit_intelligence/service/mod.rs | 346 +++++++++++++----- 12 files changed, 712 insertions(+), 175 deletions(-) create mode 100644 modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs diff --git a/entity/src/exploit_intelligence_job.rs b/entity/src/exploit_intelligence_job.rs index 1384014ac..7c7a408ec 100644 --- a/entity/src/exploit_intelligence_job.rs +++ b/entity/src/exploit_intelligence_job.rs @@ -6,6 +6,7 @@ /// [`exploit_intelligence_job_component`](super::exploit_intelligence_job_component) rows — /// even for single-component CycloneDX jobs, which have exactly one component record. use sea_orm::entity::prelude::*; +use sea_orm::prelude::Decimal; use time::OffsetDateTime; /// Lifecycle status of an Exploit Intelligence analysis job. @@ -72,6 +73,9 @@ pub struct Model { /// Expected number of components in the product. pub total_components: Option, + /// Unix time in nanos, for coordinating multi-instance job ownership. + pub heartbeat: Option, + /// Timestamp when the job was created. pub created: OffsetDateTime, diff --git a/migration/src/m0002230_create_exploit_intelligence_job.rs b/migration/src/m0002230_create_exploit_intelligence_job.rs index 0f5bfbd7b..04ec22087 100644 --- a/migration/src/m0002230_create_exploit_intelligence_job.rs +++ b/migration/src/m0002230_create_exploit_intelligence_job.rs @@ -57,6 +57,7 @@ impl MigrationTrait for Migration { .col(ColumnDef::new(ExploitIntelligenceJob::ErrorMessage).string()) .col(ColumnDef::new(ExploitIntelligenceJob::ProductId).string()) .col(ColumnDef::new(ExploitIntelligenceJob::TotalComponents).integer()) + .col(ColumnDef::new(ExploitIntelligenceJob::Heartbeat).decimal().null()) .col( ColumnDef::new(ExploitIntelligenceJob::Created) .timestamp_with_time_zone() @@ -287,6 +288,7 @@ enum ExploitIntelligenceJob { ErrorMessage, ProductId, TotalComponents, + Heartbeat, Created, Updated, } diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 51f726715..096eaae22 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -45,6 +45,7 @@ tar = { workspace = true } thiserror = { workspace = true } time = { workspace = true } tokio = { workspace = true, features = ["full"] } +tokio-util = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-futures = { workspace = true, features = ["futures-03"] } urlencoding = { workspace = true } @@ -89,7 +90,6 @@ strum = { workspace = true } tar = { workspace = true } test-context = { workspace = true } test-log = { workspace = true, features = ["log", "trace"] } -tokio-util = { workspace = true } tracing = { workspace = true, features = ["std"]} tracing-core = { workspace = true } tracing-opentelemetry = { workspace = true } diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs index 74daaed03..5f8a2838d 100644 --- a/modules/fundamental/src/exploit_intelligence/auth.rs +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -93,10 +93,10 @@ impl OidcState { // Need refresh -- acquire write lock let mut cached = self.cached.write().await; // Double-check after acquiring write lock (another task may have refreshed) - if let Some(ref tok) = *cached { - if tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) { - return Ok(tok.access_token.clone()); - } + if let Some(ref tok) = *cached + && tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) + { + return Ok(tok.access_token.clone()); } let response = self diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 7022627a4..c6fc189fd 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -62,7 +62,8 @@ pub struct JobFilter { /// /// Creates a job record and spawns a background task that submits the SBOM /// to the EI client service, polls for completion, and ingests any returned -/// VEX document. +/// VEX document. A heartbeat is maintained so other instances know the job +/// is being actively serviced. #[utoipa::path( tag = "exploit-intelligence", operation_id = "analyzeExploitIntelligence", @@ -124,14 +125,25 @@ pub async fn analyze( let job_id = job.id; let sbom_id = body.sbom_id; let vulnerability_id = body.vulnerability_id.clone(); + // The job was created with an initial heartbeat — use it for the heartbeat loop. + let initial_heartbeat = job.heartbeat; - // Spawn background task to submit and poll. + // Spawn background task to submit and poll, wrapped with heartbeat. let ei_svc = ei_service.as_ref().clone(); let ingestor_svc = ingestor.as_ref().clone(); let db_rw_clone = db_rw.as_ref().clone(); + let db_hb = db_rw.as_ref().clone(); actix_web::rt::spawn(async move { - if let Err(e) = run_analysis( + use crate::exploit_intelligence::runner::heartbeat::{self, HeartbeatError}; + use tokio_util::sync::CancellationToken; + + let token = CancellationToken::new(); + let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); + let hb_for_work = hb_state.clone(); + let token_for_work = token.clone(); + + let work = run_analysis( &ei_svc, &ingestor_svc, &db_rw_clone, @@ -140,13 +152,20 @@ pub async fn analyze( job_id, sbom_id, &vulnerability_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "exploit intelligence analysis failed"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; + hb_for_work, + token_for_work, + ); + match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + Ok(()) => {} + Err(HeartbeatError::LockLost) => { + tracing::warn!(job_id = %job_id, "heartbeat lost, another instance owns this job"); + // Do NOT call update_job_failed — we no longer own this job. + } + Err(HeartbeatError::Work(e)) => { + tracing::error!(job_id = %job_id, error = %e, "exploit intelligence analysis failed"); + // The work future already handled failure with CAS-protected updates. + // No additional update_job_failed call needed. + } } }); diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 7eb0d85ee..51c460f02 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -52,6 +52,7 @@ async fn insert_test_job( error_message: Set(error_message.map(|s| s.to_string())), product_id: Set(None), total_components: Set(None), + heartbeat: Set(None), created: Set(now), updated: Set(now), }; @@ -453,10 +454,11 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { .await; // When the service marks it as failed with a timeout error + // Pass None as heartbeat since the test job was created without one. let ei_service = test_service(); let db_rw = db::ReadWrite::new(ctx.db.clone()); ei_service - .update_job_failed(job_id, "analysis timed out after 1800s", &db_rw) + .update_job_failed(job_id, "analysis timed out after 1800s", None, &db_rw) .await?; // Then fetching the job via GET shows the failed status and timeout message diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index ae6acc957..6f5501ce2 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -1,11 +1,13 @@ use crate::exploit_intelligence::{ runner::{EiRequestError, classify_response_error}, + runner::heartbeat::SharedHeartbeat, service::ExploitIntelligenceService, }; use backon::{ExponentialBuilder, Retryable}; use futures_util::TryStreamExt; use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use std::str::FromStr; +use tokio_util::sync::CancellationToken; use trustify_common::db; use trustify_module_ingestor::service::IngestorService; use trustify_module_storage::service::StorageBackend; @@ -14,6 +16,11 @@ use uuid::Uuid; use super::polling::{poll_for_product_result, poll_for_result}; /// Background task: submit SBOM to EI client, poll for results, ingest VEX. +/// +/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. +/// It is passed through to the polling functions for CAS-protected terminal +/// status updates. +#[allow(clippy::too_many_arguments)] pub(crate) async fn run_analysis( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, @@ -23,6 +30,8 @@ pub(crate) async fn run_analysis( job_id: Uuid, sbom_id: Uuid, vulnerability_id: &str, + heartbeat: SharedHeartbeat, + token: CancellationToken, ) -> Result<(), anyhow::Error> { // Retrieve the raw SBOM document from storage (read-only – no mutations needed). let tx = db_rw @@ -130,11 +139,17 @@ pub(crate) async fn run_analysis( response.json().await?; let product_id = spdx_upload.product_id; - // Update job to running with the product ID. - ei_service - .update_job_running_with_product(job_id, &product_id, db_rw) + // Read the current heartbeat for CAS-protected update. + let current_hb = heartbeat.lock().ok().and_then(|guard| *guard); + + // Update job to running with the product ID (CAS-protected). + let applied = ei_service + .update_job_running_with_product(job_id, &product_id, current_hb, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + return Err(anyhow::anyhow!("job {job_id} ownership lost, cannot transition to running")); + } tracing::info!( job_id = %job_id, @@ -150,8 +165,9 @@ pub(crate) async fn run_analysis( config, client, job_id, - vulnerability_id, &product_id, + heartbeat, + token, ) .await } else { @@ -221,9 +237,10 @@ pub(crate) async fn run_analysis( config, client, job_id, - vulnerability_id, &scan_id, component_id, + heartbeat, + token, ) .await } diff --git a/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs b/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs new file mode 100644 index 000000000..e927105ec --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs @@ -0,0 +1,193 @@ +//! Heartbeat coordination for exploit-intelligence jobs. +//! +//! Provides optimistic-locking primitives that let multiple Trustify instances +//! claim jobs exclusively. A job is considered "actively owned" if its +//! heartbeat was updated within `2 * RATE`. + +use sea_orm::prelude::Decimal; +use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter}; +use sea_query::Expr; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use time::OffsetDateTime; +use tokio_util::sync::CancellationToken; +use trustify_common::db; +use trustify_entity::exploit_intelligence_job; +use uuid::Uuid; + +/// How often the heartbeat is refreshed. +pub const RATE: Duration = Duration::from_secs(10); + +/// Typed error returned by [`run_with_heartbeat`]. +/// +/// Distinguishes between a heartbeat loss (another instance claimed the job) +/// and a work-future error (the analysis itself failed). +pub enum HeartbeatError { + /// The heartbeat CAS failed — another instance owns this job. + /// The caller MUST NOT write to the job row after receiving this. + LockLost, + /// The work future returned an error. The work future is responsible + /// for having already called terminal status updates with CAS protection. + Work(anyhow::Error), +} + +impl std::fmt::Display for HeartbeatError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::LockLost => write!(f, "heartbeat lock lost"), + Self::Work(e) => write!(f, "work error: {e}"), + } + } +} + +impl std::fmt::Debug for HeartbeatError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::LockLost => write!(f, "HeartbeatError::LockLost"), + Self::Work(e) => write!(f, "HeartbeatError::Work({e:?})"), + } + } +} + +/// Shared state for the current heartbeat value. +/// +/// Updated by the heartbeat loop; read by the work future when performing +/// CAS-protected terminal status updates. +pub type SharedHeartbeat = Arc>>; + +/// Create a new shared heartbeat state initialised with the given value. +pub fn shared_heartbeat(initial: Option) -> SharedHeartbeat { + Arc::new(Mutex::new(initial)) +} + +/// Attempt one heartbeat update using optimistic locking. +/// +/// The caller provides the previous heartbeat value (or `None` for the first +/// beat). A CAS-style `UPDATE … WHERE heartbeat = ` ensures that two +/// instances cannot both claim the same job. +/// +/// Returns the new heartbeat value on success. +pub async fn beat( + job_id: Uuid, + old_heartbeat: Option, + db: &impl ConnectionTrait, +) -> Result { + let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); + let new_hb = Decimal::from_i128_with_scale(now, 0); + + let lock = match old_heartbeat { + Some(t) => Expr::col(exploit_intelligence_job::Column::Heartbeat).eq(t), + None => Expr::col(exploit_intelligence_job::Column::Heartbeat).is_null(), + }; + + let result = exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Heartbeat, + Expr::value(new_hb), + ) + .col_expr( + exploit_intelligence_job::Column::Updated, + Expr::value(OffsetDateTime::now_utc()), + ) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .filter(lock) + .exec(db) + .await?; + + if result.rows_affected == 1 { + Ok(new_hb) + } else { + Err(anyhow::anyhow!( + "heartbeat update affected 0 rows — lock lost" + )) + } +} + +/// Clear the heartbeat on a job so another instance can claim it. +/// +/// Not currently called — terminal status updates clear the heartbeat +/// atomically via CAS. Retained for manual cleanup scenarios. +#[allow(dead_code)] +pub async fn clear_heartbeat( + job_id: Uuid, + db: &impl ConnectionTrait, +) -> Result<(), anyhow::Error> { + exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Heartbeat, + Expr::value(Option::::None), + ) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .exec(db) + .await?; + Ok(()) +} + +/// Run a future while maintaining a heartbeat loop alongside it. +/// +/// The heartbeat is refreshed every [`RATE`] seconds. If the heartbeat +/// is lost (another instance claimed the job), the future's result is +/// discarded and [`HeartbeatError::LockLost`] is returned. When the work +/// completes, its result is propagated: `Ok(())` on success, or +/// [`HeartbeatError::Work`] on failure. +/// +/// The `heartbeat_state` mutex is updated after each successful beat so +/// that the work future can read the current heartbeat value for CAS-protected +/// terminal status updates. +/// +/// The `token` is cancelled when the heartbeat is lost, signalling the work +/// future to stop at the next checkpoint rather than relying solely on +/// `tokio::select!` drop semantics. The work future should check +/// `token.is_cancelled()` before performing side-effecting operations. +/// +/// **Important:** This function does NOT clear the heartbeat on completion. +/// The terminal status update (performed inside the work future with CAS) +/// is responsible for atomically clearing the heartbeat. +pub async fn run_with_heartbeat( + job_id: Uuid, + initial_heartbeat: Option, + db_hb: &db::ReadWrite, + heartbeat_state: SharedHeartbeat, + token: CancellationToken, + work: F, +) -> Result<(), HeartbeatError> +where + F: Future>, +{ + tokio::pin!(work); + + let mut heartbeat_val = initial_heartbeat; + let mut interval = tokio::time::interval(RATE); + // Consume the first immediate tick so we don't beat immediately. + interval.tick().await; + + loop { + tokio::select! { + result = &mut work => { + let result: Result<(), anyhow::Error> = result; + return result.map_err(HeartbeatError::Work); + } + _ = interval.tick() => { + match beat(job_id, heartbeat_val, &**db_hb).await { + Ok(hb) => { + heartbeat_val = Some(hb); + // Update shared state so the work future can read it. + if let Ok(mut guard) = heartbeat_state.lock() { + *guard = Some(hb); + } + } + Err(_) => { + tracing::warn!(job_id = %job_id, "heartbeat lost, aborting work"); + token.cancel(); + return Err(HeartbeatError::LockLost); + } + } + } + _ = token.cancelled() => { + tracing::warn!(job_id = %job_id, "cancellation token triggered, aborting work"); + return Err(HeartbeatError::LockLost); + } + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/runner/mod.rs b/modules/fundamental/src/exploit_intelligence/runner/mod.rs index 191c932ce..449e57954 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod analysis; +pub(crate) mod heartbeat; pub(crate) mod polling; pub(crate) mod recovery; diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index 156362503..a38c17837 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -1,5 +1,6 @@ use crate::exploit_intelligence::{ runner::{EiRequestError, classify_response_error}, + runner::heartbeat::SharedHeartbeat, service::ExploitIntelligenceService, }; use backon::{ConstantBuilder, Retryable}; @@ -7,6 +8,7 @@ use sea_orm::TransactionTrait; use std::cell::Cell; use std::str::FromStr; use std::time::Duration; +use tokio_util::sync::CancellationToken; use trustify_common::db; use trustify_entity::labels::Labels; use trustify_module_ingestor::service::{Cache, Format, IngestorService}; @@ -19,8 +21,13 @@ use uuid::Uuid; /// `run_analysis` (normal path) and from the startup recovery path for /// orphaned Running jobs. /// +/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. +/// It is read when performing CAS-protected terminal status updates so that +/// only the owning instance can transition the job to a terminal state. +/// /// Updates both the component record (with finding/advisory) and the parent /// job record (status only) upon completion. +#[allow(clippy::too_many_arguments)] pub(crate) async fn poll_for_result( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, @@ -28,9 +35,10 @@ pub(crate) async fn poll_for_result( config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, client: &reqwest::Client, job_id: Uuid, - _vulnerability_id: &str, scan_id: &str, component_id: Uuid, + heartbeat: SharedHeartbeat, + token: CancellationToken, ) -> Result<(), anyhow::Error> { let poll_url = format!( "{}/api/v1/reports/by-scan-id/{}", @@ -40,6 +48,7 @@ pub(crate) async fn poll_for_result( let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); + let token_for_when = token.clone(); let result = tokio::time::timeout( config.max_poll_duration, @@ -50,12 +59,12 @@ pub(crate) async fn poll_for_result( async move { let mut poll_request = cli.get(&url); if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { + let auth_token = provider.get_token().await.map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - poll_request = poll_request.bearer_auth(&token); + poll_request = poll_request.bearer_auth(&auth_token); } let resp = classify_response_error(poll_request.send().await)?; @@ -84,17 +93,22 @@ pub(crate) async fn poll_for_result( .with_delay(poll_interval) .without_max_times(), ) - .when(|e| match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true + .when(|e| { + if token_for_when.is_cancelled() { + return false; } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < max_consecutive + match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < max_consecutive + } + EiRequestError::Permanent(_) => false, } - EiRequestError::Permanent(_) => false, }) .notify(|e, dur| { if !matches!(e, EiRequestError::Transient(r) if r == "pending") { @@ -104,24 +118,43 @@ pub(crate) async fn poll_for_result( ) .await; + /// Read the current heartbeat value from the shared state. + fn read_hb(hb: &SharedHeartbeat) -> Option { + hb.lock().ok().and_then(|guard| *guard) + } + // Convert timeout/retry result to our expected shape. let poll = match result { Ok(Ok(poll)) => poll, Ok(Err(e)) => { let err = e.into_anyhow(); let msg = err.to_string(); - ei_service - .update_job_failed(job_id, &msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + let applied = ei_service + .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); - ei_service - .update_job_failed(job_id, &msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + let applied = ei_service + .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } return Err(anyhow::anyhow!("{msg}")); } }; @@ -155,6 +188,11 @@ pub(crate) async fn poll_for_result( let advisory_id = if let Some(vex) = vex_json { if !vex.is_null() { + // Check cancellation before VEX ingestion (side-effecting operation). + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before VEX ingestion")); + } + // Ingest the VEX document as an advisory with source label. let vex_bytes = serde_json::to_vec(vex)?; let mut labels = Labels::default(); @@ -192,11 +230,20 @@ pub(crate) async fn poll_for_result( .await .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + // Check cancellation before terminal job status update. + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + // Mark the parent job as completed (finding lives on the component). - ei_service - .update_job_completed(job_id, db_rw) + let applied = ei_service + .update_job_completed(job_id, read_hb(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } tracing::info!( job_id = %job_id, @@ -214,10 +261,18 @@ pub(crate) async fn poll_for_result( .unwrap_or("analysis failed") .to_string(); - ei_service - .update_job_failed(job_id, &error_msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + + let applied = ei_service + .update_job_failed(job_id, &error_msg, read_hb(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } Ok(()) } @@ -230,6 +285,10 @@ pub(crate) async fn poll_for_result( /// This is the SPDX counterpart of `poll_for_result`. It polls the product-level /// endpoint until all components are complete, then fetches each component's /// report individually. +/// +/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. +/// It is read when performing CAS-protected terminal status updates. +#[allow(clippy::too_many_arguments)] pub(crate) async fn poll_for_product_result( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, @@ -237,8 +296,9 @@ pub(crate) async fn poll_for_product_result( config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, client: &reqwest::Client, job_id: Uuid, - _vulnerability_id: &str, product_id: &str, + heartbeat: SharedHeartbeat, + token: CancellationToken, ) -> Result<(), anyhow::Error> { let product_url = format!( "{}/api/v1/products/{}", @@ -248,6 +308,7 @@ pub(crate) async fn poll_for_product_result( let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); + let token_for_when = token.clone(); let mut components_initialized = false; @@ -285,12 +346,12 @@ pub(crate) async fn poll_for_product_result( async move { let mut poll_request = cli.get(&url); if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { + let auth_token = provider.get_token().await.map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - poll_request = poll_request.bearer_auth(&token); + poll_request = poll_request.bearer_auth(&auth_token); } let resp = classify_response_error(poll_request.send().await)?; @@ -319,17 +380,22 @@ pub(crate) async fn poll_for_product_result( .with_delay(poll_interval) .without_max_times(), ) - .when(|e| match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true + .when(|e| { + if token_for_when.is_cancelled() { + return false; } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < max_consecutive + match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true + } + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < max_consecutive + } + EiRequestError::Permanent(_) => false, } - EiRequestError::Permanent(_) => false, }) .notify(|e, dur| { if !matches!(e, EiRequestError::Transient(r) if r == "pending") { @@ -339,24 +405,43 @@ pub(crate) async fn poll_for_product_result( ) .await; + /// Read the current heartbeat value from the shared state. + fn read_hb_product(hb: &SharedHeartbeat) -> Option { + hb.lock().ok().and_then(|guard| *guard) + } + // Convert timeout/retry result to our expected shape. let product = match result { Ok(Ok(product)) => product, Ok(Err(e)) => { let err = e.into_anyhow(); let msg = err.to_string(); - ei_service - .update_job_failed(job_id, &msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + let applied = ei_service + .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); - ei_service - .update_job_failed(job_id, &msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + let applied = ei_service + .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } return Err(anyhow::anyhow!("{msg}")); } }; @@ -382,11 +467,11 @@ pub(crate) async fn poll_for_product_result( let mut reports_request = client.get(&reports_url); if let Some(ref provider) = config.token_provider { - let token = provider + let auth_token = provider .get_token() .await .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - reports_request = reports_request.bearer_auth(&token); + reports_request = reports_request.bearer_auth(&auth_token); } let reports_response = reports_request @@ -485,11 +570,11 @@ pub(crate) async fn poll_for_product_result( let mut full_request = client.get(&full_report_url); if let Some(ref provider) = config.token_provider { - let token = provider + let auth_token = provider .get_token() .await .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - full_request = full_request.bearer_auth(&token); + full_request = full_request.bearer_auth(&auth_token); } let advisory_id = match full_request.send().await { @@ -505,6 +590,11 @@ pub(crate) async fn poll_for_product_result( if let Some(vex) = vex_json { if !vex.is_null() { + // Check cancellation before VEX ingestion (side-effecting operation). + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before VEX ingestion")); + } + let vex_bytes = serde_json::to_vec(vex)?; let mut labels = Labels::default(); labels @@ -564,12 +654,21 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; } + // Check cancellation before terminal job status update. + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + // Mark the parent job as completed. The aggregate finding is // computed from component records at query time. - ei_service - .update_job_completed(job_id, db_rw) + let applied = ei_service + .update_job_completed(job_id, read_hb_product(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } tracing::info!( job_id = %job_id, @@ -581,10 +680,17 @@ pub(crate) async fn poll_for_product_result( } "failed" => { let msg = format!("EI product analysis failed for product {product_id}"); - ei_service - .update_job_failed(job_id, &msg, db_rw) + if token.is_cancelled() { + return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + } + let applied = ei_service + .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + if !applied { + tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); + return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + } Ok(()) } _ => unreachable!("backon retry only returns terminal states"), diff --git a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs index c6adf31c3..ea8080f39 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs @@ -1,14 +1,17 @@ use crate::exploit_intelligence::service::ExploitIntelligenceService; +use tokio_util::sync::CancellationToken; use trustify_common::db; use trustify_module_ingestor::service::IngestorService; +use super::heartbeat::{self, HeartbeatError}; use super::polling::{poll_for_product_result, poll_for_result}; /// Recover orphaned EI jobs after a server restart. /// /// Pending jobs (never submitted) are marked as Failed. Running jobs /// (scan_id present) get their poll loops re-spawned so the EI backend -/// result is picked up transparently. +/// result is picked up transparently. Each resumed task maintains a +/// heartbeat so other instances know the job is still being serviced. pub(crate) async fn recover_and_resume( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, @@ -31,34 +34,45 @@ pub(crate) async fn recover_and_resume( for job in result.running_jobs { let job_id = job.id; - let vulnerability_id = job.vulnerability_id.clone(); let ei_svc = ei_service.clone(); let ingestor_svc = ingestor.clone(); let db_rw_clone = db_rw.clone(); let config_clone = config.clone(); let client_clone = client.clone(); + // The heartbeat was freshly set by recover_orphaned_jobs. + let initial_heartbeat = job.heartbeat; if let Some(ref product_id) = job.product_id { // Multi-component SPDX flow — resume product-level polling. let product_id = product_id.clone(); + let db_hb = db_rw.clone(); actix_web::rt::spawn(async move { tracing::info!(job_id = %job_id, product_id = %product_id, "resuming orphaned SPDX product polling"); - if let Err(e) = poll_for_product_result( + + let token = CancellationToken::new(); + let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); + let hb_for_work = hb_state.clone(); + let token_for_work = token.clone(); + + let work = poll_for_product_result( &ei_svc, &ingestor_svc, &db_rw_clone, &config_clone, &client_clone, job_id, - &vulnerability_id, &product_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; + hb_for_work, + token_for_work, + ); + match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + Ok(()) => {} + Err(HeartbeatError::LockLost) => { + tracing::warn!(job_id = %job_id, "heartbeat lost during SPDX product recovery, another instance owns this job"); + } + Err(HeartbeatError::Work(e)) => { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); + } } }); } else { @@ -68,7 +82,7 @@ pub(crate) async fn recover_and_resume( Err(e) => { tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); let _ = ei_service - .update_job_failed(job_id, &format!("recovery error: {e}"), db_rw) + .update_job_failed(job_id, &format!("recovery error: {e}"), initial_heartbeat, db_rw) .await; continue; } @@ -80,25 +94,35 @@ pub(crate) async fn recover_and_resume( if let (Some(scan_id), Some(component_id)) = (scan_id, component_id) { let scan_id = scan_id.to_string(); + let db_hb = db_rw.clone(); actix_web::rt::spawn(async move { tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); - if let Err(e) = poll_for_result( + + let token = CancellationToken::new(); + let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); + let hb_for_work = hb_state.clone(); + let token_for_work = token.clone(); + + let work = poll_for_result( &ei_svc, &ingestor_svc, &db_rw_clone, &config_clone, &client_clone, job_id, - &vulnerability_id, &scan_id, component_id, - ) - .await - { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); - let _ = ei_svc - .update_job_failed(job_id, &e.to_string(), &db_rw_clone) - .await; + hb_for_work, + token_for_work, + ); + match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + Ok(()) => {} + Err(HeartbeatError::LockLost) => { + tracing::warn!(job_id = %job_id, "heartbeat lost during recovery, another instance owns this job"); + } + Err(HeartbeatError::Work(e)) => { + tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); + } } }); } else { @@ -107,6 +131,7 @@ pub(crate) async fn recover_and_resume( .update_job_failed( job_id, "Running job missing component scan_id during recovery", + initial_heartbeat, db_rw, ) .await; diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 087803c3a..0f7d6e912 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -5,9 +5,10 @@ use crate::{ }, }; use sea_orm::{ - ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, IntoActiveModel, PaginatorTrait, - QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, - sea_query::Expr, + ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, EntityTrait, IntoActiveModel, + PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, + prelude::Decimal, + sea_query::{Expr, LockBehavior, LockType}, }; use std::collections::HashMap; use std::time::Duration; @@ -116,6 +117,19 @@ impl ExploitIntelligenceService { /// /// The caller must provide a connection (typically a transaction) and is /// responsible for committing it. + /// Create an analysis job record in the database. + /// + /// The job is created with an initial heartbeat value so it is "born owned" + /// by the creating instance. This prevents the recovery pass from marking + /// it as orphaned before the heartbeat loop has started. + /// + /// Returns the created job details. The caller is responsible for + /// spawning the background task that submits the request and polls for + /// results. The initial heartbeat value should be passed to + /// [`run_with_heartbeat`](crate::exploit_intelligence::runner::heartbeat::run_with_heartbeat). + /// + /// The caller must provide a connection (typically a transaction) and is + /// responsible for committing it. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn create_job( &self, @@ -127,6 +141,7 @@ impl ExploitIntelligenceService { let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); + let initial_hb = Decimal::from_i128_with_scale(now.unix_timestamp_nanos(), 0); let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), @@ -136,6 +151,7 @@ impl ExploitIntelligenceService { error_message: Set(None), product_id: Set(None), total_components: Set(None), + heartbeat: Set(Some(initial_hb)), created: Set(now), updated: Set(now), }; @@ -310,17 +326,16 @@ impl ExploitIntelligenceService { }; // Set report_url from component scan_id for single-component jobs. - if model.product_id.is_none() { - if let Some(scan_id) = scan_id_map.get(&model.id) { - if let Some(base) = self.ui_url() { - summary.report_url = Some(format!( - "{}/reports/component/{}/{}", - base, - urlencoding::encode(&model.vulnerability_id), - urlencoding::encode(scan_id), - )); - } - } + if model.product_id.is_none() + && let Some(scan_id) = scan_id_map.get(&model.id) + && let Some(base) = self.ui_url() + { + summary.report_url = Some(format!( + "{}/reports/component/{}/{}", + base, + urlencoding::encode(&model.vulnerability_id), + urlencoding::encode(scan_id), + )); } summary @@ -333,50 +348,101 @@ impl ExploitIntelligenceService { }) } - /// Update a job to completed. + /// Update a job to completed, using the heartbeat as a CAS condition. + /// + /// Only succeeds if the job's current heartbeat matches the provided value. + /// On success the heartbeat is cleared (set to NULL) and the status is set + /// to Completed. Returns `true` if the update was applied, `false` if the + /// CAS failed (another instance owns the job). #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_completed( &self, job_id: Uuid, + heartbeat: Option, db: &db::ReadWrite, - ) -> Result<(), Error> { - let tx = db.begin().await?; + ) -> Result { let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job::ActiveModel { - id: Set(job_id), - status: Set(ExploitIntelligenceJobStatus::Completed), - updated: Set(now), - ..Default::default() - }; + let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); + match heartbeat { + Some(hb) => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); + } + None => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); + } + } - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; - Ok(()) + let result = exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Completed), + ) + .col_expr( + exploit_intelligence_job::Column::Heartbeat, + Expr::value(Option::::None), + ) + .col_expr( + exploit_intelligence_job::Column::Updated, + Expr::value(now), + ) + .filter(filter) + .exec(&**db) + .await + .map_err(Error::from)?; + + Ok(result.rows_affected > 0) } - /// Update a job to failed with an error message. + /// Update a job to failed with an error message, using the heartbeat as a CAS condition. + /// + /// Only succeeds if the job's current heartbeat matches the provided value. + /// On success the heartbeat is cleared (set to NULL) and the status is set + /// to Failed. Returns `true` if the update was applied, `false` if the + /// CAS failed (another instance owns the job). #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_failed( &self, job_id: Uuid, error_message: &str, + heartbeat: Option, db: &db::ReadWrite, - ) -> Result<(), Error> { - let tx = db.begin().await?; + ) -> Result { let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job::ActiveModel { - id: Set(job_id), - status: Set(ExploitIntelligenceJobStatus::Failed), - error_message: Set(Some(error_message.to_string())), - updated: Set(now), - ..Default::default() - }; + let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); + match heartbeat { + Some(hb) => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); + } + None => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); + } + } - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; - Ok(()) + let result = exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Failed), + ) + .col_expr( + exploit_intelligence_job::Column::ErrorMessage, + Expr::value(Some(error_message.to_string())), + ) + .col_expr( + exploit_intelligence_job::Column::Heartbeat, + Expr::value(Option::::None), + ) + .col_expr( + exploit_intelligence_job::Column::Updated, + Expr::value(now), + ) + .filter(filter) + .exec(&**db) + .await + .map_err(Error::from)?; + + Ok(result.rows_affected > 0) } /// Find an existing job in Pending or Running state for the given SBOM and vulnerability. @@ -408,6 +474,9 @@ impl ExploitIntelligenceService { /// Create an analysis job with product_id and total_components set (multi-component SPDX flow). /// + /// The job is created with an initial heartbeat value so it is "born owned" + /// by the creating instance. + /// /// Returns the created job details. The caller is responsible for creating /// component records and spawning background tasks. #[instrument(skip_all, err(level = tracing::Level::INFO))] @@ -422,6 +491,7 @@ impl ExploitIntelligenceService { let _config = self.runtime.as_ref().ok_or(Error::Unavailable)?; let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); + let initial_hb = Decimal::from_i128_with_scale(now.unix_timestamp_nanos(), 0); let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), @@ -431,6 +501,7 @@ impl ExploitIntelligenceService { error_message: Set(None), product_id: Set(Some(product_id.to_string())), total_components: Set(Some(total_components)), + heartbeat: Set(Some(initial_hb)), created: Set(now), updated: Set(now), }; @@ -512,6 +583,9 @@ impl ExploitIntelligenceService { } /// Update a component's status to running and store the scan ID from the EI service. + /// + /// Includes a guard: only updates if the component is not already in a terminal + /// state (completed), preventing silent overwrites of finished components. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_component_running( &self, @@ -521,23 +595,37 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let tx = db.begin().await?; let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job_component::ActiveModel { - id: Set(component_id), - scan_id: Set(Some(scan_id.to_string())), - status: Set(ExploitIntelligenceJobStatus::Running), - updated: Set(now), - ..Default::default() - }; + exploit_intelligence_job_component::Entity::update_many() + .col_expr( + exploit_intelligence_job_component::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Running), + ) + .col_expr( + exploit_intelligence_job_component::Column::ScanId, + Expr::value(Some(scan_id.to_string())), + ) + .col_expr( + exploit_intelligence_job_component::Column::Updated, + Expr::value(now), + ) + .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + .filter( + exploit_intelligence_job_component::Column::Status + .ne(ExploitIntelligenceJobStatus::Completed), + ) + .exec(&**db) + .await + .map_err(Error::from)?; - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; Ok(()) } /// Update a component to completed with the analysis finding. + /// + /// Includes a guard: only updates if the component is not already completed, + /// preventing silent re-writes of terminal state. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_component_completed( &self, @@ -548,24 +636,41 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let tx = db.begin().await?; let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job_component::ActiveModel { - id: Set(component_id), - status: Set(ExploitIntelligenceJobStatus::Completed), - finding: Set(Some(finding)), - advisory_id: Set(advisory_id), - updated: Set(now), - ..Default::default() - }; + exploit_intelligence_job_component::Entity::update_many() + .col_expr( + exploit_intelligence_job_component::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Completed), + ) + .col_expr( + exploit_intelligence_job_component::Column::Finding, + Expr::value(Some(finding)), + ) + .col_expr( + exploit_intelligence_job_component::Column::AdvisoryId, + Expr::value(advisory_id), + ) + .col_expr( + exploit_intelligence_job_component::Column::Updated, + Expr::value(now), + ) + .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + .filter( + exploit_intelligence_job_component::Column::Status + .ne(ExploitIntelligenceJobStatus::Completed), + ) + .exec(&**db) + .await + .map_err(Error::from)?; - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; Ok(()) } /// Update a component to failed with an error message. + /// + /// Includes a guard: only updates if the component is not already completed, + /// preventing silent re-writes of terminal state. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_component_failed( &self, @@ -575,44 +680,77 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let tx = db.begin().await?; let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job_component::ActiveModel { - id: Set(component_id), - status: Set(ExploitIntelligenceJobStatus::Failed), - error_message: Set(Some(error_message.to_string())), - updated: Set(now), - ..Default::default() - }; + exploit_intelligence_job_component::Entity::update_many() + .col_expr( + exploit_intelligence_job_component::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Failed), + ) + .col_expr( + exploit_intelligence_job_component::Column::ErrorMessage, + Expr::value(Some(error_message.to_string())), + ) + .col_expr( + exploit_intelligence_job_component::Column::Updated, + Expr::value(now), + ) + .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + .filter( + exploit_intelligence_job_component::Column::Status + .ne(ExploitIntelligenceJobStatus::Completed), + ) + .exec(&**db) + .await + .map_err(Error::from)?; - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; Ok(()) } /// Update a job's status to running and store the product ID from the EI service. + /// + /// Uses the heartbeat as a CAS condition to ensure only the owning instance + /// can transition the job. Returns `true` if the update was applied, `false` + /// if the CAS failed (another instance owns the job). #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_running_with_product( &self, job_id: Uuid, product_id: &str, + heartbeat: Option, db: &db::ReadWrite, - ) -> Result<(), Error> { - let tx = db.begin().await?; + ) -> Result { let now = OffsetDateTime::now_utc(); - let model = exploit_intelligence_job::ActiveModel { - id: Set(job_id), - product_id: Set(Some(product_id.to_string())), - status: Set(ExploitIntelligenceJobStatus::Running), - updated: Set(now), - ..Default::default() - }; + let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); + match heartbeat { + Some(hb) => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); + } + None => { + filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); + } + } - model.update(&tx).await.map_err(Error::from)?; - tx.commit().await?; - Ok(()) + let result = exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Running), + ) + .col_expr( + exploit_intelligence_job::Column::ProductId, + Expr::value(Some(product_id.to_string())), + ) + .col_expr( + exploit_intelligence_job::Column::Updated, + Expr::value(now), + ) + .filter(filter) + .exec(&**db) + .await + .map_err(Error::from)?; + + Ok(result.rows_affected > 0) } /// Fetch all component records for a given job, ordered by creation time. @@ -645,22 +783,40 @@ impl ExploitIntelligenceService { /// **Pending** jobs (no scan_id yet) are marked as Failed because the SBOM /// was never submitted to the EI service. /// - /// **Running** jobs (scan_id is present) are returned to the caller so - /// their poll loops can be resumed — the EI service may still be working - /// on them. + /// **Running** jobs whose heartbeat is stale (or absent) are returned to + /// the caller so their poll loops can be resumed — the EI service may + /// still be working on them. Jobs with an active heartbeat are being + /// serviced by another instance and are left alone. + /// + /// `FOR UPDATE SKIP LOCKED` prevents two instances from claiming the same + /// orphaned jobs simultaneously. pub async fn recover_orphaned_jobs( &self, db: &db::ReadWrite, ) -> Result { + use crate::exploit_intelligence::runner::heartbeat; + let tx = db.begin().await?; let now = OffsetDateTime::now_utc(); - // Find all Pending or Running jobs. + // Heartbeats older than 2 × RATE are considered stale. + let stale_threshold = { + let threshold_nanos = (now - 2 * heartbeat::RATE).unix_timestamp_nanos(); + Decimal::from_i128_with_scale(threshold_nanos, 0) + }; + + // Find Pending/Running jobs that are NOT actively heartbeating. let orphaned = exploit_intelligence_job::Entity::find() .filter(exploit_intelligence_job::Column::Status.is_in([ ExploitIntelligenceJobStatus::Pending, ExploitIntelligenceJobStatus::Running, ])) + .filter( + exploit_intelligence_job::Column::Heartbeat + .is_null() + .or(exploit_intelligence_job::Column::Heartbeat.lt(stale_threshold)), + ) + .lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) .all(&tx) .await .map_err(Error::from)?; @@ -678,14 +834,26 @@ impl ExploitIntelligenceService { Please re-trigger the analysis." .to_string(), )); + active.heartbeat = Set(None); active.updated = Set(now); active.update(&tx).await.map_err(Error::from)?; pending_failed += 1; } ExploitIntelligenceJobStatus::Running => { - // Keep them in Running state — the caller will probe the - // EI backend and either resume polling or fail them. - running_jobs.push(job); + // Claim the job by setting a fresh heartbeat inside + // the same transaction, so the row lock protects us. + let hb_nanos = now.unix_timestamp_nanos(); + let new_hb = Decimal::from_i128_with_scale(hb_nanos, 0); + let mut active = job.clone().into_active_model(); + active.heartbeat = Set(Some(new_hb)); + active.updated = Set(now); + active.update(&tx).await.map_err(Error::from)?; + + // Push the model with the NEW heartbeat value so + // callers pass the correct value to run_with_heartbeat. + let mut claimed_job = job; + claimed_job.heartbeat = Some(new_hb); + running_jobs.push(claimed_job); } _ => { // Should not happen given the filter, but be safe. From 8690e40757e991322558e8fee9d75ad32313a15b Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Tue, 14 Jul 2026 13:59:32 +0100 Subject: [PATCH 19/25] fix(exploit-intelligence): replace expect() with error propagation in OIDC client builder Change TokenProvider::oidc() to return Result and propagate the error to the caller, fixing the clippy::expect_used violation that fails CI. Co-Authored-By: Claude Opus 4.6 (1M context) --- ...0002230_create_exploit_intelligence_job.rs | 35 +++++- .../src/exploit_intelligence/auth.rs | 12 +- .../src/exploit_intelligence/endpoints/mod.rs | 11 +- .../src/exploit_intelligence/model/mod.rs | 11 +- .../exploit_intelligence/runner/analysis.rs | 66 ++++++----- .../exploit_intelligence/runner/heartbeat.rs | 5 +- .../exploit_intelligence/runner/polling.rs | 104 ++++++++++++------ .../exploit_intelligence/runner/recovery.rs | 29 ++++- .../src/exploit_intelligence/service/mod.rs | 66 ++++++----- server/src/profile/api.rs | 42 +++++-- 10 files changed, 260 insertions(+), 121 deletions(-) diff --git a/migration/src/m0002230_create_exploit_intelligence_job.rs b/migration/src/m0002230_create_exploit_intelligence_job.rs index 04ec22087..dce4a2733 100644 --- a/migration/src/m0002230_create_exploit_intelligence_job.rs +++ b/migration/src/m0002230_create_exploit_intelligence_job.rs @@ -54,10 +54,20 @@ impl MigrationTrait for Migration { .not_null() .default("pending"), ) - .col(ColumnDef::new(ExploitIntelligenceJob::ErrorMessage).string()) - .col(ColumnDef::new(ExploitIntelligenceJob::ProductId).string()) - .col(ColumnDef::new(ExploitIntelligenceJob::TotalComponents).integer()) - .col(ColumnDef::new(ExploitIntelligenceJob::Heartbeat).decimal().null()) + .col( + ColumnDef::new(ExploitIntelligenceJob::ErrorMessage).string(), + ) + .col( + ColumnDef::new(ExploitIntelligenceJob::ProductId).string(), + ) + .col( + ColumnDef::new(ExploitIntelligenceJob::TotalComponents).integer(), + ) + .col( + ColumnDef::new(ExploitIntelligenceJob::Heartbeat) + .decimal() + .null(), + ) .col( ColumnDef::new(ExploitIntelligenceJob::Created) .timestamp_with_time_zone() @@ -103,6 +113,18 @@ impl MigrationTrait for Migration { ) .await?; + // Partial unique index to prevent duplicate active jobs for the same + // (sbom_id, vulnerability_id) pair. SeaORM's Index builder does not + // support partial indexes, so we use raw SQL. + manager + .get_connection() + .execute_unprepared( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_ei_job_active_dedup \ + ON exploit_intelligence_job (sbom_id, vulnerability_id) \ + WHERE status IN ('pending', 'running')", + ) + .await?; + manager .create_table( Table::create() @@ -223,6 +245,11 @@ impl MigrationTrait for Migration { ) .await?; + manager + .get_connection() + .execute_unprepared("DROP INDEX IF EXISTS idx_ei_job_active_dedup") + .await?; + manager .drop_index( Index::drop() diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs index 5f8a2838d..05c42087c 100644 --- a/modules/fundamental/src/exploit_intelligence/auth.rs +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -53,19 +53,23 @@ impl TokenProvider { /// /// Builds an internal HTTP client with sensible timeouts for token /// exchange requests. - pub fn oidc(token_url: String, client_id: String, client_secret: String) -> Self { + pub fn oidc( + token_url: String, + client_id: String, + client_secret: String, + ) -> Result { let client = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(30)) .build() - .expect("failed to build OIDC HTTP client"); - Self::Oidc(Arc::new(OidcState { + .map_err(|e| anyhow::anyhow!("failed to build OIDC HTTP client: {e}"))?; + Ok(Self::Oidc(Arc::new(OidcState { client, token_url, client_id, client_secret, cached: RwLock::new(None), - })) + }))) } /// Get a valid bearer token string. For OIDC, refreshes automatically if expired. diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index c6fc189fd..7cfce10de 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -155,7 +155,16 @@ pub async fn analyze( hb_for_work, token_for_work, ); - match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + match heartbeat::run_with_heartbeat( + job_id, + initial_heartbeat, + &db_hb, + hb_state, + token, + work, + ) + .await + { Ok(()) => {} Err(HeartbeatError::LockLost) => { tracing::warn!(job_id = %job_id, "heartbeat lost, another instance owns this job"); diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index 74f320532..a9bab88b8 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -1,3 +1,4 @@ +use sea_orm::prelude::Decimal; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use trustify_entity::exploit_intelligence_job::{ExploitIntelligenceJobStatus, Finding}; @@ -98,6 +99,11 @@ pub struct ExploitIntelligenceJobDetails { /// Timestamp when the job was last updated. #[schema(value_type = String)] pub updated: OffsetDateTime, + /// Internal: current heartbeat value for CAS operations. + /// Not exposed in the API response. + #[serde(skip)] + #[schema(ignore)] + pub heartbeat: Option, } /// Per-component analysis result within an analysis job. @@ -347,6 +353,7 @@ impl ExploitIntelligenceJobDetails { components: Vec::new(), created: model.created, updated: model.updated, + heartbeat: model.heartbeat, } } @@ -358,9 +365,7 @@ impl ExploitIntelligenceJobDetails { component_models: &[trustify_entity::exploit_intelligence_job_component::Model], ui_url: Option<&str>, ) -> Self { - let first_scan_id = component_models - .first() - .and_then(|c| c.scan_id.as_deref()); + let first_scan_id = component_models.first().and_then(|c| c.scan_id.as_deref()); let mut details = Self { report_url: compute_job_report_url(model, ui_url, first_scan_id), ..Self::from_entity(model, ui_url) diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index 6f5501ce2..aa65a744e 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -1,6 +1,6 @@ use crate::exploit_intelligence::{ - runner::{EiRequestError, classify_response_error}, runner::heartbeat::SharedHeartbeat, + runner::{EiRequestError, classify_response_error}, service::ExploitIntelligenceService, }; use backon::{ExponentialBuilder, Retryable}; @@ -103,17 +103,15 @@ pub(crate) async fn run_analysis( let cli = client.clone(); let tp = config.token_provider.clone(); async move { - let form = reqwest::multipart::Form::new() - .text("cveId", vuln_id) - .part( - "file", - reqwest::multipart::Part::bytes(sbom) - .file_name("sbom.json") - .mime_str("application/json") - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) - })?, - ); + let form = reqwest::multipart::Form::new().text("cveId", vuln_id).part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); let mut request = cli.post(&url).multipart(form); @@ -129,7 +127,11 @@ pub(crate) async fn run_analysis( classify_response_error(request.send().await) } }) - .retry(ExponentialBuilder::default().with_min_delay(config.upload_retry_delay).with_max_times(config.upload_max_retries)) + .retry( + ExponentialBuilder::default() + .with_min_delay(config.upload_retry_delay) + .with_max_times(config.upload_max_retries), + ) .when(|e| e.is_transient()) .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) .await @@ -148,7 +150,9 @@ pub(crate) async fn run_analysis( .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { - return Err(anyhow::anyhow!("job {job_id} ownership lost, cannot transition to running")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, cannot transition to running" + )); } tracing::info!( @@ -173,7 +177,10 @@ pub(crate) async fn run_analysis( } else { // CycloneDX single-component flow. // Create the component record before uploading so it is tracked. - let component_ref = sbom_model.document_id.clone().unwrap_or_else(|| sbom_model.node_id.clone()); + let component_ref = sbom_model + .document_id + .clone() + .unwrap_or_else(|| sbom_model.node_id.clone()); let component_id = ei_service .create_component(job_id, &component_ref, db_rw) .await @@ -188,17 +195,15 @@ pub(crate) async fn run_analysis( let cli = client.clone(); let tp = config.token_provider.clone(); async move { - let form = reqwest::multipart::Form::new() - .text("cveId", vuln_id) - .part( - "file", - reqwest::multipart::Part::bytes(sbom) - .file_name("sbom.json") - .mime_str("application/json") - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) - })?, - ); + let form = reqwest::multipart::Form::new().text("cveId", vuln_id).part( + "file", + reqwest::multipart::Part::bytes(sbom) + .file_name("sbom.json") + .mime_str("application/json") + .map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("mime error: {e}")) + })?, + ); let mut request = cli.post(&url).multipart(form); @@ -214,13 +219,18 @@ pub(crate) async fn run_analysis( classify_response_error(request.send().await) } }) - .retry(ExponentialBuilder::default().with_min_delay(config.upload_retry_delay).with_max_times(config.upload_max_retries)) + .retry( + ExponentialBuilder::default() + .with_min_delay(config.upload_retry_delay) + .with_max_times(config.upload_max_retries), + ) .when(|e| e.is_transient()) .notify(|e, dur| tracing::warn!(?dur, %e, "retrying EI upload")) .await .map_err(EiRequestError::into_anyhow)?; - let upload: crate::exploit_intelligence::ei_client::UploadResponse = response.json().await?; + let upload: crate::exploit_intelligence::ei_client::UploadResponse = + response.json().await?; let scan_id = upload.report_request_id.report_id; // Update component to running with scan ID. diff --git a/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs b/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs index e927105ec..5eb6a226c 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs @@ -109,10 +109,7 @@ pub async fn beat( /// Not currently called — terminal status updates clear the heartbeat /// atomically via CAS. Retained for manual cleanup scenarios. #[allow(dead_code)] -pub async fn clear_heartbeat( - job_id: Uuid, - db: &impl ConnectionTrait, -) -> Result<(), anyhow::Error> { +pub async fn clear_heartbeat(job_id: Uuid, db: &impl ConnectionTrait) -> Result<(), anyhow::Error> { exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Heartbeat, diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index a38c17837..33976e627 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -1,6 +1,6 @@ use crate::exploit_intelligence::{ - runner::{EiRequestError, classify_response_error}, runner::heartbeat::SharedHeartbeat, + runner::{EiRequestError, classify_response_error}, service::ExploitIntelligenceService, }; use backon::{ConstantBuilder, Retryable}; @@ -41,10 +41,10 @@ pub(crate) async fn poll_for_result( token: CancellationToken, ) -> Result<(), anyhow::Error> { let poll_url = format!( - "{}/api/v1/reports/by-scan-id/{}", - config.url, - urlencoding::encode(scan_id) - ); + "{}/api/v1/reports/by-scan-id/{}", + config.url, + urlencoding::encode(scan_id) + ); let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); @@ -130,7 +130,9 @@ pub(crate) async fn poll_for_result( let err = e.into_anyhow(); let msg = err.to_string(); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) @@ -138,14 +140,18 @@ pub(crate) async fn poll_for_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) @@ -153,7 +159,9 @@ pub(crate) async fn poll_for_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } return Err(anyhow::anyhow!("{msg}")); } @@ -190,7 +198,9 @@ pub(crate) async fn poll_for_result( if !vex.is_null() { // Check cancellation before VEX ingestion (side-effecting operation). if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before VEX ingestion")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before VEX ingestion" + )); } // Ingest the VEX document as an advisory with source label. @@ -232,7 +242,9 @@ pub(crate) async fn poll_for_result( // Check cancellation before terminal job status update. if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } // Mark the parent job as completed (finding lives on the component). @@ -242,7 +254,9 @@ pub(crate) async fn poll_for_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } tracing::info!( @@ -262,7 +276,9 @@ pub(crate) async fn poll_for_result( .to_string(); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service @@ -271,7 +287,9 @@ pub(crate) async fn poll_for_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } Ok(()) @@ -301,10 +319,10 @@ pub(crate) async fn poll_for_product_result( token: CancellationToken, ) -> Result<(), anyhow::Error> { let product_url = format!( - "{}/api/v1/products/{}", - config.url, - urlencoding::encode(product_id) - ); + "{}/api/v1/products/{}", + config.url, + urlencoding::encode(product_id) + ); let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); @@ -417,7 +435,9 @@ pub(crate) async fn poll_for_product_result( let err = e.into_anyhow(); let msg = err.to_string(); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) @@ -425,14 +445,18 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) @@ -440,7 +464,9 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } return Err(anyhow::anyhow!("{msg}")); } @@ -592,14 +618,17 @@ pub(crate) async fn poll_for_product_result( if !vex.is_null() { // Check cancellation before VEX ingestion (side-effecting operation). if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before VEX ingestion")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before VEX ingestion" + )); } let vex_bytes = serde_json::to_vec(vex)?; let mut labels = Labels::default(); - labels - .0 - .insert("source".to_string(), "exploit-intelligence".to_string()); + labels.0.insert( + "source".to_string(), + "exploit-intelligence".to_string(), + ); let tx = db_rw.begin().await?; let result = ingestor @@ -644,19 +673,16 @@ pub(crate) async fn poll_for_product_result( }; ei_service - .update_component_completed( - component_id, - finding, - advisory_id, - db_rw, - ) + .update_component_completed(component_id, finding, advisory_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; } // Check cancellation before terminal job status update. if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } // Mark the parent job as completed. The aggregate finding is @@ -667,7 +693,9 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } tracing::info!( @@ -681,7 +709,9 @@ pub(crate) async fn poll_for_product_result( "failed" => { let msg = format!("EI product analysis failed for product {product_id}"); if token.is_cancelled() { - return Err(anyhow::anyhow!("heartbeat lost, aborting before terminal update")); + return Err(anyhow::anyhow!( + "heartbeat lost, aborting before terminal update" + )); } let applied = ei_service .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) @@ -689,7 +719,9 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; if !applied { tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!("job {job_id} ownership lost, terminal update not applied")); + return Err(anyhow::anyhow!( + "job {job_id} ownership lost, terminal update not applied" + )); } Ok(()) } diff --git a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs index ea8080f39..1e68f7e86 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs @@ -65,7 +65,16 @@ pub(crate) async fn recover_and_resume( hb_for_work, token_for_work, ); - match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + match heartbeat::run_with_heartbeat( + job_id, + initial_heartbeat, + &db_hb, + hb_state, + token, + work, + ) + .await + { Ok(()) => {} Err(HeartbeatError::LockLost) => { tracing::warn!(job_id = %job_id, "heartbeat lost during SPDX product recovery, another instance owns this job"); @@ -82,7 +91,12 @@ pub(crate) async fn recover_and_resume( Err(e) => { tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); let _ = ei_service - .update_job_failed(job_id, &format!("recovery error: {e}"), initial_heartbeat, db_rw) + .update_job_failed( + job_id, + &format!("recovery error: {e}"), + initial_heartbeat, + db_rw, + ) .await; continue; } @@ -115,7 +129,16 @@ pub(crate) async fn recover_and_resume( hb_for_work, token_for_work, ); - match heartbeat::run_with_heartbeat(job_id, initial_heartbeat, &db_hb, hb_state, token, work).await { + match heartbeat::run_with_heartbeat( + job_id, + initial_heartbeat, + &db_hb, + hb_state, + token, + work, + ) + .await + { Ok(()) => {} Err(HeartbeatError::LockLost) => { tracing::warn!(job_id = %job_id, "heartbeat lost during recovery, another instance owns this job"); diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 0f7d6e912..6f74dfaeb 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -106,7 +106,9 @@ impl ExploitIntelligenceService { /// Returns the EI web UI base URL, if configured. pub fn ui_url(&self) -> Option<&str> { - self.runtime.as_ref().and_then(|r| r.config.ui_url.as_deref()) + self.runtime + .as_ref() + .and_then(|r| r.config.ui_url.as_deref()) } /// Create an analysis job record in the database. @@ -158,7 +160,10 @@ impl ExploitIntelligenceService { let result = model.insert(connection).await.map_err(Error::from)?; - Ok(ExploitIntelligenceJobDetails::from_entity(&result, self.ui_url())) + Ok(ExploitIntelligenceJobDetails::from_entity( + &result, + self.ui_url(), + )) } /// Fetch details of a single job by its ID. @@ -182,9 +187,18 @@ impl ExploitIntelligenceService { Some(model) => { let components = self.fetch_components(model.id, connection).await?; if components.is_empty() { - Ok(Some(ExploitIntelligenceJobDetails::from_entity(&model, self.ui_url()))) + Ok(Some(ExploitIntelligenceJobDetails::from_entity( + &model, + self.ui_url(), + ))) } else { - Ok(Some(ExploitIntelligenceJobDetails::from_entity_with_components(&model, &components, self.ui_url()))) + Ok(Some( + ExploitIntelligenceJobDetails::from_entity_with_components( + &model, + &components, + self.ui_url(), + ), + )) } } None => Ok(None), @@ -288,7 +302,9 @@ impl ExploitIntelligenceService { .select_only() .column(exploit_intelligence_job_component::Column::JobId) .column(exploit_intelligence_job_component::Column::ScanId) - .filter(exploit_intelligence_job_component::Column::JobId.is_in(job_ids_for_scan)) + .filter( + exploit_intelligence_job_component::Column::JobId.is_in(job_ids_for_scan), + ) .filter(exploit_intelligence_job_component::Column::ScanId.is_not_null()) .into_tuple() .all(connection) @@ -305,8 +321,14 @@ impl ExploitIntelligenceService { let items = results .iter() .map(|model| { - let mut summary = if let Some(&(completed, failed, total, vulnerable, not_vulnerable, uncertain)) = - counts_map.get(&model.id) + let mut summary = if let Some(&( + completed, + failed, + total, + vulnerable, + not_vulnerable, + uncertain, + )) = counts_map.get(&model.id) { let mut s = ExploitIntelligenceJobSummary::from_entity_with_counts( model, @@ -342,10 +364,7 @@ impl ExploitIntelligenceService { }) .collect(); - Ok(PaginatedResults { - items, - total, - }) + Ok(PaginatedResults { items, total }) } /// Update a job to completed, using the heartbeat as a CAS condition. @@ -382,10 +401,7 @@ impl ExploitIntelligenceService { exploit_intelligence_job::Column::Heartbeat, Expr::value(Option::::None), ) - .col_expr( - exploit_intelligence_job::Column::Updated, - Expr::value(now), - ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) .filter(filter) .exec(&**db) .await @@ -433,10 +449,7 @@ impl ExploitIntelligenceService { exploit_intelligence_job::Column::Heartbeat, Expr::value(Option::::None), ) - .col_expr( - exploit_intelligence_job::Column::Updated, - Expr::value(now), - ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) .filter(filter) .exec(&**db) .await @@ -510,7 +523,10 @@ impl ExploitIntelligenceService { let result = model.insert(&tx).await.map_err(Error::from)?; tx.commit().await?; - Ok(ExploitIntelligenceJobDetails::from_entity(&result, self.ui_url())) + Ok(ExploitIntelligenceJobDetails::from_entity( + &result, + self.ui_url(), + )) } /// Create a component record for an analysis job. @@ -741,10 +757,7 @@ impl ExploitIntelligenceService { exploit_intelligence_job::Column::ProductId, Expr::value(Some(product_id.to_string())), ) - .col_expr( - exploit_intelligence_job::Column::Updated, - Expr::value(now), - ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) .filter(filter) .exec(&**db) .await @@ -790,10 +803,7 @@ impl ExploitIntelligenceService { /// /// `FOR UPDATE SKIP LOCKED` prevents two instances from claiming the same /// orphaned jobs simultaneously. - pub async fn recover_orphaned_jobs( - &self, - db: &db::ReadWrite, - ) -> Result { + pub async fn recover_orphaned_jobs(&self, db: &db::ReadWrite) -> Result { use crate::exploit_intelligence::runner::heartbeat; let tx = db.begin().await?; diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 21f2df485..e79db8fe2 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -101,23 +101,43 @@ pub struct Run { pub exploit_intelligence_ui_url: Option, /// Polling interval for EI analysis completion (humantime, e.g. "30s"). - #[arg(long, env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL", default_value = "30s")] + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_POLL_INTERVAL", + default_value = "30s" + )] pub exploit_intelligence_poll_interval: humantime::Duration, /// Maximum duration before EI polling is considered timed out (humantime, e.g. "30m"). - #[arg(long, env = "EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION", default_value = "30m")] + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_MAX_POLL_DURATION", + default_value = "30m" + )] pub exploit_intelligence_max_poll_duration: humantime::Duration, /// Maximum number of upload retry attempts for EI analysis. - #[arg(long, env = "EXPLOIT_INTELLIGENCE_UPLOAD_MAX_RETRIES", default_value_t = 3)] + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_UPLOAD_MAX_RETRIES", + default_value_t = 3 + )] pub exploit_intelligence_upload_max_retries: usize, /// Initial delay between upload retries (humantime, exponential backoff). - #[arg(long, env = "EXPLOIT_INTELLIGENCE_UPLOAD_RETRY_DELAY", default_value = "1s")] + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_UPLOAD_RETRY_DELAY", + default_value = "1s" + )] pub exploit_intelligence_upload_retry_delay: humantime::Duration, /// Maximum consecutive poll failures before giving up. - #[arg(long, env = "EXPLOIT_INTELLIGENCE_MAX_CONSECUTIVE_POLL_FAILURES", default_value_t = 5)] + #[arg( + long, + env = "EXPLOIT_INTELLIGENCE_MAX_CONSECUTIVE_POLL_FAILURES", + default_value_t = 5 + )] pub exploit_intelligence_max_consecutive_poll_failures: u32, /// Authentication token for the Exploit Intelligence service (static token, for backward compatibility). @@ -345,7 +365,7 @@ impl InitData { }, }; - let ei_config = run.exploit_intelligence_url.map(|url| { + let ei_config = if let Some(url) = run.exploit_intelligence_url { use trustify_module_fundamental::exploit_intelligence::auth::TokenProvider; // Build token provider: prefer OIDC if all three vars are set, @@ -358,7 +378,7 @@ impl InitData { let token_provider = match oidc_args { (Some(token_url), Some(client_id), Some(client_secret)) => { - Some(TokenProvider::oidc(token_url, client_id, client_secret)) + Some(TokenProvider::oidc(token_url, client_id, client_secret)?) } (None, None, None) => run .exploit_intelligence_auth_token @@ -376,7 +396,7 @@ impl InitData { } }; - trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { + Some(trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { url, ui_url: run.exploit_intelligence_ui_url, poll_interval: run.exploit_intelligence_poll_interval.into(), @@ -385,8 +405,10 @@ impl InitData { upload_retry_delay: run.exploit_intelligence_upload_retry_delay.into(), max_consecutive_poll_failures: run.exploit_intelligence_max_consecutive_poll_failures, token_provider, - } - }); + }) + } else { + None + }; Ok(InitData { analysis: AnalysisService::new(run.analysis, db_ro.clone()), From aca7b8e78d33fb89b12c7f6c702bf24572c327fc Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 15 Jul 2026 16:55:53 +0100 Subject: [PATCH 20/25] refactor(exploit-intelligence): replace custom OIDC with existing trustify-auth infrastructure Replace the hand-rolled OIDC client_credentials implementation in exploit_intelligence/auth.rs with the existing OpenIdTokenProvider and TokenInjector from common/auth. This eliminates ~140 lines of duplicated token caching/refresh logic and replaces 7 copy-pasted token injection blocks with single-line inject_token() calls. The EI integration now uses OIDC discovery (issuer URL) instead of a raw token endpoint, and gains TLS insecure support and configurable refresh-before duration for free. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/env-vars.md | 4 +- .../src/exploit_intelligence/auth.rs | 151 ++---------------- .../exploit_intelligence/runner/analysis.rs | 25 +-- .../exploit_intelligence/runner/polling.rs | 47 +++--- .../src/exploit_intelligence/service/mod.rs | 20 ++- server/src/profile/api.rs | 133 +++++++++------ 6 files changed, 157 insertions(+), 223 deletions(-) diff --git a/docs/env-vars.md b/docs/env-vars.md index 378783fef..498fc0fb7 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -70,9 +70,11 @@ | `EXPLOIT_INTELLIGENCE_UPLOAD_RETRY_DELAY`| Initial delay between upload retries (humantime, exponential backoff) | `1s` | | `EXPLOIT_INTELLIGENCE_MAX_CONSECUTIVE_POLL_FAILURES`| Maximum consecutive poll failures before giving up | `5` | | `EXPLOIT_INTELLIGENCE_AUTH_TOKEN` | Static authentication token for the Exploit Intelligence service (used when OIDC is not configured) | | -| `EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL` | OIDC token endpoint URL for EI service authentication (client credentials flow) | | +| `EXPLOIT_INTELLIGENCE_OIDC_ISSUER_URL` | OIDC issuer URL for EI service authentication (client credentials flow with discovery) | | | `EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID` | OIDC client ID for EI service authentication | | | `EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET`| OIDC client secret for EI service authentication | | +| `EXPLOIT_INTELLIGENCE_OIDC_REFRESH_BEFORE`| Duration an EI access token must still be valid before requesting a new one | `30s` | +| `EXPLOIT_INTELLIGENCE_OIDC_TLS_INSECURE`| Allow insecure TLS connections with the EI OIDC issuer | `false` | ## Data Migration diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs index 05c42087c..713bcc316 100644 --- a/modules/fundamental/src/exploit_intelligence/auth.rs +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -1,137 +1,20 @@ use std::sync::Arc; -use std::time::Duration; -use time::OffsetDateTime; -use tokio::sync::RwLock; - -/// Provides bearer tokens for EI service authentication. -/// Supports static tokens and OIDC client credentials flow. -#[derive(Clone)] -pub enum TokenProvider { - /// A fixed bearer token (from env var / CLI arg). - Static(String), - /// OIDC client credentials grant with automatic refresh. - Oidc(Arc), -} - -pub struct OidcState { - client: reqwest::Client, - token_url: String, - client_id: String, - client_secret: String, - /// Cached token + expiry - cached: RwLock>, -} - -struct CachedToken { - access_token: String, - expires_at: OffsetDateTime, -} - -#[derive(serde::Deserialize)] -struct TokenResponse { - access_token: String, - expires_in: Option, - // token_type, scope, etc. ignored -} - -impl std::fmt::Debug for TokenProvider { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Static(_) => f.debug_tuple("Static").field(&"[redacted]").finish(), - Self::Oidc(_) => f.debug_tuple("Oidc").field(&"[...]").finish(), - } - } -} - -impl TokenProvider { - /// Create a static token provider. - pub fn static_token(token: String) -> Self { - Self::Static(token) - } - - /// Create an OIDC client credentials provider. - /// - /// Builds an internal HTTP client with sensible timeouts for token - /// exchange requests. - pub fn oidc( - token_url: String, - client_id: String, - client_secret: String, - ) -> Result { - let client = reqwest::Client::builder() - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(30)) - .build() - .map_err(|e| anyhow::anyhow!("failed to build OIDC HTTP client: {e}"))?; - Ok(Self::Oidc(Arc::new(OidcState { - client, - token_url, - client_id, - client_secret, - cached: RwLock::new(None), - }))) - } - - /// Get a valid bearer token string. For OIDC, refreshes automatically if expired. - pub async fn get_token(&self) -> Result { - match self { - Self::Static(token) => Ok(token.clone()), - Self::Oidc(state) => state.get_or_refresh().await, - } - } -} - -impl OidcState { - async fn get_or_refresh(&self) -> Result { - // Check cached token under read lock - { - let cached = self.cached.read().await; - if let Some(ref tok) = *cached { - // Refresh 30s before expiry - if tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) { - return Ok(tok.access_token.clone()); - } - } - } - - // Need refresh -- acquire write lock - let mut cached = self.cached.write().await; - // Double-check after acquiring write lock (another task may have refreshed) - if let Some(ref tok) = *cached - && tok.expires_at > OffsetDateTime::now_utc() + Duration::from_secs(30) - { - return Ok(tok.access_token.clone()); - } - - let response = self - .client - .post(&self.token_url) - .form(&[ - ("grant_type", "client_credentials"), - ("client_id", &self.client_id), - ("client_secret", &self.client_secret), - ]) - .send() - .await?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "OIDC token request failed ({status}): {body}" - )); - } - - let token_resp: TokenResponse = response.json().await?; - let expires_in = token_resp.expires_in.unwrap_or(3600); // default 1 hour - let expires_at = OffsetDateTime::now_utc() + Duration::from_secs(expires_in); - - let access_token = token_resp.access_token.clone(); - *cached = Some(CachedToken { - access_token: token_resp.access_token, - expires_at, - }); - - Ok(access_token) +use trustify_auth::client::{BearerTokenProvider, OpenIdTokenProviderConfig, TokenProvider}; + +/// Build an EI [`TokenProvider`] from OIDC configuration, falling back to a +/// static token if OIDC is not configured. +/// +/// Returns `None` when neither OIDC nor a static token is available. +pub async fn build_provider( + oidc_config: Option, + static_token: Option, +) -> anyhow::Result>> { + match oidc_config { + Some(config) => Ok(Some( + OpenIdTokenProviderConfig::new_provider(Some(config)).await?, + )), + None => Ok(static_token.map(|t| -> Arc { + Arc::new(BearerTokenProvider { token: t }) + })), } } diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index aa65a744e..9926b1423 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -8,6 +8,7 @@ use futures_util::TryStreamExt; use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; use std::str::FromStr; use tokio_util::sync::CancellationToken; +use trustify_auth::client::TokenInjector; use trustify_common::db; use trustify_module_ingestor::service::IngestorService; use trustify_module_storage::service::StorageBackend; @@ -113,16 +114,16 @@ pub(crate) async fn run_analysis( })?, ); - let mut request = cli.post(&url).multipart(form); - - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { + let request = cli + .post(&url) + .multipart(form) + .inject_token(&tp) + .await + .map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - request = request.bearer_auth(&token); - } classify_response_error(request.send().await) } @@ -205,16 +206,16 @@ pub(crate) async fn run_analysis( })?, ); - let mut request = cli.post(&url).multipart(form); - - if let Some(ref provider) = tp { - let token = provider.get_token().await.map_err(|e| { + let request = cli + .post(&url) + .multipart(form) + .inject_token(&tp) + .await + .map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - request = request.bearer_auth(&token); - } classify_response_error(request.send().await) } diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index 33976e627..6b4d69549 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -9,6 +9,7 @@ use std::cell::Cell; use std::str::FromStr; use std::time::Duration; use tokio_util::sync::CancellationToken; +use trustify_auth::client::TokenInjector; use trustify_common::db; use trustify_entity::labels::Labels; use trustify_module_ingestor::service::{Cache, Format, IngestorService}; @@ -57,15 +58,15 @@ pub(crate) async fn poll_for_result( let cli = client.clone(); let tp = config.token_provider.clone(); async move { - let mut poll_request = cli.get(&url); - if let Some(ref provider) = tp { - let auth_token = provider.get_token().await.map_err(|e| { + let poll_request = cli + .get(&url) + .inject_token(&tp) + .await + .map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - poll_request = poll_request.bearer_auth(&auth_token); - } let resp = classify_response_error(poll_request.send().await)?; @@ -362,15 +363,15 @@ pub(crate) async fn poll_for_product_result( let cli = client.clone(); let tp = config.token_provider.clone(); async move { - let mut poll_request = cli.get(&url); - if let Some(ref provider) = tp { - let auth_token = provider.get_token().await.map_err(|e| { + let poll_request = cli + .get(&url) + .inject_token(&tp) + .await + .map_err(|e| { EiRequestError::Permanent(anyhow::anyhow!( "failed to obtain auth token: {e}" )) })?; - poll_request = poll_request.bearer_auth(&auth_token); - } let resp = classify_response_error(poll_request.send().await)?; @@ -491,14 +492,11 @@ pub(crate) async fn poll_for_product_result( urlencoding::encode(product_id) ); - let mut reports_request = client.get(&reports_url); - if let Some(ref provider) = config.token_provider { - let auth_token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - reports_request = reports_request.bearer_auth(&auth_token); - } + let reports_request = client + .get(&reports_url) + .inject_token(&config.token_provider) + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; let reports_response = reports_request .send() @@ -594,14 +592,11 @@ pub(crate) async fn poll_for_product_result( urlencoding::encode(scan_id) ); - let mut full_request = client.get(&full_report_url); - if let Some(ref provider) = config.token_provider { - let auth_token = provider - .get_token() - .await - .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; - full_request = full_request.bearer_auth(&auth_token); - } + let full_request = client + .get(&full_report_url) + .inject_token(&config.token_provider) + .await + .map_err(|e| anyhow::anyhow!("failed to obtain auth token: {e}"))?; let advisory_id = match full_request.send().await { Ok(resp) if resp.status().is_success() => { diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 6f74dfaeb..31f741a54 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -11,6 +11,7 @@ use sea_orm::{ sea_query::{Expr, LockBehavior, LockType}, }; use std::collections::HashMap; +use std::sync::Arc; use std::time::Duration; use time::OffsetDateTime; use tracing::{Instrument, info_span, instrument}; @@ -23,7 +24,7 @@ use trustify_entity::exploit_intelligence_job_component; use uuid::Uuid; /// Configuration for the Exploit Intelligence client connection. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ExploitIntelligenceConfig { /// Base URL of the Exploit Intelligence client service. pub url: String, @@ -43,7 +44,22 @@ pub struct ExploitIntelligenceConfig { pub max_consecutive_poll_failures: u32, /// Optional token provider for EI service authentication. /// Supports static tokens and OIDC client credentials flow. - pub token_provider: Option, + pub token_provider: Option>, +} + +impl std::fmt::Debug for ExploitIntelligenceConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExploitIntelligenceConfig") + .field("url", &self.url) + .field("ui_url", &self.ui_url) + .field("poll_interval", &self.poll_interval) + .field("max_poll_duration", &self.max_poll_duration) + .field("upload_max_retries", &self.upload_max_retries) + .field("upload_retry_delay", &self.upload_retry_delay) + .field("max_consecutive_poll_failures", &self.max_consecutive_poll_failures) + .field("token_provider", &self.token_provider.as_ref().map(|_| "[configured]")) + .finish() + } } /// Result of the startup recovery pass over orphaned EI jobs. diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index e79db8fe2..690da7039 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -5,6 +5,7 @@ use crate::{endpoints, profile::spawn_db_check, sample_data}; use actix_web::web; use bytesize::ByteSize; use futures::FutureExt; +use trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig; use std::{env, process::ExitCode, sync::Arc}; use trustify_auth::{ auth::AuthConfigArguments, @@ -144,17 +145,8 @@ pub struct Run { #[arg(long, env = "EXPLOIT_INTELLIGENCE_AUTH_TOKEN")] pub exploit_intelligence_auth_token: Option, - /// OIDC token endpoint URL for EI service authentication (client credentials flow). - #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL")] - pub exploit_intelligence_oidc_token_url: Option, - - /// OIDC client ID for EI service authentication. - #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID")] - pub exploit_intelligence_oidc_client_id: Option, - - /// OIDC client secret for EI service authentication. - #[arg(long, env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET")] - pub exploit_intelligence_oidc_client_secret: Option, + #[command(flatten)] + pub exploit_intelligence_oidc: EiOidcArguments, // flattened commands must go last // @@ -193,6 +185,78 @@ pub struct Run { pub ui: UiConfig, } +/// Clap arguments for EI-specific OIDC credentials. +/// +/// Separate from the main Trustify OIDC config because the EI service may +/// use a different IdP or client registration. +#[derive(clap::Args, Debug, Clone)] +#[command(next_help_heading = "Exploit Intelligence OIDC")] +pub struct EiOidcArguments { + #[arg( + id = "ei_oidc_client_id", + long = "ei-oidc-client-id", + env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID" + )] + pub client_id: Option, + + #[arg( + id = "ei_oidc_client_secret", + long = "ei-oidc-client-secret", + env = "EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET" + )] + pub client_secret: Option, + + #[arg( + id = "ei_oidc_issuer_url", + long = "ei-oidc-issuer-url", + env = "EXPLOIT_INTELLIGENCE_OIDC_ISSUER_URL" + )] + pub issuer_url: Option, + + #[arg( + id = "ei_oidc_refresh_before", + long = "ei-oidc-refresh-before", + env = "EXPLOIT_INTELLIGENCE_OIDC_REFRESH_BEFORE", + default_value = "30s" + )] + pub refresh_before: humantime::Duration, + + #[arg( + id = "ei_oidc_tls_insecure", + long = "ei-oidc-tls-insecure", + env = "EXPLOIT_INTELLIGENCE_OIDC_TLS_INSECURE", + default_value = "false" + )] + pub tls_insecure: bool, +} + +impl EiOidcArguments { + fn into_config(self) -> Option { + match (self.client_id, self.client_secret, self.issuer_url) { + (Some(client_id), Some(client_secret), Some(issuer_url)) => { + Some(trustify_auth::client::OpenIdTokenProviderConfig { + client_id, + client_secret, + issuer_url, + refresh_before: self.refresh_before, + tls_insecure: self.tls_insecure, + }) + } + (None, None, None) => None, + _ => { + log::warn!( + "Incomplete EI OIDC configuration: all three of \ + EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID, \ + EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET, and \ + EXPLOIT_INTELLIGENCE_OIDC_ISSUER_URL must be set together. \ + Falling back to static token or no auth." + ); + None + } + } + } +} + mod default { use bytesize::ByteSize; use trustify_common::model::BinaryByteSize; @@ -258,9 +322,7 @@ struct InitData { config: ModuleConfig, analysis: AnalysisService, read_only: bool, - ei_config: Option< - trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig, - >, + ei_config: Option, } /// Groups all module configurations. @@ -366,37 +428,14 @@ impl InitData { }; let ei_config = if let Some(url) = run.exploit_intelligence_url { - use trustify_module_fundamental::exploit_intelligence::auth::TokenProvider; - - // Build token provider: prefer OIDC if all three vars are set, - // fall back to static token, or None. - let oidc_args = ( - run.exploit_intelligence_oidc_token_url, - run.exploit_intelligence_oidc_client_id, - run.exploit_intelligence_oidc_client_secret, - ); - - let token_provider = match oidc_args { - (Some(token_url), Some(client_id), Some(client_secret)) => { - Some(TokenProvider::oidc(token_url, client_id, client_secret)?) - } - (None, None, None) => run - .exploit_intelligence_auth_token - .map(TokenProvider::static_token), - _ => { - log::warn!( - "Incomplete EI OIDC configuration: all three of \ - EXPLOIT_INTELLIGENCE_OIDC_TOKEN_URL, \ - EXPLOIT_INTELLIGENCE_OIDC_CLIENT_ID, and \ - EXPLOIT_INTELLIGENCE_OIDC_CLIENT_SECRET must be set together. \ - Falling back to static token or no auth." - ); - run.exploit_intelligence_auth_token - .map(TokenProvider::static_token) - } - }; - - Some(trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig { + let token_provider = + trustify_module_fundamental::exploit_intelligence::auth::build_provider( + run.exploit_intelligence_oidc.into_config(), + run.exploit_intelligence_auth_token, + ) + .await?; + + Some(ExploitIntelligenceConfig { url, ui_url: run.exploit_intelligence_ui_url, poll_interval: run.exploit_intelligence_poll_interval.into(), @@ -510,9 +549,7 @@ pub(crate) struct Config { pub(crate) analysis: AnalysisService, pub(crate) auth: Option>, pub(crate) read_only: bool, - pub(crate) ei_config: Option< - trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig, - >, + pub(crate) ei_config: Option, } pub(crate) fn configure(svc: &mut utoipa_actix_web::service_config::ServiceConfig, config: Config) { From f272d7dc9187dbdeffbf5996dae128b48a3e358b Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 15 Jul 2026 17:07:48 +0100 Subject: [PATCH 21/25] refactor(exploit-intelligence): replace heartbeat with PostgreSQL job queue pattern Replace the heartbeat-based optimistic locking mechanism with a visibility-timeout job queue pattern using FOR UPDATE SKIP LOCKED, matching the approach suggested in review and used by the importer module. The POST /analyze handler now only creates a job row and returns immediately. A background worker loop dequeues eligible jobs atomically, processes them with no concurrency cap (each job is a lightweight async poll loop), and handles crash recovery automatically through expired visibility timeouts. Key design decisions: - Short initial visibility window (5 min) extended on each poll tick, so long-running analyses (hours) stay claimed while crashed jobs are reclaimed quickly - DEQUEUE_BATCH_SIZE caps per-tick claiming to share work across instances via SKIP LOCKED - Job failure atomically cascades to non-terminal components in a single transaction - Crash-recovery creates missing component records instead of silently skipping them - extend_visibility errors are logged (not silently discarded) Co-Authored-By: Claude Opus 4.6 (1M context) --- entity/src/exploit_intelligence_job.rs | 9 +- ...0002230_create_exploit_intelligence_job.rs | 23 +- modules/fundamental/src/endpoints.rs | 8 +- .../src/exploit_intelligence/auth.rs | 5 +- .../src/exploit_intelligence/endpoints/mod.rs | 87 +--- .../exploit_intelligence/endpoints/test.rs | 14 +- .../src/exploit_intelligence/mod.rs | 2 +- .../src/exploit_intelligence/model/mod.rs | 7 - .../exploit_intelligence/runner/analysis.rs | 50 +- .../exploit_intelligence/runner/heartbeat.rs | 190 -------- .../src/exploit_intelligence/runner/mod.rs | 3 +- .../exploit_intelligence/runner/polling.rs | 282 +++--------- .../exploit_intelligence/runner/recovery.rs | 164 ------- .../src/exploit_intelligence/runner/worker.rs | 94 ++++ .../src/exploit_intelligence/service/mod.rs | 428 +++++++++--------- server/src/profile/api.rs | 47 +- 16 files changed, 485 insertions(+), 928 deletions(-) delete mode 100644 modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs delete mode 100644 modules/fundamental/src/exploit_intelligence/runner/recovery.rs create mode 100644 modules/fundamental/src/exploit_intelligence/runner/worker.rs diff --git a/entity/src/exploit_intelligence_job.rs b/entity/src/exploit_intelligence_job.rs index 7c7a408ec..bee025ae4 100644 --- a/entity/src/exploit_intelligence_job.rs +++ b/entity/src/exploit_intelligence_job.rs @@ -6,7 +6,6 @@ /// [`exploit_intelligence_job_component`](super::exploit_intelligence_job_component) rows — /// even for single-component CycloneDX jobs, which have exactly one component record. use sea_orm::entity::prelude::*; -use sea_orm::prelude::Decimal; use time::OffsetDateTime; /// Lifecycle status of an Exploit Intelligence analysis job. @@ -73,8 +72,12 @@ pub struct Model { /// Expected number of components in the product. pub total_components: Option, - /// Unix time in nanos, for coordinating multi-instance job ownership. - pub heartbeat: Option, + /// Visibility timeout for the job queue pattern. When set, the job is + /// invisible to other workers until this timestamp is reached. + pub visible_at: Option, + + /// Number of times this job has been dequeued for processing. + pub retry_count: i32, /// Timestamp when the job was created. pub created: OffsetDateTime, diff --git a/migration/src/m0002230_create_exploit_intelligence_job.rs b/migration/src/m0002230_create_exploit_intelligence_job.rs index dce4a2733..6d2f0606d 100644 --- a/migration/src/m0002230_create_exploit_intelligence_job.rs +++ b/migration/src/m0002230_create_exploit_intelligence_job.rs @@ -54,19 +54,19 @@ impl MigrationTrait for Migration { .not_null() .default("pending"), ) + .col(ColumnDef::new(ExploitIntelligenceJob::ErrorMessage).string()) + .col(ColumnDef::new(ExploitIntelligenceJob::ProductId).string()) + .col(ColumnDef::new(ExploitIntelligenceJob::TotalComponents).integer()) .col( - ColumnDef::new(ExploitIntelligenceJob::ErrorMessage).string(), - ) - .col( - ColumnDef::new(ExploitIntelligenceJob::ProductId).string(), - ) - .col( - ColumnDef::new(ExploitIntelligenceJob::TotalComponents).integer(), + ColumnDef::new(ExploitIntelligenceJob::VisibleAt) + .timestamp_with_time_zone() + .null(), ) .col( - ColumnDef::new(ExploitIntelligenceJob::Heartbeat) - .decimal() - .null(), + ColumnDef::new(ExploitIntelligenceJob::RetryCount) + .integer() + .not_null() + .default(0), ) .col( ColumnDef::new(ExploitIntelligenceJob::Created) @@ -315,7 +315,8 @@ enum ExploitIntelligenceJob { ErrorMessage, ProductId, TotalComponents, - Heartbeat, + VisibleAt, + RetryCount, Created, Updated, } diff --git a/modules/fundamental/src/endpoints.rs b/modules/fundamental/src/endpoints.rs index 1f47f1004..1e525e468 100644 --- a/modules/fundamental/src/endpoints.rs +++ b/modules/fundamental/src/endpoints.rs @@ -59,13 +59,7 @@ pub fn configure( let ei_service = ExploitIntelligenceService::new(ei_config) .unwrap_or_else(|e| panic!("failed to create ExploitIntelligenceService: {e}")); - crate::exploit_intelligence::endpoints::configure( - svc, - db_rw, - db_ro, - ei_service, - ingestor_service, - ); + crate::exploit_intelligence::endpoints::configure(svc, db_rw, db_ro, ei_service); } #[derive(Clone, Debug, PartialEq, Eq, Default, ToSchema, serde::Deserialize, IntoParams)] diff --git a/modules/fundamental/src/exploit_intelligence/auth.rs b/modules/fundamental/src/exploit_intelligence/auth.rs index 713bcc316..3d5342e54 100644 --- a/modules/fundamental/src/exploit_intelligence/auth.rs +++ b/modules/fundamental/src/exploit_intelligence/auth.rs @@ -13,8 +13,7 @@ pub async fn build_provider( Some(config) => Ok(Some( OpenIdTokenProviderConfig::new_provider(Some(config)).await?, )), - None => Ok(static_token.map(|t| -> Arc { - Arc::new(BearerTokenProvider { token: t }) - })), + None => Ok(static_token + .map(|t| -> Arc { Arc::new(BearerTokenProvider { token: t }) })), } } diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs index 7cfce10de..2b4f67341 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/mod.rs @@ -8,7 +8,6 @@ use crate::{ AnalyzeRequest, AnalyzeResponse, ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, }, - runner::{analysis::run_analysis, recovery::recover_and_resume}, service::ExploitIntelligenceService, }, }; @@ -19,27 +18,15 @@ use trustify_common::{ db, model::{Paginated, PaginatedResults}, }; -use trustify_module_ingestor::service::IngestorService; use uuid::Uuid; -/// Register exploit intelligence endpoints and spawn recovery for orphaned jobs. +/// Register exploit intelligence endpoints. pub fn configure( config: &mut utoipa_actix_web::service_config::ServiceConfig, db_rw: db::ReadWrite, db_ro: db::ReadOnly, ei_service: ExploitIntelligenceService, - ingestor: IngestorService, ) { - // Spawn recovery for orphaned jobs from a previous server lifetime. - if ei_service.config().is_some() { - let ei_svc = ei_service.clone(); - let ingestor_svc = ingestor.clone(); - let db_rw_clone = db_rw.clone(); - actix_web::rt::spawn(async move { - recover_and_resume(&ei_svc, &ingestor_svc, &db_rw_clone).await; - }); - } - config .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) @@ -60,10 +47,8 @@ pub struct JobFilter { /// Trigger an Exploit Intelligence analysis for a CVE against an SBOM. /// -/// Creates a job record and spawns a background task that submits the SBOM -/// to the EI client service, polls for completion, and ingests any returned -/// VEX document. A heartbeat is maintained so other instances know the job -/// is being actively serviced. +/// Creates a job record that the background worker will pick up and +/// process. Returns immediately after persisting the job. #[utoipa::path( tag = "exploit-intelligence", operation_id = "analyzeExploitIntelligence", @@ -78,14 +63,12 @@ pub struct JobFilter { #[post("/v3/exploit-intelligence/analyze")] pub async fn analyze( ei_service: web::Data, - ingestor: web::Data, db_rw: web::Data, body: web::Json, _: Require, ) -> actix_web::Result { - let runtime = ei_service.runtime().cloned().ok_or(Error::Unavailable)?; - let config = runtime.config; - let client = runtime.client; + // Verify EI is configured before doing any work. + let _ = ei_service.runtime().ok_or(Error::Unavailable)?; // Use a single read-write transaction for the existence check, active-job // deduplication, and job creation to prevent TOCTOU race conditions where @@ -108,7 +91,7 @@ pub async fn analyze( .find_active_job(body.sbom_id, &body.vulnerability_id, &tx) .await? { - // No need to commit — no mutations were made. + // No need to commit -- no mutations were made. return Ok(HttpResponse::Ok().json(AnalyzeResponse { job_id: active_job.id, status: active_job.status, @@ -122,64 +105,8 @@ pub async fn analyze( tx.commit().await.map_err(Error::from)?; - let job_id = job.id; - let sbom_id = body.sbom_id; - let vulnerability_id = body.vulnerability_id.clone(); - // The job was created with an initial heartbeat — use it for the heartbeat loop. - let initial_heartbeat = job.heartbeat; - - // Spawn background task to submit and poll, wrapped with heartbeat. - let ei_svc = ei_service.as_ref().clone(); - let ingestor_svc = ingestor.as_ref().clone(); - let db_rw_clone = db_rw.as_ref().clone(); - let db_hb = db_rw.as_ref().clone(); - - actix_web::rt::spawn(async move { - use crate::exploit_intelligence::runner::heartbeat::{self, HeartbeatError}; - use tokio_util::sync::CancellationToken; - - let token = CancellationToken::new(); - let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); - let hb_for_work = hb_state.clone(); - let token_for_work = token.clone(); - - let work = run_analysis( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config, - &client, - job_id, - sbom_id, - &vulnerability_id, - hb_for_work, - token_for_work, - ); - match heartbeat::run_with_heartbeat( - job_id, - initial_heartbeat, - &db_hb, - hb_state, - token, - work, - ) - .await - { - Ok(()) => {} - Err(HeartbeatError::LockLost) => { - tracing::warn!(job_id = %job_id, "heartbeat lost, another instance owns this job"); - // Do NOT call update_job_failed — we no longer own this job. - } - Err(HeartbeatError::Work(e)) => { - tracing::error!(job_id = %job_id, error = %e, "exploit intelligence analysis failed"); - // The work future already handled failure with CAS-protected updates. - // No additional update_job_failed call needed. - } - } - }); - Ok(HttpResponse::Created().json(AnalyzeResponse { - job_id, + job_id: job.id, status: JobStatus::Pending, })) } diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 51c460f02..2da1beddf 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -52,7 +52,8 @@ async fn insert_test_job( error_message: Set(error_message.map(|s| s.to_string())), product_id: Set(None), total_components: Set(None), - heartbeat: Set(None), + visible_at: Set(None), + retry_count: Set(0), created: Set(now), updated: Set(now), }; @@ -411,7 +412,6 @@ async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Resu .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) - .app_data(web::Data::new(ctx.ingestor.clone())) .service(super::analyze), ) .await; @@ -427,8 +427,8 @@ async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Resu let resp = test::call_service(&app, req).await; - // Then the response returns the existing job (not a new one) - assert_eq!(resp.status(), 201); + // Then the response returns the existing job (not a new one) with 200 + assert_eq!(resp.status(), 200); let body: AnalyzeResponse = test::read_body_json(resp).await; assert_eq!(body.job_id, existing_job_id); assert_eq!(body.status, JobStatus::Pending); @@ -454,11 +454,10 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { .await; // When the service marks it as failed with a timeout error - // Pass None as heartbeat since the test job was created without one. let ei_service = test_service(); let db_rw = db::ReadWrite::new(ctx.db.clone()); ei_service - .update_job_failed(job_id, "analysis timed out after 1800s", None, &db_rw) + .update_job_failed(job_id, "analysis timed out after 1800s", &db_rw) .await?; // Then fetching the job via GET shows the failed status and timeout message @@ -573,7 +572,6 @@ async fn analyze_creates_new_job(ctx: &TrustifyContext) -> anyhow::Result<()> { .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) - .app_data(web::Data::new(ctx.ingestor.clone())) .service(super::analyze), ) .await; @@ -729,7 +727,6 @@ async fn analyze_sbom_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) - .app_data(web::Data::new(ctx.ingestor.clone())) .service(super::analyze), ) .await; @@ -767,7 +764,6 @@ async fn analyze_ei_not_configured(ctx: &TrustifyContext) -> anyhow::Result<()> .app_data(web::Data::new(db_rw)) .app_data(web::Data::new(db_ro)) .app_data(web::Data::new(ei_service)) - .app_data(web::Data::new(ctx.ingestor.clone())) .service(super::analyze), ) .await; diff --git a/modules/fundamental/src/exploit_intelligence/mod.rs b/modules/fundamental/src/exploit_intelligence/mod.rs index 930cf84be..4f768a58d 100644 --- a/modules/fundamental/src/exploit_intelligence/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/mod.rs @@ -3,5 +3,5 @@ pub mod ei_client; pub(crate) mod endpoints; pub mod model; -pub(crate) mod runner; +pub mod runner; pub mod service; diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index a9bab88b8..d3cd92e08 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -1,4 +1,3 @@ -use sea_orm::prelude::Decimal; use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use trustify_entity::exploit_intelligence_job::{ExploitIntelligenceJobStatus, Finding}; @@ -99,11 +98,6 @@ pub struct ExploitIntelligenceJobDetails { /// Timestamp when the job was last updated. #[schema(value_type = String)] pub updated: OffsetDateTime, - /// Internal: current heartbeat value for CAS operations. - /// Not exposed in the API response. - #[serde(skip)] - #[schema(ignore)] - pub heartbeat: Option, } /// Per-component analysis result within an analysis job. @@ -353,7 +347,6 @@ impl ExploitIntelligenceJobDetails { components: Vec::new(), created: model.created, updated: model.updated, - heartbeat: model.heartbeat, } } diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index 9926b1423..f31547cf5 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -1,43 +1,39 @@ use crate::exploit_intelligence::{ - runner::heartbeat::SharedHeartbeat, runner::{EiRequestError, classify_response_error}, service::ExploitIntelligenceService, }; use backon::{ExponentialBuilder, Retryable}; use futures_util::TryStreamExt; -use sea_orm::{AccessMode, ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; use std::str::FromStr; -use tokio_util::sync::CancellationToken; use trustify_auth::client::TokenInjector; use trustify_common::db; +use trustify_entity::exploit_intelligence_job; use trustify_module_ingestor::service::IngestorService; use trustify_module_storage::service::StorageBackend; -use uuid::Uuid; use super::polling::{poll_for_product_result, poll_for_result}; /// Background task: submit SBOM to EI client, poll for results, ingest VEX. /// -/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. -/// It is passed through to the polling functions for CAS-protected terminal -/// status updates. -#[allow(clippy::too_many_arguments)] +/// Accepts the full job model so the caller does not need to pass individual +/// fields. Uses `db_ro` for read-only SBOM retrieval and `db_rw` for +/// mutations. pub(crate) async fn run_analysis( ei_service: &ExploitIntelligenceService, ingestor: &IngestorService, db_rw: &db::ReadWrite, + db_ro: &db::ReadOnly, config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, client: &reqwest::Client, - job_id: Uuid, - sbom_id: Uuid, - vulnerability_id: &str, - heartbeat: SharedHeartbeat, - token: CancellationToken, + job: &exploit_intelligence_job::Model, ) -> Result<(), anyhow::Error> { - // Retrieve the raw SBOM document from storage (read-only – no mutations needed). - let tx = db_rw - .begin_with_config(None, Some(AccessMode::ReadOnly)) - .await?; + let job_id = job.id; + let sbom_id = job.sbom_id; + let vulnerability_id = &job.vulnerability_id; + + // Retrieve the raw SBOM document from storage (read-only -- no mutations needed). + let tx = db_ro.begin().await?; let sbom_model = trustify_entity::sbom::Entity::find() .filter(trustify_entity::sbom::Column::SbomId.eq(sbom_id)) .one(&tx) @@ -51,7 +47,7 @@ pub(crate) async fn run_analysis( .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; // Build storage key from source document SHA256. - // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:…". + // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:...". let sha256_with_prefix = format!("sha256:{}", source_doc.sha256); let storage_key = trustify_common::id::Id::from_str(&sha256_with_prefix) .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; @@ -142,19 +138,11 @@ pub(crate) async fn run_analysis( response.json().await?; let product_id = spdx_upload.product_id; - // Read the current heartbeat for CAS-protected update. - let current_hb = heartbeat.lock().ok().and_then(|guard| *guard); - - // Update job to running with the product ID (CAS-protected). - let applied = ei_service - .update_job_running_with_product(job_id, &product_id, current_hb, db_rw) + // Update job to running with the product ID. + ei_service + .update_job_running_with_product(job_id, &product_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, cannot transition to running" - )); - } tracing::info!( job_id = %job_id, @@ -171,8 +159,6 @@ pub(crate) async fn run_analysis( client, job_id, &product_id, - heartbeat, - token, ) .await } else { @@ -250,8 +236,6 @@ pub(crate) async fn run_analysis( job_id, &scan_id, component_id, - heartbeat, - token, ) .await } diff --git a/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs b/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs deleted file mode 100644 index 5eb6a226c..000000000 --- a/modules/fundamental/src/exploit_intelligence/runner/heartbeat.rs +++ /dev/null @@ -1,190 +0,0 @@ -//! Heartbeat coordination for exploit-intelligence jobs. -//! -//! Provides optimistic-locking primitives that let multiple Trustify instances -//! claim jobs exclusively. A job is considered "actively owned" if its -//! heartbeat was updated within `2 * RATE`. - -use sea_orm::prelude::Decimal; -use sea_orm::{ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter}; -use sea_query::Expr; -use std::future::Future; -use std::sync::{Arc, Mutex}; -use std::time::Duration; -use time::OffsetDateTime; -use tokio_util::sync::CancellationToken; -use trustify_common::db; -use trustify_entity::exploit_intelligence_job; -use uuid::Uuid; - -/// How often the heartbeat is refreshed. -pub const RATE: Duration = Duration::from_secs(10); - -/// Typed error returned by [`run_with_heartbeat`]. -/// -/// Distinguishes between a heartbeat loss (another instance claimed the job) -/// and a work-future error (the analysis itself failed). -pub enum HeartbeatError { - /// The heartbeat CAS failed — another instance owns this job. - /// The caller MUST NOT write to the job row after receiving this. - LockLost, - /// The work future returned an error. The work future is responsible - /// for having already called terminal status updates with CAS protection. - Work(anyhow::Error), -} - -impl std::fmt::Display for HeartbeatError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::LockLost => write!(f, "heartbeat lock lost"), - Self::Work(e) => write!(f, "work error: {e}"), - } - } -} - -impl std::fmt::Debug for HeartbeatError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::LockLost => write!(f, "HeartbeatError::LockLost"), - Self::Work(e) => write!(f, "HeartbeatError::Work({e:?})"), - } - } -} - -/// Shared state for the current heartbeat value. -/// -/// Updated by the heartbeat loop; read by the work future when performing -/// CAS-protected terminal status updates. -pub type SharedHeartbeat = Arc>>; - -/// Create a new shared heartbeat state initialised with the given value. -pub fn shared_heartbeat(initial: Option) -> SharedHeartbeat { - Arc::new(Mutex::new(initial)) -} - -/// Attempt one heartbeat update using optimistic locking. -/// -/// The caller provides the previous heartbeat value (or `None` for the first -/// beat). A CAS-style `UPDATE … WHERE heartbeat = ` ensures that two -/// instances cannot both claim the same job. -/// -/// Returns the new heartbeat value on success. -pub async fn beat( - job_id: Uuid, - old_heartbeat: Option, - db: &impl ConnectionTrait, -) -> Result { - let now = OffsetDateTime::now_utc().unix_timestamp_nanos(); - let new_hb = Decimal::from_i128_with_scale(now, 0); - - let lock = match old_heartbeat { - Some(t) => Expr::col(exploit_intelligence_job::Column::Heartbeat).eq(t), - None => Expr::col(exploit_intelligence_job::Column::Heartbeat).is_null(), - }; - - let result = exploit_intelligence_job::Entity::update_many() - .col_expr( - exploit_intelligence_job::Column::Heartbeat, - Expr::value(new_hb), - ) - .col_expr( - exploit_intelligence_job::Column::Updated, - Expr::value(OffsetDateTime::now_utc()), - ) - .filter(exploit_intelligence_job::Column::Id.eq(job_id)) - .filter(lock) - .exec(db) - .await?; - - if result.rows_affected == 1 { - Ok(new_hb) - } else { - Err(anyhow::anyhow!( - "heartbeat update affected 0 rows — lock lost" - )) - } -} - -/// Clear the heartbeat on a job so another instance can claim it. -/// -/// Not currently called — terminal status updates clear the heartbeat -/// atomically via CAS. Retained for manual cleanup scenarios. -#[allow(dead_code)] -pub async fn clear_heartbeat(job_id: Uuid, db: &impl ConnectionTrait) -> Result<(), anyhow::Error> { - exploit_intelligence_job::Entity::update_many() - .col_expr( - exploit_intelligence_job::Column::Heartbeat, - Expr::value(Option::::None), - ) - .filter(exploit_intelligence_job::Column::Id.eq(job_id)) - .exec(db) - .await?; - Ok(()) -} - -/// Run a future while maintaining a heartbeat loop alongside it. -/// -/// The heartbeat is refreshed every [`RATE`] seconds. If the heartbeat -/// is lost (another instance claimed the job), the future's result is -/// discarded and [`HeartbeatError::LockLost`] is returned. When the work -/// completes, its result is propagated: `Ok(())` on success, or -/// [`HeartbeatError::Work`] on failure. -/// -/// The `heartbeat_state` mutex is updated after each successful beat so -/// that the work future can read the current heartbeat value for CAS-protected -/// terminal status updates. -/// -/// The `token` is cancelled when the heartbeat is lost, signalling the work -/// future to stop at the next checkpoint rather than relying solely on -/// `tokio::select!` drop semantics. The work future should check -/// `token.is_cancelled()` before performing side-effecting operations. -/// -/// **Important:** This function does NOT clear the heartbeat on completion. -/// The terminal status update (performed inside the work future with CAS) -/// is responsible for atomically clearing the heartbeat. -pub async fn run_with_heartbeat( - job_id: Uuid, - initial_heartbeat: Option, - db_hb: &db::ReadWrite, - heartbeat_state: SharedHeartbeat, - token: CancellationToken, - work: F, -) -> Result<(), HeartbeatError> -where - F: Future>, -{ - tokio::pin!(work); - - let mut heartbeat_val = initial_heartbeat; - let mut interval = tokio::time::interval(RATE); - // Consume the first immediate tick so we don't beat immediately. - interval.tick().await; - - loop { - tokio::select! { - result = &mut work => { - let result: Result<(), anyhow::Error> = result; - return result.map_err(HeartbeatError::Work); - } - _ = interval.tick() => { - match beat(job_id, heartbeat_val, &**db_hb).await { - Ok(hb) => { - heartbeat_val = Some(hb); - // Update shared state so the work future can read it. - if let Ok(mut guard) = heartbeat_state.lock() { - *guard = Some(hb); - } - } - Err(_) => { - tracing::warn!(job_id = %job_id, "heartbeat lost, aborting work"); - token.cancel(); - return Err(HeartbeatError::LockLost); - } - } - } - _ = token.cancelled() => { - tracing::warn!(job_id = %job_id, "cancellation token triggered, aborting work"); - return Err(HeartbeatError::LockLost); - } - } - } -} diff --git a/modules/fundamental/src/exploit_intelligence/runner/mod.rs b/modules/fundamental/src/exploit_intelligence/runner/mod.rs index 449e57954..ea85730df 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/mod.rs @@ -1,7 +1,6 @@ pub(crate) mod analysis; -pub(crate) mod heartbeat; pub(crate) mod polling; -pub(crate) mod recovery; +pub mod worker; /// Error classification for EI HTTP requests. pub(crate) enum EiRequestError { diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index 6b4d69549..614a5858c 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -1,5 +1,4 @@ use crate::exploit_intelligence::{ - runner::heartbeat::SharedHeartbeat, runner::{EiRequestError, classify_response_error}, service::ExploitIntelligenceService, }; @@ -8,7 +7,6 @@ use sea_orm::TransactionTrait; use std::cell::Cell; use std::str::FromStr; use std::time::Duration; -use tokio_util::sync::CancellationToken; use trustify_auth::client::TokenInjector; use trustify_common::db; use trustify_entity::labels::Labels; @@ -18,13 +16,12 @@ use uuid::Uuid; /// Poll the EI service for an in-flight single-component analysis and ingest the result. /// /// This is the second half of the analysis lifecycle: it assumes the SBOM has -/// already been uploaded and a `scan_id` obtained. It is called both from -/// `run_analysis` (normal path) and from the startup recovery path for -/// orphaned Running jobs. +/// already been uploaded and a `scan_id` obtained. /// -/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. -/// It is read when performing CAS-protected terminal status updates so that -/// only the owning instance can transition the job to a terminal state. +/// Each successful poll tick extends the job's visibility window so that +/// other instances know this job is still being actively processed. EI +/// analysis can take hours; without extension the short initial window +/// would expire and another instance would re-claim the job. /// /// Updates both the component record (with finding/advisory) and the parent /// job record (status only) upon completion. @@ -38,8 +35,6 @@ pub(crate) async fn poll_for_result( job_id: Uuid, scan_id: &str, component_id: Uuid, - heartbeat: SharedHeartbeat, - token: CancellationToken, ) -> Result<(), anyhow::Error> { let poll_url = format!( "{}/api/v1/reports/by-scan-id/{}", @@ -49,7 +44,6 @@ pub(crate) async fn poll_for_result( let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); - let token_for_when = token.clone(); let result = tokio::time::timeout( config.max_poll_duration, @@ -57,16 +51,12 @@ pub(crate) async fn poll_for_result( let url = poll_url.clone(); let cli = client.clone(); let tp = config.token_provider.clone(); + let ei_svc = ei_service.clone(); + let db = db_rw.clone(); async move { - let poll_request = cli - .get(&url) - .inject_token(&tp) - .await - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; + let poll_request = cli.get(&url).inject_token(&tp).await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("failed to obtain auth token: {e}")) + })?; let resp = classify_response_error(poll_request.send().await)?; @@ -84,6 +74,13 @@ pub(crate) async fn poll_for_result( status = poll.status, "exploit intelligence analysis still in progress" ); + // Extend visibility so other instances know we're alive. + if let Err(e) = ei_svc + .extend_visibility(job_id, std::time::Duration::from_secs(5 * 60), &*db) + .await + { + tracing::warn!(job_id = %job_id, error = %e, "failed to extend job visibility"); + } Err(EiRequestError::Transient("pending".to_string())) } } @@ -94,22 +91,17 @@ pub(crate) async fn poll_for_result( .with_delay(poll_interval) .without_max_times(), ) - .when(|e| { - if token_for_when.is_cancelled() { - return false; + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true } - match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true - } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < max_consecutive - } - EiRequestError::Permanent(_) => false, + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < max_consecutive } + EiRequestError::Permanent(_) => false, }) .notify(|e, dur| { if !matches!(e, EiRequestError::Transient(r) if r == "pending") { @@ -119,51 +111,18 @@ pub(crate) async fn poll_for_result( ) .await; - /// Read the current heartbeat value from the shared state. - fn read_hb(hb: &SharedHeartbeat) -> Option { - hb.lock().ok().and_then(|guard| *guard) - } - // Convert timeout/retry result to our expected shape. let poll = match result { Ok(Ok(poll)) => poll, Ok(Err(e)) => { let err = e.into_anyhow(); let msg = err.to_string(); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - let applied = ei_service - .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } + ei_service.update_job_failed(job_id, &msg, db_rw).await?; return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - let applied = ei_service - .update_job_failed(job_id, &msg, read_hb(&heartbeat), db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } + ei_service.update_job_failed(job_id, &msg, db_rw).await?; return Err(anyhow::anyhow!("{msg}")); } }; @@ -197,13 +156,6 @@ pub(crate) async fn poll_for_result( let advisory_id = if let Some(vex) = vex_json { if !vex.is_null() { - // Check cancellation before VEX ingestion (side-effecting operation). - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before VEX ingestion" - )); - } - // Ingest the VEX document as an advisory with source label. let vex_bytes = serde_json::to_vec(vex)?; let mut labels = Labels::default(); @@ -241,24 +193,11 @@ pub(crate) async fn poll_for_result( .await .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; - // Check cancellation before terminal job status update. - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - // Mark the parent job as completed (finding lives on the component). - let applied = ei_service - .update_job_completed(job_id, read_hb(&heartbeat), db_rw) + ei_service + .update_job_completed(job_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } tracing::info!( job_id = %job_id, @@ -276,22 +215,10 @@ pub(crate) async fn poll_for_result( .unwrap_or("analysis failed") .to_string(); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - - let applied = ei_service - .update_job_failed(job_id, &error_msg, read_hb(&heartbeat), db_rw) + ei_service + .update_job_failed(job_id, &error_msg, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } Ok(()) } @@ -303,10 +230,8 @@ pub(crate) async fn poll_for_result( /// /// This is the SPDX counterpart of `poll_for_result`. It polls the product-level /// endpoint until all components are complete, then fetches each component's -/// report individually. -/// -/// The `heartbeat` parameter is a shared state updated by the heartbeat loop. -/// It is read when performing CAS-protected terminal status updates. +/// report individually. Like `poll_for_result`, each tick extends the job's +/// visibility window. #[allow(clippy::too_many_arguments)] pub(crate) async fn poll_for_product_result( ei_service: &ExploitIntelligenceService, @@ -316,8 +241,6 @@ pub(crate) async fn poll_for_product_result( client: &reqwest::Client, job_id: Uuid, product_id: &str, - heartbeat: SharedHeartbeat, - token: CancellationToken, ) -> Result<(), anyhow::Error> { let product_url = format!( "{}/api/v1/products/{}", @@ -327,7 +250,6 @@ pub(crate) async fn poll_for_product_result( let poll_interval = config.poll_interval.max(Duration::from_secs(5)); let max_consecutive = config.max_consecutive_poll_failures; let consecutive_failures = Cell::new(0u32); - let token_for_when = token.clone(); let mut components_initialized = false; @@ -339,7 +261,7 @@ pub(crate) async fn poll_for_product_result( // crash/restart that orphaned a running job). If they do, skip re-initialization // to avoid duplicate component rows. let existing_components = ei_service - .fetch_components(job_id, db_rw) + .fetch_components(job_id, &**db_rw) .await .map_err(|e| anyhow::anyhow!("failed to fetch existing components: {e}"))?; if !existing_components.is_empty() { @@ -362,16 +284,12 @@ pub(crate) async fn poll_for_product_result( let url = product_url.clone(); let cli = client.clone(); let tp = config.token_provider.clone(); + let ei_svc = ei_service.clone(); + let db = db_rw.clone(); async move { - let poll_request = cli - .get(&url) - .inject_token(&tp) - .await - .map_err(|e| { - EiRequestError::Permanent(anyhow::anyhow!( - "failed to obtain auth token: {e}" - )) - })?; + let poll_request = cli.get(&url).inject_token(&tp).await.map_err(|e| { + EiRequestError::Permanent(anyhow::anyhow!("failed to obtain auth token: {e}")) + })?; let resp = classify_response_error(poll_request.send().await)?; @@ -389,6 +307,12 @@ pub(crate) async fn poll_for_product_result( state = product.summary.product_state, "SPDX product analysis still in progress" ); + if let Err(e) = ei_svc + .extend_visibility(job_id, std::time::Duration::from_secs(5 * 60), &*db) + .await + { + tracing::warn!(job_id = %job_id, error = %e, "failed to extend job visibility"); + } Err(EiRequestError::Transient("pending".to_string())) } } @@ -399,22 +323,17 @@ pub(crate) async fn poll_for_product_result( .with_delay(poll_interval) .without_max_times(), ) - .when(|e| { - if token_for_when.is_cancelled() { - return false; + .when(|e| match e { + EiRequestError::Transient(reason) if reason == "pending" => { + consecutive_failures.set(0); + true } - match e { - EiRequestError::Transient(reason) if reason == "pending" => { - consecutive_failures.set(0); - true - } - EiRequestError::Transient(_) => { - let n = consecutive_failures.get() + 1; - consecutive_failures.set(n); - n < max_consecutive - } - EiRequestError::Permanent(_) => false, + EiRequestError::Transient(_) => { + let n = consecutive_failures.get() + 1; + consecutive_failures.set(n); + n < max_consecutive } + EiRequestError::Permanent(_) => false, }) .notify(|e, dur| { if !matches!(e, EiRequestError::Transient(r) if r == "pending") { @@ -424,51 +343,18 @@ pub(crate) async fn poll_for_product_result( ) .await; - /// Read the current heartbeat value from the shared state. - fn read_hb_product(hb: &SharedHeartbeat) -> Option { - hb.lock().ok().and_then(|guard| *guard) - } - // Convert timeout/retry result to our expected shape. let product = match result { Ok(Ok(product)) => product, Ok(Err(e)) => { let err = e.into_anyhow(); let msg = err.to_string(); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - let applied = ei_service - .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } + ei_service.update_job_failed(job_id, &msg, db_rw).await?; return Err(err); } Err(_elapsed) => { let msg = format!("polling timed out after {:?}", config.max_poll_duration); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - let applied = ei_service - .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } + ei_service.update_job_failed(job_id, &msg, db_rw).await?; return Err(anyhow::anyhow!("{msg}")); } }; @@ -555,7 +441,19 @@ pub(crate) async fn poll_for_product_result( } else { match scan_to_component.get(scan_id) { Some(id) => *id, - None => continue, + None => { + // Component was not tracked before the crash — create it now + // so it is not silently dropped. + let id = ei_service + .create_component(job_id, image_name, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to create component: {e}"))?; + ei_service + .update_component_running(id, scan_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + id + } } }; @@ -611,13 +509,6 @@ pub(crate) async fn poll_for_product_result( if let Some(vex) = vex_json { if !vex.is_null() { - // Check cancellation before VEX ingestion (side-effecting operation). - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before VEX ingestion" - )); - } - let vex_bytes = serde_json::to_vec(vex)?; let mut labels = Labels::default(); labels.0.insert( @@ -673,25 +564,12 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; } - // Check cancellation before terminal job status update. - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - // Mark the parent job as completed. The aggregate finding is // computed from component records at query time. - let applied = ei_service - .update_job_completed(job_id, read_hb_product(&heartbeat), db_rw) + ei_service + .update_job_completed(job_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } tracing::info!( job_id = %job_id, @@ -703,21 +581,7 @@ pub(crate) async fn poll_for_product_result( } "failed" => { let msg = format!("EI product analysis failed for product {product_id}"); - if token.is_cancelled() { - return Err(anyhow::anyhow!( - "heartbeat lost, aborting before terminal update" - )); - } - let applied = ei_service - .update_job_failed(job_id, &msg, read_hb_product(&heartbeat), db_rw) - .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; - if !applied { - tracing::warn!(job_id = %job_id, "job ownership lost, terminal update not applied"); - return Err(anyhow::anyhow!( - "job {job_id} ownership lost, terminal update not applied" - )); - } + ei_service.update_job_failed(job_id, &msg, db_rw).await?; Ok(()) } _ => unreachable!("backon retry only returns terminal states"), diff --git a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs b/modules/fundamental/src/exploit_intelligence/runner/recovery.rs deleted file mode 100644 index 1e68f7e86..000000000 --- a/modules/fundamental/src/exploit_intelligence/runner/recovery.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::exploit_intelligence::service::ExploitIntelligenceService; -use tokio_util::sync::CancellationToken; -use trustify_common::db; -use trustify_module_ingestor::service::IngestorService; - -use super::heartbeat::{self, HeartbeatError}; -use super::polling::{poll_for_product_result, poll_for_result}; - -/// Recover orphaned EI jobs after a server restart. -/// -/// Pending jobs (never submitted) are marked as Failed. Running jobs -/// (scan_id present) get their poll loops re-spawned so the EI backend -/// result is picked up transparently. Each resumed task maintains a -/// heartbeat so other instances know the job is still being serviced. -pub(crate) async fn recover_and_resume( - ei_service: &ExploitIntelligenceService, - ingestor: &IngestorService, - db_rw: &db::ReadWrite, -) { - let result = match ei_service.recover_orphaned_jobs(db_rw).await { - Ok(r) => r, - Err(e) => { - tracing::error!(error = %e, "failed to recover orphaned exploit intelligence jobs"); - return; - } - }; - - let runtime = match ei_service.runtime().cloned() { - Some(rt) => rt, - None => return, - }; - let config = runtime.config; - let client = runtime.client; - - for job in result.running_jobs { - let job_id = job.id; - let ei_svc = ei_service.clone(); - let ingestor_svc = ingestor.clone(); - let db_rw_clone = db_rw.clone(); - let config_clone = config.clone(); - let client_clone = client.clone(); - // The heartbeat was freshly set by recover_orphaned_jobs. - let initial_heartbeat = job.heartbeat; - - if let Some(ref product_id) = job.product_id { - // Multi-component SPDX flow — resume product-level polling. - let product_id = product_id.clone(); - let db_hb = db_rw.clone(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, product_id = %product_id, "resuming orphaned SPDX product polling"); - - let token = CancellationToken::new(); - let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); - let hb_for_work = hb_state.clone(); - let token_for_work = token.clone(); - - let work = poll_for_product_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &product_id, - hb_for_work, - token_for_work, - ); - match heartbeat::run_with_heartbeat( - job_id, - initial_heartbeat, - &db_hb, - hb_state, - token, - work, - ) - .await - { - Ok(()) => {} - Err(HeartbeatError::LockLost) => { - tracing::warn!(job_id = %job_id, "heartbeat lost during SPDX product recovery, another instance owns this job"); - } - Err(HeartbeatError::Work(e)) => { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned SPDX product polling"); - } - } - }); - } else { - // Single-component CycloneDX flow — look up scan_id from the component. - let components = match ei_service.fetch_components(job_id, db_rw).await { - Ok(c) => c, - Err(e) => { - tracing::error!(job_id = %job_id, error = %e, "failed to fetch components during recovery"); - let _ = ei_service - .update_job_failed( - job_id, - &format!("recovery error: {e}"), - initial_heartbeat, - db_rw, - ) - .await; - continue; - } - }; - - let first_component = components.first(); - let scan_id = first_component.and_then(|c| c.scan_id.as_deref()); - let component_id = first_component.map(|c| c.id); - - if let (Some(scan_id), Some(component_id)) = (scan_id, component_id) { - let scan_id = scan_id.to_string(); - let db_hb = db_rw.clone(); - actix_web::rt::spawn(async move { - tracing::info!(job_id = %job_id, scan_id = %scan_id, "resuming orphaned EI analysis"); - - let token = CancellationToken::new(); - let hb_state = heartbeat::shared_heartbeat(initial_heartbeat); - let hb_for_work = hb_state.clone(); - let token_for_work = token.clone(); - - let work = poll_for_result( - &ei_svc, - &ingestor_svc, - &db_rw_clone, - &config_clone, - &client_clone, - job_id, - &scan_id, - component_id, - hb_for_work, - token_for_work, - ); - match heartbeat::run_with_heartbeat( - job_id, - initial_heartbeat, - &db_hb, - hb_state, - token, - work, - ) - .await - { - Ok(()) => {} - Err(HeartbeatError::LockLost) => { - tracing::warn!(job_id = %job_id, "heartbeat lost during recovery, another instance owns this job"); - } - Err(HeartbeatError::Work(e)) => { - tracing::error!(job_id = %job_id, error = %e, "failed to resume orphaned EI analysis"); - } - } - }); - } else { - tracing::error!(job_id = %job_id, "running job has no component with scan_id, marking failed"); - let _ = ei_service - .update_job_failed( - job_id, - "Running job missing component scan_id during recovery", - initial_heartbeat, - db_rw, - ) - .await; - } - } - } -} diff --git a/modules/fundamental/src/exploit_intelligence/runner/worker.rs b/modules/fundamental/src/exploit_intelligence/runner/worker.rs new file mode 100644 index 000000000..94a012c7c --- /dev/null +++ b/modules/fundamental/src/exploit_intelligence/runner/worker.rs @@ -0,0 +1,94 @@ +//! Background worker loop for processing Exploit Intelligence jobs. +//! +//! Uses a PostgreSQL job queue pattern: each tick dequeues a bounded +//! batch of eligible jobs via `FOR UPDATE SKIP LOCKED` and sets a short +//! visibility window on each. While processing, the worker periodically +//! extends the window so other instances know the job is still alive. +//! If the worker crashes, the window expires and another instance +//! reclaims the job on its next tick. +//! +//! No semaphore or concurrency limit is needed because each job's work +//! is a lightweight async poll loop (one HTTP GET every ~30s). The batch +//! LIMIT on dequeue ensures work is shared across instances. + +use crate::exploit_intelligence::{ + runner::analysis::run_analysis, service::ExploitIntelligenceService, +}; +use std::time::Duration; +use tokio::time::MissedTickBehavior; +use trustify_common::db; +use trustify_module_ingestor::service::IngestorService; + +/// Maximum jobs to claim per tick. Keeps one instance from starving +/// others when multiple instances share the same database. +const DEQUEUE_BATCH_SIZE: u64 = 50; + +/// Run the EI worker loop, polling for and processing jobs. +/// +/// The loop runs indefinitely, dequeuing eligible jobs every +/// `poll_interval`. Each claimed job is spawned as an independent +/// async task — no concurrency cap is needed because the work is +/// I/O-bound polling (not CPU/memory intensive like importer jobs). +pub async fn run_worker( + ei_service: ExploitIntelligenceService, + ingestor: IngestorService, + db_rw: db::ReadWrite, + db_ro: db::ReadOnly, + poll_interval: Duration, +) -> anyhow::Result<()> { + let mut interval = tokio::time::interval(poll_interval); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + interval.tick().await; + + let jobs = match ei_service.dequeue_jobs(DEQUEUE_BATCH_SIZE, &db_rw).await { + Ok(j) => j, + Err(e) => { + tracing::error!(error = %e, "failed to dequeue EI jobs"); + continue; + } + }; + + // Mark retry-exhausted jobs as failed so they don't linger + // as invisible zombies (excluded from dequeue but never terminal). + if let Err(e) = ei_service.fail_exhausted_jobs(&db_rw).await { + tracing::error!(error = %e, "failed to mark exhausted EI jobs as failed"); + } + + for job in jobs { + let ei_svc = ei_service.clone(); + let ingestor_svc = ingestor.clone(); + let db_rw_clone = db_rw.clone(); + let db_ro_clone = db_ro.clone(); + let job_id = job.id; + + let runtime = match ei_svc.runtime().cloned() { + Some(rt) => rt, + None => continue, + }; + + actix_web::rt::spawn(async move { + let config = &runtime.config; + let client = &runtime.client; + + if let Err(e) = run_analysis( + &ei_svc, + &ingestor_svc, + &db_rw_clone, + &db_ro_clone, + config, + client, + &job, + ) + .await + { + tracing::error!(job_id = %job_id, error = %e, "EI analysis failed"); + let _ = ei_svc + .update_job_failed(job_id, &e.to_string(), &db_rw_clone) + .await; + } + }); + } + } +} diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 31f741a54..196b6d4b2 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -5,9 +5,8 @@ use crate::{ }, }; use sea_orm::{ - ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, EntityTrait, IntoActiveModel, - PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, - prelude::Decimal, + ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, EntityTrait, PaginatorTrait, + QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, sea_query::{Expr, LockBehavior, LockType}, }; use std::collections::HashMap; @@ -56,20 +55,18 @@ impl std::fmt::Debug for ExploitIntelligenceConfig { .field("max_poll_duration", &self.max_poll_duration) .field("upload_max_retries", &self.upload_max_retries) .field("upload_retry_delay", &self.upload_retry_delay) - .field("max_consecutive_poll_failures", &self.max_consecutive_poll_failures) - .field("token_provider", &self.token_provider.as_ref().map(|_| "[configured]")) + .field( + "max_consecutive_poll_failures", + &self.max_consecutive_poll_failures, + ) + .field( + "token_provider", + &self.token_provider.as_ref().map(|_| "[configured]"), + ) .finish() } } -/// Result of the startup recovery pass over orphaned EI jobs. -pub struct RecoveryResult { - /// Number of Pending jobs that were marked as Failed. - pub pending_failed: u64, - /// Running jobs whose poll loops should be resumed by the caller. - pub running_jobs: Vec, -} - /// Holds the EI configuration together with a shared HTTP client. /// /// These two values are always present or absent together: when EI is @@ -129,25 +126,11 @@ impl ExploitIntelligenceService { /// Create an analysis job record in the database. /// - /// Returns the created job details. The caller is responsible for - /// spawning the background task that submits the request and polls for - /// results. - /// - /// The caller must provide a connection (typically a transaction) and is - /// responsible for committing it. - /// Create an analysis job record in the database. - /// - /// The job is created with an initial heartbeat value so it is "born owned" - /// by the creating instance. This prevents the recovery pass from marking - /// it as orphaned before the heartbeat loop has started. - /// - /// Returns the created job details. The caller is responsible for - /// spawning the background task that submits the request and polls for - /// results. The initial heartbeat value should be passed to - /// [`run_with_heartbeat`](crate::exploit_intelligence::runner::heartbeat::run_with_heartbeat). + /// The job is created with `visible_at = NULL` so the background worker + /// picks it up on its next poll cycle. /// - /// The caller must provide a connection (typically a transaction) and is - /// responsible for committing it. + /// Returns the created job details. The caller must provide a connection + /// (typically a transaction) and is responsible for committing it. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn create_job( &self, @@ -159,7 +142,6 @@ impl ExploitIntelligenceService { let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); - let initial_hb = Decimal::from_i128_with_scale(now.unix_timestamp_nanos(), 0); let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), @@ -169,7 +151,8 @@ impl ExploitIntelligenceService { error_message: Set(None), product_id: Set(None), total_components: Set(None), - heartbeat: Set(Some(initial_hb)), + visible_at: Set(None), + retry_count: Set(0), created: Set(now), updated: Set(now), }; @@ -182,6 +165,139 @@ impl ExploitIntelligenceService { )) } + /// Atomically dequeue eligible jobs using `FOR UPDATE SKIP LOCKED`. + /// + /// At most `limit` jobs are claimed per call to avoid one instance + /// starving others. Each claimed job gets a short visibility window; + /// the worker must periodically call `extend_visibility` to prove it + /// is still alive. If it crashes, the window expires and another + /// instance reclaims the job on its next tick. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn dequeue_jobs( + &self, + limit: u64, + db: &db::ReadWrite, + ) -> Result, Error> { + let runtime = self.runtime.as_ref().ok_or(Error::Unavailable)?; + // Short initial window — the worker extends it while processing. + let visibility_timeout = Duration::from_secs(5 * 60); + let max_retries = runtime.config.upload_max_retries as i32; + + let tx = db.begin().await?; + let now = OffsetDateTime::now_utc(); + + let jobs = exploit_intelligence_job::Entity::find() + .filter(exploit_intelligence_job::Column::Status.is_in([ + ExploitIntelligenceJobStatus::Pending, + ExploitIntelligenceJobStatus::Running, + ])) + .filter( + Condition::any() + .add(exploit_intelligence_job::Column::VisibleAt.is_null()) + .add(exploit_intelligence_job::Column::VisibleAt.lte(now)), + ) + .filter(exploit_intelligence_job::Column::RetryCount.lt(max_retries)) + .lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) + .limit(limit) + .all(&tx) + .await + .map_err(Error::from)?; + + for job in &jobs { + exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Running), + ) + .col_expr( + exploit_intelligence_job::Column::VisibleAt, + Expr::value(Some(now + visibility_timeout)), + ) + .col_expr( + exploit_intelligence_job::Column::RetryCount, + Expr::value(job.retry_count + 1), + ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) + .filter(exploit_intelligence_job::Column::Id.eq(job.id)) + .exec(&tx) + .await + .map_err(Error::from)?; + } + + tx.commit().await?; + Ok(jobs) + } + + /// Push the visibility window forward so other instances know this job + /// is still being actively processed. Without periodic extension, a + /// long-running analysis (hours) would exceed the initial short window + /// and be re-claimed by another instance. + #[instrument(skip(self, db), err(level = tracing::Level::INFO))] + pub async fn extend_visibility( + &self, + job_id: Uuid, + extension: Duration, + db: &impl ConnectionTrait, + ) -> Result<(), Error> { + let now = OffsetDateTime::now_utc(); + exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::VisibleAt, + Expr::value(Some(now + extension)), + ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .filter( + exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), + ) + .exec(db) + .await + .map_err(Error::from)?; + Ok(()) + } + + /// Mark jobs that have exhausted their retry budget as Failed. + /// + /// Without this, retry-exhausted jobs would remain in Running/Pending + /// status forever — excluded from dequeue (retry_count >= max) but + /// never transitioned to a terminal state. + #[instrument(skip_all, err(level = tracing::Level::INFO))] + pub async fn fail_exhausted_jobs(&self, db: &db::ReadWrite) -> Result<(), Error> { + let runtime = self.runtime.as_ref().ok_or(Error::Unavailable)?; + let max_retries = runtime.config.upload_max_retries as i32; + let now = OffsetDateTime::now_utc(); + + exploit_intelligence_job::Entity::update_many() + .col_expr( + exploit_intelligence_job::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Failed), + ) + .col_expr( + exploit_intelligence_job::Column::ErrorMessage, + Expr::value(Some("max retries exhausted".to_string())), + ) + .col_expr( + exploit_intelligence_job::Column::VisibleAt, + Expr::value(Option::::None), + ) + .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) + .filter(exploit_intelligence_job::Column::Status.is_in([ + ExploitIntelligenceJobStatus::Pending, + ExploitIntelligenceJobStatus::Running, + ])) + .filter(exploit_intelligence_job::Column::RetryCount.gte(max_retries)) + .filter( + Condition::any() + .add(exploit_intelligence_job::Column::VisibleAt.is_null()) + .add(exploit_intelligence_job::Column::VisibleAt.lte(now)), + ) + .exec(&**db) + .await + .map_err(Error::from)?; + + Ok(()) + } + /// Fetch details of a single job by its ID. /// /// Component records are always loaded so that the aggregate finding and @@ -264,7 +380,7 @@ impl ExploitIntelligenceService { .map_err(Error::from)?; // Compute component counts per job via aggregate query, grouped by status and finding. - // Keyed by job_id → (completed, failed, total, vulnerable, not_vulnerable, uncertain). + // Keyed by job_id -> (completed, failed, total, vulnerable, not_vulnerable, uncertain). let job_ids: Vec = results.iter().map(|j| j.id).collect(); let mut counts_map: HashMap = HashMap::new(); @@ -383,76 +499,59 @@ impl ExploitIntelligenceService { Ok(PaginatedResults { items, total }) } - /// Update a job to completed, using the heartbeat as a CAS condition. + /// Update a job to completed. /// - /// Only succeeds if the job's current heartbeat matches the provided value. - /// On success the heartbeat is cleared (set to NULL) and the status is set - /// to Completed. Returns `true` if the update was applied, `false` if the - /// CAS failed (another instance owns the job). + /// Only transitions the job if its current status is `Running`, + /// which prevents double-completion. The visibility timeout is + /// cleared (set to NULL) on terminal state. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_completed( &self, job_id: Uuid, - heartbeat: Option, db: &db::ReadWrite, - ) -> Result { + ) -> Result<(), Error> { let now = OffsetDateTime::now_utc(); - let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); - match heartbeat { - Some(hb) => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); - } - None => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); - } - } - - let result = exploit_intelligence_job::Entity::update_many() + exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Status, Expr::value(ExploitIntelligenceJobStatus::Completed), ) .col_expr( - exploit_intelligence_job::Column::Heartbeat, - Expr::value(Option::::None), + exploit_intelligence_job::Column::VisibleAt, + Expr::value(Option::::None), ) .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(filter) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .filter( + exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), + ) .exec(&**db) .await .map_err(Error::from)?; - Ok(result.rows_affected > 0) + Ok(()) } - /// Update a job to failed with an error message, using the heartbeat as a CAS condition. + /// Update a job to failed with an error message. + /// + /// Only transitions the job if its current status is `Running`, + /// which prevents double-completion. The visibility timeout is + /// cleared (set to NULL) on terminal state. /// - /// Only succeeds if the job's current heartbeat matches the provided value. - /// On success the heartbeat is cleared (set to NULL) and the status is set - /// to Failed. Returns `true` if the update was applied, `false` if the - /// CAS failed (another instance owns the job). + /// Also cascades failure to any non-terminal components so the API + /// never shows a Failed job with Running/Pending components. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_failed( &self, job_id: Uuid, error_message: &str, - heartbeat: Option, db: &db::ReadWrite, - ) -> Result { + ) -> Result<(), Error> { let now = OffsetDateTime::now_utc(); + let tx = db.begin().await?; - let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); - match heartbeat { - Some(hb) => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); - } - None => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); - } - } - - let result = exploit_intelligence_job::Entity::update_many() + exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Status, Expr::value(ExploitIntelligenceJobStatus::Failed), @@ -462,16 +561,50 @@ impl ExploitIntelligenceService { Expr::value(Some(error_message.to_string())), ) .col_expr( - exploit_intelligence_job::Column::Heartbeat, - Expr::value(Option::::None), + exploit_intelligence_job::Column::VisibleAt, + Expr::value(Option::::None), ) .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(filter) - .exec(&**db) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .filter( + exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), + ) + .exec(&tx) + .await + .map_err(Error::from)?; + + // Cascade: fail any non-terminal components belonging to this job. + // Runs in the same transaction so a crash between the two UPDATEs + // cannot leave components in Running while the job is Failed. + exploit_intelligence_job_component::Entity::update_many() + .col_expr( + exploit_intelligence_job_component::Column::Status, + Expr::value(ExploitIntelligenceJobStatus::Failed), + ) + .col_expr( + exploit_intelligence_job_component::Column::ErrorMessage, + Expr::value(Some("parent job failed".to_string())), + ) + .col_expr( + exploit_intelligence_job_component::Column::Updated, + Expr::value(now), + ) + .filter(exploit_intelligence_job_component::Column::JobId.eq(job_id)) + .filter( + exploit_intelligence_job_component::Column::Status + .ne(ExploitIntelligenceJobStatus::Completed), + ) + .filter( + exploit_intelligence_job_component::Column::Status + .ne(ExploitIntelligenceJobStatus::Failed), + ) + .exec(&tx) .await .map_err(Error::from)?; - Ok(result.rows_affected > 0) + tx.commit().await?; + + Ok(()) } /// Find an existing job in Pending or Running state for the given SBOM and vulnerability. @@ -503,8 +636,8 @@ impl ExploitIntelligenceService { /// Create an analysis job with product_id and total_components set (multi-component SPDX flow). /// - /// The job is created with an initial heartbeat value so it is "born owned" - /// by the creating instance. + /// The job is created with `visible_at = NULL` so the background worker + /// picks it up on its next poll cycle. /// /// Returns the created job details. The caller is responsible for creating /// component records and spawning background tasks. @@ -520,7 +653,6 @@ impl ExploitIntelligenceService { let _config = self.runtime.as_ref().ok_or(Error::Unavailable)?; let now = OffsetDateTime::now_utc(); let job_id = Uuid::now_v7(); - let initial_hb = Decimal::from_i128_with_scale(now.unix_timestamp_nanos(), 0); let model = exploit_intelligence_job::ActiveModel { id: Set(job_id), @@ -530,7 +662,8 @@ impl ExploitIntelligenceService { error_message: Set(None), product_id: Set(Some(product_id.to_string())), total_components: Set(Some(total_components)), - heartbeat: Set(Some(initial_hb)), + visible_at: Set(None), + retry_count: Set(0), created: Set(now), updated: Set(now), }; @@ -741,30 +874,18 @@ impl ExploitIntelligenceService { /// Update a job's status to running and store the product ID from the EI service. /// - /// Uses the heartbeat as a CAS condition to ensure only the owning instance - /// can transition the job. Returns `true` if the update was applied, `false` - /// if the CAS failed (another instance owns the job). + /// Uses a simple `WHERE id = ?` filter. The visibility timeout ensures + /// only one worker processes a job at a time. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn update_job_running_with_product( &self, job_id: Uuid, product_id: &str, - heartbeat: Option, db: &db::ReadWrite, - ) -> Result { + ) -> Result<(), Error> { let now = OffsetDateTime::now_utc(); - let mut filter = Condition::all().add(exploit_intelligence_job::Column::Id.eq(job_id)); - match heartbeat { - Some(hb) => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.eq(hb)); - } - None => { - filter = filter.add(exploit_intelligence_job::Column::Heartbeat.is_null()); - } - } - - let result = exploit_intelligence_job::Entity::update_many() + exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Status, Expr::value(ExploitIntelligenceJobStatus::Running), @@ -774,12 +895,15 @@ impl ExploitIntelligenceService { Expr::value(Some(product_id.to_string())), ) .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(filter) + .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + .filter( + exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), + ) .exec(&**db) .await .map_err(Error::from)?; - Ok(result.rows_affected > 0) + Ok(()) } /// Fetch all component records for a given job, ordered by creation time. @@ -806,106 +930,4 @@ impl ExploitIntelligenceService { pub fn config(&self) -> Option<&ExploitIntelligenceConfig> { self.runtime.as_ref().map(|rt| &rt.config) } - - /// Recover orphaned jobs after a server restart. - /// - /// **Pending** jobs (no scan_id yet) are marked as Failed because the SBOM - /// was never submitted to the EI service. - /// - /// **Running** jobs whose heartbeat is stale (or absent) are returned to - /// the caller so their poll loops can be resumed — the EI service may - /// still be working on them. Jobs with an active heartbeat are being - /// serviced by another instance and are left alone. - /// - /// `FOR UPDATE SKIP LOCKED` prevents two instances from claiming the same - /// orphaned jobs simultaneously. - pub async fn recover_orphaned_jobs(&self, db: &db::ReadWrite) -> Result { - use crate::exploit_intelligence::runner::heartbeat; - - let tx = db.begin().await?; - let now = OffsetDateTime::now_utc(); - - // Heartbeats older than 2 × RATE are considered stale. - let stale_threshold = { - let threshold_nanos = (now - 2 * heartbeat::RATE).unix_timestamp_nanos(); - Decimal::from_i128_with_scale(threshold_nanos, 0) - }; - - // Find Pending/Running jobs that are NOT actively heartbeating. - let orphaned = exploit_intelligence_job::Entity::find() - .filter(exploit_intelligence_job::Column::Status.is_in([ - ExploitIntelligenceJobStatus::Pending, - ExploitIntelligenceJobStatus::Running, - ])) - .filter( - exploit_intelligence_job::Column::Heartbeat - .is_null() - .or(exploit_intelligence_job::Column::Heartbeat.lt(stale_threshold)), - ) - .lock_with_behavior(LockType::Update, LockBehavior::SkipLocked) - .all(&tx) - .await - .map_err(Error::from)?; - - let mut pending_failed: u64 = 0; - let mut running_jobs = Vec::new(); - - for job in orphaned { - match job.status { - ExploitIntelligenceJobStatus::Pending => { - let mut active = job.into_active_model(); - active.status = Set(ExploitIntelligenceJobStatus::Failed); - active.error_message = Set(Some( - "Server restarted before analysis was submitted. \ - Please re-trigger the analysis." - .to_string(), - )); - active.heartbeat = Set(None); - active.updated = Set(now); - active.update(&tx).await.map_err(Error::from)?; - pending_failed += 1; - } - ExploitIntelligenceJobStatus::Running => { - // Claim the job by setting a fresh heartbeat inside - // the same transaction, so the row lock protects us. - let hb_nanos = now.unix_timestamp_nanos(); - let new_hb = Decimal::from_i128_with_scale(hb_nanos, 0); - let mut active = job.clone().into_active_model(); - active.heartbeat = Set(Some(new_hb)); - active.updated = Set(now); - active.update(&tx).await.map_err(Error::from)?; - - // Push the model with the NEW heartbeat value so - // callers pass the correct value to run_with_heartbeat. - let mut claimed_job = job; - claimed_job.heartbeat = Some(new_hb); - running_jobs.push(claimed_job); - } - _ => { - // Should not happen given the filter, but be safe. - } - } - } - - tx.commit().await?; - - if pending_failed > 0 { - tracing::warn!( - pending_failed, - "marked orphaned pending exploit intelligence jobs as failed after restart" - ); - } - - if !running_jobs.is_empty() { - tracing::info!( - count = running_jobs.len(), - "found orphaned running exploit intelligence jobs to resume" - ); - } - - Ok(RecoveryResult { - pending_failed, - running_jobs, - }) - } } diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 690da7039..7943201e3 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -6,7 +6,7 @@ use actix_web::web; use bytesize::ByteSize; use futures::FutureExt; use trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig; -use std::{env, process::ExitCode, sync::Arc}; +use std::{env, process::ExitCode, sync::Arc, time::Duration}; use trustify_auth::{ auth::AuthConfigArguments, authenticator::Authenticator, @@ -472,13 +472,44 @@ impl InitData { #[allow(unused_mut)] async fn run(mut self) -> anyhow::Result<()> { - // Recovery for orphaned exploit intelligence jobs is handled inside - // the EI endpoints `configure()`, which spawns a background task. - // This avoids needing the IngestorService here (it is only available - // inside the module configuration). - let ui = Arc::new(UiResources::new(&self.ui)?); + // Build the EI worker task before the HTTP server captures `self`. + // The worker needs an IngestorService, which is also created inside + // the module configuration — build one here and clone it for the worker. + let ei_worker_task = if let Some(ref ei_config) = self.ei_config { + if !self.read_only { + let ei_service = trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceService::new( + Some(ei_config.clone()), + )?; + let ingestor_for_worker = trustify_module_ingestor::service::IngestorService::new( + Graph::new(), + self.storage.clone(), + Some(self.analysis.clone()), + ); + let db_rw = self.db_rw.clone(); + let db_ro = self.db_ro.clone(); + + Some( + async move { + trustify_module_fundamental::exploit_intelligence::runner::worker::run_worker( + ei_service, + ingestor_for_worker, + db_rw, + db_ro, + Duration::from_secs(5), // worker poll interval + ) + .await + } + .boxed_local(), + ) + } else { + None + } + } else { + None + }; + let http = { HttpServerBuilder::try_from(self.http)? .tracing(self.tracing) @@ -509,6 +540,10 @@ impl InitData { #[allow(unused_mut)] let mut tasks = vec![http]; + if let Some(worker) = ei_worker_task { + tasks.push(worker); + } + // track the embedded OIDC server task #[cfg(feature = "garage-door")] if let Some(embedded_oidc) = self.embedded_oidc.take() { From fc2f7b40bb7223a3aeb9e4c810c62c419b833679 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Wed, 15 Jul 2026 18:28:00 +0100 Subject: [PATCH 22/25] refactor(exploit-intelligence): address code style review comments Flatten EI clap args into ExploitIntelligenceArgs struct with into_config() method. Import namespaces instead of inline paths. Use SourceDocument::try_into() for storage key construction, Format::sbom_from_bytes() for content-based format detection, bytes::Bytes for zero-copy retry, and thiserror for EiRequestError. Co-Authored-By: Claude Opus 4.6 (1M context) --- modules/fundamental/Cargo.toml | 1 + .../exploit_intelligence/runner/analysis.rs | 57 +++---- .../src/exploit_intelligence/runner/mod.rs | 12 +- server/src/profile/api.rs | 141 ++++++++++-------- 4 files changed, 107 insertions(+), 104 deletions(-) diff --git a/modules/fundamental/Cargo.toml b/modules/fundamental/Cargo.toml index 096eaae22..7278d96c9 100644 --- a/modules/fundamental/Cargo.toml +++ b/modules/fundamental/Cargo.toml @@ -22,6 +22,7 @@ trustify-query-derive = { workspace = true } actix-http = { workspace = true } actix-web = { workspace = true } backon = { workspace = true } +bytes = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } cpe = { workspace = true } diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index f31547cf5..c965f600f 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -1,16 +1,17 @@ use crate::exploit_intelligence::{ + ei_client::{SpdxUploadResponse, UploadResponse}, runner::{EiRequestError, classify_response_error}, - service::ExploitIntelligenceService, + service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, }; +use crate::source_document::model::SourceDocument; use backon::{ExponentialBuilder, Retryable}; use futures_util::TryStreamExt; use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; -use std::str::FromStr; use trustify_auth::client::TokenInjector; use trustify_common::db; use trustify_entity::exploit_intelligence_job; -use trustify_module_ingestor::service::IngestorService; -use trustify_module_storage::service::StorageBackend; +use trustify_module_ingestor::service::{Format, IngestorService}; +use trustify_module_storage::service::{StorageBackend, StorageKey}; use super::polling::{poll_for_product_result, poll_for_result}; @@ -24,7 +25,7 @@ pub(crate) async fn run_analysis( ingestor: &IngestorService, db_rw: &db::ReadWrite, db_ro: &db::ReadOnly, - config: &crate::exploit_intelligence::service::ExploitIntelligenceConfig, + config: &ExploitIntelligenceConfig, client: &reqwest::Client, job: &exploit_intelligence_job::Model, ) -> Result<(), anyhow::Error> { @@ -46,16 +47,12 @@ pub(crate) async fn run_analysis( .await? .ok_or_else(|| anyhow::anyhow!("source document not found for SBOM: {sbom_id}"))?; - // Build storage key from source document SHA256. - // The DB stores the raw hex digest without algorithm prefix, but Id::from_str expects "sha256:...". - let sha256_with_prefix = format!("sha256:{}", source_doc.sha256); - let storage_key = trustify_common::id::Id::from_str(&sha256_with_prefix) - .map_err(|e| anyhow::anyhow!("invalid sha256: {e}"))?; - let storage_key: trustify_module_storage::service::StorageKey = storage_key + // Reuse the SourceDocument model's TryInto implementation, + // which handles the "sha256:" prefix and fallback to sha384/sha512. + let source_doc_model = SourceDocument::from_entity(&source_doc); + let storage_key: StorageKey = source_doc_model .try_into() - .map_err(|e: trustify_module_storage::service::StorageKeyError| { - anyhow::anyhow!("storage key error: {e}") - })?; + .map_err(|e| anyhow::anyhow!("storage key error: {e}"))?; let stream = ingestor .storage() @@ -75,19 +72,13 @@ pub(crate) async fn run_analysis( drop(tx); - // Detect SBOM format from labels. - let sbom_format = match sbom_model.labels.0.get("type") { - Some(t) => t.as_str(), - None => { - tracing::warn!( - sbom_id = %sbom_id, - "SBOM has no 'type' label, defaulting to CycloneDX format" - ); - "cyclonedx" - } - }; + // Detect SBOM format from content bytes rather than labels. + let format = Format::sbom_from_bytes(&sbom_bytes) + .map_err(|e| anyhow::anyhow!("failed to detect SBOM format: {e}"))?; + let is_spdx = matches!(format, Format::SPDX); - let is_spdx = sbom_format == "spdx"; + // Wrap in Bytes for O(1) cloning inside the retry closures. + let sbom_data = bytes::Bytes::from(sbom_bytes); if is_spdx { // SPDX multi-component flow: upload to /api/v1/products/upload-spdx @@ -95,14 +86,14 @@ pub(crate) async fn run_analysis( let response = (|| { let url = upload_url.clone(); - let sbom = sbom_bytes.clone(); + let sbom = sbom_data.clone(); let vuln_id = vulnerability_id.to_string(); let cli = client.clone(); let tp = config.token_provider.clone(); async move { let form = reqwest::multipart::Form::new().text("cveId", vuln_id).part( "file", - reqwest::multipart::Part::bytes(sbom) + reqwest::multipart::Part::bytes(sbom.to_vec()) .file_name("sbom.json") .mime_str("application/json") .map_err(|e| { @@ -134,8 +125,7 @@ pub(crate) async fn run_analysis( .await .map_err(EiRequestError::into_anyhow)?; - let spdx_upload: crate::exploit_intelligence::ei_client::SpdxUploadResponse = - response.json().await?; + let spdx_upload: SpdxUploadResponse = response.json().await?; let product_id = spdx_upload.product_id; // Update job to running with the product ID. @@ -177,14 +167,14 @@ pub(crate) async fn run_analysis( let response = (|| { let url = upload_url.clone(); - let sbom = sbom_bytes.clone(); + let sbom = sbom_data.clone(); let vuln_id = vulnerability_id.to_string(); let cli = client.clone(); let tp = config.token_provider.clone(); async move { let form = reqwest::multipart::Form::new().text("cveId", vuln_id).part( "file", - reqwest::multipart::Part::bytes(sbom) + reqwest::multipart::Part::bytes(sbom.to_vec()) .file_name("sbom.json") .mime_str("application/json") .map_err(|e| { @@ -216,8 +206,7 @@ pub(crate) async fn run_analysis( .await .map_err(EiRequestError::into_anyhow)?; - let upload: crate::exploit_intelligence::ei_client::UploadResponse = - response.json().await?; + let upload: UploadResponse = response.json().await?; let scan_id = upload.report_request_id.report_id; // Update component to running with scan ID. diff --git a/modules/fundamental/src/exploit_intelligence/runner/mod.rs b/modules/fundamental/src/exploit_intelligence/runner/mod.rs index ea85730df..6050b1cbe 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/mod.rs @@ -3,10 +3,13 @@ pub(crate) mod polling; pub mod worker; /// Error classification for EI HTTP requests. +#[derive(Debug, thiserror::Error)] pub(crate) enum EiRequestError { /// Retryable failure (network timeout, 5xx, etc.) + #[error("transient: {0}")] Transient(String), /// Fatal failure (auth error, 4xx, parse error) + #[error("permanent: {0}")] Permanent(anyhow::Error), } @@ -23,15 +26,6 @@ impl EiRequestError { } } -impl std::fmt::Display for EiRequestError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Transient(msg) => write!(f, "transient: {msg}"), - Self::Permanent(err) => write!(f, "permanent: {err}"), - } - } -} - /// Classify an HTTP response or reqwest error into an EiRequestError. pub(crate) fn classify_response_error( result: Result, diff --git a/server/src/profile/api.rs b/server/src/profile/api.rs index 7943201e3..bfac3326b 100644 --- a/server/src/profile/api.rs +++ b/server/src/profile/api.rs @@ -5,7 +5,6 @@ use crate::{endpoints, profile::spawn_db_check, sample_data}; use actix_web::web; use bytesize::ByteSize; use futures::FutureExt; -use trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceConfig; use std::{env, process::ExitCode, sync::Arc, time::Duration}; use trustify_auth::{ auth::AuthConfigArguments, @@ -33,6 +32,11 @@ use trustify_infrastructure::{ otel::{Metrics as OtelMetrics, Tracing}, }; use trustify_module_analysis::{config::AnalysisConfig, service::AnalysisService}; +use trustify_module_fundamental::exploit_intelligence::{ + auth::build_provider, + runner::worker::run_worker, + service::{ExploitIntelligenceConfig, ExploitIntelligenceService}, +}; use trustify_module_ingestor::graph::Graph; use trustify_module_storage::{config::StorageConfig, service::dispatch::DispatchBackend}; use trustify_module_ui::{UI, endpoints::UiResources}; @@ -93,6 +97,51 @@ pub struct Run { )] pub scan_limit: BinaryByteSize, + /// Exploit Intelligence configuration + #[command(flatten)] + pub exploit_intelligence: ExploitIntelligenceArgs, + + // flattened commands must go last + // + /// Analysis configuration + #[command(flatten)] + pub analysis: AnalysisConfig, + + /// Database configuration + #[command(flatten)] + pub database: Database, + + /// Read-only database configuration (optional, falls back to primary database) + #[command(flatten)] + pub database_ro: DatabaseReadOnly, + + /// Location of the storage + #[command(flatten)] + pub storage: StorageConfig, + + #[command(flatten)] + pub infra: InfrastructureConfig, + + #[command(flatten)] + pub auth: AuthConfigArguments, + + #[command(flatten)] + pub http: HttpServerConfig, + + #[command(flatten)] + pub swagger_ui_oidc: SwaggerUiOidcConfig, + + #[command(flatten)] + pub pagination: PaginationConfig, + + #[command(flatten)] + pub ui: UiConfig, +} + +/// All Exploit Intelligence CLI arguments. +#[derive(clap::Args, Debug)] +#[command(next_help_heading = "Exploit Intelligence")] +pub struct ExploitIntelligenceArgs { /// Base URL of the Exploit Intelligence client service. #[arg(long, env = "EXPLOIT_INTELLIGENCE_URL")] pub exploit_intelligence_url: Option, @@ -146,43 +195,35 @@ pub struct Run { pub exploit_intelligence_auth_token: Option, #[command(flatten)] - pub exploit_intelligence_oidc: EiOidcArguments, - - // flattened commands must go last - // - /// Analysis configuration - #[command(flatten)] - pub analysis: AnalysisConfig, - - /// Database configuration - #[command(flatten)] - pub database: Database, - - /// Read-only database configuration (optional, falls back to primary database) - #[command(flatten)] - pub database_ro: DatabaseReadOnly, - - /// Location of the storage - #[command(flatten)] - pub storage: StorageConfig, - - #[command(flatten)] - pub infra: InfrastructureConfig, - - #[command(flatten)] - pub auth: AuthConfigArguments, - - #[command(flatten)] - pub http: HttpServerConfig, - - #[command(flatten)] - pub swagger_ui_oidc: SwaggerUiOidcConfig, + pub oidc: EiOidcArguments, +} - #[command(flatten)] - pub pagination: PaginationConfig, +impl ExploitIntelligenceArgs { + /// Convert CLI arguments into an optional `ExploitIntelligenceConfig`. + /// + /// Returns `None` when no EI URL is configured (feature disabled). + pub async fn into_config(self) -> Result, anyhow::Error> { + let Some(url) = self.exploit_intelligence_url else { + return Ok(None); + }; - #[command(flatten)] - pub ui: UiConfig, + let token_provider = build_provider( + self.oidc.into_config(), + self.exploit_intelligence_auth_token, + ) + .await?; + + Ok(Some(ExploitIntelligenceConfig { + url, + ui_url: self.exploit_intelligence_ui_url, + poll_interval: self.exploit_intelligence_poll_interval.into(), + max_poll_duration: self.exploit_intelligence_max_poll_duration.into(), + upload_max_retries: self.exploit_intelligence_upload_max_retries, + upload_retry_delay: self.exploit_intelligence_upload_retry_delay.into(), + max_consecutive_poll_failures: self.exploit_intelligence_max_consecutive_poll_failures, + token_provider, + })) + } } /// Clap arguments for EI-specific OIDC credentials. @@ -427,27 +468,7 @@ impl InitData { }, }; - let ei_config = if let Some(url) = run.exploit_intelligence_url { - let token_provider = - trustify_module_fundamental::exploit_intelligence::auth::build_provider( - run.exploit_intelligence_oidc.into_config(), - run.exploit_intelligence_auth_token, - ) - .await?; - - Some(ExploitIntelligenceConfig { - url, - ui_url: run.exploit_intelligence_ui_url, - poll_interval: run.exploit_intelligence_poll_interval.into(), - max_poll_duration: run.exploit_intelligence_max_poll_duration.into(), - upload_max_retries: run.exploit_intelligence_upload_max_retries, - upload_retry_delay: run.exploit_intelligence_upload_retry_delay.into(), - max_consecutive_poll_failures: run.exploit_intelligence_max_consecutive_poll_failures, - token_provider, - }) - } else { - None - }; + let ei_config = run.exploit_intelligence.into_config().await?; Ok(InitData { analysis: AnalysisService::new(run.analysis, db_ro.clone()), @@ -479,9 +500,7 @@ impl InitData { // the module configuration — build one here and clone it for the worker. let ei_worker_task = if let Some(ref ei_config) = self.ei_config { if !self.read_only { - let ei_service = trustify_module_fundamental::exploit_intelligence::service::ExploitIntelligenceService::new( - Some(ei_config.clone()), - )?; + let ei_service = ExploitIntelligenceService::new(Some(ei_config.clone()))?; let ingestor_for_worker = trustify_module_ingestor::service::IngestorService::new( Graph::new(), self.storage.clone(), @@ -492,7 +511,7 @@ impl InitData { Some( async move { - trustify_module_fundamental::exploit_intelligence::runner::worker::run_worker( + run_worker( ei_service, ingestor_for_worker, db_rw, From 4f7662154e0863bc903504777e73dcfacdc5a0c5 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 16 Jul 2026 11:08:41 +0100 Subject: [PATCH 23/25] chore: bump migration num and address more comments --- migration/src/lib.rs | 4 ++-- ...002250_create_exploit_intelligence_job.rs} | 0 .../exploit_intelligence/runner/analysis.rs | 24 ++++++++++--------- 3 files changed, 15 insertions(+), 13 deletions(-) rename migration/src/{m0002230_create_exploit_intelligence_job.rs => m0002250_create_exploit_intelligence_job.rs} (100%) diff --git a/migration/src/lib.rs b/migration/src/lib.rs index acc6bd056..da5917bf4 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -66,7 +66,7 @@ mod m0002210_sbom_node_name_index; mod m0002220_drop_qualified_purl_gist_indexes; mod m0002230_sle_license_id_index; mod m0002240_product_version_sbom_index; -mod m0002230_create_exploit_intelligence_job; +mod m0002250_create_exploit_intelligence_job; pub trait MigratorExt: Send { fn build_migrations() -> Migrations; @@ -148,7 +148,7 @@ impl MigratorExt for Migrator { .normal(m0002220_drop_qualified_purl_gist_indexes::Migration) .normal(m0002230_sle_license_id_index::Migration) .normal(m0002240_product_version_sbom_index::Migration) - .normal(m0002230_create_exploit_intelligence_job::Migration) + .normal(m0002250_create_exploit_intelligence_job::Migration) } } diff --git a/migration/src/m0002230_create_exploit_intelligence_job.rs b/migration/src/m0002250_create_exploit_intelligence_job.rs similarity index 100% rename from migration/src/m0002230_create_exploit_intelligence_job.rs rename to migration/src/m0002250_create_exploit_intelligence_job.rs diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index c965f600f..b7d3f4d45 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -10,7 +10,7 @@ use sea_orm::{ColumnTrait, EntityTrait, QueryFilter}; use trustify_auth::client::TokenInjector; use trustify_common::db; use trustify_entity::exploit_intelligence_job; -use trustify_module_ingestor::service::{Format, IngestorService}; +use trustify_module_ingestor::service::{DocumentDetector, Format, IngestorService}; use trustify_module_storage::service::{StorageBackend, StorageKey}; use super::polling::{poll_for_product_result, poll_for_result}; @@ -61,26 +61,28 @@ pub(crate) async fn run_analysis( .map_err(|e| anyhow::anyhow!("storage retrieve error: {e}"))? .ok_or_else(|| anyhow::anyhow!("SBOM document not found in storage"))?; - // Collect the stream into bytes. let sbom_bytes: Vec = stream - .try_fold(Vec::new(), |mut acc, chunk| async move { - acc.extend_from_slice(&chunk); - Ok(acc) - }) + .try_fold( + Vec::with_capacity(source_doc.size as usize), + |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }, + ) .await .map_err(|e| anyhow::anyhow!("failed to read SBOM stream: {e}"))?; drop(tx); - // Detect SBOM format from content bytes rather than labels. - let format = Format::sbom_from_bytes(&sbom_bytes) - .map_err(|e| anyhow::anyhow!("failed to detect SBOM format: {e}"))?; - let is_spdx = matches!(format, Format::SPDX); + // Detect SBOM format from content rather than labels (which users can modify). + let format = DocumentDetector::detect(&sbom_bytes) + .map_err(|e| anyhow::anyhow!("failed to detect SBOM format: {e}"))? + .format(); // Wrap in Bytes for O(1) cloning inside the retry closures. let sbom_data = bytes::Bytes::from(sbom_bytes); - if is_spdx { + if format == Format::SPDX { // SPDX multi-component flow: upload to /api/v1/products/upload-spdx let upload_url = format!("{}/api/v1/products/upload-spdx", config.url); From b05411048ec948e8a765a0359a20c9da2bdece35 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Thu, 16 Jul 2026 13:29:21 +0100 Subject: [PATCH 24/25] fix(exploit-intelligence): use ActiveModel for single-row updates and fix enum casts Switch single-row status updates from update_many (which requires explicit CAST for PostgreSQL enum types) to Entity::update(ActiveModel) which handles enum serialization correctly via DeriveActiveEnum. Add AllowNotUpdated trait with ok_if_not_updated() to silently handle status guard mismatches (another worker already transitioned the record). Add LocalSet wrapper to the worker loop for actix_web::rt spawn compatibility. Consolidate dequeue_jobs into a single UPDATE with SQL-level retry_count increment. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/exploit_intelligence/runner/worker.rs | 22 ++ .../src/exploit_intelligence/service/mod.rs | 197 +++++++++--------- 2 files changed, 118 insertions(+), 101 deletions(-) diff --git a/modules/fundamental/src/exploit_intelligence/runner/worker.rs b/modules/fundamental/src/exploit_intelligence/runner/worker.rs index 94a012c7c..c1e393977 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/worker.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/worker.rs @@ -15,6 +15,7 @@ use crate::exploit_intelligence::{ runner::analysis::run_analysis, service::ExploitIntelligenceService, }; use std::time::Duration; +use tokio::task::LocalSet; use tokio::time::MissedTickBehavior; use trustify_common::db; use trustify_module_ingestor::service::IngestorService; @@ -29,12 +30,33 @@ const DEQUEUE_BATCH_SIZE: u64 = 50; /// `poll_interval`. Each claimed job is spawned as an independent /// async task — no concurrency cap is needed because the work is /// I/O-bound polling (not CPU/memory intensive like importer jobs). +/// +/// Wraps the inner loop in a `LocalSet` because spawned tasks use +/// `actix_web::rt::spawn` (which requires a local task context). pub async fn run_worker( ei_service: ExploitIntelligenceService, ingestor: IngestorService, db_rw: db::ReadWrite, db_ro: db::ReadOnly, poll_interval: Duration, +) -> anyhow::Result<()> { + LocalSet::new() + .run_until(run_worker_inner( + ei_service, + ingestor, + db_rw, + db_ro, + poll_interval, + )) + .await +} + +async fn run_worker_inner( + ei_service: ExploitIntelligenceService, + ingestor: IngestorService, + db_rw: db::ReadWrite, + db_ro: db::ReadOnly, + poll_interval: Duration, ) -> anyhow::Result<()> { let mut interval = tokio::time::interval(poll_interval); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index 196b6d4b2..aa98ac87e 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -4,8 +4,9 @@ use crate::{ ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, }, }; +use sea_orm::sea_query::SimpleExpr; use sea_orm::{ - ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, EntityTrait, PaginatorTrait, + ActiveModelTrait, ColumnTrait, Condition, ConnectionTrait, DbErr, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, TransactionTrait, sea_query::{Expr, LockBehavior, LockType}, }; @@ -22,6 +23,36 @@ use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStat use trustify_entity::exploit_intelligence_job_component; use uuid::Uuid; +/// Cast a job status string to the PostgreSQL enum type. +/// `update_many` with `Expr::value()` sends a plain text value, which +/// PostgreSQL rejects because the column is a custom enum. This helper +/// wraps the value in an explicit CAST. +fn status_expr(status: ExploitIntelligenceJobStatus) -> SimpleExpr { + let s = match status { + ExploitIntelligenceJobStatus::Pending => "pending", + ExploitIntelligenceJobStatus::Running => "running", + ExploitIntelligenceJobStatus::Completed => "completed", + ExploitIntelligenceJobStatus::Failed => "failed", + }; + Expr::cust_with_values("CAST($1 AS exploit_intelligence_job_status)", [s]) +} + +/// Extension for guarded updates where 0 rows affected (status guard +/// mismatch) is expected — another worker may have already transitioned +/// the record. +trait AllowNotUpdated { + fn ok_if_not_updated(self) -> Result<(), Error>; +} + +impl AllowNotUpdated for Result { + fn ok_if_not_updated(self) -> Result<(), Error> { + match self { + Ok(_) | Err(DbErr::RecordNotUpdated) => Ok(()), + Err(e) => Err(Error::from(e)), + } + } +} + /// Configuration for the Exploit Intelligence client connection. #[derive(Clone)] pub struct ExploitIntelligenceConfig { @@ -203,11 +234,12 @@ impl ExploitIntelligenceService { .await .map_err(Error::from)?; - for job in &jobs { + if !jobs.is_empty() { + let job_ids: Vec = jobs.iter().map(|j| j.id).collect(); exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Running), + status_expr(ExploitIntelligenceJobStatus::Running), ) .col_expr( exploit_intelligence_job::Column::VisibleAt, @@ -215,10 +247,10 @@ impl ExploitIntelligenceService { ) .col_expr( exploit_intelligence_job::Column::RetryCount, - Expr::value(job.retry_count + 1), + Expr::col(exploit_intelligence_job::Column::RetryCount).add(1), ) .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(exploit_intelligence_job::Column::Id.eq(job.id)) + .filter(exploit_intelligence_job::Column::Id.is_in(job_ids)) .exec(&tx) .await .map_err(Error::from)?; @@ -270,7 +302,7 @@ impl ExploitIntelligenceService { exploit_intelligence_job::Entity::update_many() .col_expr( exploit_intelligence_job::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Failed), + status_expr(ExploitIntelligenceJobStatus::Failed), ) .col_expr( exploit_intelligence_job::Column::ErrorMessage, @@ -510,25 +542,21 @@ impl ExploitIntelligenceService { job_id: Uuid, db: &db::ReadWrite, ) -> Result<(), Error> { - let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + status: Set(ExploitIntelligenceJobStatus::Completed), + visible_at: Set(None), + updated: Set(OffsetDateTime::now_utc()), + ..Default::default() + }; - exploit_intelligence_job::Entity::update_many() - .col_expr( - exploit_intelligence_job::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Completed), - ) - .col_expr( - exploit_intelligence_job::Column::VisibleAt, - Expr::value(Option::::None), - ) - .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + exploit_intelligence_job::Entity::update(model) .filter( exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), ) .exec(&**db) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; Ok(()) } @@ -551,27 +579,22 @@ impl ExploitIntelligenceService { let now = OffsetDateTime::now_utc(); let tx = db.begin().await?; - exploit_intelligence_job::Entity::update_many() - .col_expr( - exploit_intelligence_job::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Failed), - ) - .col_expr( - exploit_intelligence_job::Column::ErrorMessage, - Expr::value(Some(error_message.to_string())), - ) - .col_expr( - exploit_intelligence_job::Column::VisibleAt, - Expr::value(Option::::None), - ) - .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + status: Set(ExploitIntelligenceJobStatus::Failed), + error_message: Set(Some(error_message.to_string())), + visible_at: Set(None), + updated: Set(now), + ..Default::default() + }; + + exploit_intelligence_job::Entity::update(model) .filter( exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), ) .exec(&tx) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; // Cascade: fail any non-terminal components belonging to this job. // Runs in the same transaction so a crash between the two UPDATEs @@ -579,7 +602,7 @@ impl ExploitIntelligenceService { exploit_intelligence_job_component::Entity::update_many() .col_expr( exploit_intelligence_job_component::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Failed), + status_expr(ExploitIntelligenceJobStatus::Failed), ) .col_expr( exploit_intelligence_job_component::Column::ErrorMessage, @@ -760,29 +783,22 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + status: Set(ExploitIntelligenceJobStatus::Running), + scan_id: Set(Some(scan_id.to_string())), + updated: Set(OffsetDateTime::now_utc()), + ..Default::default() + }; - exploit_intelligence_job_component::Entity::update_many() - .col_expr( - exploit_intelligence_job_component::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Running), - ) - .col_expr( - exploit_intelligence_job_component::Column::ScanId, - Expr::value(Some(scan_id.to_string())), - ) - .col_expr( - exploit_intelligence_job_component::Column::Updated, - Expr::value(now), - ) - .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + exploit_intelligence_job_component::Entity::update(model) .filter( exploit_intelligence_job_component::Column::Status .ne(ExploitIntelligenceJobStatus::Completed), ) .exec(&**db) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; Ok(()) } @@ -801,33 +817,23 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + status: Set(ExploitIntelligenceJobStatus::Completed), + finding: Set(Some(finding)), + advisory_id: Set(advisory_id), + updated: Set(OffsetDateTime::now_utc()), + ..Default::default() + }; - exploit_intelligence_job_component::Entity::update_many() - .col_expr( - exploit_intelligence_job_component::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Completed), - ) - .col_expr( - exploit_intelligence_job_component::Column::Finding, - Expr::value(Some(finding)), - ) - .col_expr( - exploit_intelligence_job_component::Column::AdvisoryId, - Expr::value(advisory_id), - ) - .col_expr( - exploit_intelligence_job_component::Column::Updated, - Expr::value(now), - ) - .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + exploit_intelligence_job_component::Entity::update(model) .filter( exploit_intelligence_job_component::Column::Status .ne(ExploitIntelligenceJobStatus::Completed), ) .exec(&**db) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; Ok(()) } @@ -845,29 +851,22 @@ impl ExploitIntelligenceService { ) -> Result<(), Error> { use trustify_entity::exploit_intelligence_job_component; - let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job_component::ActiveModel { + id: Set(component_id), + status: Set(ExploitIntelligenceJobStatus::Failed), + error_message: Set(Some(error_message.to_string())), + updated: Set(OffsetDateTime::now_utc()), + ..Default::default() + }; - exploit_intelligence_job_component::Entity::update_many() - .col_expr( - exploit_intelligence_job_component::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Failed), - ) - .col_expr( - exploit_intelligence_job_component::Column::ErrorMessage, - Expr::value(Some(error_message.to_string())), - ) - .col_expr( - exploit_intelligence_job_component::Column::Updated, - Expr::value(now), - ) - .filter(exploit_intelligence_job_component::Column::Id.eq(component_id)) + exploit_intelligence_job_component::Entity::update(model) .filter( exploit_intelligence_job_component::Column::Status .ne(ExploitIntelligenceJobStatus::Completed), ) .exec(&**db) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; Ok(()) } @@ -883,25 +882,21 @@ impl ExploitIntelligenceService { product_id: &str, db: &db::ReadWrite, ) -> Result<(), Error> { - let now = OffsetDateTime::now_utc(); + let model = exploit_intelligence_job::ActiveModel { + id: Set(job_id), + status: Set(ExploitIntelligenceJobStatus::Running), + product_id: Set(Some(product_id.to_string())), + updated: Set(OffsetDateTime::now_utc()), + ..Default::default() + }; - exploit_intelligence_job::Entity::update_many() - .col_expr( - exploit_intelligence_job::Column::Status, - Expr::value(ExploitIntelligenceJobStatus::Running), - ) - .col_expr( - exploit_intelligence_job::Column::ProductId, - Expr::value(Some(product_id.to_string())), - ) - .col_expr(exploit_intelligence_job::Column::Updated, Expr::value(now)) - .filter(exploit_intelligence_job::Column::Id.eq(job_id)) + exploit_intelligence_job::Entity::update(model) .filter( exploit_intelligence_job::Column::Status.eq(ExploitIntelligenceJobStatus::Running), ) .exec(&**db) .await - .map_err(Error::from)?; + .ok_if_not_updated()?; Ok(()) } From e51e839c54597c1a2e57eb45011bbad20a963181 Mon Sep 17 00:00:00 2001 From: Noah Santschi-Cooney Date: Fri, 17 Jul 2026 16:13:24 +0100 Subject: [PATCH 25/25] feat(exploit-intelligence): add excluded component tracking, improve error handling and job completion logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `excluded` boolean column to job components to distinguish EI-excluded components from genuinely failed ones - Add `excluded_components` count to job summary and detail API models - Mark jobs as completed (not failed) when all components are excluded (matches EI service's own semantics) - EI-side analysis failures (failed/expired poll status) now mark the component as failed while the job completes — the orchestration succeeded, only the analysis outcome is negative - Include response body in error messages (truncated, HTML filtered) with operation label for debuggability - Fix ReportError deserialization (EI returns error as object, not string) - Fix aggregate finding: genuinely failed components produce null finding instead of misleading "uncertain" - Fix test fixtures to use real SBOM records (FK constraint satisfaction) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/exploit_intelligence_job_component.rs | 4 + ...0002250_create_exploit_intelligence_job.rs | 7 ++ .../src/exploit_intelligence/ei_client.rs | 9 +- .../exploit_intelligence/endpoints/test.rs | 63 ++++++------- .../src/exploit_intelligence/model/mod.rs | 76 +++++++++++----- .../exploit_intelligence/runner/analysis.rs | 4 +- .../src/exploit_intelligence/runner/mod.rs | 29 ++++-- .../exploit_intelligence/runner/polling.rs | 78 ++++++++++++---- .../src/exploit_intelligence/service/mod.rs | 90 +++++++++---------- 9 files changed, 236 insertions(+), 124 deletions(-) diff --git a/entity/src/exploit_intelligence_job_component.rs b/entity/src/exploit_intelligence_job_component.rs index 0dfcaf9e9..ccc3406f3 100644 --- a/entity/src/exploit_intelligence_job_component.rs +++ b/entity/src/exploit_intelligence_job_component.rs @@ -35,6 +35,10 @@ pub struct Model { /// FK to advisory — set when a VEX document is ingested from the result. pub advisory_id: Option, + /// Whether this component was excluded by the EI service (e.g., + /// unsupported ecosystem) rather than genuinely failing. + pub excluded: bool, + /// Error details for failed analyses. pub error_message: Option, diff --git a/migration/src/m0002250_create_exploit_intelligence_job.rs b/migration/src/m0002250_create_exploit_intelligence_job.rs index 6d2f0606d..2660834d4 100644 --- a/migration/src/m0002250_create_exploit_intelligence_job.rs +++ b/migration/src/m0002250_create_exploit_intelligence_job.rs @@ -161,6 +161,12 @@ impl MigrationTrait for Migration { .custom(EiFinding::Table), ) .col(ColumnDef::new(ExploitIntelligenceJobComponent::AdvisoryId).uuid()) + .col( + ColumnDef::new(ExploitIntelligenceJobComponent::Excluded) + .boolean() + .not_null() + .default(false), + ) .col(ColumnDef::new(ExploitIntelligenceJobComponent::ErrorMessage).string()) .col( ColumnDef::new(ExploitIntelligenceJobComponent::Created) @@ -331,6 +337,7 @@ enum ExploitIntelligenceJobComponent { Status, Finding, AdvisoryId, + Excluded, ErrorMessage, Created, Updated, diff --git a/modules/fundamental/src/exploit_intelligence/ei_client.rs b/modules/fundamental/src/exploit_intelligence/ei_client.rs index 8c89a5868..1bd20afc7 100644 --- a/modules/fundamental/src/exploit_intelligence/ei_client.rs +++ b/modules/fundamental/src/exploit_intelligence/ei_client.rs @@ -23,7 +23,14 @@ pub struct PollResponse { #[derive(Debug, Deserialize)] pub struct Report { pub output: Option, - pub error: Option, + pub error: Option, +} + +#[derive(Debug, Deserialize)] +pub struct ReportError { + #[serde(rename = "type")] + pub error_type: Option, + pub message: Option, } #[derive(Debug, Deserialize)] diff --git a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs index 2da1beddf..6dc1b8ed9 100644 --- a/modules/fundamental/src/exploit_intelligence/endpoints/test.rs +++ b/modules/fundamental/src/exploit_intelligence/endpoints/test.rs @@ -19,6 +19,16 @@ use trustify_entity::exploit_intelligence_job::{self, ExploitIntelligenceJobStat use trustify_test_context::{TrustifyContext, app::TestApp}; use uuid::Uuid; +/// Ingest an SBOM document and return its UUID. +async fn ingest_and_parse_sbom_id(ctx: &TrustifyContext, path: &str) -> Uuid { + let result = ctx.ingest_document(path).await.expect("ingest test SBOM"); + let id = result + .id + .strip_prefix("urn:uuid:") + .unwrap_or(&result.id); + Uuid::parse_str(id).expect("parse sbom UUID") +} + /// Helper to create a test service with a fake EI backend URL. fn test_service() -> ExploitIntelligenceService { ExploitIntelligenceService::new(Some(ExploitIntelligenceConfig { @@ -83,6 +93,7 @@ async fn insert_test_component( status: Set(status), finding: Set(finding), advisory_id: Set(None), + excluded: Set(false), error_message: Set(None), created: Set(now), updated: Set(now), @@ -100,11 +111,12 @@ async fn insert_test_component( #[test(actix_web::test)] async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a completed job with a component record in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-1234", ExploitIntelligenceJobStatus::Completed, None, @@ -155,11 +167,12 @@ async fn get_job_completed(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn get_job_pending(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a pending job record in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-5678", ExploitIntelligenceJobStatus::Pending, None, @@ -201,11 +214,12 @@ async fn get_job_pending(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn get_job_failed(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a failed job record in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-9999", ExploitIntelligenceJobStatus::Failed, Some("analysis timed out"), @@ -275,11 +289,12 @@ async fn get_job_not_found(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn list_jobs_paginated(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given three job records in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; for i in 0..3 { insert_test_job( &ctx.db, Uuid::now_v7(), - Uuid::nil(), + sbom_id, &format!("CVE-2024-{i:04}"), ExploitIntelligenceJobStatus::Pending, None, @@ -320,8 +335,8 @@ async fn list_jobs_paginated(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given jobs for two different SBOMs - let target_sbom_id = Uuid::now_v7(); - let other_sbom_id = Uuid::now_v7(); + let target_sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; + let other_sbom_id = ingest_and_parse_sbom_id(ctx, "zookeeper-3.9.2-cyclonedx.json").await; insert_test_job( &ctx.db, @@ -380,15 +395,7 @@ async fn list_jobs_filter_by_sbom_id(ctx: &TrustifyContext) -> anyhow::Result<() #[test(actix_web::test)] async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a real SBOM in the database - let ingest_result = ctx - .ingest_document("cyclonedx/application.cdx.json") - .await?; - let sbom_id = Uuid::parse_str( - ingest_result - .id - .strip_prefix("urn:uuid:") - .expect("sbom id should have urn:uuid: prefix"), - )?; + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; // And an existing pending job for that SBOM and CVE let existing_job_id = Uuid::now_v7(); @@ -442,11 +449,12 @@ async fn analyze_deduplicates_active_jobs(ctx: &TrustifyContext) -> anyhow::Resu #[test(actix_web::test)] async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a running job record in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-TIMEOUT", ExploitIntelligenceJobStatus::Running, None, @@ -497,11 +505,12 @@ async fn get_job_timeout_failure(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a completed job with a vulnerable component + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-VULN", ExploitIntelligenceJobStatus::Completed, None, @@ -552,15 +561,7 @@ async fn get_job_finding_without_advisory(ctx: &TrustifyContext) -> anyhow::Resu #[test(actix_web::test)] async fn analyze_creates_new_job(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a real SBOM in the database - let ingest_result = ctx - .ingest_document("cyclonedx/application.cdx.json") - .await?; - let sbom_id = Uuid::parse_str( - ingest_result - .id - .strip_prefix("urn:uuid:") - .expect("sbom id should have urn:uuid: prefix"), - )?; + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let ei_service = test_service(); let db_rw = db::ReadWrite::new(ctx.db.clone()); @@ -603,11 +604,12 @@ async fn analyze_creates_new_job(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn get_job_running(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given a running job record in the database + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; let job_id = Uuid::now_v7(); insert_test_job( &ctx.db, job_id, - Uuid::nil(), + sbom_id, "CVE-2024-RUN", ExploitIntelligenceJobStatus::Running, None, @@ -650,10 +652,11 @@ async fn get_job_running(ctx: &TrustifyContext) -> anyhow::Result<()> { #[test(actix_web::test)] async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> { // Given jobs with different statuses + let sbom_id = ingest_and_parse_sbom_id(ctx, "cyclonedx/application.cdx.json").await; insert_test_job( &ctx.db, Uuid::now_v7(), - Uuid::nil(), + sbom_id, "CVE-2024-S001", ExploitIntelligenceJobStatus::Completed, None, @@ -663,7 +666,7 @@ async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> insert_test_job( &ctx.db, Uuid::now_v7(), - Uuid::nil(), + sbom_id, "CVE-2024-S002", ExploitIntelligenceJobStatus::Pending, None, @@ -673,7 +676,7 @@ async fn list_jobs_filter_by_status(ctx: &TrustifyContext) -> anyhow::Result<()> insert_test_job( &ctx.db, Uuid::now_v7(), - Uuid::nil(), + sbom_id, "CVE-2024-S003", ExploitIntelligenceJobStatus::Completed, None, diff --git a/modules/fundamental/src/exploit_intelligence/model/mod.rs b/modules/fundamental/src/exploit_intelligence/model/mod.rs index d3cd92e08..a73de2a2e 100644 --- a/modules/fundamental/src/exploit_intelligence/model/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/model/mod.rs @@ -35,6 +35,8 @@ pub struct ExploitIntelligenceJobSummary { pub status: JobStatus, /// Analysis result from Exploit Intelligence. pub finding: Option, + /// Error details when the job has failed. + pub error_message: Option, /// Link to the EI human-readable justification report. pub report_url: Option, /// EI product ID for multi-component SPDX flow. @@ -43,8 +45,10 @@ pub struct ExploitIntelligenceJobSummary { pub total_components: Option, /// Number of completed components (computed at query time, not stored). pub completed_components: Option, - /// Number of failed/excluded components (computed at query time, not stored). + /// Number of genuinely failed components (computed at query time, not stored). pub failed_components: Option, + /// Number of components excluded by the EI service (computed at query time, not stored). + pub excluded_components: Option, /// Number of completed components with Vulnerable finding (computed at query time). pub vulnerable_components: Option, /// Number of completed components with NotVulnerable finding (computed at query time). @@ -82,8 +86,10 @@ pub struct ExploitIntelligenceJobDetails { pub total_components: Option, /// Number of completed components (computed at query time, not stored). pub completed_components: Option, - /// Number of failed/excluded components (computed at query time, not stored). + /// Number of genuinely failed components (computed at query time, not stored). pub failed_components: Option, + /// Number of components excluded by the EI service (computed at query time, not stored). + pub excluded_components: Option, /// Number of completed components with Vulnerable finding (computed at query time). pub vulnerable_components: Option, /// Number of completed components with NotVulnerable finding (computed at query time). @@ -100,6 +106,19 @@ pub struct ExploitIntelligenceJobDetails { pub updated: OffsetDateTime, } +/// Aggregated component counts for a job, used to populate summary fields +/// and compute the overall finding. +#[derive(Debug, Clone, Default)] +pub struct ComponentCounts { + pub completed: i32, + pub failed: i32, + pub excluded: i32, + pub total: i32, + pub vulnerable: i32, + pub not_vulnerable: i32, + pub uncertain: i32, +} + /// Per-component analysis result within an analysis job. #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct ComponentResult { @@ -115,6 +134,9 @@ pub struct ComponentResult { pub report_url: Option, /// FK to advisory — set when a VEX document is ingested from the result. pub advisory_id: Option, + /// Whether this component was excluded by the EI service (e.g., + /// unsupported ecosystem) rather than genuinely failing. + pub excluded: bool, /// Error details for failed component analyses. pub error_message: Option, /// Timestamp when the component record was created. @@ -210,6 +232,7 @@ impl ComponentResult { finding: model.finding.map(JobFinding::from), report_url, advisory_id: model.advisory_id, + excluded: model.excluded, error_message: model.error_message.clone(), created: model.created, updated: model.updated, @@ -267,11 +290,13 @@ impl ExploitIntelligenceJobSummary { vulnerability_id: model.vulnerability_id.clone(), status: model.status.into(), finding: None, + error_message: model.error_message.clone(), report_url: compute_job_report_url(model, ui_url, None), product_id: model.product_id.clone(), total_components: model.total_components, completed_components: None, failed_components: None, + excluded_components: None, vulnerable_components: None, not_vulnerable_components: None, uncertain_components: None, @@ -289,29 +314,31 @@ impl ExploitIntelligenceJobSummary { pub fn from_entity_with_counts( model: &trustify_entity::exploit_intelligence_job::Model, ui_url: Option<&str>, - completed_components: Option, - failed_components: Option, - vulnerable_components: Option, - not_vulnerable_components: Option, - uncertain_components: Option, + counts: &ComponentCounts, ) -> Self { let mut summary = Self::from_entity(model, ui_url); - summary.completed_components = completed_components; - summary.failed_components = failed_components; - summary.vulnerable_components = vulnerable_components; - summary.not_vulnerable_components = not_vulnerable_components; - summary.uncertain_components = uncertain_components; + summary.completed_components = Some(counts.completed); + summary.failed_components = Some(counts.failed); + summary.excluded_components = Some(counts.excluded); + summary.vulnerable_components = Some(counts.vulnerable); + summary.not_vulnerable_components = Some(counts.not_vulnerable); + summary.uncertain_components = Some(counts.uncertain); // Compute aggregate finding from component counts. - let vuln = vulnerable_components.unwrap_or(0); - let not_vuln = not_vulnerable_components.unwrap_or(0); - let completed = completed_components.unwrap_or(0); - - if vuln > 0 { + // If any component is vulnerable, the whole job is vulnerable. + // If some components genuinely failed (not excluded), we can't be + // sure what they would have found — the aggregate is uncertain. + // Only if all analysed components are not_vulnerable is the job + // confidently not_vulnerable. + if counts.vulnerable > 0 { summary.finding = Some(JobFinding::Vulnerable); - } else if completed > 0 && not_vuln == completed { + } else if counts.failed > 0 { + // Some components genuinely failed — we can't confidently + // aggregate, so leave finding as None. The UI can use + // failed_components > 0 to explain why. + } else if counts.completed > 0 && counts.not_vulnerable == counts.completed { summary.finding = Some(JobFinding::NotVulnerable); - } else if completed > 0 { + } else if counts.completed > 0 { summary.finding = Some(JobFinding::Uncertain); } // else: no completed components yet, finding stays None @@ -341,6 +368,7 @@ impl ExploitIntelligenceJobDetails { total_components: model.total_components, completed_components: None, failed_components: None, + excluded_components: None, vulnerable_components: None, not_vulnerable_components: None, uncertain_components: None, @@ -370,12 +398,13 @@ impl ExploitIntelligenceJobDetails { .collect(); let completed = component_models .iter() - .filter(|c| c.status == ExploitIntelligenceJobStatus::Completed) + .filter(|c| c.status == ExploitIntelligenceJobStatus::Completed && !c.excluded) .count() as i32; let failed = component_models .iter() - .filter(|c| c.status == ExploitIntelligenceJobStatus::Failed) + .filter(|c| c.status == ExploitIntelligenceJobStatus::Failed && !c.excluded) .count() as i32; + let excluded = component_models.iter().filter(|c| c.excluded).count() as i32; let vulnerable = component_models .iter() .filter(|c| c.finding == Some(Finding::Vulnerable)) @@ -390,13 +419,16 @@ impl ExploitIntelligenceJobDetails { .count() as i32; details.completed_components = Some(completed); details.failed_components = Some(failed); + details.excluded_components = Some(excluded); details.vulnerable_components = Some(vulnerable); details.not_vulnerable_components = Some(not_vulnerable); details.uncertain_components = Some(uncertain); - // Compute aggregate finding from components. + // Compute aggregate finding from components (same logic as summary). if vulnerable > 0 { details.finding = Some(JobFinding::Vulnerable); + } else if failed > 0 { + // Some components genuinely failed — leave finding as None. } else if completed > 0 && not_vulnerable == completed { details.finding = Some(JobFinding::NotVulnerable); } else if completed > 0 { diff --git a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs index b7d3f4d45..6261d6c07 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/analysis.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/analysis.rs @@ -114,7 +114,7 @@ pub(crate) async fn run_analysis( )) })?; - classify_response_error(request.send().await) + classify_response_error("SPDX upload", request.send().await).await } }) .retry( @@ -195,7 +195,7 @@ pub(crate) async fn run_analysis( )) })?; - classify_response_error(request.send().await) + classify_response_error("CycloneDX upload", request.send().await).await } }) .retry( diff --git a/modules/fundamental/src/exploit_intelligence/runner/mod.rs b/modules/fundamental/src/exploit_intelligence/runner/mod.rs index 6050b1cbe..7997c3f5c 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/mod.rs @@ -27,7 +27,8 @@ impl EiRequestError { } /// Classify an HTTP response or reqwest error into an EiRequestError. -pub(crate) fn classify_response_error( +pub(crate) async fn classify_response_error( + operation: &str, result: Result, ) -> Result { match result { @@ -36,20 +37,34 @@ pub(crate) fn classify_response_error( if status.is_success() { Ok(resp) } else { + let body = resp.text().await.unwrap_or_default(); + let body = body.trim(); + let detail = if body.is_empty() || body.starts_with('<') { + format!("{operation}: HTTP {status}") + } else if body.len() <= 512 { + format!("{operation}: HTTP {status}: {body}") + } else { + let end = body + .char_indices() + .map(|(i, _)| i) + .find(|&i| i >= 512) + .unwrap_or(body.len()); + format!("{operation}: HTTP {status}: {}", &body[..end]) + }; match status.as_u16() { - 408 | 429 | 502 | 503 | 504 => Err(EiRequestError::Transient(format!( - "transient HTTP status {status}" - ))), - _ => Err(EiRequestError::Permanent(anyhow::anyhow!("HTTP {status}"))), + 408 | 429 | 502 | 503 | 504 => Err(EiRequestError::Transient(detail)), + _ => Err(EiRequestError::Permanent(anyhow::anyhow!("{detail}"))), } } } Err(err) => { if err.is_connect() || err.is_timeout() || err.is_request() { - Err(EiRequestError::Transient(format!("network error: {err}"))) + Err(EiRequestError::Transient(format!( + "{operation}: network error: {err}" + ))) } else { Err(EiRequestError::Permanent(anyhow::anyhow!( - "network error: {err}" + "{operation}: network error: {err}" ))) } } diff --git a/modules/fundamental/src/exploit_intelligence/runner/polling.rs b/modules/fundamental/src/exploit_intelligence/runner/polling.rs index 614a5858c..f51ee541c 100644 --- a/modules/fundamental/src/exploit_intelligence/runner/polling.rs +++ b/modules/fundamental/src/exploit_intelligence/runner/polling.rs @@ -9,6 +9,7 @@ use std::str::FromStr; use std::time::Duration; use trustify_auth::client::TokenInjector; use trustify_common::db; +use trustify_entity::exploit_intelligence_job::Finding; use trustify_entity::labels::Labels; use trustify_module_ingestor::service::{Cache, Format, IngestorService}; use uuid::Uuid; @@ -58,7 +59,8 @@ pub(crate) async fn poll_for_result( EiRequestError::Permanent(anyhow::anyhow!("failed to obtain auth token: {e}")) })?; - let resp = classify_response_error(poll_request.send().await)?; + let resp = + classify_response_error("poll scan result", poll_request.send().await).await?; let poll: crate::exploit_intelligence::ei_client::PollResponse = resp.json().await.map_err(|e| { @@ -131,7 +133,6 @@ pub(crate) async fn poll_for_result( match poll.status.as_str() { "completed" => { // Extract finding from the analysis result. - use trustify_entity::exploit_intelligence_job::Finding; let finding = poll .report .as_ref() @@ -208,15 +209,24 @@ pub(crate) async fn poll_for_result( Ok(()) } "failed" | "expired" => { + // EI completed its lifecycle but the analysis itself failed. + // Mark the component as failed (not the job — the job's + // orchestration succeeded, only the analysis outcome is negative). let error_msg = poll .report .as_ref() - .and_then(|r| r.error.as_deref()) + .and_then(|r| r.error.as_ref()) + .and_then(|e| e.message.as_deref()) .unwrap_or("analysis failed") .to_string(); ei_service - .update_job_failed(job_id, &error_msg, db_rw) + .update_component_failed(component_id, &error_msg, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; + + ei_service + .update_job_completed(job_id, db_rw) .await .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; @@ -291,7 +301,9 @@ pub(crate) async fn poll_for_product_result( EiRequestError::Permanent(anyhow::anyhow!("failed to obtain auth token: {e}")) })?; - let resp = classify_response_error(poll_request.send().await)?; + let resp = + classify_response_error("poll product status", poll_request.send().await) + .await?; let product: crate::exploit_intelligence::ei_client::ProductStatusResponse = resp.json().await.map_err(|e| { @@ -421,8 +433,6 @@ pub(crate) async fn poll_for_product_result( "processing per-component reports for SPDX product" ); - use trustify_entity::exploit_intelligence_job::Finding; - for entry in &component_reports { let scan_id = &entry.scan_id; let image_name = &entry.image_name; @@ -564,18 +574,52 @@ pub(crate) async fn poll_for_product_result( .map_err(|e| anyhow::anyhow!("failed to update component: {e}"))?; } - // Mark the parent job as completed. The aggregate finding is - // computed from component records at query time. - ei_service - .update_job_completed(job_id, db_rw) + // Check whether any component actually has a finding. + // If every component was excluded or failed, mark the job + // as failed rather than misleadingly "completed". + let components = ei_service + .fetch_components(job_id, &**db_rw) .await - .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + .map_err(|e| anyhow::anyhow!("failed to fetch components: {e}"))?; - tracing::info!( - job_id = %job_id, - product_id = %product_id, - "SPDX product analysis completed" - ); + let has_finding = components.iter().any(|c| { + c.status == trustify_entity::exploit_intelligence_job::ExploitIntelligenceJobStatus::Completed + && c.finding.is_some() + }); + let all_excluded = !components.is_empty() && components.iter().all(|c| c.excluded); + + if has_finding || all_excluded { + ei_service + .update_job_completed(job_id, db_rw) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::info!( + job_id = %job_id, + product_id = %product_id, + all_excluded, + "SPDX product analysis completed" + ); + } else { + let failed_not_excluded = components.iter().filter(|c| !c.excluded).count(); + ei_service + .update_job_failed( + job_id, + &format!( + "no components produced findings ({} total, {failed_not_excluded} failed)", + components.len() + ), + db_rw, + ) + .await + .map_err(|e| anyhow::anyhow!("failed to update job: {e}"))?; + + tracing::warn!( + job_id = %job_id, + product_id = %product_id, + "SPDX product analysis failed — no components have findings" + ); + } Ok(()) } diff --git a/modules/fundamental/src/exploit_intelligence/service/mod.rs b/modules/fundamental/src/exploit_intelligence/service/mod.rs index aa98ac87e..9ffc6d02d 100644 --- a/modules/fundamental/src/exploit_intelligence/service/mod.rs +++ b/modules/fundamental/src/exploit_intelligence/service/mod.rs @@ -1,7 +1,7 @@ use crate::{ Error, exploit_intelligence::model::{ - ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, + ComponentCounts, ExploitIntelligenceJobDetails, ExploitIntelligenceJobSummary, JobStatus, }, }; use sea_orm::sea_query::SimpleExpr; @@ -411,43 +411,52 @@ impl ExploitIntelligenceService { .await .map_err(Error::from)?; - // Compute component counts per job via aggregate query, grouped by status and finding. - // Keyed by job_id -> (completed, failed, total, vulnerable, not_vulnerable, uncertain). + // Compute component counts per job via aggregate query, grouped by + // status, finding, and excluded flag. let job_ids: Vec = results.iter().map(|j| j.id).collect(); - let mut counts_map: HashMap = HashMap::new(); + let mut counts_map: HashMap = HashMap::new(); if !job_ids.is_empty() { - let rows: Vec<(Uuid, ExploitIntelligenceJobStatus, Option, i64)> = - exploit_intelligence_job_component::Entity::find() - .select_only() - .column(exploit_intelligence_job_component::Column::JobId) - .column(exploit_intelligence_job_component::Column::Status) - .column(exploit_intelligence_job_component::Column::Finding) - .expr(Expr::col(exploit_intelligence_job_component::Column::Id).count()) - .filter(exploit_intelligence_job_component::Column::JobId.is_in(job_ids)) - .group_by(exploit_intelligence_job_component::Column::JobId) - .group_by(exploit_intelligence_job_component::Column::Status) - .group_by(exploit_intelligence_job_component::Column::Finding) - .into_tuple() - .all(connection) - .instrument(info_span!("count exploit intelligence job components")) - .await - .map_err(Error::from)?; + let rows: Vec<( + Uuid, + ExploitIntelligenceJobStatus, + Option, + bool, + i64, + )> = exploit_intelligence_job_component::Entity::find() + .select_only() + .column(exploit_intelligence_job_component::Column::JobId) + .column(exploit_intelligence_job_component::Column::Status) + .column(exploit_intelligence_job_component::Column::Finding) + .column(exploit_intelligence_job_component::Column::Excluded) + .expr(Expr::col(exploit_intelligence_job_component::Column::Id).count()) + .filter(exploit_intelligence_job_component::Column::JobId.is_in(job_ids)) + .group_by(exploit_intelligence_job_component::Column::JobId) + .group_by(exploit_intelligence_job_component::Column::Status) + .group_by(exploit_intelligence_job_component::Column::Finding) + .group_by(exploit_intelligence_job_component::Column::Excluded) + .into_tuple() + .all(connection) + .instrument(info_span!("count exploit intelligence job components")) + .await + .map_err(Error::from)?; - for (job_id, status, finding, count) in rows { - let entry = counts_map.entry(job_id).or_insert((0, 0, 0, 0, 0, 0)); + for (job_id, status, finding, excluded, count) in rows { + let entry = counts_map.entry(job_id).or_default(); let count = count as i32; - entry.2 += count; // total - if status == ExploitIntelligenceJobStatus::Completed { - entry.0 += count; // completed + entry.total += count; + if excluded { + entry.excluded += count; + } else if status == ExploitIntelligenceJobStatus::Completed { + entry.completed += count; match finding { - Some(Finding::Vulnerable) => entry.3 += count, - Some(Finding::NotVulnerable) => entry.4 += count, - Some(Finding::Uncertain) => entry.5 += count, + Some(Finding::Vulnerable) => entry.vulnerable += count, + Some(Finding::NotVulnerable) => entry.not_vulnerable += count, + Some(Finding::Uncertain) => entry.uncertain += count, _ => {} } } else if status == ExploitIntelligenceJobStatus::Failed { - entry.1 += count; // failed + entry.failed += count; } } } @@ -485,26 +494,14 @@ impl ExploitIntelligenceService { let items = results .iter() .map(|model| { - let mut summary = if let Some(&( - completed, - failed, - total, - vulnerable, - not_vulnerable, - uncertain, - )) = counts_map.get(&model.id) - { + let mut summary = if let Some(c) = counts_map.get(&model.id) { let mut s = ExploitIntelligenceJobSummary::from_entity_with_counts( model, self.ui_url(), - Some(completed), - Some(failed), - Some(vulnerable), - Some(not_vulnerable), - Some(uncertain), + c, ); if s.total_components.is_none() { - s.total_components = Some(total); + s.total_components = Some(c.total); } s } else { @@ -722,6 +719,7 @@ impl ExploitIntelligenceService { status: Set(ExploitIntelligenceJobStatus::Pending), finding: Set(None), advisory_id: Set(None), + excluded: Set(false), error_message: Set(None), created: Set(now), updated: Set(now), @@ -734,9 +732,10 @@ impl ExploitIntelligenceService { Ok(component_id) } - /// Create a component record marked as excluded (Failed with an error message). + /// Create a component record marked as excluded by the EI service. /// /// Used for components that cannot be analysed (e.g., unsupported ecosystem). + /// These are distinct from genuinely failed components. #[instrument(skip_all, err(level = tracing::Level::INFO))] pub async fn create_excluded_component( &self, @@ -758,6 +757,7 @@ impl ExploitIntelligenceService { status: Set(ExploitIntelligenceJobStatus::Failed), finding: Set(None), advisory_id: Set(None), + excluded: Set(true), error_message: Set(Some(error_message.to_string())), created: Set(now), updated: Set(now),