From ce9d8a5ca939685685a4b28352ec8f489102f60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 22:08:54 +0200 Subject: [PATCH 01/21] Add Thinking budget to LLM agent config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces an optional extended-thinking token budget on LLMAgentConfig so agents that benefit from reasoning over multi-part files can opt in via JSON config without a code change. ResolveAgent propagates the default like the other model parameters. Only providers and models supported by pkg/agent.WithThinking will honour the value. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- ...scription_status.go => evidence_assessment_status.go} | 0 ...scription_worker.go => evidence_assessment_worker.go} | 0 pkg/probodconfig/llm_config.go | 9 +++++++++ 3 files changed, 9 insertions(+) rename pkg/coredata/{evidence_description_status.go => evidence_assessment_status.go} (100%) rename pkg/probo/{evidence_description_worker.go => evidence_assessment_worker.go} (100%) diff --git a/pkg/coredata/evidence_description_status.go b/pkg/coredata/evidence_assessment_status.go similarity index 100% rename from pkg/coredata/evidence_description_status.go rename to pkg/coredata/evidence_assessment_status.go diff --git a/pkg/probo/evidence_description_worker.go b/pkg/probo/evidence_assessment_worker.go similarity index 100% rename from pkg/probo/evidence_description_worker.go rename to pkg/probo/evidence_assessment_worker.go diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index 2dda1df55b..9677b1a88d 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -29,6 +29,11 @@ type ( ModelName string `json:"model-name"` Temperature *float64 `json:"temperature"` MaxTokens *int `json:"max-tokens"` + // Thinking is the extended-thinking budget in tokens for agents + // that opt in. Leave nil to disable extended thinking; set to 0 + // to explicitly disable via config. Only a few providers and + // models support this; see pkg/agent/WithThinking. + Thinking *int `json:"thinking"` } // EvidenceDescriberConfig holds worker-side tuning for the evidence @@ -109,5 +114,9 @@ func (c *AgentsConfig) ResolveAgent(agent LLMAgentConfig) LLMAgentConfig { agent.MaxTokens = new(*c.Default.MaxTokens) } + if agent.Thinking == nil { + agent.Thinking = c.Default.Thinking + } + return agent } From 26eb1512f69b3f972ef7caab5aec8813d6c46683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 22:09:27 +0200 Subject: [PATCH 02/21] Describe evidence with structured assessment output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evidence describer now produces a typed EvidenceAssessment (system, setting, scope, captured_at, frameworks, issues, confidence, readable, rejection_reason, summary) instead of a free-text description. The worker persists the full record as JSONB in evidences.assessment and derives Evidence.Description from Assessment.Summary, preserving the existing GraphQL contract. The column description_status is renamed to assessment_status (and the backing enum type alongside it) because the worker now tracks the status of the whole assessment, not just a description. The existing four-state machine (PENDING -> PROCESSING -> COMPLETED | FAILED) and stale-recovery behaviour are unchanged. Agent construction follows the pattern introduced by pkg/vetting: a memoised output schema decorated with an enum on the confidence field, and an optional extended-thinking budget threaded through LLMAgentConfig. Frontend surface is deliberately unchanged; exposing the new fields in GraphQL or the CLI is left as a follow-up. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence.go | 177 +++++++++++-------- pkg/coredata/evidence_assessment.go | 75 ++++++++ pkg/coredata/evidence_assessment_status.go | 48 ++--- pkg/coredata/evidence_assessment_test.go | 67 +++++++ pkg/coredata/migrations/20260424T637521Z.sql | 24 +++ pkg/evidencedescriber/assessment.go | 147 +++++++++++++++ pkg/evidencedescriber/assessment_test.go | 58 ++++++ pkg/evidencedescriber/evidencedescriber.go | 82 --------- pkg/evidencedescriber/prompt.txt | 40 +++-- pkg/probo/evidence_assessment_worker.go | 55 +++--- pkg/probo/evidence_service.go | 16 +- pkg/probo/measure_service.go | 18 +- pkg/probod/probod.go | 36 ++-- 13 files changed, 592 insertions(+), 251 deletions(-) create mode 100644 pkg/coredata/evidence_assessment.go create mode 100644 pkg/coredata/evidence_assessment_test.go create mode 100644 pkg/coredata/migrations/20260424T637521Z.sql create mode 100644 pkg/evidencedescriber/assessment.go create mode 100644 pkg/evidencedescriber/assessment_test.go delete mode 100644 pkg/evidencedescriber/evidencedescriber.go diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index 47a44ac542..4a7000844e 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -31,20 +31,21 @@ import ( type ( Evidence struct { - ID gid.GID `db:"id"` - OrganizationID gid.GID `db:"organization_id"` - MeasureID gid.GID `db:"measure_id"` - TaskID *gid.GID `db:"task_id"` - State EvidenceState `db:"state"` - ReferenceID string `db:"reference_id"` - Type EvidenceType `db:"type"` - URL string `db:"url"` - EvidenceFileId *gid.GID `db:"evidence_file_id"` - Description *string `db:"description"` - DescriptionStatus EvidenceDescriptionStatus `db:"description_status"` - DescriptionProcessingStartedAt *time.Time `db:"description_processing_started_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID gid.GID `db:"id"` + OrganizationID gid.GID `db:"organization_id"` + MeasureID gid.GID `db:"measure_id"` + TaskID *gid.GID `db:"task_id"` + State EvidenceState `db:"state"` + ReferenceID string `db:"reference_id"` + Type EvidenceType `db:"type"` + URL string `db:"url"` + EvidenceFileId *gid.GID `db:"evidence_file_id"` + Description *string `db:"description"` + Assessment jsonRawMessageOrNull `db:"assessment"` + AssessmentStatus EvidenceAssessmentStatus `db:"assessment_status"` + AssessmentProcessingStartedAt *time.Time `db:"assessment_processing_started_at"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } Evidences []*Evidence @@ -99,6 +100,16 @@ func (e *Evidence) AuthorizationAttributes( return attrsByID, nil } +// assessmentArg returns the value to bind for the `assessment` JSONB +// column. pgx rejects empty byte slices as invalid JSON, so an empty +// Assessment is sent as NULL. +func (e Evidence) assessmentArg() any { + if len(e.Assessment) == 0 { + return nil + } + return []byte(e.Assessment) +} + func (e Evidence) Upsert( ctx context.Context, conn pg.Querier, @@ -117,8 +128,9 @@ INSERT INTO url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at ) @@ -133,8 +145,9 @@ VALUES ( @url, @evidence_file_id, @description, - @description_status, - @description_processing_started_at, + @assessment, + @assessment_status, + @assessment_processing_started_at, @created_at, @updated_at ) @@ -146,20 +159,21 @@ WHERE evidences.state = 'REQUESTED'; ` args := pgx.StrictNamedArgs{ - "tenant_id": scope.GetTenantID(), - "evidence_id": e.ID, - "measure_id": e.MeasureID, - "task_id": e.TaskID, - "reference_id": e.ReferenceID, - "evidence_file_id": e.EvidenceFileId, - "created_at": e.CreatedAt, - "updated_at": e.UpdatedAt, - "state": e.State, - "type": e.Type, - "url": e.URL, - "description": e.Description, - "description_status": e.DescriptionStatus, - "description_processing_started_at": e.DescriptionProcessingStartedAt, + "tenant_id": scope.GetTenantID(), + "evidence_id": e.ID, + "measure_id": e.MeasureID, + "task_id": e.TaskID, + "reference_id": e.ReferenceID, + "evidence_file_id": e.EvidenceFileId, + "created_at": e.CreatedAt, + "updated_at": e.UpdatedAt, + "state": e.State, + "type": e.Type, + "url": e.URL, + "description": e.Description, + "assessment": e.assessmentArg(), + "assessment_status": e.AssessmentStatus, + "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } _, err := conn.Exec(ctx, q, args) @@ -185,8 +199,9 @@ INSERT INTO url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at ) @@ -202,29 +217,31 @@ VALUES ( @url, @evidence_file_id, @description, - @description_status, - @description_processing_started_at, + @assessment, + @assessment_status, + @assessment_processing_started_at, @created_at, @updated_at ) ` args := pgx.StrictNamedArgs{ - "tenant_id": scope.GetTenantID(), - "evidence_id": e.ID, - "organization_id": e.OrganizationID, - "measure_id": e.MeasureID, - "task_id": e.TaskID, - "reference_id": e.ReferenceID, - "evidence_file_id": e.EvidenceFileId, - "created_at": e.CreatedAt, - "updated_at": e.UpdatedAt, - "state": e.State, - "type": e.Type, - "url": e.URL, - "description": e.Description, - "description_status": e.DescriptionStatus, - "description_processing_started_at": e.DescriptionProcessingStartedAt, + "tenant_id": scope.GetTenantID(), + "evidence_id": e.ID, + "organization_id": e.OrganizationID, + "measure_id": e.MeasureID, + "task_id": e.TaskID, + "reference_id": e.ReferenceID, + "evidence_file_id": e.EvidenceFileId, + "created_at": e.CreatedAt, + "updated_at": e.UpdatedAt, + "state": e.State, + "type": e.Type, + "url": e.URL, + "description": e.Description, + "assessment": e.assessmentArg(), + "assessment_status": e.AssessmentStatus, + "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } _, err := conn.Exec(ctx, q, args) @@ -259,8 +276,9 @@ SELECT url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at FROM @@ -343,8 +361,9 @@ SELECT url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at FROM @@ -428,8 +447,9 @@ SELECT url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at FROM @@ -475,8 +495,9 @@ SET evidence_file_id = @evidence_file_id, url = @url, description = @description, - description_status = @description_status, - description_processing_started_at = @description_processing_started_at, + assessment = @assessment, + assessment_status = @assessment_status, + assessment_processing_started_at = @assessment_processing_started_at, updated_at = @updated_at WHERE %s @@ -486,15 +507,16 @@ WHERE q = fmt.Sprintf(q, scope.SQLFragment()) args := pgx.StrictNamedArgs{ - "evidence_id": e.ID, - "type": e.Type, - "state": e.State, - "evidence_file_id": e.EvidenceFileId, - "url": e.URL, - "description": e.Description, - "description_status": e.DescriptionStatus, - "description_processing_started_at": e.DescriptionProcessingStartedAt, - "updated_at": e.UpdatedAt, + "evidence_id": e.ID, + "type": e.Type, + "state": e.State, + "evidence_file_id": e.EvidenceFileId, + "url": e.URL, + "description": e.Description, + "assessment": e.assessmentArg(), + "assessment_status": e.AssessmentStatus, + "assessment_processing_started_at": e.AssessmentProcessingStartedAt, + "updated_at": e.UpdatedAt, } maps.Copy(args, scope.SQLArguments()) @@ -529,7 +551,7 @@ WHERE return nil } -func (e *Evidence) LoadNextPendingDescriptionForUpdateSkipLocked( +func (e *Evidence) LoadNextPendingAssessmentForUpdateSkipLocked( ctx context.Context, conn pg.Tx, ) error { @@ -545,14 +567,15 @@ SELECT url, evidence_file_id, description, - description_status, - description_processing_started_at, + assessment, + assessment_status, + assessment_processing_started_at, created_at, updated_at FROM evidences WHERE - description_status = 'PENDING' + assessment_status = 'PENDING' AND evidence_file_id IS NOT NULL ORDER BY created_at ASC @@ -579,7 +602,7 @@ FOR UPDATE SKIP LOCKED; return nil } -func ResetStaleDescriptionProcessing( +func ResetStaleAssessmentProcessing( ctx context.Context, conn pg.Querier, staleAfter time.Duration, @@ -587,11 +610,11 @@ func ResetStaleDescriptionProcessing( q := ` UPDATE evidences SET - description_status = 'PENDING', - description_processing_started_at = NULL + assessment_status = 'PENDING', + assessment_processing_started_at = NULL WHERE - description_status = 'PROCESSING' - AND description_processing_started_at < $1; + assessment_status = 'PROCESSING' + AND assessment_processing_started_at < $1; ` _, err := conn.Exec(ctx, q, time.Now().Add(-staleAfter)) diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go new file mode 100644 index 0000000000..69b8aea330 --- /dev/null +++ b/pkg/coredata/evidence_assessment.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "encoding/json" + "fmt" +) + +// EvidenceAssessment is the structured output produced by the evidence +// describer agent. It is persisted as JSONB in evidences.assessment and +// is the canonical form; Evidence.Description is a one-sentence summary +// derived from Assessment.Summary. +type EvidenceAssessment struct { + Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows; no markdown, no preamble. When readable is false this field must restate the rejection reason so downstream systems displaying only the summary still inform the user."` + System string `json:"system" jsonschema:"Tool or platform shown (e.g. 'Google Workspace', 'GitHub', 'AWS IAM'); empty string if not identifiable."` + Setting string `json:"setting" jsonschema:"Specific configuration, feature, or state demonstrated; empty string if not identifiable."` + Scope string `json:"scope" jsonschema:"Who or what the evidence applies to (e.g. 'organization-wide', 'all users', 'repository acme/foo'); empty string if not stated."` + CapturedAt string `json:"captured_at" jsonschema:"ISO-8601 date or datetime visible on the file; empty string if no date is shown."` + Language string `json:"language" jsonschema:"BCP-47 language tag of the visible text (e.g. 'en', 'fr'); empty string if unclear."` + Frameworks []string `json:"frameworks" jsonschema:"Compliance frameworks the evidence is plausibly relevant to (e.g. 'SOC2', 'ISO27001'); empty when unclear."` + Issues []string `json:"issues" jsonschema:"Quality problems observed: redacted fields, crop, stale date, low resolution, etc."` + Confidence string `json:"confidence" jsonschema:"Overall confidence in the System / Setting / Scope identification; one of HIGH, MEDIUM, LOW."` + Readable bool `json:"readable" jsonschema:"True if the file is readable compliance evidence; false if unreadable or off-topic."` + RejectionReason string `json:"rejection_reason" jsonschema:"Set only when readable is false; explains why the file is not usable evidence."` +} + +// EvidenceAssessmentConfidenceEnum is the canonical set of allowed +// EvidenceAssessment.Confidence values. Kept in a package-level slice +// because jsonschema struct tags are free-form descriptions and cannot +// encode enum constraints directly; consumers building an output schema +// inject it post-marshal (see pkg/evidencedescriber). +var EvidenceAssessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} + +// SetAssessment marshals a typed assessment into Evidence.Assessment. +// Passing nil clears the field. This is the only supported way to write +// the assessment column from outside the coredata package; the backing +// field type is unexported. +func (e *Evidence) SetAssessment(a *EvidenceAssessment) error { + if a == nil { + e.Assessment = nil + return nil + } + data, err := json.Marshal(a) + if err != nil { + return fmt.Errorf("cannot marshal evidence assessment: %w", err) + } + e.Assessment = data + return nil +} + +// GetAssessment unmarshals Evidence.Assessment into a typed +// EvidenceAssessment. Returns (nil, nil) when the column is NULL. +func (e *Evidence) GetAssessment() (*EvidenceAssessment, error) { + if len(e.Assessment) == 0 { + return nil, nil + } + var a EvidenceAssessment + if err := json.Unmarshal(e.Assessment, &a); err != nil { + return nil, fmt.Errorf("cannot unmarshal evidence assessment: %w", err) + } + return &a, nil +} diff --git a/pkg/coredata/evidence_assessment_status.go b/pkg/coredata/evidence_assessment_status.go index 64c3d10a47..4bc1d4082c 100644 --- a/pkg/coredata/evidence_assessment_status.go +++ b/pkg/coredata/evidence_assessment_status.go @@ -20,56 +20,56 @@ import ( ) type ( - EvidenceDescriptionStatus string + EvidenceAssessmentStatus string ) const ( - EvidenceDescriptionStatusPending EvidenceDescriptionStatus = "PENDING" - EvidenceDescriptionStatusProcessing EvidenceDescriptionStatus = "PROCESSING" - EvidenceDescriptionStatusCompleted EvidenceDescriptionStatus = "COMPLETED" - EvidenceDescriptionStatusFailed EvidenceDescriptionStatus = "FAILED" + EvidenceAssessmentStatusPending EvidenceAssessmentStatus = "PENDING" + EvidenceAssessmentStatusProcessing EvidenceAssessmentStatus = "PROCESSING" + EvidenceAssessmentStatusCompleted EvidenceAssessmentStatus = "COMPLETED" + EvidenceAssessmentStatusFailed EvidenceAssessmentStatus = "FAILED" ) var ( - _ fmt.Stringer = EvidenceDescriptionStatus("") - _ encoding.TextMarshaler = EvidenceDescriptionStatus("") - _ encoding.TextUnmarshaler = (*EvidenceDescriptionStatus)(nil) + _ fmt.Stringer = EvidenceAssessmentStatus("") + _ encoding.TextMarshaler = EvidenceAssessmentStatus("") + _ encoding.TextUnmarshaler = (*EvidenceAssessmentStatus)(nil) ) -func EvidenceDescriptionStatuses() []EvidenceDescriptionStatus { - return []EvidenceDescriptionStatus{ - EvidenceDescriptionStatusPending, - EvidenceDescriptionStatusProcessing, - EvidenceDescriptionStatusCompleted, - EvidenceDescriptionStatusFailed, +func EvidenceAssessmentStatuses() []EvidenceAssessmentStatus { + return []EvidenceAssessmentStatus{ + EvidenceAssessmentStatusPending, + EvidenceAssessmentStatusProcessing, + EvidenceAssessmentStatusCompleted, + EvidenceAssessmentStatusFailed, } } -func (v EvidenceDescriptionStatus) IsValid() bool { +func (v EvidenceAssessmentStatus) IsValid() bool { switch v { case - EvidenceDescriptionStatusPending, - EvidenceDescriptionStatusProcessing, - EvidenceDescriptionStatusCompleted, - EvidenceDescriptionStatusFailed: + EvidenceAssessmentStatusPending, + EvidenceAssessmentStatusProcessing, + EvidenceAssessmentStatusCompleted, + EvidenceAssessmentStatusFailed: return true } return false } -func (v EvidenceDescriptionStatus) String() string { +func (v EvidenceAssessmentStatus) String() string { return string(v) } -func (v EvidenceDescriptionStatus) MarshalText() ([]byte, error) { +func (v EvidenceAssessmentStatus) MarshalText() ([]byte, error) { return []byte(v.String()), nil } -func (v *EvidenceDescriptionStatus) UnmarshalText(text []byte) error { - val := EvidenceDescriptionStatus(text) +func (v *EvidenceAssessmentStatus) UnmarshalText(text []byte) error { + val := EvidenceAssessmentStatus(text) if !val.IsValid() { - return fmt.Errorf("invalid EvidenceDescriptionStatus value: %q", string(text)) + return fmt.Errorf("invalid EvidenceAssessmentStatus value: %q", string(text)) } *v = val diff --git a/pkg/coredata/evidence_assessment_test.go b/pkg/coredata/evidence_assessment_test.go new file mode 100644 index 0000000000..2a9ce1cbd1 --- /dev/null +++ b/pkg/coredata/evidence_assessment_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvidence_SetAndGetAssessment_RoundTrip(t *testing.T) { + t.Parallel() + + in := &EvidenceAssessment{ + Summary: "Google Workspace admin console showing enforced 2SV for all users.", + System: "Google Workspace", + Setting: "enforced 2-step verification", + Scope: "organization-wide", + Language: "en", + Frameworks: []string{"SOC2", "ISO27001"}, + Issues: []string{}, + Confidence: "HIGH", + Readable: true, + } + + var e Evidence + require.NoError(t, e.SetAssessment(in)) + require.NotEmpty(t, e.Assessment, "Assessment raw bytes should be populated") + + out, err := e.GetAssessment() + require.NoError(t, err) + require.NotNil(t, out) + assert.Equal(t, in, out) +} + +func TestEvidence_SetAssessment_Nil_ClearsField(t *testing.T) { + t.Parallel() + + e := Evidence{} + require.NoError(t, e.SetAssessment(&EvidenceAssessment{Summary: "stub"})) + require.NotEmpty(t, e.Assessment) + + require.NoError(t, e.SetAssessment(nil)) + assert.Empty(t, e.Assessment, "nil input should clear the raw bytes") +} + +func TestEvidence_GetAssessment_EmptyReturnsNil(t *testing.T) { + t.Parallel() + + var e Evidence + got, err := e.GetAssessment() + require.NoError(t, err) + assert.Nil(t, got) +} diff --git a/pkg/coredata/migrations/20260424T637521Z.sql b/pkg/coredata/migrations/20260424T637521Z.sql new file mode 100644 index 0000000000..bfd542537f --- /dev/null +++ b/pkg/coredata/migrations/20260424T637521Z.sql @@ -0,0 +1,24 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TABLE evidences + ADD COLUMN assessment JSONB NULL; + +ALTER TABLE evidences + RENAME COLUMN description_status TO assessment_status; + +ALTER TABLE evidences + RENAME COLUMN description_processing_started_at TO assessment_processing_started_at; + +ALTER TYPE evidence_description_status RENAME TO evidence_assessment_status; diff --git a/pkg/evidencedescriber/assessment.go b/pkg/evidencedescriber/assessment.go new file mode 100644 index 0000000000..8f71f7f1e4 --- /dev/null +++ b/pkg/evidencedescriber/assessment.go @@ -0,0 +1,147 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package evidencedescriber + +import ( + "context" + _ "embed" + "encoding/json" + "fmt" + + "go.gearno.de/kit/log" + "go.probo.inc/probo/pkg/agent" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/llm" +) + +//go:embed prompt.txt +var systemPrompt string + +type ( + Config struct { + Client *llm.Client + Model string + Temp float64 + MaxTokens int + // Thinking is the extended-thinking budget in tokens. 0 disables + // extended thinking entirely. + Thinking int + Logger *log.Logger + } + + Describer struct { + cfg Config + outputType *agent.OutputType + } +) + +// New builds a Describer. The structured output type is decorated once +// (enum on confidence) and cached on the Describer so every call reuses +// the same schema. +func New(cfg Config) (*Describer, error) { + ot, err := assessmentOutputType() + if err != nil { + return nil, fmt.Errorf("cannot build evidence assessment output type: %w", err) + } + + return &Describer{cfg: cfg, outputType: ot}, nil +} + +// Describe runs the evidence describer agent against a single uploaded +// file and returns a structured assessment. The worker persists the +// result as JSONB and derives Evidence.Description from Summary. +func (d *Describer) Describe( + ctx context.Context, + filename string, + mimeType string, + fileBase64 string, +) (*coredata.EvidenceAssessment, error) { + opts := []agent.Option{ + agent.WithInstructions(systemPrompt), + agent.WithModel(d.cfg.Model), + agent.WithTemperature(d.cfg.Temp), + agent.WithMaxTokens(d.cfg.MaxTokens), + agent.WithOutputType(d.outputType), + } + if d.cfg.Thinking > 0 { + opts = append(opts, agent.WithThinking(d.cfg.Thinking)) + } + if d.cfg.Logger != nil { + opts = append(opts, agent.WithLogger(d.cfg.Logger)) + } + + ag := agent.New("evidence_describer", d.cfg.Client, opts...) + + result, err := ag.Run(ctx, []llm.Message{ + { + Role: llm.RoleUser, + Parts: []llm.Part{ + llm.TextPart{Text: fmt.Sprintf("Filename: %s", filename)}, + llm.FilePart{ + Data: fileBase64, + MimeType: mimeType, + Filename: filename, + }, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("cannot describe evidence: %w", err) + } + + var a coredata.EvidenceAssessment + if err := json.Unmarshal([]byte(result.FinalMessage().Text()), &a); err != nil { + return nil, fmt.Errorf("cannot parse evidence assessment: %w", err) + } + + return &a, nil +} + +// assessmentOutputType builds the EvidenceAssessment structured output +// type and decorates its JSON Schema with an explicit enum on +// `confidence`. jsonschema-go reads struct tags only as free-form +// descriptions, so the enum cannot be encoded in the tag itself; +// see pkg/vetting/assessment.go for the same pattern applied to +// VendorInfo. +func assessmentOutputType() (*agent.OutputType, error) { + ot, err := agent.NewOutputType[coredata.EvidenceAssessment]("evidence_assessment") + if err != nil { + return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) + } + + var schema map[string]any + if err := json.Unmarshal(ot.Schema, &schema); err != nil { + return nil, fmt.Errorf("cannot unmarshal evidence assessment schema: %w", err) + } + + properties, ok := schema["properties"].(map[string]any) + if !ok { + return nil, fmt.Errorf("evidence assessment schema has no properties") + } + + confidence, ok := properties["confidence"].(map[string]any) + if !ok { + return nil, fmt.Errorf("evidence assessment schema has no confidence property") + } + confidence["enum"] = coredata.EvidenceAssessmentConfidenceEnum + + decorated, err := json.Marshal(schema) + if err != nil { + return nil, fmt.Errorf("cannot marshal decorated evidence assessment schema: %w", err) + } + ot.Schema = decorated + + return ot, nil +} diff --git a/pkg/evidencedescriber/assessment_test.go b/pkg/evidencedescriber/assessment_test.go new file mode 100644 index 0000000000..f08ad101b0 --- /dev/null +++ b/pkg/evidencedescriber/assessment_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +// White-box test (package evidencedescriber, not _test) so it can reach +// the unexported assessmentOutputType helper. + +package evidencedescriber + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.probo.inc/probo/pkg/coredata" +) + +// TestAssessmentOutputType_DecoratesConfidenceEnum guards the +// schema-mutation trick used to inject an enum into the generated JSON +// schema. If jsonschema-go ever reshapes the "properties" block or +// stops preserving the confidence field path, this test will fail +// loudly instead of silently shipping an un-constrained schema. +func TestAssessmentOutputType_DecoratesConfidenceEnum(t *testing.T) { + t.Parallel() + + outputType, err := assessmentOutputType() + require.NoError(t, err) + require.NotNil(t, outputType) + + var schema map[string]any + require.NoError(t, json.Unmarshal(outputType.Schema, &schema)) + + properties, ok := schema["properties"].(map[string]any) + require.True(t, ok, "schema has no properties block") + + confidence, ok := properties["confidence"].(map[string]any) + require.True(t, ok, "schema has no confidence property") + + enumRaw, ok := confidence["enum"].([]any) + require.True(t, ok, "confidence has no enum array") + + actual := make([]string, len(enumRaw)) + for i, v := range enumRaw { + actual[i] = v.(string) + } + assert.Equal(t, coredata.EvidenceAssessmentConfidenceEnum, actual) +} diff --git a/pkg/evidencedescriber/evidencedescriber.go b/pkg/evidencedescriber/evidencedescriber.go deleted file mode 100644 index db7d552169..0000000000 --- a/pkg/evidencedescriber/evidencedescriber.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2026 Probo Inc . -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -// PERFORMANCE OF THIS SOFTWARE. - -package evidencedescriber - -import ( - "context" - _ "embed" - "fmt" - - "go.probo.inc/probo/pkg/agent" - "go.probo.inc/probo/pkg/llm" -) - -//go:embed prompt.txt -var systemPrompt string - -type ( - Config struct { - Model string - Temp float64 - MaxTokens int - } - - Describer struct { - client *llm.Client - config Config - } -) - -func New(client *llm.Client, cfg Config) *Describer { - return &Describer{ - client: client, - config: cfg, - } -} - -func (d *Describer) Describe(ctx context.Context, filename string, mimeType string, fileBase64 string) (*string, error) { - ag := agent.New( - "evidence_describer", - d.client, - agent.WithInstructions(systemPrompt), - agent.WithModel(d.config.Model), - agent.WithTemperature(d.config.Temp), - agent.WithMaxTokens(d.config.MaxTokens), - ) - - result, err := ag.Run( - ctx, - []llm.Message{ - { - Role: llm.RoleUser, - Parts: []llm.Part{ - llm.TextPart{Text: fmt.Sprintf("Filename: %s", filename)}, - llm.FilePart{ - Data: fileBase64, - MimeType: mimeType, - Filename: filename, - }, - }, - }, - }, - ) - if err != nil { - return nil, fmt.Errorf("cannot describe evidence: %w", err) - } - - text := result.FinalMessage().Text() - - return &text, nil -} diff --git a/pkg/evidencedescriber/prompt.txt b/pkg/evidencedescriber/prompt.txt index 049b0da72a..5baf18c9ba 100644 --- a/pkg/evidencedescriber/prompt.txt +++ b/pkg/evidencedescriber/prompt.txt @@ -1,17 +1,35 @@ -You are an ISO/SOC auditor writing evidence descriptions for a compliance review. +You are an ISO/SOC auditor producing a structured record for one piece of compliance evidence (a screenshot, PDF, or document export). Your reader is another auditor; write for a compliance review, not a user. -Input: a single image of an evidence file (screenshot or document export). +Output a single JSON object that conforms exactly to the provided schema. Emit nothing outside the object — no prose, no markdown, no code fences, no preamble. -Output: plain text, 1–2 sentences. No markdown, no line breaks, no labels, no preamble. +Field rules, in the order a reader will care about them: -Include these elements if clearly present; omit any that are not: -— System: the tool or platform shown (e.g. GitHub, Google Workspace, AWS). -— Setting: the specific configuration, feature, or state demonstrated. -— Scope: who or what it applies to (e.g. organization-wide, all users, a specific repository). +- summary: one sentence, two at most. Plain text. No markdown, no line breaks, no labels, no filenames, no image-quality comments, no phrases like "this screenshot shows". Use the language of the document. When `readable` is false, `summary` must restate the rejection reason in a human-readable sentence so that downstream systems displaying only the summary still inform the reader. -Use the language of the document. Do not include greetings, caveats, file names, image quality comments, or phrases like "This screenshot shows." Never guess — if something is not clearly visible, leave it out. +- system: the tool or platform visible on the file (e.g. "Google Workspace", "GitHub", "AWS IAM"). Empty string if not clearly identifiable — never guess. -If the image is unreadable or is not compliance evidence, respond with exactly: "Unable to describe: unreadable or not recognized as compliance evidence." +- setting: the specific configuration, feature, or state demonstrated (e.g. "enforced 2-step verification", "branch protection requiring review"). Empty string if not clearly identifiable. -Example — good: "Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted." -Example — bad: "This shows Google security settings." \ No newline at end of file +- scope: who or what the setting applies to (e.g. "organization-wide", "all users", "repository acme/foo", "production environment"). Empty string if not stated. + +- captured_at: ISO-8601 date or datetime if visible on the file (timestamps, "last updated" labels, status dates). Empty string if no date is shown. + +- language: BCP-47 tag of the visible text ("en", "fr", "de"). Empty string if unclear. + +- frameworks: compliance frameworks the evidence is plausibly relevant to. Only include a framework when it is explicitly named on the file (e.g. "SOC2 report summary") or strongly implied by a recognisable domain artifact (e.g. an AWS Config dashboard for SOC2). Empty array when unclear — do not speculate. + +- issues: quality problems observed on the file itself: redactions covering key fields, cropped content, low resolution, stale date, illegible screenshot. Empty array when the file is clean. + +- confidence: HIGH only when all three of `system`, `setting`, and `scope` are unambiguous on the file. MEDIUM when at most one of the three is inferred or partially visible. LOW otherwise. The schema enforces the enum; pick one. + +- readable: true when the file is a usable piece of compliance evidence. false when the file is unreadable (blank, corrupt, mis-OCRed), off-topic (an invoice, a marketing page, a cat picture), or when the content is compliance-adjacent but too ambiguous to describe (e.g. a blurred screenshot of an unlabeled panel). + +- rejection_reason: populated only when `readable` is false. Explain why the file is not usable evidence in one short sentence. When `readable` is true this field must be empty. + +Do not guess. If a field cannot be determined from the file alone, leave it empty rather than inferring it from the filename, from the surrounding metadata, or from general knowledge about the platform. + +Example — readable: +{"summary":"Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted.","system":"Google Workspace","setting":"enforced 2-step verification","scope":"organization-wide","captured_at":"","language":"en","frameworks":["SOC2","ISO27001"],"issues":[],"confidence":"HIGH","readable":true,"rejection_reason":""} + +Example — unreadable: +{"summary":"Unable to describe: image is blank and contains no recognisable compliance evidence.","system":"","setting":"","scope":"","captured_at":"","language":"","frameworks":[],"issues":[],"confidence":"LOW","readable":false,"rejection_reason":"Image is blank and contains no recognisable compliance evidence."} diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 85db1a001d..77a65e8add 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -29,7 +29,7 @@ import ( ) type ( - evidenceDescriptionHandler struct { + evidenceAssessmentHandler struct { pg *pg.Client fileManager *filemanager.Service describer *evidencedescriber.Describer @@ -37,17 +37,17 @@ type ( staleAfter time.Duration } - EvidenceDescriptionWorkerConfig struct { + EvidenceAssessmentWorkerConfig struct { StaleAfter time.Duration } ) -func NewEvidenceDescriptionWorker( +func NewEvidenceAssessmentWorker( pgClient *pg.Client, fileManager *filemanager.Service, describer *evidencedescriber.Describer, logger *log.Logger, - cfg EvidenceDescriptionWorkerConfig, + cfg EvidenceAssessmentWorkerConfig, opts ...worker.Option, ) *worker.Worker[coredata.Evidence] { staleAfter := cfg.StaleAfter @@ -55,7 +55,7 @@ func NewEvidenceDescriptionWorker( staleAfter = 5 * time.Minute } - h := &evidenceDescriptionHandler{ + h := &evidenceAssessmentHandler{ pg: pgClient, fileManager: fileManager, describer: describer, @@ -64,26 +64,26 @@ func NewEvidenceDescriptionWorker( } return worker.New( - "evidence-description-worker", + "evidence-assessment-worker", h, logger, opts..., ) } -func (h *evidenceDescriptionHandler) Claim(ctx context.Context) (coredata.Evidence, error) { +func (h *evidenceAssessmentHandler) Claim(ctx context.Context) (coredata.Evidence, error) { var evidence coredata.Evidence if err := h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - if err := evidence.LoadNextPendingDescriptionForUpdateSkipLocked(ctx, tx); err != nil { + if err := evidence.LoadNextPendingAssessmentForUpdateSkipLocked(ctx, tx); err != nil { return err } now := time.Now() - evidence.DescriptionStatus = coredata.EvidenceDescriptionStatusProcessing - evidence.DescriptionProcessingStartedAt = &now + evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusProcessing + evidence.AssessmentProcessingStartedAt = &now evidence.UpdatedAt = now if err := evidence.Update(ctx, tx, coredata.NewNoScope()); err != nil { @@ -103,17 +103,17 @@ func (h *evidenceDescriptionHandler) Claim(ctx context.Context) (coredata.Eviden return evidence, nil } -func (h *evidenceDescriptionHandler) Process(ctx context.Context, evidence coredata.Evidence) error { - if err := h.describeAndCommit(ctx, &evidence); err != nil { +func (h *evidenceAssessmentHandler) Process(ctx context.Context, evidence coredata.Evidence) error { + if err := h.assessAndCommit(ctx, &evidence); err != nil { h.logger.ErrorCtx( ctx, - "evidence description worker failure", + "evidence assessment worker failure", log.Error(err), log.String("evidence_id", evidence.ID.String()), ) if err := h.failEvidence(ctx, &evidence); err != nil { - h.logger.ErrorCtx(ctx, "cannot mark evidence description as failed", log.Error(err)) + h.logger.ErrorCtx(ctx, "cannot mark evidence assessment as failed", log.Error(err)) } return err @@ -122,12 +122,12 @@ func (h *evidenceDescriptionHandler) Process(ctx context.Context, evidence cored return nil } -func (h *evidenceDescriptionHandler) RecoverStale(ctx context.Context) error { +func (h *evidenceAssessmentHandler) RecoverStale(ctx context.Context) error { return h.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - if err := coredata.ResetStaleDescriptionProcessing(ctx, conn, h.staleAfter); err != nil { - return fmt.Errorf("cannot reset stale description processing: %w", err) + if err := coredata.ResetStaleAssessmentProcessing(ctx, conn, h.staleAfter); err != nil { + return fmt.Errorf("cannot reset stale assessment processing: %w", err) } return nil @@ -135,7 +135,7 @@ func (h *evidenceDescriptionHandler) RecoverStale(ctx context.Context) error { ) } -func (h *evidenceDescriptionHandler) describeAndCommit( +func (h *evidenceAssessmentHandler) assessAndCommit( ctx context.Context, evidence *coredata.Evidence, ) error { @@ -165,17 +165,22 @@ func (h *evidenceDescriptionHandler) describeAndCommit( return fmt.Errorf("cannot download file: %w", err) } - description, err := h.describer.Describe(ctx, file.FileName, mimeType, base64Data) + assessment, err := h.describer.Describe(ctx, file.FileName, mimeType, base64Data) if err != nil { return fmt.Errorf("cannot describe evidence: %w", err) } + if err := evidence.SetAssessment(assessment); err != nil { + return err + } + return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - evidence.Description = description - evidence.DescriptionStatus = coredata.EvidenceDescriptionStatusCompleted - evidence.DescriptionProcessingStartedAt = nil + summary := assessment.Summary + evidence.Description = &summary + evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusCompleted + evidence.AssessmentProcessingStartedAt = nil evidence.UpdatedAt = time.Now() if err := evidence.Update(ctx, tx, scope); err != nil { @@ -187,7 +192,7 @@ func (h *evidenceDescriptionHandler) describeAndCommit( ) } -func (h *evidenceDescriptionHandler) failEvidence( +func (h *evidenceAssessmentHandler) failEvidence( ctx context.Context, evidence *coredata.Evidence, ) error { @@ -196,8 +201,8 @@ func (h *evidenceDescriptionHandler) failEvidence( return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - evidence.DescriptionStatus = coredata.EvidenceDescriptionStatusFailed - evidence.DescriptionProcessingStartedAt = nil + evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusFailed + evidence.AssessmentProcessingStartedAt = nil evidence.UpdatedAt = time.Now() if err := evidence.Update(ctx, tx, scope); err != nil { diff --git a/pkg/probo/evidence_service.go b/pkg/probo/evidence_service.go index ce345b364d..e7c388df3c 100644 --- a/pkg/probo/evidence_service.go +++ b/pkg/probo/evidence_service.go @@ -91,14 +91,14 @@ func (s EvidenceService) UploadMeasureEvidence( } evidence := &coredata.Evidence{ - ID: evidenceID, - MeasureID: req.MeasureID, - State: coredata.EvidenceStateFulfilled, - ReferenceID: "custom-evidence-" + referenceID.String(), - Type: coredata.EvidenceTypeFile, - DescriptionStatus: coredata.EvidenceDescriptionStatusPending, - CreatedAt: now, - UpdatedAt: now, + ID: evidenceID, + MeasureID: req.MeasureID, + State: coredata.EvidenceStateFulfilled, + ReferenceID: "custom-evidence-" + referenceID.String(), + Type: coredata.EvidenceTypeFile, + AssessmentStatus: coredata.EvidenceAssessmentStatusPending, + CreatedAt: now, + UpdatedAt: now, } err = s.svc.pg.WithTx( diff --git a/pkg/probo/measure_service.go b/pkg/probo/measure_service.go index 8c75ede4b1..bf7f64d61b 100644 --- a/pkg/probo/measure_service.go +++ b/pkg/probo/measure_service.go @@ -426,15 +426,15 @@ func (s MeasureService) Import( evidenceDescription := req.Measures[i].Tasks[j].RequestedEvidences[k].Name evidence := &coredata.Evidence{ - State: coredata.EvidenceStateRequested, - ID: evidenceID, - TaskID: &task.ID, - ReferenceID: req.Measures[i].Tasks[j].RequestedEvidences[k].ReferenceID, - Type: req.Measures[i].Tasks[j].RequestedEvidences[k].Type, - Description: &evidenceDescription, - DescriptionStatus: coredata.EvidenceDescriptionStatusPending, - CreatedAt: now, - UpdatedAt: now, + State: coredata.EvidenceStateRequested, + ID: evidenceID, + TaskID: &task.ID, + ReferenceID: req.Measures[i].Tasks[j].RequestedEvidences[k].ReferenceID, + Type: req.Measures[i].Tasks[j].RequestedEvidences[k].Type, + Description: &evidenceDescription, + AssessmentStatus: coredata.EvidenceAssessmentStatusPending, + CreatedAt: now, + UpdatedAt: now, } if err := evidence.Upsert(ctx, tx, scope); err != nil { diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index ad90f24a4c..d73b242da4 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -838,31 +838,37 @@ func (impl *Implm) Run( }, ) - evidenceDescriber := evidencedescriber.New( - evidenceDescriberLLMClient, - evidencedescriber.Config{ - Model: evidenceDescriberAgentCfg.ModelName, - Temp: ref.UnrefOrZero(evidenceDescriberAgentCfg.Temperature), - MaxTokens: ref.UnrefOrZero(evidenceDescriberAgentCfg.MaxTokens), - }, - ) - evidenceDescriptionWorker := probo.NewEvidenceDescriptionWorker( + evidenceDescriberCfg := evidencedescriber.Config{ + Client: evidenceDescriberLLMClient, + Model: evidenceDescriberAgentCfg.ModelName, + Temp: ref.UnrefOrZero(evidenceDescriberAgentCfg.Temperature), + MaxTokens: ref.UnrefOrZero(evidenceDescriberAgentCfg.MaxTokens), + Logger: l.Named("evidence-describer"), + } + if evidenceDescriberAgentCfg.Thinking != nil { + evidenceDescriberCfg.Thinking = *evidenceDescriberAgentCfg.Thinking + } + evidenceDescriber, err := evidencedescriber.New(evidenceDescriberCfg) + if err != nil { + return fmt.Errorf("cannot build evidence describer: %w", err) + } + evidenceAssessmentWorker := probo.NewEvidenceAssessmentWorker( pgClient, fileManagerService, evidenceDescriber, - l.Named("evidence-description-worker"), - probo.EvidenceDescriptionWorkerConfig{ + l.Named("evidence-assessment-worker"), + probo.EvidenceAssessmentWorkerConfig{ StaleAfter: time.Duration(impl.cfg.EvidenceDescriber.StaleAfter) * time.Second, }, worker.WithInterval(time.Duration(impl.cfg.EvidenceDescriber.Interval)*time.Second), worker.WithMaxConcurrency(impl.cfg.EvidenceDescriber.MaxConcurrency), ) - evidenceDescriptionWorkerCtx, stopEvidenceDescriptionWorker := context.WithCancel(context.Background()) + evidenceAssessmentWorkerCtx, stopEvidenceAssessmentWorker := context.WithCancel(context.Background()) wg.Go( func() { - if err := evidenceDescriptionWorker.Run(evidenceDescriptionWorkerCtx); err != nil { - cancel(fmt.Errorf("evidence description worker crashed: %w", err)) + if err := evidenceAssessmentWorker.Run(evidenceAssessmentWorkerCtx); err != nil { + cancel(fmt.Errorf("evidence assessment worker crashed: %w", err)) } }, ) @@ -920,7 +926,7 @@ func (impl *Implm) Run( stopCommonPatternEnrichmentWorker() stopMailingListWorker() stopVettingWorker() - stopEvidenceDescriptionWorker() + stopEvidenceAssessmentWorker() stopDocumentPDFWorker() stopExportJobExporter() stopAccessReviewWorker() From 205f2d9b7354d11addfdad0ac5e2d02589795018 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 22:24:14 +0200 Subject: [PATCH 03/21] Extract OutputType.DecorateEnum helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The enum-injection workaround for jsonschema-go was duplicated between pkg/vetting and the new evidence describer. Move it to pkg/agent as a method on OutputType so every agent that needs string enums on structured outputs can opt in with one call. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/agent/output_type.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/vetting/assessment.go | 30 ++++-------------------------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/pkg/agent/output_type.go b/pkg/agent/output_type.go index d99c132ed0..f4cbd6d111 100644 --- a/pkg/agent/output_type.go +++ b/pkg/agent/output_type.go @@ -50,3 +50,39 @@ func (o *OutputType) responseFormat() *llm.ResponseFormat { }, } } + +// DecorateEnum injects explicit `enum` constraints on top-level +// properties of the schema. jsonschema-go only reads struct tags as +// free-form descriptions, so enums cannot be encoded on the tag itself; +// callers use this helper after NewOutputType to lock down string +// fields whose allowed values live in a package-level slice. +func (o *OutputType) DecorateEnum(enums map[string][]string) error { + if len(enums) == 0 { + return nil + } + + var schema map[string]any + if err := json.Unmarshal(o.Schema, &schema); err != nil { + return fmt.Errorf("cannot unmarshal output type schema: %w", err) + } + + properties, ok := schema["properties"].(map[string]any) + if !ok { + return fmt.Errorf("output type schema has no properties") + } + + for field, values := range enums { + prop, ok := properties[field].(map[string]any) + if !ok { + return fmt.Errorf("output type schema has no %q property", field) + } + prop["enum"] = values + } + + decorated, err := json.Marshal(schema) + if err != nil { + return fmt.Errorf("cannot marshal decorated output type schema: %w", err) + } + o.Schema = decorated + return nil +} diff --git a/pkg/vetting/assessment.go b/pkg/vetting/assessment.go index a9413f06f2..c17c846fa5 100644 --- a/pkg/vetting/assessment.go +++ b/pkg/vetting/assessment.go @@ -314,35 +314,13 @@ func thirdPartyInfoOutputType() (*agent.OutputType, error) { return nil, fmt.Errorf("cannot create thirdParty info output type: %w", err) } - var schema map[string]any - if err := json.Unmarshal(outputType.Schema, &schema); err != nil { - return nil, fmt.Errorf("cannot unmarshal thirdParty info schema: %w", err) - } - - properties, ok := schema["properties"].(map[string]any) - if !ok { - return nil, fmt.Errorf("thirdParty info schema has no properties") - } - - enums := map[string][]string{ + if err := outputType.DecorateEnum(map[string][]string{ "category": thirdPartyCategoryEnum, "third_party_type": thirdPartyTypeEnum, + }); err != nil { + return nil, fmt.Errorf("cannot decorate thirdParty info schema: %w", err) } - for field, values := range enums { - prop, ok := properties[field].(map[string]any) - if !ok { - return nil, fmt.Errorf("thirdParty info schema has no %q property", field) - } - - prop["enum"] = values - } - - decorated, err := json.Marshal(schema) - if err != nil { - return nil, fmt.Errorf("cannot marshal decorated thirdParty info schema: %w", err) - } - - strict, err := enforceStrictJSONSchema(decorated) + strict, err := enforceStrictJSONSchema(outputType.Schema) if err != nil { return nil, fmt.Errorf("cannot enforce strict thirdParty info schema: %w", err) } From 5b44245d131ab0d6bd428deffee13f54c1510d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 22:24:20 +0200 Subject: [PATCH 04/21] Add Arg method to jsonRawMessageOrNull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pgx rejects empty byte slices as invalid JSON, so callers need to send SQL NULL when the raw message is empty. The guard was inlined in Connector.Insert / Connector.Update and added again as a one-off method on Evidence. Collapse all three sites onto a single Arg method on the underlying type. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/connector.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/coredata/connector.go b/pkg/coredata/connector.go index 403577f68f..ce7ac53095 100644 --- a/pkg/coredata/connector.go +++ b/pkg/coredata/connector.go @@ -59,6 +59,16 @@ func (j *jsonRawMessageOrNull) Scan(src any) error { } } +// Arg returns the value to bind for a JSONB named argument. pgx rejects +// empty byte slices as invalid JSON, so an empty/nil value is sent as +// SQL NULL. +func (j jsonRawMessageOrNull) Arg() any { + if len(j) == 0 { + return nil + } + return []byte(j) +} + type ( Connector struct { ID gid.GID `db:"id"` @@ -389,18 +399,13 @@ INSERT INTO connectors ( return fmt.Errorf("cannot encrypt connection: %w", err) } - var settingsArg any - if len(c.RawSettings) > 0 { - settingsArg = []byte(c.RawSettings) - } - args := pgx.StrictNamedArgs{ "id": c.ID, "tenant_id": scope.GetTenantID(), "organization_id": c.OrganizationID, "provider": c.Provider, "protocol": c.Protocol, - "settings": settingsArg, + "settings": c.RawSettings.Arg(), "encrypted_connection": encryptedConnection, "created_at": c.CreatedAt, "updated_at": c.UpdatedAt, @@ -613,14 +618,9 @@ WHERE return fmt.Errorf("cannot encrypt connection: %w", err) } - var settingsArg any - if len(c.RawSettings) > 0 { - settingsArg = []byte(c.RawSettings) - } - args := pgx.StrictNamedArgs{ "id": c.ID, - "settings": settingsArg, + "settings": c.RawSettings.Arg(), "encrypted_connection": encryptedConnection, "updated_at": c.UpdatedAt, } From 8ab2d3bf772b01032a6ca30da279a64fb7dec385 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 22:24:31 +0200 Subject: [PATCH 05/21] Move EvidenceAssessment into the describer package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit coredata owns the DB round-trip, not the LLM-domain schema. The EvidenceAssessment struct and its confidence enum belong with the agent that produces them. coredata exposes generic SetAssessment(any) / AssessmentInto(dst any) helpers that work over json.RawMessage, keeping the persistence layer schema- agnostic. Also: - Scope the PENDING -> PROCESSING transition in the worker by evidence.ID instead of NoScope, matching the later Update calls on the same row. - Wrap the 'no file attached' and the three pgx Exec errors in evidence.go (Upsert, Update, ResetStaleAssessmentProcessing) with cannot-prefixed messages, as required by the error convention. These were pre-existing bare returns preserved from the old evidence_description_worker; cleaning them up now since the same functions are touched here. - Replace the Evidence.assessmentArg helper with the shared Assessment.Arg method from the previous commit. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence.go | 31 ++++++------ pkg/coredata/evidence_assessment.go | 53 ++++++-------------- pkg/coredata/evidence_assessment_test.go | 37 +++++++------- pkg/evidencedescriber/assessment.go | 63 ++++++++++++------------ pkg/evidencedescriber/assessment_test.go | 3 +- pkg/probo/evidence_assessment_worker.go | 4 +- 6 files changed, 83 insertions(+), 108 deletions(-) diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index 4a7000844e..3a56323b6d 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -100,16 +100,6 @@ func (e *Evidence) AuthorizationAttributes( return attrsByID, nil } -// assessmentArg returns the value to bind for the `assessment` JSONB -// column. pgx rejects empty byte slices as invalid JSON, so an empty -// Assessment is sent as NULL. -func (e Evidence) assessmentArg() any { - if len(e.Assessment) == 0 { - return nil - } - return []byte(e.Assessment) -} - func (e Evidence) Upsert( ctx context.Context, conn pg.Querier, @@ -171,13 +161,16 @@ WHERE evidences.state = 'REQUESTED'; "type": e.Type, "url": e.URL, "description": e.Description, - "assessment": e.assessmentArg(), + "assessment": e.Assessment.Arg(), "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } _, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot upsert evidence: %w", err) + } - return err + return nil } func (e Evidence) Insert( @@ -239,7 +232,7 @@ VALUES ( "type": e.Type, "url": e.URL, "description": e.Description, - "assessment": e.assessmentArg(), + "assessment": e.Assessment.Arg(), "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } @@ -513,7 +506,7 @@ WHERE "evidence_file_id": e.EvidenceFileId, "url": e.URL, "description": e.Description, - "assessment": e.assessmentArg(), + "assessment": e.Assessment.Arg(), "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, "updated_at": e.UpdatedAt, @@ -521,8 +514,11 @@ WHERE maps.Copy(args, scope.SQLArguments()) _, err := conn.Exec(ctx, q, args) + if err != nil { + return fmt.Errorf("cannot update evidence: %w", err) + } - return err + return nil } func (e Evidence) Delete( @@ -618,6 +614,9 @@ WHERE ` _, err := conn.Exec(ctx, q, time.Now().Add(-staleAfter)) + if err != nil { + return fmt.Errorf("cannot reset stale assessment processing: %w", err) + } - return err + return nil } diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go index 69b8aea330..286da1df34 100644 --- a/pkg/coredata/evidence_assessment.go +++ b/pkg/coredata/evidence_assessment.go @@ -19,41 +19,17 @@ import ( "fmt" ) -// EvidenceAssessment is the structured output produced by the evidence -// describer agent. It is persisted as JSONB in evidences.assessment and -// is the canonical form; Evidence.Description is a one-sentence summary -// derived from Assessment.Summary. -type EvidenceAssessment struct { - Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows; no markdown, no preamble. When readable is false this field must restate the rejection reason so downstream systems displaying only the summary still inform the user."` - System string `json:"system" jsonschema:"Tool or platform shown (e.g. 'Google Workspace', 'GitHub', 'AWS IAM'); empty string if not identifiable."` - Setting string `json:"setting" jsonschema:"Specific configuration, feature, or state demonstrated; empty string if not identifiable."` - Scope string `json:"scope" jsonschema:"Who or what the evidence applies to (e.g. 'organization-wide', 'all users', 'repository acme/foo'); empty string if not stated."` - CapturedAt string `json:"captured_at" jsonschema:"ISO-8601 date or datetime visible on the file; empty string if no date is shown."` - Language string `json:"language" jsonschema:"BCP-47 language tag of the visible text (e.g. 'en', 'fr'); empty string if unclear."` - Frameworks []string `json:"frameworks" jsonschema:"Compliance frameworks the evidence is plausibly relevant to (e.g. 'SOC2', 'ISO27001'); empty when unclear."` - Issues []string `json:"issues" jsonschema:"Quality problems observed: redacted fields, crop, stale date, low resolution, etc."` - Confidence string `json:"confidence" jsonschema:"Overall confidence in the System / Setting / Scope identification; one of HIGH, MEDIUM, LOW."` - Readable bool `json:"readable" jsonschema:"True if the file is readable compliance evidence; false if unreadable or off-topic."` - RejectionReason string `json:"rejection_reason" jsonschema:"Set only when readable is false; explains why the file is not usable evidence."` -} - -// EvidenceAssessmentConfidenceEnum is the canonical set of allowed -// EvidenceAssessment.Confidence values. Kept in a package-level slice -// because jsonschema struct tags are free-form descriptions and cannot -// encode enum constraints directly; consumers building an output schema -// inject it post-marshal (see pkg/evidencedescriber). -var EvidenceAssessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} - // SetAssessment marshals a typed assessment into Evidence.Assessment. -// Passing nil clears the field. This is the only supported way to write -// the assessment column from outside the coredata package; the backing -// field type is unexported. -func (e *Evidence) SetAssessment(a *EvidenceAssessment) error { - if a == nil { +// Passing nil clears the field. The shape of the assessment is defined +// by its producer (see pkg/evidencedescriber.EvidenceAssessment); this +// package is intentionally agnostic about the schema and only owns the +// raw JSONB round-trip. +func (e *Evidence) SetAssessment(v any) error { + if v == nil { e.Assessment = nil return nil } - data, err := json.Marshal(a) + data, err := json.Marshal(v) if err != nil { return fmt.Errorf("cannot marshal evidence assessment: %w", err) } @@ -61,15 +37,14 @@ func (e *Evidence) SetAssessment(a *EvidenceAssessment) error { return nil } -// GetAssessment unmarshals Evidence.Assessment into a typed -// EvidenceAssessment. Returns (nil, nil) when the column is NULL. -func (e *Evidence) GetAssessment() (*EvidenceAssessment, error) { +// AssessmentInto unmarshals Evidence.Assessment into dst. It is a no-op +// when the column is NULL/empty, leaving dst untouched. +func (e *Evidence) AssessmentInto(dst any) error { if len(e.Assessment) == 0 { - return nil, nil + return nil } - var a EvidenceAssessment - if err := json.Unmarshal(e.Assessment, &a); err != nil { - return nil, fmt.Errorf("cannot unmarshal evidence assessment: %w", err) + if err := json.Unmarshal(e.Assessment, dst); err != nil { + return fmt.Errorf("cannot unmarshal evidence assessment: %w", err) } - return &a, nil + return nil } diff --git a/pkg/coredata/evidence_assessment_test.go b/pkg/coredata/evidence_assessment_test.go index 2a9ce1cbd1..3572bc7c32 100644 --- a/pkg/coredata/evidence_assessment_test.go +++ b/pkg/coredata/evidence_assessment_test.go @@ -21,47 +21,48 @@ import ( "github.com/stretchr/testify/require" ) -func TestEvidence_SetAndGetAssessment_RoundTrip(t *testing.T) { +// A representative payload shape. The coredata layer is schema-agnostic +// so any JSON-serialisable struct should round-trip. +type testAssessmentPayload struct { + Summary string `json:"summary"` + Confidence string `json:"confidence"` + Frameworks []string `json:"frameworks"` +} + +func TestEvidence_SetAssessment_ReadBack(t *testing.T) { t.Parallel() - in := &EvidenceAssessment{ + in := &testAssessmentPayload{ Summary: "Google Workspace admin console showing enforced 2SV for all users.", - System: "Google Workspace", - Setting: "enforced 2-step verification", - Scope: "organization-wide", - Language: "en", - Frameworks: []string{"SOC2", "ISO27001"}, - Issues: []string{}, Confidence: "HIGH", - Readable: true, + Frameworks: []string{"SOC2", "ISO27001"}, } var e Evidence require.NoError(t, e.SetAssessment(in)) require.NotEmpty(t, e.Assessment, "Assessment raw bytes should be populated") - out, err := e.GetAssessment() - require.NoError(t, err) - require.NotNil(t, out) - assert.Equal(t, in, out) + var out testAssessmentPayload + require.NoError(t, e.AssessmentInto(&out)) + assert.Equal(t, *in, out) } func TestEvidence_SetAssessment_Nil_ClearsField(t *testing.T) { t.Parallel() e := Evidence{} - require.NoError(t, e.SetAssessment(&EvidenceAssessment{Summary: "stub"})) + require.NoError(t, e.SetAssessment(&testAssessmentPayload{Summary: "stub"})) require.NotEmpty(t, e.Assessment) require.NoError(t, e.SetAssessment(nil)) assert.Empty(t, e.Assessment, "nil input should clear the raw bytes") } -func TestEvidence_GetAssessment_EmptyReturnsNil(t *testing.T) { +func TestEvidence_AssessmentInto_EmptyIsNoOp(t *testing.T) { t.Parallel() var e Evidence - got, err := e.GetAssessment() - require.NoError(t, err) - assert.Nil(t, got) + out := testAssessmentPayload{Summary: "unchanged"} + require.NoError(t, e.AssessmentInto(&out)) + assert.Equal(t, "unchanged", out.Summary, "empty column should leave dst untouched") } diff --git a/pkg/evidencedescriber/assessment.go b/pkg/evidencedescriber/assessment.go index 8f71f7f1e4..b706a8cb81 100644 --- a/pkg/evidencedescriber/assessment.go +++ b/pkg/evidencedescriber/assessment.go @@ -22,14 +22,36 @@ import ( "go.gearno.de/kit/log" "go.probo.inc/probo/pkg/agent" - "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/llm" ) //go:embed prompt.txt var systemPrompt string +// EvidenceAssessmentConfidenceEnum is the canonical set of allowed +// values for EvidenceAssessment.Confidence. It is injected into the +// generated JSON schema via agent.OutputType.DecorateEnum because +// jsonschema struct tags cannot encode enum constraints directly. +var evidenceAssessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} + type ( + // EvidenceAssessment is the structured output produced by the + // evidence describer. The worker persists it as JSONB on the + // evidences table and derives Evidence.Description from Summary. + EvidenceAssessment struct { + Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows; no markdown, no preamble. When readable is false this field must restate the rejection reason so downstream systems displaying only the summary still inform the user."` + System string `json:"system" jsonschema:"Tool or platform shown (e.g. 'Google Workspace', 'GitHub', 'AWS IAM'); empty string if not identifiable."` + Setting string `json:"setting" jsonschema:"Specific configuration, feature, or state demonstrated; empty string if not identifiable."` + Scope string `json:"scope" jsonschema:"Who or what the evidence applies to (e.g. 'organization-wide', 'all users', 'repository acme/foo'); empty string if not stated."` + CapturedAt string `json:"captured_at" jsonschema:"ISO-8601 date or datetime visible on the file; empty string if no date is shown."` + Language string `json:"language" jsonschema:"BCP-47 language tag of the visible text (e.g. 'en', 'fr'); empty string if unclear."` + Frameworks []string `json:"frameworks" jsonschema:"Compliance frameworks the evidence is plausibly relevant to (e.g. 'SOC2', 'ISO27001'); empty when unclear."` + Issues []string `json:"issues" jsonschema:"Quality problems observed: redacted fields, crop, stale date, low resolution, etc."` + Confidence string `json:"confidence" jsonschema:"Overall confidence in the System / Setting / Scope identification; one of HIGH, MEDIUM, LOW."` + Readable bool `json:"readable" jsonschema:"True if the file is readable compliance evidence; false if unreadable or off-topic."` + RejectionReason string `json:"rejection_reason" jsonschema:"Set only when readable is false; explains why the file is not usable evidence."` + } + Config struct { Client *llm.Client Model string @@ -67,7 +89,7 @@ func (d *Describer) Describe( filename string, mimeType string, fileBase64 string, -) (*coredata.EvidenceAssessment, error) { +) (*EvidenceAssessment, error) { opts := []agent.Option{ agent.WithInstructions(systemPrompt), agent.WithModel(d.cfg.Model), @@ -101,7 +123,7 @@ func (d *Describer) Describe( return nil, fmt.Errorf("cannot describe evidence: %w", err) } - var a coredata.EvidenceAssessment + var a EvidenceAssessment if err := json.Unmarshal([]byte(result.FinalMessage().Text()), &a); err != nil { return nil, fmt.Errorf("cannot parse evidence assessment: %w", err) } @@ -110,38 +132,17 @@ func (d *Describer) Describe( } // assessmentOutputType builds the EvidenceAssessment structured output -// type and decorates its JSON Schema with an explicit enum on -// `confidence`. jsonschema-go reads struct tags only as free-form -// descriptions, so the enum cannot be encoded in the tag itself; -// see pkg/vetting/assessment.go for the same pattern applied to -// VendorInfo. +// type and decorates its JSON Schema with an explicit enum on the +// `confidence` field. func assessmentOutputType() (*agent.OutputType, error) { - ot, err := agent.NewOutputType[coredata.EvidenceAssessment]("evidence_assessment") + ot, err := agent.NewOutputType[EvidenceAssessment]("evidence_assessment") if err != nil { return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) } - - var schema map[string]any - if err := json.Unmarshal(ot.Schema, &schema); err != nil { - return nil, fmt.Errorf("cannot unmarshal evidence assessment schema: %w", err) - } - - properties, ok := schema["properties"].(map[string]any) - if !ok { - return nil, fmt.Errorf("evidence assessment schema has no properties") - } - - confidence, ok := properties["confidence"].(map[string]any) - if !ok { - return nil, fmt.Errorf("evidence assessment schema has no confidence property") - } - confidence["enum"] = coredata.EvidenceAssessmentConfidenceEnum - - decorated, err := json.Marshal(schema) - if err != nil { - return nil, fmt.Errorf("cannot marshal decorated evidence assessment schema: %w", err) + if err := ot.DecorateEnum(map[string][]string{ + "confidence": evidenceAssessmentConfidenceEnum, + }); err != nil { + return nil, fmt.Errorf("cannot decorate evidence assessment schema: %w", err) } - ot.Schema = decorated - return ot, nil } diff --git a/pkg/evidencedescriber/assessment_test.go b/pkg/evidencedescriber/assessment_test.go index f08ad101b0..0f9dcfb976 100644 --- a/pkg/evidencedescriber/assessment_test.go +++ b/pkg/evidencedescriber/assessment_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.probo.inc/probo/pkg/coredata" ) // TestAssessmentOutputType_DecoratesConfidenceEnum guards the @@ -54,5 +53,5 @@ func TestAssessmentOutputType_DecoratesConfidenceEnum(t *testing.T) { for i, v := range enumRaw { actual[i] = v.(string) } - assert.Equal(t, coredata.EvidenceAssessmentConfidenceEnum, actual) + assert.Equal(t, evidenceAssessmentConfidenceEnum, actual) } diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 77a65e8add..e80ff20279 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -86,7 +86,7 @@ func (h *evidenceAssessmentHandler) Claim(ctx context.Context) (coredata.Evidenc evidence.AssessmentProcessingStartedAt = &now evidence.UpdatedAt = now - if err := evidence.Update(ctx, tx, coredata.NewNoScope()); err != nil { + if err := evidence.Update(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)); err != nil { return fmt.Errorf("cannot update evidence: %w", err) } @@ -140,7 +140,7 @@ func (h *evidenceAssessmentHandler) assessAndCommit( evidence *coredata.Evidence, ) error { if evidence.EvidenceFileId == nil { - return fmt.Errorf("evidence %s has no file", evidence.ID) + return fmt.Errorf("cannot assess evidence %s: no file attached", evidence.ID) } scope := coredata.NewScopeFromObjectID(evidence.ID) From 7e3ca9afa51fd0d315e83a973edef97cfafef25c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 23:30:28 +0200 Subject: [PATCH 06/21] Scope evidence FAILED transition to status columns only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assessAndCommit mutates evidence.Description and evidence.Assessment on the shared pointer before writing. If the commit transaction errors out, Process falls through to failEvidence, which used to call the full Evidence.Update and therefore wrote the in-memory mutations (new assessment, new description) back with FAILED status. Replace that with a targeted MarkAssessmentFailed statement that only updates assessment_status, assessment_processing_started_at, and updated_at, so the failure path can never overwrite the canonical columns with partial results. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence.go | 39 +++++++++++++++++++++++++ pkg/probo/evidence_assessment_worker.go | 17 +++-------- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index 3a56323b6d..c603a49f58 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -598,6 +598,45 @@ FOR UPDATE SKIP LOCKED; return nil } +// MarkAssessmentFailed transitions the evidence row to FAILED without +// touching description, assessment, or any other column. The worker's +// in-memory evidence pointer may carry mutations from a partially +// successful assess-and-commit (e.g. a new assessment that could not be +// persisted); using the full Update method would write those back with +// FAILED status. This targeted statement guarantees the failure path +// only changes state-transition columns. +func (e Evidence) MarkAssessmentFailed( + ctx context.Context, + conn pg.Tx, + scope Scoper, +) error { + q := ` +UPDATE + evidences +SET + assessment_status = @assessment_status, + assessment_processing_started_at = NULL, + updated_at = @updated_at +WHERE + %s + AND id = @evidence_id +` + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{ + "evidence_id": e.ID, + "assessment_status": EvidenceAssessmentStatusFailed, + "updated_at": time.Now(), + } + maps.Copy(args, scope.SQLArguments()) + + if _, err := conn.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot mark evidence assessment failed: %w", err) + } + return nil +} + func ResetStaleAssessmentProcessing( ctx context.Context, conn pg.Querier, diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index e80ff20279..36dafde2fb 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -170,13 +170,12 @@ func (h *evidenceAssessmentHandler) assessAndCommit( return fmt.Errorf("cannot describe evidence: %w", err) } - if err := evidence.SetAssessment(assessment); err != nil { - return err - } - return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { + if err := evidence.SetAssessment(assessment); err != nil { + return err + } summary := assessment.Summary evidence.Description = &summary evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusCompleted @@ -201,15 +200,7 @@ func (h *evidenceAssessmentHandler) failEvidence( return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusFailed - evidence.AssessmentProcessingStartedAt = nil - - evidence.UpdatedAt = time.Now() - if err := evidence.Update(ctx, tx, scope); err != nil { - return fmt.Errorf("cannot update evidence: %w", err) - } - - return nil + return evidence.MarkAssessmentFailed(ctx, tx, scope) }, ) } From 0171cbf59ecbc493ab4a25d958f279b7df2b716d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 23:30:34 +0200 Subject: [PATCH 07/21] Clear evidence assessment on typed-nil inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A typed-nil pointer (e.g. (*evidencedescriber.EvidenceAssessment)(nil)) wrapped in an interface does not compare equal to untyped nil, so the fast path in SetAssessment was missing it. json.Marshal would then emit the literal 'null', which we would store as 4 bytes of JSONB instead of SQL NULL. Check the marshalled output for "null" and clear the column in that case too. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence_assessment.go | 15 +++++++++++---- pkg/coredata/evidence_assessment_test.go | 14 +++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go index 286da1df34..deb6cd04c5 100644 --- a/pkg/coredata/evidence_assessment.go +++ b/pkg/coredata/evidence_assessment.go @@ -20,10 +20,13 @@ import ( ) // SetAssessment marshals a typed assessment into Evidence.Assessment. -// Passing nil clears the field. The shape of the assessment is defined -// by its producer (see pkg/evidencedescriber.EvidenceAssessment); this -// package is intentionally agnostic about the schema and only owns the -// raw JSONB round-trip. +// Passing nil (untyped or typed) clears the field; we catch the typed +// case by checking the marshalled output for the JSON literal "null" +// so callers cannot accidentally persist a null JSONB value instead of +// SQL NULL. The shape of the assessment is defined by its producer +// (see pkg/evidencedescriber.EvidenceAssessment); this package is +// intentionally agnostic about the schema and only owns the raw JSONB +// round-trip. func (e *Evidence) SetAssessment(v any) error { if v == nil { e.Assessment = nil @@ -33,6 +36,10 @@ func (e *Evidence) SetAssessment(v any) error { if err != nil { return fmt.Errorf("cannot marshal evidence assessment: %w", err) } + if string(data) == "null" { + e.Assessment = nil + return nil + } e.Assessment = data return nil } diff --git a/pkg/coredata/evidence_assessment_test.go b/pkg/coredata/evidence_assessment_test.go index 3572bc7c32..1b8670c782 100644 --- a/pkg/coredata/evidence_assessment_test.go +++ b/pkg/coredata/evidence_assessment_test.go @@ -55,7 +55,19 @@ func TestEvidence_SetAssessment_Nil_ClearsField(t *testing.T) { require.NotEmpty(t, e.Assessment) require.NoError(t, e.SetAssessment(nil)) - assert.Empty(t, e.Assessment, "nil input should clear the raw bytes") + assert.Empty(t, e.Assessment, "untyped nil should clear the raw bytes") +} + +func TestEvidence_SetAssessment_TypedNil_ClearsField(t *testing.T) { + t.Parallel() + + e := Evidence{} + require.NoError(t, e.SetAssessment(&testAssessmentPayload{Summary: "stub"})) + require.NotEmpty(t, e.Assessment) + + var typedNil *testAssessmentPayload + require.NoError(t, e.SetAssessment(typedNil)) + assert.Empty(t, e.Assessment, "typed-nil pointer should clear, not persist JSON null") } func TestEvidence_AssessmentInto_EmptyIsNoOp(t *testing.T) { From 017b62ad20b9aa90ea5b3fa39502ef621fa2a282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 23:30:40 +0200 Subject: [PATCH 08/21] Remove speculative frameworks from prompt example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readable example listed ['SOC2', 'ISO27001'] for a Google Workspace 2SV screenshot that names no framework on the file. This contradicted the prompt's own 'do not speculate' rule for the frameworks field and risked training the agent to hallucinate framework tags on any security-looking screenshot. Empty the array in the example and add a parenthetical reminder that explains why. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/evidencedescriber/prompt.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/evidencedescriber/prompt.txt b/pkg/evidencedescriber/prompt.txt index 5baf18c9ba..79979bc685 100644 --- a/pkg/evidencedescriber/prompt.txt +++ b/pkg/evidencedescriber/prompt.txt @@ -28,8 +28,8 @@ Field rules, in the order a reader will care about them: Do not guess. If a field cannot be determined from the file alone, leave it empty rather than inferring it from the filename, from the surrounding metadata, or from general knowledge about the platform. -Example — readable: -{"summary":"Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted.","system":"Google Workspace","setting":"enforced 2-step verification","scope":"organization-wide","captured_at":"","language":"en","frameworks":["SOC2","ISO27001"],"issues":[],"confidence":"HIGH","readable":true,"rejection_reason":""} +Example — readable (no frameworks named on the file, so the array is empty): +{"summary":"Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted.","system":"Google Workspace","setting":"enforced 2-step verification","scope":"organization-wide","captured_at":"","language":"en","frameworks":[],"issues":[],"confidence":"HIGH","readable":true,"rejection_reason":""} Example — unreadable: {"summary":"Unable to describe: image is blank and contains no recognisable compliance evidence.","system":"","setting":"","scope":"","captured_at":"","language":"","frameworks":[],"issues":[],"confidence":"LOW","readable":false,"rejection_reason":"Image is blank and contains no recognisable compliance evidence."} From 43205f78da12f7fc452f7eefceca7b7c080dc1ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 23:40:33 +0200 Subject: [PATCH 09/21] Rename evidencedescriber to evidenceassessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The package, agent name, struct (Describer -> Assessor), method (Describe -> Assess), config key (evidence-describer -> evidence-assessor), and worker log scope all still said 'describer' while the output is EvidenceAssessment and the column is evidences.assessment. Align the vocabulary end-to-end so a grep for 'describer' finds nothing. Also: - s/EvidenceFileId/EvidenceFileID/ for Go acronym convention. - Move MarkAssessmentFailed from a method on Evidence (used only e.ID) to a top-level helper taking the ID directly. - Detect typed-nil in Evidence.SetAssessment via reflect so a (*T)(nil) caller no longer persists JSON 'null' instead of SQL NULL. Rename AssessmentInto -> GetAssessment. - Pass evidence by value to assessAndCommit so a failed commit cannot leak partial in-memory mutations to failEvidence. - Trim the prompt: field-by-field descriptions already live on the jsonschema struct tags and were duplicated verbatim. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/bootstrap/builder.go | 4 +- pkg/bootstrap/builder_test.go | 16 ++-- pkg/coredata/evidence.go | 25 +++--- pkg/coredata/evidence_assessment.go | 39 +++++---- pkg/coredata/evidence_assessment_test.go | 6 +- .../assessment.go | 81 +++++++++---------- .../assessment_test.go | 6 +- pkg/evidenceassessor/prompt.txt | 21 +++++ pkg/evidencedescriber/prompt.txt | 35 -------- pkg/probo/evidence_assessment_worker.go | 35 ++++---- pkg/probo/evidence_service.go | 2 +- pkg/probo/framework_service.go | 4 +- pkg/probod/aliases.go | 2 +- pkg/probod/probod.go | 32 ++++---- pkg/probodconfig/config.go | 2 +- pkg/probodconfig/llm_config.go | 22 ++--- pkg/server/api/console/v1/types/evidence.go | 4 +- 17 files changed, 166 insertions(+), 170 deletions(-) rename pkg/{evidencedescriber => evidenceassessor}/assessment.go (59%) rename pkg/{evidencedescriber => evidenceassessor}/assessment_test.go (92%) create mode 100644 pkg/evidenceassessor/prompt.txt delete mode 100644 pkg/evidencedescriber/prompt.txt diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 42a729e386..1b383d9cfe 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -200,7 +200,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { Temperature: b.getEnvFloatPtr("AGENT_PROBO_TEMPERATURE"), MaxTokens: b.getEnvIntPtr("AGENT_PROBO_MAX_TOKENS"), }, - EvidenceDescriber: probodconfig.LLMAgentConfig{ + EvidenceAssessor: probodconfig.LLMAgentConfig{ Provider: b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_PROVIDER", ""), ModelName: b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_MODEL_NAME", ""), Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"), @@ -247,7 +247,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { ESign: probodconfig.ESignConfig{ TSAURL: b.getEnvOrDefault("ESIGN_TSA_URL", "http://timestamp.digicert.com"), }, - EvidenceDescriber: probodconfig.EvidenceDescriberConfig{ + EvidenceAssessor: probodconfig.EvidenceAssessmentConfig{ Interval: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_INTERVAL", 10), StaleAfter: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_STALE_AFTER", 300), MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_MAX_CONCURRENCY", 10), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index 3529489425..d0297deebf 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -211,10 +211,10 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Empty(t, cfg.Probod.Agents.Probo.ModelName) assert.Nil(t, cfg.Probod.Agents.Probo.Temperature) assert.Nil(t, cfg.Probod.Agents.Probo.MaxTokens) - assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.Provider) - assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.ModelName) - assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.Temperature) - assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.MaxTokens) + assert.Empty(t, cfg.Probod.Agents.EvidenceAssessor.Provider) + assert.Empty(t, cfg.Probod.Agents.EvidenceAssessor.ModelName) + assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.Temperature) + assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.MaxTokens) assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.Provider) assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.ModelName) assert.Nil(t, cfg.Probod.Agents.ThirdPartyVetter.Temperature) @@ -433,10 +433,10 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Empty(t, cfg.Probod.Agents.Probo.Provider) assert.Empty(t, cfg.Probod.Agents.Probo.ModelName) // Agents — evidence-describer overrides - assert.Equal(t, "anthropic", cfg.Probod.Agents.EvidenceDescriber.Provider) - assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceDescriber.ModelName) - assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceDescriber.Temperature) - assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceDescriber.MaxTokens) + assert.Equal(t, "anthropic", cfg.Probod.Agents.EvidenceAssessor.Provider) + assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceAssessor.ModelName) + assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceAssessor.Temperature) + assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceAssessor.MaxTokens) // Agents — third-party-vetter overrides assert.Equal(t, "openai", cfg.Probod.Agents.ThirdPartyVetter.Provider) assert.Equal(t, "gpt-4o", cfg.Probod.Agents.ThirdPartyVetter.ModelName) diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index c603a49f58..9c9fcc855a 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -39,7 +39,7 @@ type ( ReferenceID string `db:"reference_id"` Type EvidenceType `db:"type"` URL string `db:"url"` - EvidenceFileId *gid.GID `db:"evidence_file_id"` + EvidenceFileID *gid.GID `db:"evidence_file_id"` Description *string `db:"description"` Assessment jsonRawMessageOrNull `db:"assessment"` AssessmentStatus EvidenceAssessmentStatus `db:"assessment_status"` @@ -154,7 +154,7 @@ WHERE evidences.state = 'REQUESTED'; "measure_id": e.MeasureID, "task_id": e.TaskID, "reference_id": e.ReferenceID, - "evidence_file_id": e.EvidenceFileId, + "evidence_file_id": e.EvidenceFileID, "created_at": e.CreatedAt, "updated_at": e.UpdatedAt, "state": e.State, @@ -225,7 +225,7 @@ VALUES ( "measure_id": e.MeasureID, "task_id": e.TaskID, "reference_id": e.ReferenceID, - "evidence_file_id": e.EvidenceFileId, + "evidence_file_id": e.EvidenceFileID, "created_at": e.CreatedAt, "updated_at": e.UpdatedAt, "state": e.State, @@ -503,7 +503,7 @@ WHERE "evidence_id": e.ID, "type": e.Type, "state": e.State, - "evidence_file_id": e.EvidenceFileId, + "evidence_file_id": e.EvidenceFileID, "url": e.URL, "description": e.Description, "assessment": e.Assessment.Arg(), @@ -598,17 +598,16 @@ FOR UPDATE SKIP LOCKED; return nil } -// MarkAssessmentFailed transitions the evidence row to FAILED without -// touching description, assessment, or any other column. The worker's -// in-memory evidence pointer may carry mutations from a partially -// successful assess-and-commit (e.g. a new assessment that could not be -// persisted); using the full Update method would write those back with -// FAILED status. This targeted statement guarantees the failure path -// only changes state-transition columns. -func (e Evidence) MarkAssessmentFailed( +// MarkEvidenceAssessmentFailed transitions the evidence row to FAILED +// without touching description, assessment, or any other column. It +// only needs the evidence ID, so it is a top-level func rather than a +// method on Evidence — callers should not need to load the row first +// when all they have is the ID. +func MarkEvidenceAssessmentFailed( ctx context.Context, conn pg.Tx, scope Scoper, + evidenceID gid.GID, ) error { q := ` UPDATE @@ -625,7 +624,7 @@ WHERE q = fmt.Sprintf(q, scope.SQLFragment()) args := pgx.StrictNamedArgs{ - "evidence_id": e.ID, + "evidence_id": evidenceID, "assessment_status": EvidenceAssessmentStatusFailed, "updated_at": time.Now(), } diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go index deb6cd04c5..33d1a2be8e 100644 --- a/pkg/coredata/evidence_assessment.go +++ b/pkg/coredata/evidence_assessment.go @@ -17,18 +17,18 @@ package coredata import ( "encoding/json" "fmt" + "reflect" ) // SetAssessment marshals a typed assessment into Evidence.Assessment. -// Passing nil (untyped or typed) clears the field; we catch the typed -// case by checking the marshalled output for the JSON literal "null" -// so callers cannot accidentally persist a null JSONB value instead of -// SQL NULL. The shape of the assessment is defined by its producer -// (see pkg/evidencedescriber.EvidenceAssessment); this package is -// intentionally agnostic about the schema and only owns the raw JSONB -// round-trip. +// Passing an untyped or typed nil clears the field; the typed case is +// detected via reflection so callers cannot accidentally persist JSON +// null instead of SQL NULL. The shape of the assessment is defined by +// its producer (see pkg/evidenceassessor.EvidenceAssessment); this +// package is intentionally agnostic about the schema and only owns the +// raw JSONB round-trip. func (e *Evidence) SetAssessment(v any) error { - if v == nil { + if isNil(v) { e.Assessment = nil return nil } @@ -36,17 +36,13 @@ func (e *Evidence) SetAssessment(v any) error { if err != nil { return fmt.Errorf("cannot marshal evidence assessment: %w", err) } - if string(data) == "null" { - e.Assessment = nil - return nil - } e.Assessment = data return nil } -// AssessmentInto unmarshals Evidence.Assessment into dst. It is a no-op +// GetAssessment unmarshals Evidence.Assessment into dst. It is a no-op // when the column is NULL/empty, leaving dst untouched. -func (e *Evidence) AssessmentInto(dst any) error { +func (e *Evidence) GetAssessment(dst any) error { if len(e.Assessment) == 0 { return nil } @@ -55,3 +51,18 @@ func (e *Evidence) AssessmentInto(dst any) error { } return nil } + +// isNil returns true for untyped-nil and for typed-nil pointers, maps, +// slices, channels, funcs, and interfaces. Plain struct values, zero +// ints, and empty strings are not considered nil. +func isNil(v any) bool { + if v == nil { + return true + } + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Pointer, reflect.Map, reflect.Slice, reflect.Chan, reflect.Func, reflect.Interface: + return rv.IsNil() + } + return false +} diff --git a/pkg/coredata/evidence_assessment_test.go b/pkg/coredata/evidence_assessment_test.go index 1b8670c782..c7126bb905 100644 --- a/pkg/coredata/evidence_assessment_test.go +++ b/pkg/coredata/evidence_assessment_test.go @@ -43,7 +43,7 @@ func TestEvidence_SetAssessment_ReadBack(t *testing.T) { require.NotEmpty(t, e.Assessment, "Assessment raw bytes should be populated") var out testAssessmentPayload - require.NoError(t, e.AssessmentInto(&out)) + require.NoError(t, e.GetAssessment(&out)) assert.Equal(t, *in, out) } @@ -70,11 +70,11 @@ func TestEvidence_SetAssessment_TypedNil_ClearsField(t *testing.T) { assert.Empty(t, e.Assessment, "typed-nil pointer should clear, not persist JSON null") } -func TestEvidence_AssessmentInto_EmptyIsNoOp(t *testing.T) { +func TestEvidence_GetAssessment_EmptyIsNoOp(t *testing.T) { t.Parallel() var e Evidence out := testAssessmentPayload{Summary: "unchanged"} - require.NoError(t, e.AssessmentInto(&out)) + require.NoError(t, e.GetAssessment(&out)) assert.Equal(t, "unchanged", out.Summary, "empty column should leave dst untouched") } diff --git a/pkg/evidencedescriber/assessment.go b/pkg/evidenceassessor/assessment.go similarity index 59% rename from pkg/evidencedescriber/assessment.go rename to pkg/evidenceassessor/assessment.go index b706a8cb81..a81ab2ce82 100644 --- a/pkg/evidencedescriber/assessment.go +++ b/pkg/evidenceassessor/assessment.go @@ -12,7 +12,7 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -package evidencedescriber +package evidenceassessor import ( "context" @@ -28,28 +28,28 @@ import ( //go:embed prompt.txt var systemPrompt string -// EvidenceAssessmentConfidenceEnum is the canonical set of allowed -// values for EvidenceAssessment.Confidence. It is injected into the -// generated JSON schema via agent.OutputType.DecorateEnum because -// jsonschema struct tags cannot encode enum constraints directly. -var evidenceAssessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} +// assessmentConfidenceEnum is the canonical set of allowed values for +// EvidenceAssessment.Confidence. Injected into the generated JSON +// schema via agent.OutputType.DecorateEnum because jsonschema struct +// tags cannot encode enum constraints directly. +var assessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} type ( // EvidenceAssessment is the structured output produced by the - // evidence describer. The worker persists it as JSONB on the + // evidence assessor. The worker persists it as JSONB on the // evidences table and derives Evidence.Description from Summary. EvidenceAssessment struct { - Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows; no markdown, no preamble. When readable is false this field must restate the rejection reason so downstream systems displaying only the summary still inform the user."` - System string `json:"system" jsonschema:"Tool or platform shown (e.g. 'Google Workspace', 'GitHub', 'AWS IAM'); empty string if not identifiable."` - Setting string `json:"setting" jsonschema:"Specific configuration, feature, or state demonstrated; empty string if not identifiable."` - Scope string `json:"scope" jsonschema:"Who or what the evidence applies to (e.g. 'organization-wide', 'all users', 'repository acme/foo'); empty string if not stated."` - CapturedAt string `json:"captured_at" jsonschema:"ISO-8601 date or datetime visible on the file; empty string if no date is shown."` - Language string `json:"language" jsonschema:"BCP-47 language tag of the visible text (e.g. 'en', 'fr'); empty string if unclear."` - Frameworks []string `json:"frameworks" jsonschema:"Compliance frameworks the evidence is plausibly relevant to (e.g. 'SOC2', 'ISO27001'); empty when unclear."` - Issues []string `json:"issues" jsonschema:"Quality problems observed: redacted fields, crop, stale date, low resolution, etc."` - Confidence string `json:"confidence" jsonschema:"Overall confidence in the System / Setting / Scope identification; one of HIGH, MEDIUM, LOW."` - Readable bool `json:"readable" jsonschema:"True if the file is readable compliance evidence; false if unreadable or off-topic."` - RejectionReason string `json:"rejection_reason" jsonschema:"Set only when readable is false; explains why the file is not usable evidence."` + Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows. When readable is false this field must restate the rejection reason."` + System string `json:"system" jsonschema:"Tool or platform visible on the file; empty string if not clearly identifiable."` + Setting string `json:"setting" jsonschema:"Specific configuration or state demonstrated; empty string if not clearly identifiable."` + Scope string `json:"scope" jsonschema:"Who or what the setting applies to; empty string if not stated."` + CapturedAt string `json:"captured_at" jsonschema:"ISO-8601 date or datetime visible on the file; empty string when no date is shown."` + Language string `json:"language" jsonschema:"BCP-47 language tag of the visible text; empty when unclear."` + Frameworks []string `json:"frameworks" jsonschema:"Compliance frameworks explicitly named on the file; empty when unclear."` + Issues []string `json:"issues" jsonschema:"Quality problems on the file itself; empty when the file is clean."` + Confidence string `json:"confidence" jsonschema:"One of HIGH, MEDIUM, LOW."` + Readable bool `json:"readable" jsonschema:"True if the file is a usable piece of compliance evidence."` + RejectionReason string `json:"rejection_reason" jsonschema:"Populated only when readable is false; empty otherwise."` } Config struct { @@ -63,28 +63,27 @@ type ( Logger *log.Logger } - Describer struct { + Assessor struct { cfg Config outputType *agent.OutputType } ) -// New builds a Describer. The structured output type is decorated once -// (enum on confidence) and cached on the Describer so every call reuses -// the same schema. -func New(cfg Config) (*Describer, error) { +// New builds an Assessor. The structured output type is decorated once +// (enum on confidence) and cached so every call reuses the same schema. +func New(cfg Config) (*Assessor, error) { ot, err := assessmentOutputType() if err != nil { return nil, fmt.Errorf("cannot build evidence assessment output type: %w", err) } - return &Describer{cfg: cfg, outputType: ot}, nil + return &Assessor{cfg: cfg, outputType: ot}, nil } -// Describe runs the evidence describer agent against a single uploaded +// Assess runs the evidence assessor agent against a single uploaded // file and returns a structured assessment. The worker persists the // result as JSONB and derives Evidence.Description from Summary. -func (d *Describer) Describe( +func (a *Assessor) Assess( ctx context.Context, filename string, mimeType string, @@ -92,19 +91,19 @@ func (d *Describer) Describe( ) (*EvidenceAssessment, error) { opts := []agent.Option{ agent.WithInstructions(systemPrompt), - agent.WithModel(d.cfg.Model), - agent.WithTemperature(d.cfg.Temp), - agent.WithMaxTokens(d.cfg.MaxTokens), - agent.WithOutputType(d.outputType), + agent.WithModel(a.cfg.Model), + agent.WithTemperature(a.cfg.Temp), + agent.WithMaxTokens(a.cfg.MaxTokens), + agent.WithOutputType(a.outputType), } - if d.cfg.Thinking > 0 { - opts = append(opts, agent.WithThinking(d.cfg.Thinking)) + if a.cfg.Thinking > 0 { + opts = append(opts, agent.WithThinking(a.cfg.Thinking)) } - if d.cfg.Logger != nil { - opts = append(opts, agent.WithLogger(d.cfg.Logger)) + if a.cfg.Logger != nil { + opts = append(opts, agent.WithLogger(a.cfg.Logger)) } - ag := agent.New("evidence_describer", d.cfg.Client, opts...) + ag := agent.New("evidence_assessor", a.cfg.Client, opts...) result, err := ag.Run(ctx, []llm.Message{ { @@ -120,27 +119,27 @@ func (d *Describer) Describe( }, }) if err != nil { - return nil, fmt.Errorf("cannot describe evidence: %w", err) + return nil, fmt.Errorf("cannot assess evidence: %w", err) } - var a EvidenceAssessment - if err := json.Unmarshal([]byte(result.FinalMessage().Text()), &a); err != nil { + var out EvidenceAssessment + if err := json.Unmarshal([]byte(result.FinalMessage().Text()), &out); err != nil { return nil, fmt.Errorf("cannot parse evidence assessment: %w", err) } - return &a, nil + return &out, nil } // assessmentOutputType builds the EvidenceAssessment structured output // type and decorates its JSON Schema with an explicit enum on the -// `confidence` field. +// confidence field. func assessmentOutputType() (*agent.OutputType, error) { ot, err := agent.NewOutputType[EvidenceAssessment]("evidence_assessment") if err != nil { return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) } if err := ot.DecorateEnum(map[string][]string{ - "confidence": evidenceAssessmentConfidenceEnum, + "confidence": assessmentConfidenceEnum, }); err != nil { return nil, fmt.Errorf("cannot decorate evidence assessment schema: %w", err) } diff --git a/pkg/evidencedescriber/assessment_test.go b/pkg/evidenceassessor/assessment_test.go similarity index 92% rename from pkg/evidencedescriber/assessment_test.go rename to pkg/evidenceassessor/assessment_test.go index 0f9dcfb976..83ffd40e18 100644 --- a/pkg/evidencedescriber/assessment_test.go +++ b/pkg/evidenceassessor/assessment_test.go @@ -12,10 +12,10 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -// White-box test (package evidencedescriber, not _test) so it can reach +// White-box test (package evidenceassessor, not _test) so it can reach // the unexported assessmentOutputType helper. -package evidencedescriber +package evidenceassessor import ( "encoding/json" @@ -53,5 +53,5 @@ func TestAssessmentOutputType_DecoratesConfidenceEnum(t *testing.T) { for i, v := range enumRaw { actual[i] = v.(string) } - assert.Equal(t, evidenceAssessmentConfidenceEnum, actual) + assert.Equal(t, assessmentConfidenceEnum, actual) } diff --git a/pkg/evidenceassessor/prompt.txt b/pkg/evidenceassessor/prompt.txt new file mode 100644 index 0000000000..e04c352222 --- /dev/null +++ b/pkg/evidenceassessor/prompt.txt @@ -0,0 +1,21 @@ +You are an ISO/SOC auditor producing a structured record for one piece of compliance evidence (a screenshot, PDF, or document export). Your reader is another auditor; write for a compliance review, not a user. + +Output a single JSON object that conforms exactly to the provided schema. Emit nothing outside the object — no prose, no markdown, no code fences, no preamble. + +Rules that go beyond the schema: + +- Do not speculate. If a field cannot be determined from the file alone, leave it empty rather than inferring it from the filename, from the surrounding metadata, or from general knowledge about the platform. This applies especially to `frameworks`: only list a framework when it is explicitly named on the file, or strongly implied by a recognisable domain artifact (e.g. an AWS Config dashboard for SOC2). + +- `summary` is the only free-text field a downstream system is guaranteed to display, so it must be self-sufficient: one sentence (two at most), plain text, no markdown, no line breaks, no filenames, no image-quality comments, no phrases like "this screenshot shows". Use the language of the document. + +- When `readable` is false, `summary` must restate the rejection reason in a human-readable sentence. `rejection_reason` is populated in that case and must be empty when `readable` is true. + +- `confidence` = HIGH only when all three of `system`, `setting`, and `scope` are unambiguous on the file. MEDIUM when at most one is inferred or partially visible. LOW otherwise. + +- `readable` = false when the file is unreadable (blank, corrupt, mis-OCRed), off-topic (an invoice, a marketing page, a cat picture), or when the content is compliance-adjacent but too ambiguous to describe. + +Example — readable (no frameworks named on the file, so the array is empty): +{"summary":"Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted.","system":"Google Workspace","setting":"enforced 2-step verification","scope":"organization-wide","captured_at":"","language":"en","frameworks":[],"issues":[],"confidence":"HIGH","readable":true,"rejection_reason":""} + +Example — unreadable: +{"summary":"Unable to describe: image is blank and contains no recognisable compliance evidence.","system":"","setting":"","scope":"","captured_at":"","language":"","frameworks":[],"issues":[],"confidence":"LOW","readable":false,"rejection_reason":"Image is blank and contains no recognisable compliance evidence."} diff --git a/pkg/evidencedescriber/prompt.txt b/pkg/evidencedescriber/prompt.txt deleted file mode 100644 index 79979bc685..0000000000 --- a/pkg/evidencedescriber/prompt.txt +++ /dev/null @@ -1,35 +0,0 @@ -You are an ISO/SOC auditor producing a structured record for one piece of compliance evidence (a screenshot, PDF, or document export). Your reader is another auditor; write for a compliance review, not a user. - -Output a single JSON object that conforms exactly to the provided schema. Emit nothing outside the object — no prose, no markdown, no code fences, no preamble. - -Field rules, in the order a reader will care about them: - -- summary: one sentence, two at most. Plain text. No markdown, no line breaks, no labels, no filenames, no image-quality comments, no phrases like "this screenshot shows". Use the language of the document. When `readable` is false, `summary` must restate the rejection reason in a human-readable sentence so that downstream systems displaying only the summary still inform the reader. - -- system: the tool or platform visible on the file (e.g. "Google Workspace", "GitHub", "AWS IAM"). Empty string if not clearly identifiable — never guess. - -- setting: the specific configuration, feature, or state demonstrated (e.g. "enforced 2-step verification", "branch protection requiring review"). Empty string if not clearly identifiable. - -- scope: who or what the setting applies to (e.g. "organization-wide", "all users", "repository acme/foo", "production environment"). Empty string if not stated. - -- captured_at: ISO-8601 date or datetime if visible on the file (timestamps, "last updated" labels, status dates). Empty string if no date is shown. - -- language: BCP-47 tag of the visible text ("en", "fr", "de"). Empty string if unclear. - -- frameworks: compliance frameworks the evidence is plausibly relevant to. Only include a framework when it is explicitly named on the file (e.g. "SOC2 report summary") or strongly implied by a recognisable domain artifact (e.g. an AWS Config dashboard for SOC2). Empty array when unclear — do not speculate. - -- issues: quality problems observed on the file itself: redactions covering key fields, cropped content, low resolution, stale date, illegible screenshot. Empty array when the file is clean. - -- confidence: HIGH only when all three of `system`, `setting`, and `scope` are unambiguous on the file. MEDIUM when at most one of the three is inferred or partially visible. LOW otherwise. The schema enforces the enum; pick one. - -- readable: true when the file is a usable piece of compliance evidence. false when the file is unreadable (blank, corrupt, mis-OCRed), off-topic (an invoice, a marketing page, a cat picture), or when the content is compliance-adjacent but too ambiguous to describe (e.g. a blurred screenshot of an unlabeled panel). - -- rejection_reason: populated only when `readable` is false. Explain why the file is not usable evidence in one short sentence. When `readable` is true this field must be empty. - -Do not guess. If a field cannot be determined from the file alone, leave it empty rather than inferring it from the filename, from the surrounding metadata, or from general knowledge about the platform. - -Example — readable (no frameworks named on the file, so the array is empty): -{"summary":"Google Workspace admin console showing enforced 2-step verification for all users in the organization, with no exceptions permitted.","system":"Google Workspace","setting":"enforced 2-step verification","scope":"organization-wide","captured_at":"","language":"en","frameworks":[],"issues":[],"confidence":"HIGH","readable":true,"rejection_reason":""} - -Example — unreadable: -{"summary":"Unable to describe: image is blank and contains no recognisable compliance evidence.","system":"","setting":"","scope":"","captured_at":"","language":"","frameworks":[],"issues":[],"confidence":"LOW","readable":false,"rejection_reason":"Image is blank and contains no recognisable compliance evidence."} diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 36dafde2fb..b2d532a0fd 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -24,15 +24,16 @@ import ( "go.gearno.de/kit/pg" "go.gearno.de/kit/worker" "go.probo.inc/probo/pkg/coredata" - "go.probo.inc/probo/pkg/evidencedescriber" + "go.probo.inc/probo/pkg/evidenceassessor" "go.probo.inc/probo/pkg/filemanager" + "go.probo.inc/probo/pkg/gid" ) type ( evidenceAssessmentHandler struct { pg *pg.Client fileManager *filemanager.Service - describer *evidencedescriber.Describer + assessor *evidenceassessor.Assessor logger *log.Logger staleAfter time.Duration } @@ -45,7 +46,7 @@ type ( func NewEvidenceAssessmentWorker( pgClient *pg.Client, fileManager *filemanager.Service, - describer *evidencedescriber.Describer, + assessor *evidenceassessor.Assessor, logger *log.Logger, cfg EvidenceAssessmentWorkerConfig, opts ...worker.Option, @@ -58,7 +59,7 @@ func NewEvidenceAssessmentWorker( h := &evidenceAssessmentHandler{ pg: pgClient, fileManager: fileManager, - describer: describer, + assessor: assessor, logger: logger, staleAfter: staleAfter, } @@ -104,7 +105,7 @@ func (h *evidenceAssessmentHandler) Claim(ctx context.Context) (coredata.Evidenc } func (h *evidenceAssessmentHandler) Process(ctx context.Context, evidence coredata.Evidence) error { - if err := h.assessAndCommit(ctx, &evidence); err != nil { + if err := h.assessAndCommit(ctx, evidence); err != nil { h.logger.ErrorCtx( ctx, "evidence assessment worker failure", @@ -112,7 +113,7 @@ func (h *evidenceAssessmentHandler) Process(ctx context.Context, evidence coreda log.String("evidence_id", evidence.ID.String()), ) - if err := h.failEvidence(ctx, &evidence); err != nil { + if err := h.failEvidence(ctx, evidence.ID); err != nil { h.logger.ErrorCtx(ctx, "cannot mark evidence assessment as failed", log.Error(err)) } @@ -135,11 +136,14 @@ func (h *evidenceAssessmentHandler) RecoverStale(ctx context.Context) error { ) } +// assessAndCommit deliberately takes evidence by value; mutations made +// inside the transaction stay local, so a failed commit cannot leak +// partial state to the subsequent failEvidence call. func (h *evidenceAssessmentHandler) assessAndCommit( ctx context.Context, - evidence *coredata.Evidence, + evidence coredata.Evidence, ) error { - if evidence.EvidenceFileId == nil { + if evidence.EvidenceFileID == nil { return fmt.Errorf("cannot assess evidence %s: no file attached", evidence.ID) } @@ -150,7 +154,7 @@ func (h *evidenceAssessmentHandler) assessAndCommit( if err := h.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - if err := file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileId); err != nil { + if err := file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileID); err != nil { return fmt.Errorf("cannot load file: %w", err) } @@ -165,9 +169,9 @@ func (h *evidenceAssessmentHandler) assessAndCommit( return fmt.Errorf("cannot download file: %w", err) } - assessment, err := h.describer.Describe(ctx, file.FileName, mimeType, base64Data) + assessment, err := h.assessor.Assess(ctx, file.FileName, mimeType, base64Data) if err != nil { - return fmt.Errorf("cannot describe evidence: %w", err) + return fmt.Errorf("cannot assess evidence: %w", err) } return h.pg.WithTx( @@ -191,16 +195,13 @@ func (h *evidenceAssessmentHandler) assessAndCommit( ) } -func (h *evidenceAssessmentHandler) failEvidence( - ctx context.Context, - evidence *coredata.Evidence, -) error { - scope := coredata.NewScopeFromObjectID(evidence.ID) +func (h *evidenceAssessmentHandler) failEvidence(ctx context.Context, evidenceID gid.GID) error { + scope := coredata.NewScopeFromObjectID(evidenceID) return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - return evidence.MarkAssessmentFailed(ctx, tx, scope) + return coredata.MarkEvidenceAssessmentFailed(ctx, tx, scope, evidenceID) }, ) } diff --git a/pkg/probo/evidence_service.go b/pkg/probo/evidence_service.go index e7c388df3c..060de1e45b 100644 --- a/pkg/probo/evidence_service.go +++ b/pkg/probo/evidence_service.go @@ -130,7 +130,7 @@ func (s EvidenceService) UploadMeasureEvidence( } evidence.OrganizationID = measure.OrganizationID - evidence.EvidenceFileId = &file.ID + evidence.EvidenceFileID = &file.ID evidence.MeasureID = req.MeasureID if err := evidence.Insert(ctx, conn, scope); err != nil { diff --git a/pkg/probo/framework_service.go b/pkg/probo/framework_service.go index f949d6f740..3096a25515 100644 --- a/pkg/probo/framework_service.go +++ b/pkg/probo/framework_service.go @@ -248,12 +248,12 @@ func (s FrameworkService) Export( for _, evidence := range evidences { if evidence.Type != coredata.EvidenceTypeFile || evidence.State != coredata.EvidenceStateFulfilled || - evidence.EvidenceFileId == nil { + evidence.EvidenceFileID == nil { continue } evidence_file := &coredata.File{} - if err := evidence_file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileId); err != nil { + if err := evidence_file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileID); err != nil { return fmt.Errorf("cannot load evidence file: %w", err) } diff --git a/pkg/probod/aliases.go b/pkg/probod/aliases.go index d2a30d28df..c1f14994d4 100644 --- a/pkg/probod/aliases.go +++ b/pkg/probod/aliases.go @@ -39,7 +39,7 @@ type ( ACMEConfig = probodconfig.ACMEConfig LLMProviderConfig = probodconfig.LLMProviderConfig LLMAgentConfig = probodconfig.LLMAgentConfig - EvidenceDescriberConfig = probodconfig.EvidenceDescriberConfig + EvidenceAssessmentConfig = probodconfig.EvidenceAssessmentConfig ThirdPartyVettingWorkerConfig = probodconfig.ThirdPartyVettingWorkerConfig AgentsConfig = probodconfig.AgentsConfig diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index d73b242da4..e04251ce4c 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -55,7 +55,7 @@ import ( "go.probo.inc/probo/pkg/crypto/passwdhash" pemutil "go.probo.inc/probo/pkg/crypto/pem" "go.probo.inc/probo/pkg/esign" - "go.probo.inc/probo/pkg/evidencedescriber" + "go.probo.inc/probo/pkg/evidenceassessor" "go.probo.inc/probo/pkg/file" "go.probo.inc/probo/pkg/filemanager" "go.probo.inc/probo/pkg/filesign" @@ -173,7 +173,7 @@ func New() *Implm { TSAURL: "http://timestamp.digicert.com", }, Branding: true, - EvidenceDescriber: EvidenceDescriberConfig{ + EvidenceAssessor: EvidenceAssessmentConfig{ Interval: 10, StaleAfter: 300, MaxConcurrency: 10, @@ -310,7 +310,7 @@ func (impl *Implm) Run( return err } - evidenceDescriberAgentCfg, evidenceDescriberLLMClient, err := impl.resolveAgentClient("evidence-describer", impl.cfg.Agents.EvidenceDescriber, l, tp, r) + evidenceAssessorAgentCfg, evidenceAssessorLLMClient, err := impl.resolveAgentClient("evidence-assessor", impl.cfg.Agents.EvidenceAssessor, l, tp, r) if err != nil { return err } @@ -838,30 +838,30 @@ func (impl *Implm) Run( }, ) - evidenceDescriberCfg := evidencedescriber.Config{ - Client: evidenceDescriberLLMClient, - Model: evidenceDescriberAgentCfg.ModelName, - Temp: ref.UnrefOrZero(evidenceDescriberAgentCfg.Temperature), - MaxTokens: ref.UnrefOrZero(evidenceDescriberAgentCfg.MaxTokens), - Logger: l.Named("evidence-describer"), + evidenceAssessorCfg := evidenceassessor.Config{ + Client: evidenceAssessorLLMClient, + Model: evidenceAssessorAgentCfg.ModelName, + Temp: ref.UnrefOrZero(evidenceAssessorAgentCfg.Temperature), + MaxTokens: ref.UnrefOrZero(evidenceAssessorAgentCfg.MaxTokens), + Logger: l.Named("evidence-assessor"), } - if evidenceDescriberAgentCfg.Thinking != nil { - evidenceDescriberCfg.Thinking = *evidenceDescriberAgentCfg.Thinking + if evidenceAssessorAgentCfg.Thinking != nil { + evidenceAssessorCfg.Thinking = *evidenceAssessorAgentCfg.Thinking } - evidenceDescriber, err := evidencedescriber.New(evidenceDescriberCfg) + evidenceAssessor, err := evidenceassessor.New(evidenceAssessorCfg) if err != nil { return fmt.Errorf("cannot build evidence describer: %w", err) } evidenceAssessmentWorker := probo.NewEvidenceAssessmentWorker( pgClient, fileManagerService, - evidenceDescriber, + evidenceAssessor, l.Named("evidence-assessment-worker"), probo.EvidenceAssessmentWorkerConfig{ - StaleAfter: time.Duration(impl.cfg.EvidenceDescriber.StaleAfter) * time.Second, + StaleAfter: time.Duration(impl.cfg.EvidenceAssessor.StaleAfter) * time.Second, }, - worker.WithInterval(time.Duration(impl.cfg.EvidenceDescriber.Interval)*time.Second), - worker.WithMaxConcurrency(impl.cfg.EvidenceDescriber.MaxConcurrency), + worker.WithInterval(time.Duration(impl.cfg.EvidenceAssessor.Interval)*time.Second), + worker.WithMaxConcurrency(impl.cfg.EvidenceAssessor.MaxConcurrency), ) evidenceAssessmentWorkerCtx, stopEvidenceAssessmentWorker := context.WithCancel(context.Background()) diff --git a/pkg/probodconfig/config.go b/pkg/probodconfig/config.go index 2334d8d6fa..6cb9d7545d 100644 --- a/pkg/probodconfig/config.go +++ b/pkg/probodconfig/config.go @@ -59,7 +59,7 @@ type ( Notifications NotificationsConfig `json:"notifications"` Connectors []ConnectorConfig `json:"connectors"` Agents AgentsConfig `json:"llm"` - EvidenceDescriber EvidenceDescriberConfig `json:"evidence-describer"` + EvidenceAssessor EvidenceAssessmentConfig `json:"evidence-assessor"` ThirdPartyVetting ThirdPartyVettingWorkerConfig `json:"third-party-vetting-worker"` TrackerMappingWorker TrackerMappingWorkerConfig `json:"tracker-mapping-worker"` diff --git a/pkg/probodconfig/llm_config.go b/pkg/probodconfig/llm_config.go index 9677b1a88d..9969d0cf7c 100644 --- a/pkg/probodconfig/llm_config.go +++ b/pkg/probodconfig/llm_config.go @@ -36,10 +36,10 @@ type ( Thinking *int `json:"thinking"` } - // EvidenceDescriberConfig holds worker-side tuning for the evidence - // description background worker. LLM parameters for the same worker - // live under AgentsConfig.EvidenceDescriber. - EvidenceDescriberConfig struct { + // EvidenceAssessmentConfig holds worker-side tuning for the evidence + // assessment background worker. LLM parameters for the same worker + // live under AgentsConfig.EvidenceAssessor. + EvidenceAssessmentConfig struct { Interval int `json:"interval"` // seconds between polls StaleAfter int `json:"stale-after"` // seconds before a claim is recycled MaxConcurrency int `json:"max-concurrency"` @@ -89,13 +89,13 @@ type ( // settings. Default is used as a fallback when an agent-specific field // is zero-valued. AgentsConfig struct { - Providers map[string]LLMProviderConfig `json:"providers"` - Default LLMAgentConfig `json:"defaults"` - Probo LLMAgentConfig `json:"probo"` - EvidenceDescriber LLMAgentConfig `json:"evidence-describer"` - ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` - TrackerMapping LLMAgentConfig `json:"tracker-mapping"` - Tools AgentToolsConfig `json:"tools"` + Providers map[string]LLMProviderConfig `json:"providers"` + Default LLMAgentConfig `json:"defaults"` + Probo LLMAgentConfig `json:"probo"` + EvidenceAssessor LLMAgentConfig `json:"evidence-assessor"` + ThirdPartyVetter LLMAgentConfig `json:"third-party-vetter"` + TrackerMapping LLMAgentConfig `json:"tracker-mapping"` + Tools AgentToolsConfig `json:"tools"` } ) diff --git a/pkg/server/api/console/v1/types/evidence.go b/pkg/server/api/console/v1/types/evidence.go index ea4c8f9842..20e4396238 100644 --- a/pkg/server/api/console/v1/types/evidence.go +++ b/pkg/server/api/console/v1/types/evidence.go @@ -81,9 +81,9 @@ func NewEvidence(e *coredata.Evidence) *Evidence { UpdatedAt: e.UpdatedAt, } - if e.EvidenceFileId != nil { + if e.EvidenceFileID != nil { evidence.File = &File{ - ID: *e.EvidenceFileId, + ID: *e.EvidenceFileID, } } From 5fa78faa1230d7db0af4f53c05b7d3d5ae0570d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 24 Apr 2026 23:40:39 +0200 Subject: [PATCH 10/21] Split evidence assessment migration by concern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single migration bundled an additive ADD COLUMN, two RENAME COLUMN statements, and a RENAME TYPE. Unrelated DDL changes in one file make rollback and history review harder. Split into three files, each with a single concern: - add evidences.assessment column - rename description_status / description_processing_started_at - rename the evidence_description_status enum type Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/migrations/20260424T418273Z.sql | 16 ++++++++++++++++ ...20260424T637521Z.sql => 20260424T527194Z.sql} | 5 ----- pkg/coredata/migrations/20260424T603849Z.sql | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 pkg/coredata/migrations/20260424T418273Z.sql rename pkg/coredata/migrations/{20260424T637521Z.sql => 20260424T527194Z.sql} (87%) create mode 100644 pkg/coredata/migrations/20260424T603849Z.sql diff --git a/pkg/coredata/migrations/20260424T418273Z.sql b/pkg/coredata/migrations/20260424T418273Z.sql new file mode 100644 index 0000000000..7f2245be64 --- /dev/null +++ b/pkg/coredata/migrations/20260424T418273Z.sql @@ -0,0 +1,16 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TABLE evidences + ADD COLUMN assessment JSONB NULL; diff --git a/pkg/coredata/migrations/20260424T637521Z.sql b/pkg/coredata/migrations/20260424T527194Z.sql similarity index 87% rename from pkg/coredata/migrations/20260424T637521Z.sql rename to pkg/coredata/migrations/20260424T527194Z.sql index bfd542537f..eb14f9337d 100644 --- a/pkg/coredata/migrations/20260424T637521Z.sql +++ b/pkg/coredata/migrations/20260424T527194Z.sql @@ -12,13 +12,8 @@ -- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -- PERFORMANCE OF THIS SOFTWARE. -ALTER TABLE evidences - ADD COLUMN assessment JSONB NULL; - ALTER TABLE evidences RENAME COLUMN description_status TO assessment_status; ALTER TABLE evidences RENAME COLUMN description_processing_started_at TO assessment_processing_started_at; - -ALTER TYPE evidence_description_status RENAME TO evidence_assessment_status; diff --git a/pkg/coredata/migrations/20260424T603849Z.sql b/pkg/coredata/migrations/20260424T603849Z.sql new file mode 100644 index 0000000000..291e1b2f81 --- /dev/null +++ b/pkg/coredata/migrations/20260424T603849Z.sql @@ -0,0 +1,15 @@ +-- Copyright (c) 2026 Probo Inc . +-- +-- Permission to use, copy, modify, and/or distribute this software for any +-- purpose with or without fee is hereby granted, provided that the above +-- copyright notice and this permission notice appear in all copies. +-- +-- THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +-- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +-- AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +-- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +-- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +-- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +-- PERFORMANCE OF THIS SOFTWARE. + +ALTER TYPE evidence_description_status RENAME TO evidence_assessment_status; From 106bbea0980deeb17516ad155a3b292b7e1d1180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 09:45:16 +0200 Subject: [PATCH 11/21] Bind jsonRawMessageOrNull via driver.Valuer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implementing the standard database/sql/driver.Valuer interface lets pgx convert empty JSONB values to SQL NULL automatically, removing the per-call-site Arg() footgun where forgetting the helper would let pgx reject the empty []byte as invalid JSON. Aligns with the existing pattern used by every enum type in this package. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/connector.go | 17 +++++++++-------- pkg/coredata/evidence.go | 6 +++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/coredata/connector.go b/pkg/coredata/connector.go index ce7ac53095..1640cbaf46 100644 --- a/pkg/coredata/connector.go +++ b/pkg/coredata/connector.go @@ -16,6 +16,7 @@ package coredata import ( "context" + "database/sql/driver" "encoding/json" "errors" "fmt" @@ -59,14 +60,14 @@ func (j *jsonRawMessageOrNull) Scan(src any) error { } } -// Arg returns the value to bind for a JSONB named argument. pgx rejects -// empty byte slices as invalid JSON, so an empty/nil value is sent as -// SQL NULL. -func (j jsonRawMessageOrNull) Arg() any { +// Value implements database/sql/driver.Valuer so pgx binds the column +// without an explicit helper at every call site. pgx rejects empty byte +// slices as invalid JSON, so an empty/nil value is sent as SQL NULL. +func (j jsonRawMessageOrNull) Value() (driver.Value, error) { if len(j) == 0 { - return nil + return nil, nil } - return []byte(j) + return []byte(j), nil } type ( @@ -405,7 +406,7 @@ INSERT INTO connectors ( "organization_id": c.OrganizationID, "provider": c.Provider, "protocol": c.Protocol, - "settings": c.RawSettings.Arg(), + "settings": c.RawSettings, "encrypted_connection": encryptedConnection, "created_at": c.CreatedAt, "updated_at": c.UpdatedAt, @@ -620,7 +621,7 @@ WHERE args := pgx.StrictNamedArgs{ "id": c.ID, - "settings": c.RawSettings.Arg(), + "settings": c.RawSettings, "encrypted_connection": encryptedConnection, "updated_at": c.UpdatedAt, } diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index 9c9fcc855a..7e5cf68bc0 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -161,7 +161,7 @@ WHERE evidences.state = 'REQUESTED'; "type": e.Type, "url": e.URL, "description": e.Description, - "assessment": e.Assessment.Arg(), + "assessment": e.Assessment, "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } @@ -232,7 +232,7 @@ VALUES ( "type": e.Type, "url": e.URL, "description": e.Description, - "assessment": e.Assessment.Arg(), + "assessment": e.Assessment, "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } @@ -506,7 +506,7 @@ WHERE "evidence_file_id": e.EvidenceFileID, "url": e.URL, "description": e.Description, - "assessment": e.Assessment.Arg(), + "assessment": e.Assessment, "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, "updated_at": e.UpdatedAt, From b2009753a0288607e5951739bcf079ad1c3630d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 09:45:22 +0200 Subject: [PATCH 12/21] Reuse Evidence.Update for the FAILED transition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker already has a loaded Evidence at the failure path, so the bespoke MarkEvidenceAssessmentFailed SQL was redundant with Evidence.Update. Setting the three status fields on the Claim-state value copy and calling Update keeps one write path. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence.go | 38 ------------------------- pkg/probo/evidence_assessment_worker.go | 14 +++++---- 2 files changed, 9 insertions(+), 43 deletions(-) diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index 7e5cf68bc0..b4779ed1c7 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -598,44 +598,6 @@ FOR UPDATE SKIP LOCKED; return nil } -// MarkEvidenceAssessmentFailed transitions the evidence row to FAILED -// without touching description, assessment, or any other column. It -// only needs the evidence ID, so it is a top-level func rather than a -// method on Evidence — callers should not need to load the row first -// when all they have is the ID. -func MarkEvidenceAssessmentFailed( - ctx context.Context, - conn pg.Tx, - scope Scoper, - evidenceID gid.GID, -) error { - q := ` -UPDATE - evidences -SET - assessment_status = @assessment_status, - assessment_processing_started_at = NULL, - updated_at = @updated_at -WHERE - %s - AND id = @evidence_id -` - - q = fmt.Sprintf(q, scope.SQLFragment()) - - args := pgx.StrictNamedArgs{ - "evidence_id": evidenceID, - "assessment_status": EvidenceAssessmentStatusFailed, - "updated_at": time.Now(), - } - maps.Copy(args, scope.SQLArguments()) - - if _, err := conn.Exec(ctx, q, args); err != nil { - return fmt.Errorf("cannot mark evidence assessment failed: %w", err) - } - return nil -} - func ResetStaleAssessmentProcessing( ctx context.Context, conn pg.Querier, diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index b2d532a0fd..34046bd833 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -26,7 +26,6 @@ import ( "go.probo.inc/probo/pkg/coredata" "go.probo.inc/probo/pkg/evidenceassessor" "go.probo.inc/probo/pkg/filemanager" - "go.probo.inc/probo/pkg/gid" ) type ( @@ -113,7 +112,7 @@ func (h *evidenceAssessmentHandler) Process(ctx context.Context, evidence coreda log.String("evidence_id", evidence.ID.String()), ) - if err := h.failEvidence(ctx, evidence.ID); err != nil { + if err := h.failEvidence(ctx, evidence); err != nil { h.logger.ErrorCtx(ctx, "cannot mark evidence assessment as failed", log.Error(err)) } @@ -195,13 +194,18 @@ func (h *evidenceAssessmentHandler) assessAndCommit( ) } -func (h *evidenceAssessmentHandler) failEvidence(ctx context.Context, evidenceID gid.GID) error { - scope := coredata.NewScopeFromObjectID(evidenceID) +// failEvidence transitions the row to FAILED via Evidence.Update. +// evidence is taken by value; assessAndCommit also takes a value copy, +// so this struct still reflects the row as Claim left it (PROCESSING). +func (h *evidenceAssessmentHandler) failEvidence(ctx context.Context, evidence coredata.Evidence) error { + evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusFailed + evidence.AssessmentProcessingStartedAt = nil + evidence.UpdatedAt = time.Now() return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - return coredata.MarkEvidenceAssessmentFailed(ctx, tx, scope, evidenceID) + return evidence.Update(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)) }, ) } From 17ab7838306bac17d767cd237cef92c47be3ae96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 09:45:26 +0200 Subject: [PATCH 13/21] Drop reflection from SetAssessment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Marshalling the value first and rejecting the literal "null" output clears the field for both untyped-nil and typed-nil pointers without the reflect.Kind switch. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence_assessment.go | 34 ++++++++++------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go index 33d1a2be8e..3adf027bf0 100644 --- a/pkg/coredata/evidence_assessment.go +++ b/pkg/coredata/evidence_assessment.go @@ -17,18 +17,17 @@ package coredata import ( "encoding/json" "fmt" - "reflect" ) // SetAssessment marshals a typed assessment into Evidence.Assessment. -// Passing an untyped or typed nil clears the field; the typed case is -// detected via reflection so callers cannot accidentally persist JSON -// null instead of SQL NULL. The shape of the assessment is defined by -// its producer (see pkg/evidenceassessor.EvidenceAssessment); this -// package is intentionally agnostic about the schema and only owns the -// raw JSONB round-trip. +// Passing nil — or any value that marshals to JSON null, including a +// typed-nil pointer — clears the field rather than persisting JSON +// null. The shape of the assessment is defined by its producer (see +// pkg/evidenceassessor.EvidenceAssessment); this package is +// intentionally agnostic about the schema and only owns the raw JSONB +// round-trip. func (e *Evidence) SetAssessment(v any) error { - if isNil(v) { + if v == nil { e.Assessment = nil return nil } @@ -36,6 +35,10 @@ func (e *Evidence) SetAssessment(v any) error { if err != nil { return fmt.Errorf("cannot marshal evidence assessment: %w", err) } + if string(data) == "null" { + e.Assessment = nil + return nil + } e.Assessment = data return nil } @@ -51,18 +54,3 @@ func (e *Evidence) GetAssessment(dst any) error { } return nil } - -// isNil returns true for untyped-nil and for typed-nil pointers, maps, -// slices, channels, funcs, and interfaces. Plain struct values, zero -// ints, and empty strings are not considered nil. -func isNil(v any) bool { - if v == nil { - return true - } - rv := reflect.ValueOf(v) - switch rv.Kind() { - case reflect.Pointer, reflect.Map, reflect.Slice, reflect.Chan, reflect.Func, reflect.Interface: - return rv.IsNil() - } - return false -} From be5073be3b1dba83d1add0038f54012cfe5e5a01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 09:46:08 +0200 Subject: [PATCH 14/21] Match vetting agent.Run formatting in evidence assessor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spell out the variable name and break Run's ctx and message slice across lines, matching the layout used by the vendor info extractor in pkg/vetting. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/evidenceassessor/assessment.go | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/evidenceassessor/assessment.go b/pkg/evidenceassessor/assessment.go index a81ab2ce82..fee4c2d430 100644 --- a/pkg/evidenceassessor/assessment.go +++ b/pkg/evidenceassessor/assessment.go @@ -103,21 +103,24 @@ func (a *Assessor) Assess( opts = append(opts, agent.WithLogger(a.cfg.Logger)) } - ag := agent.New("evidence_assessor", a.cfg.Client, opts...) - - result, err := ag.Run(ctx, []llm.Message{ - { - Role: llm.RoleUser, - Parts: []llm.Part{ - llm.TextPart{Text: fmt.Sprintf("Filename: %s", filename)}, - llm.FilePart{ - Data: fileBase64, - MimeType: mimeType, - Filename: filename, + assessmentAgent := agent.New("evidence_assessor", a.cfg.Client, opts...) + + result, err := assessmentAgent.Run( + ctx, + []llm.Message{ + { + Role: llm.RoleUser, + Parts: []llm.Part{ + llm.TextPart{Text: fmt.Sprintf("Filename: %s", filename)}, + llm.FilePart{ + Data: fileBase64, + MimeType: mimeType, + Filename: filename, + }, }, }, }, - }) + ) if err != nil { return nil, fmt.Errorf("cannot assess evidence: %w", err) } From 94c49e203e99efc568b9f148516e854d7363c2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 09:46:14 +0200 Subject: [PATCH 15/21] Simplify DecorateEnum to a single field per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The map-of-fields signature was overkill for the common single-enum case and forced an awkward single-key literal at call sites. Take field name and values as positional args; chain calls when more than one field needs decorating. Also folds the per-package output-type helper inline since the call is now one line. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/agent/output_type.go | 25 ++++++++------------- pkg/evidenceassessor/assessment.go | 29 +++++++------------------ pkg/evidenceassessor/assessment_test.go | 23 +++++++++----------- pkg/vetting/assessment.go | 9 ++++---- 4 files changed, 32 insertions(+), 54 deletions(-) diff --git a/pkg/agent/output_type.go b/pkg/agent/output_type.go index f4cbd6d111..0d5c064fb3 100644 --- a/pkg/agent/output_type.go +++ b/pkg/agent/output_type.go @@ -51,16 +51,11 @@ func (o *OutputType) responseFormat() *llm.ResponseFormat { } } -// DecorateEnum injects explicit `enum` constraints on top-level -// properties of the schema. jsonschema-go only reads struct tags as -// free-form descriptions, so enums cannot be encoded on the tag itself; -// callers use this helper after NewOutputType to lock down string -// fields whose allowed values live in a package-level slice. -func (o *OutputType) DecorateEnum(enums map[string][]string) error { - if len(enums) == 0 { - return nil - } - +// DecorateEnum injects an explicit `enum` constraint on a single +// top-level property of the schema. jsonschema-go reads struct tags as +// free-form descriptions only, so enums cannot be encoded on the tag +// itself; callers chain one call per enum field after NewOutputType. +func (o *OutputType) DecorateEnum(field string, values []string) error { var schema map[string]any if err := json.Unmarshal(o.Schema, &schema); err != nil { return fmt.Errorf("cannot unmarshal output type schema: %w", err) @@ -71,13 +66,11 @@ func (o *OutputType) DecorateEnum(enums map[string][]string) error { return fmt.Errorf("output type schema has no properties") } - for field, values := range enums { - prop, ok := properties[field].(map[string]any) - if !ok { - return fmt.Errorf("output type schema has no %q property", field) - } - prop["enum"] = values + prop, ok := properties[field].(map[string]any) + if !ok { + return fmt.Errorf("output type schema has no %q property", field) } + prop["enum"] = values decorated, err := json.Marshal(schema) if err != nil { diff --git a/pkg/evidenceassessor/assessment.go b/pkg/evidenceassessor/assessment.go index fee4c2d430..2f2108990f 100644 --- a/pkg/evidenceassessor/assessment.go +++ b/pkg/evidenceassessor/assessment.go @@ -28,11 +28,11 @@ import ( //go:embed prompt.txt var systemPrompt string -// assessmentConfidenceEnum is the canonical set of allowed values for +// confidenceEnum is the canonical set of allowed values for // EvidenceAssessment.Confidence. Injected into the generated JSON // schema via agent.OutputType.DecorateEnum because jsonschema struct // tags cannot encode enum constraints directly. -var assessmentConfidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} +var confidenceEnum = []string{"HIGH", "MEDIUM", "LOW"} type ( // EvidenceAssessment is the structured output produced by the @@ -72,12 +72,15 @@ type ( // New builds an Assessor. The structured output type is decorated once // (enum on confidence) and cached so every call reuses the same schema. func New(cfg Config) (*Assessor, error) { - ot, err := assessmentOutputType() + outputType, err := agent.NewOutputType[EvidenceAssessment]("evidence_assessment") if err != nil { - return nil, fmt.Errorf("cannot build evidence assessment output type: %w", err) + return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) + } + if err := outputType.DecorateEnum("confidence", confidenceEnum); err != nil { + return nil, fmt.Errorf("cannot decorate evidence assessment schema: %w", err) } - return &Assessor{cfg: cfg, outputType: ot}, nil + return &Assessor{cfg: cfg, outputType: outputType}, nil } // Assess runs the evidence assessor agent against a single uploaded @@ -132,19 +135,3 @@ func (a *Assessor) Assess( return &out, nil } - -// assessmentOutputType builds the EvidenceAssessment structured output -// type and decorates its JSON Schema with an explicit enum on the -// confidence field. -func assessmentOutputType() (*agent.OutputType, error) { - ot, err := agent.NewOutputType[EvidenceAssessment]("evidence_assessment") - if err != nil { - return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) - } - if err := ot.DecorateEnum(map[string][]string{ - "confidence": assessmentConfidenceEnum, - }); err != nil { - return nil, fmt.Errorf("cannot decorate evidence assessment schema: %w", err) - } - return ot, nil -} diff --git a/pkg/evidenceassessor/assessment_test.go b/pkg/evidenceassessor/assessment_test.go index 83ffd40e18..2eecaa6c9d 100644 --- a/pkg/evidenceassessor/assessment_test.go +++ b/pkg/evidenceassessor/assessment_test.go @@ -12,9 +12,6 @@ // OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR // PERFORMANCE OF THIS SOFTWARE. -// White-box test (package evidenceassessor, not _test) so it can reach -// the unexported assessmentOutputType helper. - package evidenceassessor import ( @@ -25,20 +22,20 @@ import ( "github.com/stretchr/testify/require" ) -// TestAssessmentOutputType_DecoratesConfidenceEnum guards the -// schema-mutation trick used to inject an enum into the generated JSON -// schema. If jsonschema-go ever reshapes the "properties" block or -// stops preserving the confidence field path, this test will fail -// loudly instead of silently shipping an un-constrained schema. -func TestAssessmentOutputType_DecoratesConfidenceEnum(t *testing.T) { +// TestNew_DecoratesConfidenceEnum guards the schema-mutation trick used +// to inject an enum into the generated JSON schema. If jsonschema-go +// ever reshapes the "properties" block or stops preserving the +// confidence field path, this test will fail loudly instead of silently +// shipping an un-constrained schema. +func TestNew_DecoratesConfidenceEnum(t *testing.T) { t.Parallel() - outputType, err := assessmentOutputType() + assessor, err := New(Config{}) require.NoError(t, err) - require.NotNil(t, outputType) + require.NotNil(t, assessor.outputType) var schema map[string]any - require.NoError(t, json.Unmarshal(outputType.Schema, &schema)) + require.NoError(t, json.Unmarshal(assessor.outputType.Schema, &schema)) properties, ok := schema["properties"].(map[string]any) require.True(t, ok, "schema has no properties block") @@ -53,5 +50,5 @@ func TestAssessmentOutputType_DecoratesConfidenceEnum(t *testing.T) { for i, v := range enumRaw { actual[i] = v.(string) } - assert.Equal(t, assessmentConfidenceEnum, actual) + assert.Equal(t, confidenceEnum, actual) } diff --git a/pkg/vetting/assessment.go b/pkg/vetting/assessment.go index c17c846fa5..a3d435f1fc 100644 --- a/pkg/vetting/assessment.go +++ b/pkg/vetting/assessment.go @@ -314,10 +314,11 @@ func thirdPartyInfoOutputType() (*agent.OutputType, error) { return nil, fmt.Errorf("cannot create thirdParty info output type: %w", err) } - if err := outputType.DecorateEnum(map[string][]string{ - "category": thirdPartyCategoryEnum, - "third_party_type": thirdPartyTypeEnum, - }); err != nil { + if err := outputType.DecorateEnum("category", thirdPartyCategoryEnum); err != nil { + return nil, fmt.Errorf("cannot decorate thirdParty info schema: %w", err) + } + + if err := outputType.DecorateEnum("third_party_type", thirdPartyTypeEnum); err != nil { return nil, fmt.Errorf("cannot decorate thirdParty info schema: %w", err) } strict, err := enforceStrictJSONSchema(outputType.Schema) From 6892945e54bec016bde932abe9ce6a8d2607f878 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 11:25:23 +0200 Subject: [PATCH 16/21] Narrow the FAILED transition to status columns only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reusing Evidence.Update wrote every column from a snapshot held across the LLM call, which would clobber any concurrent edit to description, assessment, url, etc. landed in that window. Add an Evidence.MarkAssessmentFailed method that touches only the three status columns; the worker calls it instead. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/coredata/evidence.go | 36 +++++++++++++++++++++++++ pkg/probo/evidence_assessment_worker.go | 9 +------ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index b4779ed1c7..deaca77977 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -521,6 +521,42 @@ WHERE return nil } +// SetAssessmentFailed transitions the row to FAILED, touching only +// the three status columns. Avoids clobbering concurrent edits to +// description/assessment/url/etc. with a snapshot held across the +// preceding LLM call. +func (e Evidence) SetAssessmentFailed( + ctx context.Context, + conn pg.Tx, + scope Scoper, +) error { + q := ` +UPDATE + evidences +SET + assessment_status = @assessment_status, + assessment_processing_started_at = NULL, + updated_at = @updated_at +WHERE + %s + AND id = @evidence_id +` + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{ + "evidence_id": e.ID, + "assessment_status": EvidenceAssessmentStatusFailed, + "updated_at": time.Now(), + } + maps.Copy(args, scope.SQLArguments()) + + if _, err := conn.Exec(ctx, q, args); err != nil { + return fmt.Errorf("cannot mark evidence assessment failed: %w", err) + } + return nil +} + func (e Evidence) Delete( ctx context.Context, conn pg.Tx, diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 34046bd833..5d9d6d978e 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -194,18 +194,11 @@ func (h *evidenceAssessmentHandler) assessAndCommit( ) } -// failEvidence transitions the row to FAILED via Evidence.Update. -// evidence is taken by value; assessAndCommit also takes a value copy, -// so this struct still reflects the row as Claim left it (PROCESSING). func (h *evidenceAssessmentHandler) failEvidence(ctx context.Context, evidence coredata.Evidence) error { - evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusFailed - evidence.AssessmentProcessingStartedAt = nil - evidence.UpdatedAt = time.Now() - return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - return evidence.Update(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)) + return evidence.SetAssessmentFailed(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)) }, ) } From 268a19d303212052197540d28bcd380f15c1b309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Fri, 1 May 2026 14:14:16 +0200 Subject: [PATCH 17/21] Update stale describer wording in error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leftover from the evidencedescriber → evidenceassessor rename: the error wrapping the constructor failure still said "describer". Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/probod/probod.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index e04251ce4c..3102b5b0d5 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -850,7 +850,7 @@ func (impl *Implm) Run( } evidenceAssessor, err := evidenceassessor.New(evidenceAssessorCfg) if err != nil { - return fmt.Errorf("cannot build evidence describer: %w", err) + return fmt.Errorf("cannot build evidence assessor: %w", err) } evidenceAssessmentWorker := probo.NewEvidenceAssessmentWorker( pgClient, From 11af4fb81b2c0a21e2576b2f1d95e1c154b05774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Thu, 4 Jun 2026 22:54:44 +0200 Subject: [PATCH 18/21] Refresh LLM model registry from OpenRouter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerate pkg/llm/registry_gen.go from the live OpenRouter catalog so the current model lineup is available -- notably Claude Opus 4.8 (plus 4.8-fast and 4.7) and the GPT-5.x family. The previous snapshot topped out at Claude Opus 4.6 / GPT-5.4. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/llm/registry_gen.go | 1020 +++++++++++++++------------------------ 1 file changed, 380 insertions(+), 640 deletions(-) diff --git a/pkg/llm/registry_gen.go b/pkg/llm/registry_gen.go index b74bcd7ada..38ddc76307 100644 --- a/pkg/llm/registry_gen.go +++ b/pkg/llm/registry_gen.go @@ -14,11 +14,311 @@ // Code generated by genmodels; DO NOT EDIT. // Source: https://openrouter.ai/api/v1/models -// Generated: 2026-04-15T06:30:41Z +// Generated: 2026-06-04T20:22:07Z package llm var generatedModels = map[string]ModelDefinition{ + "anthropic/claude-opus-4.8-fast": { + Name: "Anthropic: Claude Opus 4.8 (Fast)", + ContextLength: 1000000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: false, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "anthropic/claude-opus-4.8": { + Name: "Anthropic: Claude Opus 4.8", + ContextLength: 1000000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: false, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "x-ai/grok-build-0.1": { + Name: "xAI: Grok Build 0.1", + ContextLength: 256000, + MaxOutputTokens: 0, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: false, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "google/gemini-3.5-flash": { + Name: "Google: Gemini 3.5 Flash", + ContextLength: 1048576, + MaxOutputTokens: 65536, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "anthropic/claude-opus-4.7-fast": { + Name: "Anthropic: Claude Opus 4.7 (Fast)", + ContextLength: 1000000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: false, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "google/gemini-3.1-flash-lite": { + Name: "Google: Gemini 3.1 Flash Lite", + ContextLength: 1048576, + MaxOutputTokens: 65536, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "openai/gpt-chat-latest": { + Name: "OpenAI: GPT Chat Latest", + ContextLength: 400000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: false, + }, + }, + "x-ai/grok-4.3": { + Name: "xAI: Grok 4.3", + ContextLength: 1000000, + MaxOutputTokens: 0, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: false, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "mistralai/mistral-medium-3-5": { + Name: "Mistral: Mistral Medium 3.5", + ContextLength: 262144, + MaxOutputTokens: 0, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: false, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "openai/gpt-5.5-pro": { + Name: "OpenAI: GPT-5.5 Pro", + ContextLength: 1050000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: false, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "openai/gpt-5.5": { + Name: "OpenAI: GPT-5.5", + ContextLength: 1050000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: false, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "deepseek/deepseek-v4-pro": { + Name: "DeepSeek: DeepSeek V4 Pro", + ContextLength: 1048576, + MaxOutputTokens: 384000, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: true, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "deepseek/deepseek-v4-flash": { + Name: "DeepSeek: DeepSeek V4 Flash", + ContextLength: 1048576, + MaxOutputTokens: 131072, + Supports: SupportedParameters{ + Temperature: true, + TopP: true, + TopK: true, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "openai/gpt-5.4-image-2": { + Name: "OpenAI: GPT-5.4 Image 2", + ContextLength: 272000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: true, + PresencePenalty: true, + Stop: true, + Seed: true, + MaxTokens: true, + ToolChoice: false, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, + "anthropic/claude-opus-4.7": { + Name: "Anthropic: Claude Opus 4.7", + ContextLength: 1000000, + MaxOutputTokens: 128000, + Supports: SupportedParameters{ + Temperature: false, + TopP: false, + TopK: false, + FrequencyPenalty: false, + PresencePenalty: false, + Stop: true, + Seed: false, + MaxTokens: true, + ToolChoice: true, + ParallelToolCalls: false, + ResponseFormat: true, + StructuredOutputs: true, + Reasoning: true, + }, + }, "anthropic/claude-opus-4.6-fast": { Name: "Anthropic: Claude Opus 4.6 (Fast)", ContextLength: 1000000, @@ -89,7 +389,7 @@ var generatedModels = map[string]ModelDefinition{ TopK: false, FrequencyPenalty: false, PresencePenalty: false, - Stop: false, + Stop: true, Seed: true, MaxTokens: true, ToolChoice: true, @@ -102,7 +402,7 @@ var generatedModels = map[string]ModelDefinition{ "google/gemma-4-31b-it": { Name: "Google: Gemma 4 31B", ContextLength: 262144, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -341,7 +641,7 @@ var generatedModels = map[string]ModelDefinition{ }, "google/gemini-3.1-flash-image-preview": { Name: "Google: Nano Banana 2 (Gemini 3.1 Flash Image Preview)", - ContextLength: 65536, + ContextLength: 131072, MaxOutputTokens: 65536, Supports: SupportedParameters{ Temperature: true, @@ -361,7 +661,7 @@ var generatedModels = map[string]ModelDefinition{ }, "google/gemini-3.1-pro-preview-customtools": { Name: "Google: Gemini 3.1 Pro Preview Custom Tools", - ContextLength: 1048576, + ContextLength: 1048756, MaxOutputTokens: 65536, Supports: SupportedParameters{ Temperature: true, @@ -539,30 +839,10 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "mistralai/mistral-small-creative": { - Name: "Mistral: Mistral Small Creative", - ContextLength: 32768, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: false, - TopP: false, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: false, - MaxTokens: false, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, - Reasoning: false, - }, - }, "openai/gpt-5.2-chat": { Name: "OpenAI: GPT-5.2 Chat", ContextLength: 128000, - MaxOutputTokens: 32000, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: false, TopP: false, @@ -759,30 +1039,10 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "deepseek/deepseek-v3.2-speciale": { - Name: "DeepSeek: DeepSeek V3.2 Speciale", - ContextLength: 163840, - MaxOutputTokens: 163840, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, "deepseek/deepseek-v3.2": { Name: "DeepSeek: DeepSeek V3.2", - ContextLength: 163840, - MaxOutputTokens: 0, + ContextLength: 131072, + MaxOutputTokens: 64000, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -839,26 +1099,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "x-ai/grok-4.1-fast": { - Name: "xAI: Grok 4.1 Fast", - ContextLength: 2000000, - MaxOutputTokens: 30000, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, "openai/gpt-5.1": { Name: "OpenAI: GPT-5.1", ContextLength: 400000, @@ -882,7 +1122,7 @@ var generatedModels = map[string]ModelDefinition{ "openai/gpt-5.1-chat": { Name: "OpenAI: GPT-5.1 Chat", ContextLength: 128000, - MaxOutputTokens: 16384, + MaxOutputTokens: 32000, Supports: SupportedParameters{ Temperature: false, TopP: false, @@ -922,7 +1162,7 @@ var generatedModels = map[string]ModelDefinition{ "openai/gpt-5.1-codex-mini": { Name: "OpenAI: GPT-5.1-Codex-Mini", ContextLength: 400000, - MaxOutputTokens: 128000, + MaxOutputTokens: 100000, Supports: SupportedParameters{ Temperature: false, TopP: false, @@ -1032,7 +1272,7 @@ var generatedModels = map[string]ModelDefinition{ Stop: true, Seed: true, MaxTokens: true, - ToolChoice: true, + ToolChoice: false, ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, @@ -1072,7 +1312,7 @@ var generatedModels = map[string]ModelDefinition{ Stop: true, Seed: true, MaxTokens: true, - ToolChoice: true, + ToolChoice: false, ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, @@ -1182,67 +1422,7 @@ var generatedModels = map[string]ModelDefinition{ "deepseek/deepseek-v3.2-exp": { Name: "DeepSeek: DeepSeek V3.2 Exp", ContextLength: 163840, - MaxOutputTokens: 65536, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, - "google/gemini-2.5-flash-lite-preview-09-2025": { - Name: "Google: Gemini 2.5 Flash Lite Preview 09-2025", - ContextLength: 1048576, - MaxOutputTokens: 65535, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, - "openai/gpt-5-codex": { - Name: "OpenAI: GPT-5 Codex", - ContextLength: 400000, - MaxOutputTokens: 128000, - Supports: SupportedParameters{ - Temperature: false, - TopP: false, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, - "deepseek/deepseek-v3.1-terminus": { - Name: "DeepSeek: DeepSeek V3.1 Terminus", - ContextLength: 163840, - MaxOutputTokens: 0, + MaxOutputTokens: 65536, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -1259,17 +1439,17 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "x-ai/grok-4-fast": { - Name: "xAI: Grok 4 Fast", - ContextLength: 2000000, - MaxOutputTokens: 30000, + "google/gemini-2.5-flash-lite-preview-09-2025": { + Name: "Google: Gemini 2.5 Flash Lite Preview 09-2025", + ContextLength: 1048576, + MaxOutputTokens: 65535, Supports: SupportedParameters{ Temperature: true, TopP: true, TopK: false, FrequencyPenalty: false, PresencePenalty: false, - Stop: false, + Stop: true, Seed: true, MaxTokens: true, ToolChoice: true, @@ -1279,17 +1459,17 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "x-ai/grok-code-fast-1": { - Name: "xAI: Grok Code Fast 1", - ContextLength: 256000, - MaxOutputTokens: 10000, + "openai/gpt-5-codex": { + Name: "OpenAI: GPT-5 Codex", + ContextLength: 400000, + MaxOutputTokens: 128000, Supports: SupportedParameters{ - Temperature: true, - TopP: true, + Temperature: false, + TopP: false, TopK: false, FrequencyPenalty: false, PresencePenalty: false, - Stop: true, + Stop: false, Seed: true, MaxTokens: true, ToolChoice: true, @@ -1299,10 +1479,10 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "deepseek/deepseek-chat-v3.1": { - Name: "DeepSeek: DeepSeek V3.1", - ContextLength: 32768, - MaxOutputTokens: 7168, + "deepseek/deepseek-v3.1-terminus": { + Name: "DeepSeek: DeepSeek V3.1 Terminus", + ContextLength: 163840, + MaxOutputTokens: 32768, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -1319,14 +1499,14 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "openai/gpt-4o-audio-preview": { - Name: "OpenAI: GPT-4o Audio", - ContextLength: 128000, - MaxOutputTokens: 16384, + "deepseek/deepseek-chat-v3.1": { + Name: "DeepSeek: DeepSeek V3.1", + ContextLength: 163840, + MaxOutputTokens: 32768, Supports: SupportedParameters{ Temperature: true, TopP: true, - TopK: false, + TopK: true, FrequencyPenalty: true, PresencePenalty: true, Stop: true, @@ -1336,7 +1516,7 @@ var generatedModels = map[string]ModelDefinition{ ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, - Reasoning: false, + Reasoning: true, }, }, "mistralai/mistral-medium-3.1": { @@ -1579,90 +1759,10 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "mistralai/devstral-medium": { - Name: "Mistral: Devstral Medium", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, - "mistralai/devstral-small": { - Name: "Mistral: Devstral Small 1.1", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, - "x-ai/grok-4": { - Name: "xAI: Grok 4", - ContextLength: 256000, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, - "google/gemma-3n-e2b-it:free": { - Name: "Google: Gemma 3n 2B (free)", - ContextLength: 8192, - MaxOutputTokens: 2048, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, "mistralai/mistral-small-3.2-24b-instruct": { Name: "Mistral: Mistral Small 3.2 24B", ContextLength: 128000, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -1739,46 +1839,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "x-ai/grok-3-mini": { - Name: "xAI: Grok 3 Mini", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: true, - }, - }, - "x-ai/grok-3": { - Name: "xAI: Grok 3", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "google/gemini-2.5-pro-preview": { Name: "Google: Gemini 2.5 Pro Preview 06-05", ContextLength: 1048576, @@ -1802,7 +1862,7 @@ var generatedModels = map[string]ModelDefinition{ "deepseek/deepseek-r1-0528": { Name: "DeepSeek: R1 0528", ContextLength: 163840, - MaxOutputTokens: 0, + MaxOutputTokens: 32768, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -1826,7 +1886,7 @@ var generatedModels = map[string]ModelDefinition{ Supports: SupportedParameters{ Temperature: true, TopP: true, - TopK: true, + TopK: false, FrequencyPenalty: false, PresencePenalty: false, Stop: true, @@ -1859,26 +1919,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "google/gemma-3n-e4b-it:free": { - Name: "Google: Gemma 3n 4B (free)", - ContextLength: 8192, - MaxOutputTokens: 2048, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, "google/gemma-3n-e4b-it": { Name: "Google: Gemma 3n 4B", ContextLength: 32768, @@ -1942,7 +1982,7 @@ var generatedModels = map[string]ModelDefinition{ "meta-llama/llama-guard-4-12b": { Name: "Meta: Llama Guard 4 12B", ContextLength: 163840, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -2079,46 +2119,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "x-ai/grok-3-mini-beta": { - Name: "xAI: Grok 3 Mini Beta", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: true, - }, - }, - "x-ai/grok-3-beta": { - Name: "xAI: Grok 3 Beta", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, "meta-llama/llama-4-maverick": { Name: "Meta: Llama 4 Maverick", ContextLength: 1048576, @@ -2141,7 +2141,7 @@ var generatedModels = map[string]ModelDefinition{ }, "meta-llama/llama-4-scout": { Name: "Meta: Llama 4 Scout", - ContextLength: 327680, + ContextLength: 10000000, MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, @@ -2162,7 +2162,7 @@ var generatedModels = map[string]ModelDefinition{ "deepseek/deepseek-chat-v3-0324": { Name: "DeepSeek: DeepSeek V3 0324", ContextLength: 163840, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -2176,7 +2176,7 @@ var generatedModels = map[string]ModelDefinition{ ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, - Reasoning: true, + Reasoning: false, }, }, "openai/o1-pro": { @@ -2202,39 +2202,19 @@ var generatedModels = map[string]ModelDefinition{ "mistralai/mistral-small-3.1-24b-instruct": { Name: "Mistral: Mistral Small 3.1 24B", ContextLength: 128000, - MaxOutputTokens: 0, + MaxOutputTokens: 128000, Supports: SupportedParameters{ Temperature: true, TopP: true, TopK: true, FrequencyPenalty: true, PresencePenalty: true, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, - Reasoning: false, - }, - }, - "google/gemma-3-4b-it:free": { - Name: "Google: Gemma 3 4B (free)", - ContextLength: 32768, - MaxOutputTokens: 8192, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, Stop: true, Seed: true, MaxTokens: true, ToolChoice: false, ParallelToolCalls: false, - ResponseFormat: true, + ResponseFormat: false, StructuredOutputs: false, Reasoning: false, }, @@ -2242,47 +2222,27 @@ var generatedModels = map[string]ModelDefinition{ "google/gemma-3-4b-it": { Name: "Google: Gemma 3 4B", ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, - "google/gemma-3-12b-it:free": { - Name: "Google: Gemma 3 12B (free)", - ContextLength: 32768, - MaxOutputTokens: 8192, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, + TopP: true, + TopK: true, + FrequencyPenalty: true, + PresencePenalty: true, Stop: true, Seed: true, MaxTokens: true, ToolChoice: false, ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, + ResponseFormat: true, + StructuredOutputs: true, Reasoning: false, }, }, "google/gemma-3-12b-it": { Name: "Google: Gemma 3 12B", ContextLength: 131072, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -2292,7 +2252,7 @@ var generatedModels = map[string]ModelDefinition{ Stop: true, Seed: true, MaxTokens: true, - ToolChoice: false, + ToolChoice: true, ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, @@ -2339,26 +2299,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "google/gemma-3-27b-it:free": { - Name: "Google: Gemma 3 27B (free)", - ContextLength: 131072, - MaxOutputTokens: 8192, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, "google/gemma-3-27b-it": { Name: "Google: Gemma 3 27B", ContextLength: 131072, @@ -2372,7 +2312,7 @@ var generatedModels = map[string]ModelDefinition{ Stop: true, Seed: true, MaxTokens: true, - ToolChoice: false, + ToolChoice: true, ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: true, @@ -2439,66 +2379,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "google/gemini-2.0-flash-lite-001": { - Name: "Google: Gemini 2.0 Flash Lite", - ContextLength: 1048576, - MaxOutputTokens: 8192, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, - "anthropic/claude-3.7-sonnet": { - Name: "Anthropic: Claude 3.7 Sonnet", - ContextLength: 200000, - MaxOutputTokens: 128000, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: false, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, - Reasoning: true, - }, - }, - "anthropic/claude-3.7-sonnet:thinking": { - Name: "Anthropic: Claude 3.7 Sonnet (thinking)", - ContextLength: 200000, - MaxOutputTokens: 64000, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: false, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, - Reasoning: true, - }, - }, "mistralai/mistral-saba": { Name: "Mistral: Saba", ContextLength: 32768, @@ -2522,14 +2402,14 @@ var generatedModels = map[string]ModelDefinition{ "meta-llama/llama-guard-3-8b": { Name: "Llama Guard 3 8B", ContextLength: 131072, - MaxOutputTokens: 0, + MaxOutputTokens: 131072, Supports: SupportedParameters{ Temperature: true, TopP: true, TopK: true, FrequencyPenalty: true, PresencePenalty: true, - Stop: false, + Stop: true, Seed: true, MaxTokens: true, ToolChoice: false, @@ -2559,26 +2439,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: true, }, }, - "google/gemini-2.0-flash-001": { - Name: "Google: Gemini 2.0 Flash", - ContextLength: 1048576, - MaxOutputTokens: 8192, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: false, - PresencePenalty: false, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "openai/o3-mini": { Name: "OpenAI: o3 Mini", ContextLength: 200000, @@ -2615,13 +2475,13 @@ var generatedModels = map[string]ModelDefinition{ ToolChoice: false, ParallelToolCalls: false, ResponseFormat: true, - StructuredOutputs: false, + StructuredOutputs: true, Reasoning: false, }, }, "deepseek/deepseek-r1-distill-qwen-32b": { Name: "DeepSeek: R1 Distill Qwen 32B", - ContextLength: 32768, + ContextLength: 128000, MaxOutputTokens: 32768, Supports: SupportedParameters{ Temperature: true, @@ -2630,7 +2490,7 @@ var generatedModels = map[string]ModelDefinition{ FrequencyPenalty: true, PresencePenalty: true, Stop: true, - Seed: false, + Seed: true, MaxTokens: true, ToolChoice: false, ParallelToolCalls: false, @@ -2681,7 +2541,7 @@ var generatedModels = map[string]ModelDefinition{ }, "deepseek/deepseek-r1": { Name: "DeepSeek: R1", - ContextLength: 64000, + ContextLength: 163840, MaxOutputTokens: 16000, Supports: SupportedParameters{ Temperature: true, @@ -2694,15 +2554,15 @@ var generatedModels = map[string]ModelDefinition{ MaxTokens: true, ToolChoice: true, ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, + ResponseFormat: true, + StructuredOutputs: true, Reasoning: true, }, }, "deepseek/deepseek-chat": { Name: "DeepSeek: DeepSeek V3", - ContextLength: 163840, - MaxOutputTokens: 163840, + ContextLength: 131072, + MaxOutputTokens: 16000, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -2715,7 +2575,7 @@ var generatedModels = map[string]ModelDefinition{ ToolChoice: true, ParallelToolCalls: false, ResponseFormat: true, - StructuredOutputs: false, + StructuredOutputs: true, Reasoning: false, }, }, @@ -2741,7 +2601,7 @@ var generatedModels = map[string]ModelDefinition{ }, "meta-llama/llama-3.3-70b-instruct:free": { Name: "Meta: Llama 3.3 70B Instruct (free)", - ContextLength: 65536, + ContextLength: 131072, MaxOutputTokens: 0, Supports: SupportedParameters{ Temperature: true, @@ -2859,26 +2719,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "mistralai/mistral-large-2411": { - Name: "Mistral Large 2411", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "mistralai/mistral-large-2407": { Name: "Mistral Large 2407", ContextLength: 131072, @@ -2899,26 +2739,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "mistralai/pixtral-large-2411": { - Name: "Mistral: Pixtral Large 2411", - ContextLength: 131072, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "anthropic/claude-3.5-haiku": { Name: "Anthropic: Claude 3.5 Haiku", ContextLength: 200000, @@ -2961,15 +2781,15 @@ var generatedModels = map[string]ModelDefinition{ }, "meta-llama/llama-3.2-3b-instruct": { Name: "Meta: Llama 3.2 3B Instruct", - ContextLength: 80000, - MaxOutputTokens: 0, + ContextLength: 131072, + MaxOutputTokens: 80000, Supports: SupportedParameters{ Temperature: true, TopP: true, TopK: true, FrequencyPenalty: true, PresencePenalty: true, - Stop: false, + Stop: true, Seed: true, MaxTokens: true, ToolChoice: false, @@ -2981,15 +2801,15 @@ var generatedModels = map[string]ModelDefinition{ }, "meta-llama/llama-3.2-1b-instruct": { Name: "Meta: Llama 3.2 1B Instruct", - ContextLength: 60000, - MaxOutputTokens: 0, + ContextLength: 131072, + MaxOutputTokens: 60000, Supports: SupportedParameters{ Temperature: true, TopP: true, TopK: true, FrequencyPenalty: true, PresencePenalty: true, - Stop: false, + Stop: true, Seed: true, MaxTokens: true, ToolChoice: false, @@ -3041,7 +2861,7 @@ var generatedModels = map[string]ModelDefinition{ }, "meta-llama/llama-3.1-8b-instruct": { Name: "Meta: Llama 3.1 8B Instruct", - ContextLength: 16384, + ContextLength: 131072, MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, @@ -3062,7 +2882,7 @@ var generatedModels = map[string]ModelDefinition{ "meta-llama/llama-3.1-70b-instruct": { Name: "Meta: Llama 3.1 70B Instruct", ContextLength: 131072, - MaxOutputTokens: 0, + MaxOutputTokens: 16384, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -3075,14 +2895,14 @@ var generatedModels = map[string]ModelDefinition{ ToolChoice: true, ParallelToolCalls: false, ResponseFormat: true, - StructuredOutputs: false, + StructuredOutputs: true, Reasoning: false, }, }, "mistralai/mistral-nemo": { Name: "Mistral: Mistral Nemo", ContextLength: 131072, - MaxOutputTokens: 16384, + MaxOutputTokens: 0, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -3150,7 +2970,7 @@ var generatedModels = map[string]ModelDefinition{ FrequencyPenalty: true, PresencePenalty: true, Stop: true, - Seed: false, + Seed: true, MaxTokens: true, ToolChoice: false, ParallelToolCalls: false, @@ -3199,30 +3019,10 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "openai/gpt-4o:extended": { - Name: "OpenAI: GPT-4o (extended)", - ContextLength: 128000, - MaxOutputTokens: 64000, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "meta-llama/llama-3-8b-instruct": { Name: "Meta: Llama 3 8B Instruct", ContextLength: 8192, - MaxOutputTokens: 16384, + MaxOutputTokens: 8192, Supports: SupportedParameters{ Temperature: true, TopP: true, @@ -3232,7 +3032,7 @@ var generatedModels = map[string]ModelDefinition{ Stop: true, Seed: true, MaxTokens: true, - ToolChoice: true, + ToolChoice: false, ParallelToolCalls: false, ResponseFormat: true, StructuredOutputs: false, @@ -3255,7 +3055,7 @@ var generatedModels = map[string]ModelDefinition{ ToolChoice: false, ParallelToolCalls: false, ResponseFormat: false, - StructuredOutputs: false, + StructuredOutputs: true, Reasoning: false, }, }, @@ -3379,26 +3179,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "mistralai/mixtral-8x7b-instruct": { - Name: "Mistral: Mixtral 8x7B Instruct", - ContextLength: 32768, - MaxOutputTokens: 16384, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: false, - Reasoning: false, - }, - }, "openai/gpt-4-1106-preview": { Name: "OpenAI: GPT-4 Turbo (older v1106)", ContextLength: 128000, @@ -3439,26 +3219,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "mistralai/mistral-7b-instruct-v0.1": { - Name: "Mistral: Mistral 7B Instruct v0.1", - ContextLength: 2824, - MaxOutputTokens: 0, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: true, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: false, - Seed: true, - MaxTokens: true, - ToolChoice: false, - ParallelToolCalls: false, - ResponseFormat: false, - StructuredOutputs: false, - Reasoning: false, - }, - }, "openai/gpt-3.5-turbo-16k": { Name: "OpenAI: GPT-3.5 Turbo 16k", ContextLength: 16385, @@ -3479,26 +3239,6 @@ var generatedModels = map[string]ModelDefinition{ Reasoning: false, }, }, - "openai/gpt-4-0314": { - Name: "OpenAI: GPT-4 (older v0314)", - ContextLength: 8191, - MaxOutputTokens: 4096, - Supports: SupportedParameters{ - Temperature: true, - TopP: true, - TopK: false, - FrequencyPenalty: true, - PresencePenalty: true, - Stop: true, - Seed: true, - MaxTokens: true, - ToolChoice: true, - ParallelToolCalls: false, - ResponseFormat: true, - StructuredOutputs: true, - Reasoning: false, - }, - }, "openai/gpt-4": { Name: "OpenAI: GPT-4", ContextLength: 8191, From 78caced27322bbac8b5e1e09b51d48cb8ae99acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Thu, 4 Jun 2026 22:54:58 +0200 Subject: [PATCH 19/21] Fix evidence assessor correctness and add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close a stale-recovery race on the assessment state machine. The PROCESSING-only guard on SetAssessmentFailed / SetAssessmentCompleted could let a worker whose claim had been recycled clobber a fresh worker's live claim, leaving the row stuck FAILED and dropping a good result. Both terminal transitions now also match the claim's assessment_processing_started_at, so a superseded claim is a no-op; SetAssessmentFailed returns whether it transitioned and the worker logs the superseded case. Stop the assessor from sending a temperature alongside extended thinking. Anthropic rejects that combination and the assessor was the only agent setting both, so every Anthropic run would 400. Temperature is now sent only when thinking is off, mirroring pkg/vetting. Add coredata integration tests covering the claim-ownership guard, the happy-path transitions and stale recovery (skipped without a test database). Also document that legacy COMPLETED rows carry a NULL assessment, correct the now-stale strict-schema comment in pkg/vetting, de-duplicate the worker's file-load error wrap, and refresh a doc example that still referenced the removed evidencedescriber package. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- contrib/claude/file-naming.md | 2 +- pkg/coredata/evidence.go | 100 ++++- .../evidence_assessment_state_test.go | 346 ++++++++++++++++++ .../evidence_assessment_status_test.go | 78 ++++ pkg/coredata/migrations/20260424T418273Z.sql | 6 + pkg/evidenceassessor/assessment.go | 45 ++- pkg/probo/evidence_assessment_worker.go | 44 ++- pkg/vetting/openai_schema.go | 9 +- 8 files changed, 601 insertions(+), 29 deletions(-) create mode 100644 pkg/coredata/evidence_assessment_state_test.go create mode 100644 pkg/coredata/evidence_assessment_status_test.go diff --git a/contrib/claude/file-naming.md b/contrib/claude/file-naming.md index 1dadcf969a..a2456dc24e 100644 --- a/contrib/claude/file-naming.md +++ b/contrib/claude/file-naming.md @@ -76,7 +76,7 @@ auth). In that case the file keeps its service name and the agent is built inline: ``` -pkg/evidencedescriber/evidencedescriber.go -- single-file describer service +pkg/evidenceassessor/assessment.go -- single-file evidence assessment service pkg/vetting/assessment.go -- third-party assessment service ``` diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index deaca77977..eb910b5357 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -521,15 +521,32 @@ WHERE return nil } -// SetAssessmentFailed transitions the row to FAILED, touching only -// the three status columns. Avoids clobbering concurrent edits to -// description/assessment/url/etc. with a snapshot held across the -// preceding LLM call. +// SetAssessmentFailed transitions the row to FAILED, touching only the +// two status columns so it never clobbers concurrent edits to +// description/assessment/url/etc. made while the LLM call was running. +// +// It guards on BOTH the PROCESSING status AND the claim's +// assessment_processing_started_at. The started_at predicate is what makes +// a late failure safe: if stale-recovery recycled this claim and another +// worker has since re-claimed the row with a fresh timestamp, the receiver +// still carries the original timestamp, so the UPDATE matches no row and is +// a no-op rather than flipping the new owner's in-flight claim to FAILED. +// It returns whether a row was transitioned so the caller can distinguish a +// real failure from a superseded one. The terminal status guard +// additionally prevents resurrecting an already COMPLETED/FAILED row. +// +// Note FAILED is terminal: no path re-queues it (the worker claims only +// PENDING and stale-recovery resets only PROCESSING), so a transient error +// permanently parks the evidence until it is re-triggered externally. This +// matches the prior describer behaviour and is intentional. +// +// Callers must carry the claim's AssessmentProcessingStartedAt on the +// receiver (Claim sets it). func (e Evidence) SetAssessmentFailed( ctx context.Context, conn pg.Tx, scope Scoper, -) error { +) (bool, error) { q := ` UPDATE evidences @@ -540,21 +557,82 @@ SET WHERE %s AND id = @evidence_id + AND assessment_status = @expected_status + AND assessment_processing_started_at = @expected_started_at ` q = fmt.Sprintf(q, scope.SQLFragment()) args := pgx.StrictNamedArgs{ - "evidence_id": e.ID, - "assessment_status": EvidenceAssessmentStatusFailed, - "updated_at": time.Now(), + "evidence_id": e.ID, + "assessment_status": EvidenceAssessmentStatusFailed, + "expected_status": EvidenceAssessmentStatusProcessing, + "expected_started_at": e.AssessmentProcessingStartedAt, + "updated_at": time.Now(), } maps.Copy(args, scope.SQLArguments()) - if _, err := conn.Exec(ctx, q, args); err != nil { - return fmt.Errorf("cannot mark evidence assessment failed: %w", err) + tag, err := conn.Exec(ctx, q, args) + if err != nil { + return false, fmt.Errorf("cannot mark evidence assessment failed: %w", err) } - return nil + + return tag.RowsAffected() > 0, nil +} + +// SetAssessmentCompleted records a successful assessment, writing only +// the assessment-owned columns (assessment, description) and +// transitioning PROCESSING -> COMPLETED. Like SetAssessmentFailed it +// leaves user-editable columns (type/state/url/...) untouched so an edit +// made while the assessment was running is preserved, and it guards on +// both the PROCESSING status and the claim's +// assessment_processing_started_at. The guard makes the write idempotent +// and claim-scoped: if stale-recovery recycled the claim and another +// worker re-claimed (fresh started_at) or already finished it, no row +// matches and the method reports updated=false so the caller can drop the +// superseded result instead of clobbering the fresh one. Callers must +// populate Assessment and Description and carry the claim's +// AssessmentProcessingStartedAt on the receiver before calling. +func (e Evidence) SetAssessmentCompleted( + ctx context.Context, + conn pg.Tx, + scope Scoper, +) (bool, error) { + q := ` +UPDATE + evidences +SET + assessment = @assessment, + description = @description, + assessment_status = @assessment_status, + assessment_processing_started_at = NULL, + updated_at = @updated_at +WHERE + %s + AND id = @evidence_id + AND assessment_status = @expected_status + AND assessment_processing_started_at = @expected_started_at +` + + q = fmt.Sprintf(q, scope.SQLFragment()) + + args := pgx.StrictNamedArgs{ + "evidence_id": e.ID, + "assessment": e.Assessment, + "description": e.Description, + "assessment_status": EvidenceAssessmentStatusCompleted, + "expected_status": EvidenceAssessmentStatusProcessing, + "expected_started_at": e.AssessmentProcessingStartedAt, + "updated_at": time.Now(), + } + maps.Copy(args, scope.SQLArguments()) + + tag, err := conn.Exec(ctx, q, args) + if err != nil { + return false, fmt.Errorf("cannot mark evidence assessment completed: %w", err) + } + + return tag.RowsAffected() > 0, nil } func (e Evidence) Delete( diff --git a/pkg/coredata/evidence_assessment_state_test.go b/pkg/coredata/evidence_assessment_state_test.go new file mode 100644 index 0000000000..36632f3b61 --- /dev/null +++ b/pkg/coredata/evidence_assessment_state_test.go @@ -0,0 +1,346 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.gearno.de/kit/pg" + "go.probo.inc/probo/pkg/coredata" + "go.probo.inc/probo/pkg/gid" +) + +// These integration tests exercise the assessment status state machine — the +// concurrency-sensitive guards the evidence-assessment worker relies on. They +// reuse newTestPgClient (access_entry_upsert_test.go) and skip when +// PROBO_TEST_PG_URL is unset, so `make test` stays a pure unit run. +// +// They lock in particular the claim-ownership guard on the terminal +// transitions: a worker whose claim was recycled by stale recovery and +// re-claimed by another worker must NOT clobber the new owner's PROCESSING +// row when its late SetAssessmentCompleted/SetAssessmentFailed fires. +// +// LoadNextPendingAssessmentForUpdateSkipLocked is intentionally not asserted +// here: it scans globally (no tenant scope), so it cannot be isolated against +// other rows in a shared parallel test database; the worker integration path +// covers its selection. + +type assessmentEvidenceFixture struct { + scope *coredata.Scope + organizationID gid.GID + measureID gid.GID +} + +// seedAssessmentFixture bootstraps the organization and measure that an +// evidence row's FKs require. +func seedAssessmentFixture(t *testing.T, ctx context.Context, client *pg.Client) assessmentEvidenceFixture { + t.Helper() + + tenantID := gid.NewTenantID() + scope := coredata.NewScope(tenantID) + organizationID := gid.New(tenantID, coredata.OrganizationEntityType) + measureID := gid.New(tenantID, coredata.MeasureEntityType) + now := time.Now().UTC().Truncate(time.Microsecond) + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + org := &coredata.Organization{ + ID: organizationID, + TenantID: tenantID, + Name: "Evidence Assessment Test Org", + CreatedAt: now, + UpdatedAt: now, + } + if err := org.Insert(ctx, tx); err != nil { + return err + } + + measure := &coredata.Measure{ + ID: measureID, + OrganizationID: organizationID, + Category: "Test", + Name: "Evidence Assessment Test Measure", + State: coredata.MeasureStateNotImplemented, + ReferenceID: "measure-assessment-" + measureID.String(), + CreatedAt: now, + UpdatedAt: now, + } + + return measure.Insert(ctx, tx, scope) + })) + + t.Cleanup(func() { + _ = client.WithTx(context.Background(), func(ctx context.Context, tx pg.Tx) error { + if _, err := tx.Exec(ctx, `DELETE FROM evidences WHERE measure_id = $1`, measureID); err != nil { + return err + } + if _, err := tx.Exec(ctx, `DELETE FROM measures WHERE id = $1`, measureID); err != nil { + return err + } + if _, err := tx.Exec(ctx, `DELETE FROM organizations WHERE id = $1`, organizationID); err != nil { + return err + } + + return nil + }) + }) + + return assessmentEvidenceFixture{ + scope: scope, + organizationID: organizationID, + measureID: measureID, + } +} + +// insertPendingEvidence inserts a PENDING evidence with a file attached. +func (fx assessmentEvidenceFixture) insertPendingEvidence( + t *testing.T, + ctx context.Context, + client *pg.Client, + referenceID string, + createdAt time.Time, +) gid.GID { + t.Helper() + + tenantID := fx.scope.GetTenantID() + evidenceID := gid.New(tenantID, coredata.EvidenceEntityType) + fileID := gid.New(tenantID, coredata.FileEntityType) + + evidence := coredata.Evidence{ + ID: evidenceID, + OrganizationID: fx.organizationID, + MeasureID: fx.measureID, + State: coredata.EvidenceStateFulfilled, + ReferenceID: referenceID, + Type: coredata.EvidenceTypeFile, + EvidenceFileID: &fileID, + AssessmentStatus: coredata.EvidenceAssessmentStatusPending, + CreatedAt: createdAt, + UpdatedAt: createdAt, + } + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + return evidence.Insert(ctx, tx, fx.scope) + })) + + return evidenceID +} + +func loadEvidence( + t *testing.T, + ctx context.Context, + client *pg.Client, + scope coredata.Scoper, + id gid.GID, +) coredata.Evidence { + t.Helper() + + var e coredata.Evidence + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return e.LoadByID(ctx, conn, scope, id) + })) + + return e +} + +// claimEvidence transitions a row to PROCESSING with the given started_at, +// mirroring what the worker's Claim does, and returns the claimed snapshot +// (carrying that started_at) so the caller can drive a terminal transition. +func claimEvidence( + t *testing.T, + ctx context.Context, + client *pg.Client, + scope coredata.Scoper, + id gid.GID, + startedAt time.Time, +) coredata.Evidence { + t.Helper() + + e := loadEvidence(t, ctx, client, scope, id) + e.AssessmentStatus = coredata.EvidenceAssessmentStatusProcessing + e.AssessmentProcessingStartedAt = &startedAt + e.UpdatedAt = startedAt + + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + return e.Update(ctx, tx, scope) + })) + + return e +} + +func TestEvidence_SetAssessmentCompleted_TransitionsAndPreservesUserColumns(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedAssessmentFixture(t, ctx, client) + + t0 := time.Now().UTC().Truncate(time.Microsecond) + id := fx.insertPendingEvidence(t, ctx, client, "ev-complete-"+t0.Format(time.RFC3339Nano), t0) + + claimedAt := t0.Add(time.Minute) + claimed := claimEvidence(t, ctx, client, fx.scope, id, claimedAt) + + // Simulate a concurrent user edit to a user-owned column while the + // assessment ran; the terminal write must not clobber it. + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + _, err := tx.Exec(ctx, `UPDATE evidences SET url = $1 WHERE id = $2`, "https://edited.example", id) + return err + })) + + summary := "Admin console shows enforced MFA." + claimed.Description = &summary + require.NoError(t, claimed.SetAssessment(map[string]any{"summary": summary, "confidence": "HIGH"})) + + var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + updated, err = claimed.SetAssessmentCompleted(ctx, tx, fx.scope) + return err + })) + assert.True(t, updated, "completing a PROCESSING row owned by this claim must update it") + + loaded := loadEvidence(t, ctx, client, fx.scope, id) + assert.Equal(t, coredata.EvidenceAssessmentStatusCompleted, loaded.AssessmentStatus) + require.NotNil(t, loaded.Description) + assert.Equal(t, summary, *loaded.Description) + assert.NotEmpty(t, loaded.Assessment) + assert.Nil(t, loaded.AssessmentProcessingStartedAt) + assert.Equal(t, "https://edited.example", loaded.URL, "a user edit during the run must be preserved") +} + +func TestEvidence_SetAssessmentCompleted_NoOpOnSupersededClaim(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedAssessmentFixture(t, ctx, client) + + t0 := time.Now().UTC().Truncate(time.Microsecond) + id := fx.insertPendingEvidence(t, ctx, client, "ev-superseded-complete-"+t0.Format(time.RFC3339Nano), t0) + + // Worker A claims the row. + firstClaim := claimEvidence(t, ctx, client, fx.scope, id, t0.Add(time.Minute)) + + // Stale recovery recycled A's claim and worker B re-claimed it with a + // fresh started_at — the row is now owned by B. + secondClaimAt := t0.Add(3 * time.Minute) + _ = claimEvidence(t, ctx, client, fx.scope, id, secondClaimAt) + + // Worker A finishes late and tries to commit its result. + summary := "stale result from A" + firstClaim.Description = &summary + require.NoError(t, firstClaim.SetAssessment(map[string]any{"summary": summary})) + + var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + updated, err = firstClaim.SetAssessmentCompleted(ctx, tx, fx.scope) + return err + })) + assert.False(t, updated, "a superseded claim must not overwrite the re-claimed row") + + loaded := loadEvidence(t, ctx, client, fx.scope, id) + assert.Equal(t, coredata.EvidenceAssessmentStatusProcessing, loaded.AssessmentStatus, "row stays owned by the re-claim") + assert.Empty(t, loaded.Assessment, "the stale result must not be persisted") + require.NotNil(t, loaded.AssessmentProcessingStartedAt) + assert.WithinDuration(t, secondClaimAt, *loaded.AssessmentProcessingStartedAt, time.Second) +} + +func TestEvidence_SetAssessmentFailed_TransitionsOwningClaim(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedAssessmentFixture(t, ctx, client) + + t0 := time.Now().UTC().Truncate(time.Microsecond) + id := fx.insertPendingEvidence(t, ctx, client, "ev-fail-"+t0.Format(time.RFC3339Nano), t0) + + claimed := claimEvidence(t, ctx, client, fx.scope, id, t0.Add(time.Minute)) + + var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + updated, err = claimed.SetAssessmentFailed(ctx, tx, fx.scope) + return err + })) + assert.True(t, updated, "failing a PROCESSING row owned by this claim must update it") + + loaded := loadEvidence(t, ctx, client, fx.scope, id) + assert.Equal(t, coredata.EvidenceAssessmentStatusFailed, loaded.AssessmentStatus) + assert.Nil(t, loaded.AssessmentProcessingStartedAt) +} + +func TestEvidence_SetAssessmentFailed_NoOpOnSupersededClaim(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedAssessmentFixture(t, ctx, client) + + t0 := time.Now().UTC().Truncate(time.Microsecond) + id := fx.insertPendingEvidence(t, ctx, client, "ev-superseded-fail-"+t0.Format(time.RFC3339Nano), t0) + + // Worker A claims, then B re-claims with a fresh started_at. + firstClaim := claimEvidence(t, ctx, client, fx.scope, id, t0.Add(time.Minute)) + secondClaimAt := t0.Add(3 * time.Minute) + _ = claimEvidence(t, ctx, client, fx.scope, id, secondClaimAt) + + // Worker A's late failure must not flip B's live claim to FAILED. + var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { + var err error + updated, err = firstClaim.SetAssessmentFailed(ctx, tx, fx.scope) + return err + })) + assert.False(t, updated, "a stale worker's failure must not clobber the re-claimed row") + + loaded := loadEvidence(t, ctx, client, fx.scope, id) + assert.Equal(t, coredata.EvidenceAssessmentStatusProcessing, loaded.AssessmentStatus, "row must remain PROCESSING under the new owner") + require.NotNil(t, loaded.AssessmentProcessingStartedAt) + assert.WithinDuration(t, secondClaimAt, *loaded.AssessmentProcessingStartedAt, time.Second) +} + +func TestResetStaleAssessmentProcessing_ResetsOnlyStaleClaims(t *testing.T) { + t.Parallel() + + client := newTestPgClient(t) + ctx := context.Background() + fx := seedAssessmentFixture(t, ctx, client) + + now := time.Now().UTC().Truncate(time.Microsecond) + staleID := fx.insertPendingEvidence(t, ctx, client, "ev-stale-"+now.Format(time.RFC3339Nano), now.Add(-time.Hour)) + freshID := fx.insertPendingEvidence(t, ctx, client, "ev-fresh-"+now.Format(time.RFC3339Nano), now.Add(-time.Hour)) + + // One claim started long ago (stale), one started now (fresh). + _ = claimEvidence(t, ctx, client, fx.scope, staleID, now.Add(-10*time.Minute)) + _ = claimEvidence(t, ctx, client, fx.scope, freshID, now) + + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { + return coredata.ResetStaleAssessmentProcessing(ctx, conn, 5*time.Minute) + })) + + stale := loadEvidence(t, ctx, client, fx.scope, staleID) + assert.Equal(t, coredata.EvidenceAssessmentStatusPending, stale.AssessmentStatus, "a stale claim is recycled to PENDING") + assert.Nil(t, stale.AssessmentProcessingStartedAt) + + fresh := loadEvidence(t, ctx, client, fx.scope, freshID) + assert.Equal(t, coredata.EvidenceAssessmentStatusProcessing, fresh.AssessmentStatus, "a fresh claim is left running") + require.NotNil(t, fresh.AssessmentProcessingStartedAt) +} diff --git a/pkg/coredata/evidence_assessment_status_test.go b/pkg/coredata/evidence_assessment_status_test.go new file mode 100644 index 0000000000..5279b3efc7 --- /dev/null +++ b/pkg/coredata/evidence_assessment_status_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2026 Probo Inc . +// +// Permission to use, copy, modify, and/or distribute this software for any +// purpose with or without fee is hereby granted, provided that the above +// copyright notice and this permission notice appear in all copies. +// +// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +// PERFORMANCE OF THIS SOFTWARE. + +package coredata + +import "testing" + +func TestEvidenceAssessmentStatusIsValid(t *testing.T) { + t.Parallel() + + for _, value := range EvidenceAssessmentStatuses() { + if !value.IsValid() { + t.Fatalf("IsValid() = false for %q", value) + } + } + + if EvidenceAssessmentStatus("BOGUS").IsValid() { + t.Fatal("IsValid() = true for invalid value") + } +} + +func TestEvidenceAssessmentStatusUnmarshalText(t *testing.T) { + t.Parallel() + + for _, value := range EvidenceAssessmentStatuses() { + t.Run(string(value), func(t *testing.T) { + t.Parallel() + + var got EvidenceAssessmentStatus + if err := got.UnmarshalText([]byte(value)); err != nil { + t.Fatalf("UnmarshalText(%q) returned error: %v", value, err) + } + + if got != value { + t.Fatalf("UnmarshalText(%q) = %q, want %q", value, got, value) + } + }) + } + + t.Run("invalid", func(t *testing.T) { + t.Parallel() + + var got EvidenceAssessmentStatus + if err := got.UnmarshalText([]byte("BOGUS")); err == nil { + t.Fatal("UnmarshalText(BOGUS) expected error") + } + }) +} + +func TestEvidenceAssessmentStatusMarshalText(t *testing.T) { + t.Parallel() + + for _, value := range EvidenceAssessmentStatuses() { + t.Run(string(value), func(t *testing.T) { + t.Parallel() + + got, err := value.MarshalText() + if err != nil { + t.Fatalf("MarshalText() returned error: %v", err) + } + + if string(got) != value.String() { + t.Fatalf("MarshalText() = %q, want %q", string(got), value.String()) + } + }) + } +} diff --git a/pkg/coredata/migrations/20260424T418273Z.sql b/pkg/coredata/migrations/20260424T418273Z.sql index 7f2245be64..fe1e0bea9e 100644 --- a/pkg/coredata/migrations/20260424T418273Z.sql +++ b/pkg/coredata/migrations/20260424T418273Z.sql @@ -12,5 +12,11 @@ -- OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -- PERFORMANCE OF THIS SOFTWARE. +-- Nullable with no default: the assessor writes structured output here on +-- completion. Evidences that were already COMPLETED before this column +-- existed keep a NULL assessment and are not re-enqueued (the worker claims +-- only PENDING rows); this is intentional. GetAssessment is a no-op on NULL, +-- so readers must treat a NULL assessment as "not assessed", independent of +-- assessment_status. ALTER TABLE evidences ADD COLUMN assessment JSONB NULL; diff --git a/pkg/evidenceassessor/assessment.go b/pkg/evidenceassessor/assessment.go index 2f2108990f..d163c964de 100644 --- a/pkg/evidenceassessor/assessment.go +++ b/pkg/evidenceassessor/assessment.go @@ -19,12 +19,27 @@ import ( _ "embed" "encoding/json" "fmt" + "time" "go.gearno.de/kit/log" "go.probo.inc/probo/pkg/agent" "go.probo.inc/probo/pkg/llm" ) +// AssessmentTimeout is the hard upper bound on the assessor's single LLM +// turn. The assessor performs one tool-less structured-output turn, so +// this is far tighter than pkg/vetting's multi-turn budget. It exists so a +// stalled provider connection cannot pin a worker concurrency slot forever +// (the kit worker holds the slot until Process returns). Note it bounds +// only the LLM call, not the surrounding file download or commit, so it +// does not by itself guarantee Process finishes before the worker's +// stale-recovery window (EvidenceAssessmentConfig.StaleAfter, default 300s) +// elapses; correctness under a stale re-claim is enforced by the +// claim-ownership guard on the terminal transitions (see +// coredata.Evidence.SetAssessmentCompleted / SetAssessmentFailed), not by +// this timeout. +const AssessmentTimeout = 4 * time.Minute + //go:embed prompt.txt var systemPrompt string @@ -38,6 +53,12 @@ type ( // EvidenceAssessment is the structured output produced by the // evidence assessor. The worker persists it as JSONB on the // evidences table and derives Evidence.Description from Summary. + // + // Only Summary is surfaced to users today (through Description); the + // remaining structured fields (confidence, readable, issues, + // frameworks, ...) are stored as deliberate groundwork for a later + // PR that exposes them on the GraphQL/MCP Evidence type. They are + // intentionally write-only at the API boundary for now. EvidenceAssessment struct { Summary string `json:"summary" jsonschema:"One plain-text sentence (two at most) summarising what the evidence shows. When readable is false this field must restate the rejection reason."` System string `json:"system" jsonschema:"Tool or platform visible on the file; empty string if not clearly identifiable."` @@ -92,15 +113,28 @@ func (a *Assessor) Assess( mimeType string, fileBase64 string, ) (*EvidenceAssessment, error) { + // Detach from the caller so an unrelated cancellation cannot abort a + // committed assessment, but impose a hard deadline so a stalled + // provider response cannot run forever and pin the worker slot. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), AssessmentTimeout) + defer cancel() + opts := []agent.Option{ agent.WithInstructions(systemPrompt), agent.WithModel(a.cfg.Model), - agent.WithTemperature(a.cfg.Temp), agent.WithMaxTokens(a.cfg.MaxTokens), agent.WithOutputType(a.outputType), } if a.cfg.Thinking > 0 { + // Extended thinking and a custom temperature are mutually + // exclusive on Anthropic (the Messages API rejects any temperature + // other than 1 when thinking is enabled), and OpenAI reasoning + // models ignore temperature anyway, so only send a temperature + // when thinking is off. Mirrors pkg/vetting, which sets thinking + // without a temperature for the same reason. opts = append(opts, agent.WithThinking(a.cfg.Thinking)) + } else { + opts = append(opts, agent.WithTemperature(a.cfg.Temp)) } if a.cfg.Logger != nil { opts = append(opts, agent.WithLogger(a.cfg.Logger)) @@ -128,9 +162,14 @@ func (a *Assessor) Assess( return nil, fmt.Errorf("cannot assess evidence: %w", err) } + text := result.FinalMessage().Text() + var out EvidenceAssessment - if err := json.Unmarshal([]byte(result.FinalMessage().Text()), &out); err != nil { - return nil, fmt.Errorf("cannot parse evidence assessment: %w", err) + if err := json.Unmarshal([]byte(text), &out); err != nil { + // Surface the output size (not the content, which may carry + // sensitive evidence detail) so a MaxTokens-truncated response is + // distinguishable from other parse failures in logs. + return nil, fmt.Errorf("cannot parse evidence assessment (%d bytes of output): %w", len(text), err) } return &out, nil diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 5d9d6d978e..642a8a6917 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -153,14 +153,10 @@ func (h *evidenceAssessmentHandler) assessAndCommit( if err := h.pg.WithConn( ctx, func(ctx context.Context, conn pg.Querier) error { - if err := file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileID); err != nil { - return fmt.Errorf("cannot load file: %w", err) - } - - return nil + return file.LoadByID(ctx, conn, scope, *evidence.EvidenceFileID) }, ); err != nil { - return fmt.Errorf("cannot load file: %w", err) + return fmt.Errorf("cannot load evidence file: %w", err) } base64Data, mimeType, err := h.fileManager.GetFileBase64(ctx, &file) @@ -181,12 +177,19 @@ func (h *evidenceAssessmentHandler) assessAndCommit( } summary := assessment.Summary evidence.Description = &summary - evidence.AssessmentStatus = coredata.EvidenceAssessmentStatusCompleted - evidence.AssessmentProcessingStartedAt = nil - evidence.UpdatedAt = time.Now() - if err := evidence.Update(ctx, tx, scope); err != nil { - return fmt.Errorf("cannot update evidence: %w", err) + updated, err := evidence.SetAssessmentCompleted(ctx, tx, scope) + if err != nil { + return err + } + if !updated { + // The claim was recycled by stale recovery and finished by + // another worker; drop this result rather than clobber it. + h.logger.WarnCtx( + ctx, + "evidence assessment claim superseded, dropping result", + log.String("evidence_id", evidence.ID.String()), + ) } return nil @@ -198,7 +201,24 @@ func (h *evidenceAssessmentHandler) failEvidence(ctx context.Context, evidence c return h.pg.WithTx( ctx, func(ctx context.Context, tx pg.Tx) error { - return evidence.SetAssessmentFailed(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)) + updated, err := evidence.SetAssessmentFailed(ctx, tx, coredata.NewScopeFromObjectID(evidence.ID)) + if err != nil { + return err + } + + if !updated { + // The claim was recycled by stale recovery and re-claimed (or + // already finished) by another worker; the claim-scoped guard + // matched no row, so we leave the status to the current owner + // rather than forcing FAILED over its in-flight claim. + h.logger.WarnCtx( + ctx, + "evidence assessment failure superseded, leaving status to current owner", + log.String("evidence_id", evidence.ID.String()), + ) + } + + return nil }, ) } diff --git a/pkg/vetting/openai_schema.go b/pkg/vetting/openai_schema.go index a27046d0bc..8dae55c79e 100644 --- a/pkg/vetting/openai_schema.go +++ b/pkg/vetting/openai_schema.go @@ -34,8 +34,13 @@ type strictFunctionTool[P any] struct { } // jsonSchemaForTool builds an OpenAI strict-mode JSON schema for vetting tools -// and structured outputs. OpenAI requires every property in required and -// additionalProperties=false; the shared agent schema generator does not. +// and structured outputs. The shared agent schema generator already emits +// additionalProperties=false and a complete required list on every struct and +// array-of-struct object, so enforceStrictJSONSchema only adds the remaining +// strict-mode normalisation the generator skips: it sorts required and locks +// down map[string]T fields (which serialise to a non-false additionalProperties). +// Output types that are flat or map-free (e.g. evidenceassessor.EvidenceAssessment) +// are already strict-valid straight from agent.NewOutputType and do not need this. func jsonSchemaForTool[T any]() (json.RawMessage, error) { outputType, err := agent.NewOutputType[T]("_") if err != nil { From a28df40c253e80aebfcfc3dbe108702772010ba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Thu, 4 Jun 2026 22:55:11 +0200 Subject: [PATCH 20/21] Rename evidence assessor env vars and pin model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish the describer -> assessor rename on the operator surface. The env vars feeding the renamed config were still named for the describer (AGENT_EVIDENCE_DESCRIBER_* and EVIDENCE_DESCRIBER_*), so an operator using the assessor names was silently ignored. They are now AGENT_EVIDENCE_ASSESSOR_* / EVIDENCE_ASSESSOR_*, with matching .env.example entries (fixing the missing AGENT_ prefix) and bootstrap coverage for the worker-tuning block, which was previously untested. Pin the assessor to a current vision-capable model (gpt-5.4-mini) rather than inheriting the generic AGENT_DEFAULT (gpt-4o). Evidence assessment is vision-first -- it reads screenshots, PDFs and console exports -- so a stale default directly hurt assessment quality. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- .env.example | 8 ++++++-- pkg/bootstrap/builder.go | 19 ++++++++++++------- pkg/bootstrap/builder_test.go | 28 ++++++++++++++++++++-------- pkg/probod/third_party_vetter.go | 2 +- 4 files changed, 39 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index cd343a8437..2b21b915f6 100644 --- a/.env.example +++ b/.env.example @@ -71,8 +71,12 @@ # ── LLM-backed workers (inherit from LLM defaults when unset) ───────── # PROBO_AGENT_PROVIDER=anthropic # PROBO_AGENT_MODEL_NAME=claude-sonnet-4-6 -# EVIDENCE_DESCRIBER_PROVIDER=openai -# EVIDENCE_DESCRIBER_MODEL_NAME=gpt-4o-mini +# AGENT_EVIDENCE_ASSESSOR_PROVIDER=openai # inherits AGENT_DEFAULT_PROVIDER when unset +# AGENT_EVIDENCE_ASSESSOR_MODEL_NAME=gpt-5.4-mini # default; vision-capable model for evidence assessment +# AGENT_EVIDENCE_ASSESSOR_THINKING=2048 # extended-thinking budget; pair with a thinking-capable model (e.g. claude-sonnet-4-6) +# EVIDENCE_ASSESSOR_INTERVAL=10 +# EVIDENCE_ASSESSOR_STALE_AFTER=300 +# EVIDENCE_ASSESSOR_MAX_CONCURRENCY=10 # AGENT_THIRD_PARTY_VETTER_PROVIDER=openai # inherits AGENT_DEFAULT_PROVIDER when unset # AGENT_THIRD_PARTY_VETTER_MODEL_NAME=gpt-4o # inherits AGENT_DEFAULT_MODEL_NAME when unset # THIRD_PARTY_VETTING_INTERVAL=10 diff --git a/pkg/bootstrap/builder.go b/pkg/bootstrap/builder.go index 1b383d9cfe..1704901cde 100644 --- a/pkg/bootstrap/builder.go +++ b/pkg/bootstrap/builder.go @@ -193,6 +193,7 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { ModelName: b.getEnvOrDefault("AGENT_DEFAULT_MODEL_NAME", "gpt-4o"), Temperature: new(b.getEnvFloatOrDefault("AGENT_DEFAULT_TEMPERATURE", 0.1)), MaxTokens: new(b.getEnvIntOrDefault("AGENT_DEFAULT_MAX_TOKENS", 4096)), + Thinking: b.getEnvIntPtr("AGENT_DEFAULT_THINKING"), }, Probo: probodconfig.LLMAgentConfig{ Provider: b.getEnvOrDefault("AGENT_PROBO_PROVIDER", ""), @@ -201,10 +202,14 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { MaxTokens: b.getEnvIntPtr("AGENT_PROBO_MAX_TOKENS"), }, EvidenceAssessor: probodconfig.LLMAgentConfig{ - Provider: b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_PROVIDER", ""), - ModelName: b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_MODEL_NAME", ""), - Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"), - MaxTokens: b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"), + Provider: b.getEnvOrDefault("AGENT_EVIDENCE_ASSESSOR_PROVIDER", ""), + // Evidence assessment is vision-first (it reads screenshots / + // PDFs / console exports), so it pins a current vision-capable + // model rather than inheriting the generic AGENT_DEFAULT model. + ModelName: b.getEnvOrDefault("AGENT_EVIDENCE_ASSESSOR_MODEL_NAME", "gpt-5.4-mini"), + Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_ASSESSOR_TEMPERATURE"), + MaxTokens: b.getEnvIntPtr("AGENT_EVIDENCE_ASSESSOR_MAX_TOKENS"), + Thinking: b.getEnvIntPtr("AGENT_EVIDENCE_ASSESSOR_THINKING"), }, ThirdPartyVetter: probodconfig.LLMAgentConfig{ Provider: b.getEnvOrDefault("AGENT_THIRD_PARTY_VETTER_PROVIDER", ""), @@ -248,9 +253,9 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) { TSAURL: b.getEnvOrDefault("ESIGN_TSA_URL", "http://timestamp.digicert.com"), }, EvidenceAssessor: probodconfig.EvidenceAssessmentConfig{ - Interval: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_INTERVAL", 10), - StaleAfter: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_STALE_AFTER", 300), - MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_MAX_CONCURRENCY", 10), + Interval: b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_INTERVAL", 10), + StaleAfter: b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_STALE_AFTER", 300), + MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_MAX_CONCURRENCY", 10), }, ThirdPartyVetting: probodconfig.ThirdPartyVettingWorkerConfig{ Interval: b.getEnvIntOrDefault("THIRD_PARTY_VETTING_INTERVAL", 10), diff --git a/pkg/bootstrap/builder_test.go b/pkg/bootstrap/builder_test.go index d0297deebf..7ddf5c8b44 100644 --- a/pkg/bootstrap/builder_test.go +++ b/pkg/bootstrap/builder_test.go @@ -212,9 +212,10 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Nil(t, cfg.Probod.Agents.Probo.Temperature) assert.Nil(t, cfg.Probod.Agents.Probo.MaxTokens) assert.Empty(t, cfg.Probod.Agents.EvidenceAssessor.Provider) - assert.Empty(t, cfg.Probod.Agents.EvidenceAssessor.ModelName) + assert.Equal(t, "gpt-5.4-mini", cfg.Probod.Agents.EvidenceAssessor.ModelName) assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.Temperature) assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.MaxTokens) + assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.Thinking) assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.Provider) assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.ModelName) assert.Nil(t, cfg.Probod.Agents.ThirdPartyVetter.Temperature) @@ -238,6 +239,9 @@ func TestBuilder_Build_Defaults(t *testing.T) { assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval) assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter) assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency) + assert.Equal(t, 10, cfg.Probod.EvidenceAssessor.Interval) + assert.Equal(t, 300, cfg.Probod.EvidenceAssessor.StaleAfter) + assert.Equal(t, 10, cfg.Probod.EvidenceAssessor.MaxConcurrency) // Custom domains config assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval) @@ -322,11 +326,12 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["AGENT_DEFAULT_MODEL_NAME"] = "gpt-4-turbo" env["AGENT_DEFAULT_TEMPERATURE"] = "0.5" env["AGENT_DEFAULT_MAX_TOKENS"] = "8192" - // Agents — evidence-describer override - env["AGENT_EVIDENCE_DESCRIBER_PROVIDER"] = "anthropic" - env["AGENT_EVIDENCE_DESCRIBER_MODEL_NAME"] = "claude-sonnet-4-20250514" - env["AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"] = "0.2" - env["AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"] = "4096" + // Agents — evidence-assessor override + env["AGENT_EVIDENCE_ASSESSOR_PROVIDER"] = "anthropic" + env["AGENT_EVIDENCE_ASSESSOR_MODEL_NAME"] = "claude-sonnet-4-6" + env["AGENT_EVIDENCE_ASSESSOR_TEMPERATURE"] = "0.2" + env["AGENT_EVIDENCE_ASSESSOR_MAX_TOKENS"] = "4096" + env["AGENT_EVIDENCE_ASSESSOR_THINKING"] = "2048" // Agents — third-party-vetter override env["AGENT_THIRD_PARTY_VETTER_PROVIDER"] = "openai" env["AGENT_THIRD_PARTY_VETTER_MODEL_NAME"] = "gpt-4o" @@ -351,6 +356,9 @@ func TestBuilder_Build_CustomValues(t *testing.T) { env["THIRD_PARTY_VETTING_INTERVAL"] = "15" env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800" env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2" + env["EVIDENCE_ASSESSOR_INTERVAL"] = "20" + env["EVIDENCE_ASSESSOR_STALE_AFTER"] = "600" + env["EVIDENCE_ASSESSOR_MAX_CONCURRENCY"] = "5" // Custom domains env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53" env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----" @@ -432,11 +440,12 @@ func TestBuilder_Build_CustomValues(t *testing.T) { // Agents — probo inherits default (no overrides set) assert.Empty(t, cfg.Probod.Agents.Probo.Provider) assert.Empty(t, cfg.Probod.Agents.Probo.ModelName) - // Agents — evidence-describer overrides + // Agents — evidence-assessor overrides assert.Equal(t, "anthropic", cfg.Probod.Agents.EvidenceAssessor.Provider) - assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceAssessor.ModelName) + assert.Equal(t, "claude-sonnet-4-6", cfg.Probod.Agents.EvidenceAssessor.ModelName) assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceAssessor.Temperature) assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceAssessor.MaxTokens) + assert.Equal(t, new(2048), cfg.Probod.Agents.EvidenceAssessor.Thinking) // Agents — third-party-vetter overrides assert.Equal(t, "openai", cfg.Probod.Agents.ThirdPartyVetter.Provider) assert.Equal(t, "gpt-4o", cfg.Probod.Agents.ThirdPartyVetter.ModelName) @@ -461,6 +470,9 @@ func TestBuilder_Build_CustomValues(t *testing.T) { assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval) assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter) assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency) + assert.Equal(t, 20, cfg.Probod.EvidenceAssessor.Interval) + assert.Equal(t, 600, cfg.Probod.EvidenceAssessor.StaleAfter) + assert.Equal(t, 5, cfg.Probod.EvidenceAssessor.MaxConcurrency) // Custom domains assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr) assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey) diff --git a/pkg/probod/third_party_vetter.go b/pkg/probod/third_party_vetter.go index ba041676a6..85f5c43a00 100644 --- a/pkg/probod/third_party_vetter.go +++ b/pkg/probod/third_party_vetter.go @@ -24,7 +24,7 @@ import ( // buildThirdPartyVetter wires the third-party vetting agent. Unset // third-party-vetter fields inherit from the default agent config -// (AGENT_DEFAULT_*), same as evidence-describer and probo. +// (AGENT_DEFAULT_*), same as evidence-assessor and probo. func (impl *Implm) buildThirdPartyVetter( l *log.Logger, tp trace.TracerProvider, From 7783c311a9a4f57f0ce800652afcc7dfa44182eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Sibiril?= <81782+aureliensibiril@users.noreply.github.com> Date: Thu, 4 Jun 2026 23:10:02 +0200 Subject: [PATCH 21/21] Satisfy golangci-lint wsl_v5 and govet lostcancel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rebase onto main pulled in a stricter .golangci.yml (wsl_v5), which flagged missing blank lines across the evidence assessor changes; add the required whitespace. Also build the evidence assessor before the worker context cancels are created, so evidenceassessor.New's fallible early-return path no longer trips govet's lostcancel on those cancels. Signed-off-by: Aurélien Sibiril <81782+aureliensibiril@users.noreply.github.com> --- pkg/agent/output_type.go | 3 ++ pkg/coredata/connector.go | 1 + pkg/coredata/evidence.go | 1 + pkg/coredata/evidence_assessment.go | 6 ++++ .../evidence_assessment_state_test.go | 15 +++++++++ pkg/coredata/evidence_assessment_test.go | 1 + pkg/evidenceassessor/assessment.go | 2 ++ pkg/evidenceassessor/assessment_test.go | 1 + pkg/probo/evidence_assessment_worker.go | 2 ++ pkg/probod/probod.go | 33 +++++++++++-------- pkg/vetting/assessment.go | 1 + 11 files changed, 52 insertions(+), 14 deletions(-) diff --git a/pkg/agent/output_type.go b/pkg/agent/output_type.go index 0d5c064fb3..df36272ebc 100644 --- a/pkg/agent/output_type.go +++ b/pkg/agent/output_type.go @@ -70,12 +70,15 @@ func (o *OutputType) DecorateEnum(field string, values []string) error { if !ok { return fmt.Errorf("output type schema has no %q property", field) } + prop["enum"] = values decorated, err := json.Marshal(schema) if err != nil { return fmt.Errorf("cannot marshal decorated output type schema: %w", err) } + o.Schema = decorated + return nil } diff --git a/pkg/coredata/connector.go b/pkg/coredata/connector.go index 1640cbaf46..3f0b452815 100644 --- a/pkg/coredata/connector.go +++ b/pkg/coredata/connector.go @@ -67,6 +67,7 @@ func (j jsonRawMessageOrNull) Value() (driver.Value, error) { if len(j) == 0 { return nil, nil } + return []byte(j), nil } diff --git a/pkg/coredata/evidence.go b/pkg/coredata/evidence.go index eb910b5357..c033496a75 100644 --- a/pkg/coredata/evidence.go +++ b/pkg/coredata/evidence.go @@ -165,6 +165,7 @@ WHERE evidences.state = 'REQUESTED'; "assessment_status": e.AssessmentStatus, "assessment_processing_started_at": e.AssessmentProcessingStartedAt, } + _, err := conn.Exec(ctx, q, args) if err != nil { return fmt.Errorf("cannot upsert evidence: %w", err) diff --git a/pkg/coredata/evidence_assessment.go b/pkg/coredata/evidence_assessment.go index 3adf027bf0..0210efdc35 100644 --- a/pkg/coredata/evidence_assessment.go +++ b/pkg/coredata/evidence_assessment.go @@ -31,15 +31,19 @@ func (e *Evidence) SetAssessment(v any) error { e.Assessment = nil return nil } + data, err := json.Marshal(v) if err != nil { return fmt.Errorf("cannot marshal evidence assessment: %w", err) } + if string(data) == "null" { e.Assessment = nil return nil } + e.Assessment = data + return nil } @@ -49,8 +53,10 @@ func (e *Evidence) GetAssessment(dst any) error { if len(e.Assessment) == 0 { return nil } + if err := json.Unmarshal(e.Assessment, dst); err != nil { return fmt.Errorf("cannot unmarshal evidence assessment: %w", err) } + return nil } diff --git a/pkg/coredata/evidence_assessment_state_test.go b/pkg/coredata/evidence_assessment_state_test.go index 36632f3b61..036b2d6516 100644 --- a/pkg/coredata/evidence_assessment_state_test.go +++ b/pkg/coredata/evidence_assessment_state_test.go @@ -89,9 +89,11 @@ func seedAssessmentFixture(t *testing.T, ctx context.Context, client *pg.Client) if _, err := tx.Exec(ctx, `DELETE FROM evidences WHERE measure_id = $1`, measureID); err != nil { return err } + if _, err := tx.Exec(ctx, `DELETE FROM measures WHERE id = $1`, measureID); err != nil { return err } + if _, err := tx.Exec(ctx, `DELETE FROM organizations WHERE id = $1`, organizationID); err != nil { return err } @@ -151,6 +153,7 @@ func loadEvidence( t.Helper() var e coredata.Evidence + require.NoError(t, client.WithConn(ctx, func(ctx context.Context, conn pg.Querier) error { return e.LoadByID(ctx, conn, scope, id) })) @@ -208,9 +211,12 @@ func TestEvidence_SetAssessmentCompleted_TransitionsAndPreservesUserColumns(t *t require.NoError(t, claimed.SetAssessment(map[string]any{"summary": summary, "confidence": "HIGH"})) var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { var err error + updated, err = claimed.SetAssessmentCompleted(ctx, tx, fx.scope) + return err })) assert.True(t, updated, "completing a PROCESSING row owned by this claim must update it") @@ -248,9 +254,12 @@ func TestEvidence_SetAssessmentCompleted_NoOpOnSupersededClaim(t *testing.T) { require.NoError(t, firstClaim.SetAssessment(map[string]any{"summary": summary})) var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { var err error + updated, err = firstClaim.SetAssessmentCompleted(ctx, tx, fx.scope) + return err })) assert.False(t, updated, "a superseded claim must not overwrite the re-claimed row") @@ -275,9 +284,12 @@ func TestEvidence_SetAssessmentFailed_TransitionsOwningClaim(t *testing.T) { claimed := claimEvidence(t, ctx, client, fx.scope, id, t0.Add(time.Minute)) var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { var err error + updated, err = claimed.SetAssessmentFailed(ctx, tx, fx.scope) + return err })) assert.True(t, updated, "failing a PROCESSING row owned by this claim must update it") @@ -304,9 +316,12 @@ func TestEvidence_SetAssessmentFailed_NoOpOnSupersededClaim(t *testing.T) { // Worker A's late failure must not flip B's live claim to FAILED. var updated bool + require.NoError(t, client.WithTx(ctx, func(ctx context.Context, tx pg.Tx) error { var err error + updated, err = firstClaim.SetAssessmentFailed(ctx, tx, fx.scope) + return err })) assert.False(t, updated, "a stale worker's failure must not clobber the re-claimed row") diff --git a/pkg/coredata/evidence_assessment_test.go b/pkg/coredata/evidence_assessment_test.go index c7126bb905..82865c9b94 100644 --- a/pkg/coredata/evidence_assessment_test.go +++ b/pkg/coredata/evidence_assessment_test.go @@ -74,6 +74,7 @@ func TestEvidence_GetAssessment_EmptyIsNoOp(t *testing.T) { t.Parallel() var e Evidence + out := testAssessmentPayload{Summary: "unchanged"} require.NoError(t, e.GetAssessment(&out)) assert.Equal(t, "unchanged", out.Summary, "empty column should leave dst untouched") diff --git a/pkg/evidenceassessor/assessment.go b/pkg/evidenceassessor/assessment.go index d163c964de..741f88664c 100644 --- a/pkg/evidenceassessor/assessment.go +++ b/pkg/evidenceassessor/assessment.go @@ -97,6 +97,7 @@ func New(cfg Config) (*Assessor, error) { if err != nil { return nil, fmt.Errorf("cannot create evidence assessment output type: %w", err) } + if err := outputType.DecorateEnum("confidence", confidenceEnum); err != nil { return nil, fmt.Errorf("cannot decorate evidence assessment schema: %w", err) } @@ -136,6 +137,7 @@ func (a *Assessor) Assess( } else { opts = append(opts, agent.WithTemperature(a.cfg.Temp)) } + if a.cfg.Logger != nil { opts = append(opts, agent.WithLogger(a.cfg.Logger)) } diff --git a/pkg/evidenceassessor/assessment_test.go b/pkg/evidenceassessor/assessment_test.go index 2eecaa6c9d..9c4498d435 100644 --- a/pkg/evidenceassessor/assessment_test.go +++ b/pkg/evidenceassessor/assessment_test.go @@ -50,5 +50,6 @@ func TestNew_DecoratesConfidenceEnum(t *testing.T) { for i, v := range enumRaw { actual[i] = v.(string) } + assert.Equal(t, confidenceEnum, actual) } diff --git a/pkg/probo/evidence_assessment_worker.go b/pkg/probo/evidence_assessment_worker.go index 642a8a6917..a7bc1bf041 100644 --- a/pkg/probo/evidence_assessment_worker.go +++ b/pkg/probo/evidence_assessment_worker.go @@ -175,6 +175,7 @@ func (h *evidenceAssessmentHandler) assessAndCommit( if err := evidence.SetAssessment(assessment); err != nil { return err } + summary := assessment.Summary evidence.Description = &summary @@ -182,6 +183,7 @@ func (h *evidenceAssessmentHandler) assessAndCommit( if err != nil { return err } + if !updated { // The claim was recycled by stale recovery and finished by // another worker; drop this result rather than clobber it. diff --git a/pkg/probod/probod.go b/pkg/probod/probod.go index 3102b5b0d5..7264f7aa8f 100644 --- a/pkg/probod/probod.go +++ b/pkg/probod/probod.go @@ -632,6 +632,25 @@ func (impl *Implm) Run( return fmt.Errorf("cannot create server: %w", err) } + // Build the evidence assessor before any worker cancel func is created + // so its fallible construction can return early without tripping govet's + // lostcancel on the context cancels defined further below. + evidenceAssessorCfg := evidenceassessor.Config{ + Client: evidenceAssessorLLMClient, + Model: evidenceAssessorAgentCfg.ModelName, + Temp: ref.UnrefOrZero(evidenceAssessorAgentCfg.Temperature), + MaxTokens: ref.UnrefOrZero(evidenceAssessorAgentCfg.MaxTokens), + Logger: l.Named("evidence-assessor"), + } + if evidenceAssessorAgentCfg.Thinking != nil { + evidenceAssessorCfg.Thinking = *evidenceAssessorAgentCfg.Thinking + } + + evidenceAssessor, err := evidenceassessor.New(evidenceAssessorCfg) + if err != nil { + return fmt.Errorf("cannot build evidence assessor: %w", err) + } + apiServerCtx, stopApiServer := context.WithCancel(context.Background()) defer stopApiServer() @@ -838,20 +857,6 @@ func (impl *Implm) Run( }, ) - evidenceAssessorCfg := evidenceassessor.Config{ - Client: evidenceAssessorLLMClient, - Model: evidenceAssessorAgentCfg.ModelName, - Temp: ref.UnrefOrZero(evidenceAssessorAgentCfg.Temperature), - MaxTokens: ref.UnrefOrZero(evidenceAssessorAgentCfg.MaxTokens), - Logger: l.Named("evidence-assessor"), - } - if evidenceAssessorAgentCfg.Thinking != nil { - evidenceAssessorCfg.Thinking = *evidenceAssessorAgentCfg.Thinking - } - evidenceAssessor, err := evidenceassessor.New(evidenceAssessorCfg) - if err != nil { - return fmt.Errorf("cannot build evidence assessor: %w", err) - } evidenceAssessmentWorker := probo.NewEvidenceAssessmentWorker( pgClient, fileManagerService, diff --git a/pkg/vetting/assessment.go b/pkg/vetting/assessment.go index a3d435f1fc..f04e113b1c 100644 --- a/pkg/vetting/assessment.go +++ b/pkg/vetting/assessment.go @@ -321,6 +321,7 @@ func thirdPartyInfoOutputType() (*agent.OutputType, error) { if err := outputType.DecorateEnum("third_party_type", thirdPartyTypeEnum); err != nil { return nil, fmt.Errorf("cannot decorate thirdParty info schema: %w", err) } + strict, err := enforceStrictJSONSchema(outputType.Schema) if err != nil { return nil, fmt.Errorf("cannot enforce strict thirdParty info schema: %w", err)