Describe evidence with structured assessment output

.env.example

@@ -71,8 +71,12 @@
 # ── LLM-backed workers (inherit from LLM defaults when unset) ─────────
 # PROBO_AGENT_PROVIDER=anthropic
 # PROBO_AGENT_MODEL_NAME=claude-sonnet-4-6
-# EVIDENCE_DESCRIBER_PROVIDER=openai
-# EVIDENCE_DESCRIBER_MODEL_NAME=gpt-4o-mini
+# AGENT_EVIDENCE_ASSESSOR_PROVIDER=openai  # inherits AGENT_DEFAULT_PROVIDER when unset
+# AGENT_EVIDENCE_ASSESSOR_MODEL_NAME=gpt-5.4-mini  # default; vision-capable model for evidence assessment
+# AGENT_EVIDENCE_ASSESSOR_THINKING=2048  # extended-thinking budget; pair with a thinking-capable model (e.g. claude-sonnet-4-6)
+# EVIDENCE_ASSESSOR_INTERVAL=10
+# EVIDENCE_ASSESSOR_STALE_AFTER=300
+# EVIDENCE_ASSESSOR_MAX_CONCURRENCY=10
 # AGENT_THIRD_PARTY_VETTER_PROVIDER=openai  # inherits AGENT_DEFAULT_PROVIDER when unset
 # AGENT_THIRD_PARTY_VETTER_MODEL_NAME=gpt-4o  # inherits AGENT_DEFAULT_MODEL_NAME when unset
 # THIRD_PARTY_VETTING_INTERVAL=10

contrib/claude/file-naming.md

@@ -76,7 +76,7 @@ auth). In that case the file keeps its service name and the agent is built
 inline:
 
 ```
-pkg/evidencedescriber/evidencedescriber.go   -- single-file describer service
+pkg/evidenceassessor/assessment.go            -- single-file evidence assessment service
 pkg/vetting/assessment.go                     -- third-party assessment service
 ```
 

pkg/agent/output_type.go

@@ -50,3 +50,35 @@ func (o *OutputType) responseFormat() *llm.ResponseFormat {
 		},
 	}
 }
+
+// DecorateEnum injects an explicit `enum` constraint on a single
+// top-level property of the schema. jsonschema-go reads struct tags as
+// free-form descriptions only, so enums cannot be encoded on the tag
+// itself; callers chain one call per enum field after NewOutputType.
+func (o *OutputType) DecorateEnum(field string, values []string) error {
+	var schema map[string]any
+	if err := json.Unmarshal(o.Schema, &schema); err != nil {
+		return fmt.Errorf("cannot unmarshal output type schema: %w", err)
+	}
+
+	properties, ok := schema["properties"].(map[string]any)
+	if !ok {
+		return fmt.Errorf("output type schema has no properties")
+	}
+
+	prop, ok := properties[field].(map[string]any)
+	if !ok {
+		return fmt.Errorf("output type schema has no %q property", field)
+	}
+
+	prop["enum"] = values
+
+	decorated, err := json.Marshal(schema)
+	if err != nil {
+		return fmt.Errorf("cannot marshal decorated output type schema: %w", err)
+	}
+
+	o.Schema = decorated
+
+	return nil
+}

pkg/bootstrap/builder.go

@@ -193,18 +193,23 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 					ModelName:   b.getEnvOrDefault("AGENT_DEFAULT_MODEL_NAME", "gpt-4o"),
 					Temperature: new(b.getEnvFloatOrDefault("AGENT_DEFAULT_TEMPERATURE", 0.1)),
 					MaxTokens:   new(b.getEnvIntOrDefault("AGENT_DEFAULT_MAX_TOKENS", 4096)),
+					Thinking:    b.getEnvIntPtr("AGENT_DEFAULT_THINKING"),
 				},
 				Probo: probodconfig.LLMAgentConfig{
 					Provider:    b.getEnvOrDefault("AGENT_PROBO_PROVIDER", ""),
 					ModelName:   b.getEnvOrDefault("AGENT_PROBO_MODEL_NAME", ""),
 					Temperature: b.getEnvFloatPtr("AGENT_PROBO_TEMPERATURE"),
 					MaxTokens:   b.getEnvIntPtr("AGENT_PROBO_MAX_TOKENS"),
 				},
-				EvidenceDescriber: probodconfig.LLMAgentConfig{
-					Provider:    b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_PROVIDER", ""),
-					ModelName:   b.getEnvOrDefault("AGENT_EVIDENCE_DESCRIBER_MODEL_NAME", ""),
-					Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"),
-					MaxTokens:   b.getEnvIntPtr("AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"),
+				EvidenceAssessor: probodconfig.LLMAgentConfig{
+					Provider: b.getEnvOrDefault("AGENT_EVIDENCE_ASSESSOR_PROVIDER", ""),
+					// Evidence assessment is vision-first (it reads screenshots /
+					// PDFs / console exports), so it pins a current vision-capable
+					// model rather than inheriting the generic AGENT_DEFAULT model.
+					ModelName:   b.getEnvOrDefault("AGENT_EVIDENCE_ASSESSOR_MODEL_NAME", "gpt-5.4-mini"),
+					Temperature: b.getEnvFloatPtr("AGENT_EVIDENCE_ASSESSOR_TEMPERATURE"),
+					MaxTokens:   b.getEnvIntPtr("AGENT_EVIDENCE_ASSESSOR_MAX_TOKENS"),
+					Thinking:    b.getEnvIntPtr("AGENT_EVIDENCE_ASSESSOR_THINKING"),
 				},
 				ThirdPartyVetter: probodconfig.LLMAgentConfig{
 					Provider:    b.getEnvOrDefault("AGENT_THIRD_PARTY_VETTER_PROVIDER", ""),
@@ -247,10 +252,10 @@ func (b *Builder) Build() (*probodconfig.FullConfig, error) {
 			ESign: probodconfig.ESignConfig{
 				TSAURL: b.getEnvOrDefault("ESIGN_TSA_URL", "http://timestamp.digicert.com"),
 			},
-			EvidenceDescriber: probodconfig.EvidenceDescriberConfig{
-				Interval:       b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_INTERVAL", 10),
-				StaleAfter:     b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_STALE_AFTER", 300),
-				MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_DESCRIBER_MAX_CONCURRENCY", 10),
+			EvidenceAssessor: probodconfig.EvidenceAssessmentConfig{
+				Interval:       b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_INTERVAL", 10),
+				StaleAfter:     b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_STALE_AFTER", 300),
+				MaxConcurrency: b.getEnvIntOrDefault("EVIDENCE_ASSESSOR_MAX_CONCURRENCY", 10),
 			},
 			ThirdPartyVetting: probodconfig.ThirdPartyVettingWorkerConfig{
 				Interval:       b.getEnvIntOrDefault("THIRD_PARTY_VETTING_INTERVAL", 10),

pkg/bootstrap/builder_test.go

@@ -211,10 +211,11 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Empty(t, cfg.Probod.Agents.Probo.ModelName)
 	assert.Nil(t, cfg.Probod.Agents.Probo.Temperature)
 	assert.Nil(t, cfg.Probod.Agents.Probo.MaxTokens)
-	assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.Provider)
-	assert.Empty(t, cfg.Probod.Agents.EvidenceDescriber.ModelName)
-	assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.Temperature)
-	assert.Nil(t, cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
+	assert.Empty(t, cfg.Probod.Agents.EvidenceAssessor.Provider)
+	assert.Equal(t, "gpt-5.4-mini", cfg.Probod.Agents.EvidenceAssessor.ModelName)
+	assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.Temperature)
+	assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.MaxTokens)
+	assert.Nil(t, cfg.Probod.Agents.EvidenceAssessor.Thinking)
 	assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.Provider)
 	assert.Empty(t, cfg.Probod.Agents.ThirdPartyVetter.ModelName)
 	assert.Nil(t, cfg.Probod.Agents.ThirdPartyVetter.Temperature)
@@ -238,6 +239,9 @@ func TestBuilder_Build_Defaults(t *testing.T) {
 	assert.Equal(t, 10, cfg.Probod.ThirdPartyVetting.Interval)
 	assert.Equal(t, 1500, cfg.Probod.ThirdPartyVetting.StaleAfter)
 	assert.Equal(t, 1, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
+	assert.Equal(t, 10, cfg.Probod.EvidenceAssessor.Interval)
+	assert.Equal(t, 300, cfg.Probod.EvidenceAssessor.StaleAfter)
+	assert.Equal(t, 10, cfg.Probod.EvidenceAssessor.MaxConcurrency)
 
 	// Custom domains config
 	assert.Equal(t, 3600, cfg.Probod.CustomDomains.RenewalInterval)
@@ -322,11 +326,12 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["AGENT_DEFAULT_MODEL_NAME"] = "gpt-4-turbo"
 	env["AGENT_DEFAULT_TEMPERATURE"] = "0.5"
 	env["AGENT_DEFAULT_MAX_TOKENS"] = "8192"
-	// Agents — evidence-describer override
-	env["AGENT_EVIDENCE_DESCRIBER_PROVIDER"] = "anthropic"
-	env["AGENT_EVIDENCE_DESCRIBER_MODEL_NAME"] = "claude-sonnet-4-20250514"
-	env["AGENT_EVIDENCE_DESCRIBER_TEMPERATURE"] = "0.2"
-	env["AGENT_EVIDENCE_DESCRIBER_MAX_TOKENS"] = "4096"
+	// Agents — evidence-assessor override
+	env["AGENT_EVIDENCE_ASSESSOR_PROVIDER"] = "anthropic"
+	env["AGENT_EVIDENCE_ASSESSOR_MODEL_NAME"] = "claude-sonnet-4-6"
+	env["AGENT_EVIDENCE_ASSESSOR_TEMPERATURE"] = "0.2"
+	env["AGENT_EVIDENCE_ASSESSOR_MAX_TOKENS"] = "4096"
+	env["AGENT_EVIDENCE_ASSESSOR_THINKING"] = "2048"
 	// Agents — third-party-vetter override
 	env["AGENT_THIRD_PARTY_VETTER_PROVIDER"] = "openai"
 	env["AGENT_THIRD_PARTY_VETTER_MODEL_NAME"] = "gpt-4o"
@@ -351,6 +356,9 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	env["THIRD_PARTY_VETTING_INTERVAL"] = "15"
 	env["THIRD_PARTY_VETTING_STALE_AFTER"] = "1800"
 	env["THIRD_PARTY_VETTING_MAX_CONCURRENCY"] = "2"
+	env["EVIDENCE_ASSESSOR_INTERVAL"] = "20"
+	env["EVIDENCE_ASSESSOR_STALE_AFTER"] = "600"
+	env["EVIDENCE_ASSESSOR_MAX_CONCURRENCY"] = "5"
 	// Custom domains
 	env["CUSTOM_DOMAINS_RESOLVER_ADDR"] = "1.1.1.1:53"
 	env["ACME_ACCOUNT_KEY"] = "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----"
@@ -432,11 +440,12 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	// Agents — probo inherits default (no overrides set)
 	assert.Empty(t, cfg.Probod.Agents.Probo.Provider)
 	assert.Empty(t, cfg.Probod.Agents.Probo.ModelName)
-	// Agents — evidence-describer overrides
-	assert.Equal(t, "anthropic", cfg.Probod.Agents.EvidenceDescriber.Provider)
-	assert.Equal(t, "claude-sonnet-4-20250514", cfg.Probod.Agents.EvidenceDescriber.ModelName)
-	assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceDescriber.Temperature)
-	assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceDescriber.MaxTokens)
+	// Agents — evidence-assessor overrides
+	assert.Equal(t, "anthropic", cfg.Probod.Agents.EvidenceAssessor.Provider)
+	assert.Equal(t, "claude-sonnet-4-6", cfg.Probod.Agents.EvidenceAssessor.ModelName)
+	assert.Equal(t, new(0.2), cfg.Probod.Agents.EvidenceAssessor.Temperature)
+	assert.Equal(t, new(4096), cfg.Probod.Agents.EvidenceAssessor.MaxTokens)
+	assert.Equal(t, new(2048), cfg.Probod.Agents.EvidenceAssessor.Thinking)
 	// Agents — third-party-vetter overrides
 	assert.Equal(t, "openai", cfg.Probod.Agents.ThirdPartyVetter.Provider)
 	assert.Equal(t, "gpt-4o", cfg.Probod.Agents.ThirdPartyVetter.ModelName)
@@ -461,6 +470,9 @@ func TestBuilder_Build_CustomValues(t *testing.T) {
 	assert.Equal(t, 15, cfg.Probod.ThirdPartyVetting.Interval)
 	assert.Equal(t, 1800, cfg.Probod.ThirdPartyVetting.StaleAfter)
 	assert.Equal(t, 2, cfg.Probod.ThirdPartyVetting.MaxConcurrency)
+	assert.Equal(t, 20, cfg.Probod.EvidenceAssessor.Interval)
+	assert.Equal(t, 600, cfg.Probod.EvidenceAssessor.StaleAfter)
+	assert.Equal(t, 5, cfg.Probod.EvidenceAssessor.MaxConcurrency)
 	// Custom domains
 	assert.Equal(t, "1.1.1.1:53", cfg.Probod.CustomDomains.ResolverAddr)
 	assert.Equal(t, "-----BEGIN EC PRIVATE KEY-----\ntest\n-----END EC PRIVATE KEY-----", cfg.Probod.CustomDomains.ACME.AccountKey)

pkg/coredata/connector.go

@@ -16,6 +16,7 @@ package coredata
 
 import (
 	"context"
+	"database/sql/driver"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -59,6 +60,17 @@ func (j *jsonRawMessageOrNull) Scan(src any) error {
 	}
 }
 
+// Value implements database/sql/driver.Valuer so pgx binds the column
+// without an explicit helper at every call site. pgx rejects empty byte
+// slices as invalid JSON, so an empty/nil value is sent as SQL NULL.
+func (j jsonRawMessageOrNull) Value() (driver.Value, error) {
+	if len(j) == 0 {
+		return nil, nil
+	}
+
+	return []byte(j), nil
+}
+
 type (
 	Connector struct {
 		ID                  gid.GID              `db:"id"`
@@ -389,18 +401,13 @@ INSERT INTO connectors (
 		return fmt.Errorf("cannot encrypt connection: %w", err)
 	}
 
-	var settingsArg any
-	if len(c.RawSettings) > 0 {
-		settingsArg = []byte(c.RawSettings)
-	}
-
 	args := pgx.StrictNamedArgs{
 		"id":                   c.ID,
 		"tenant_id":            scope.GetTenantID(),
 		"organization_id":      c.OrganizationID,
 		"provider":             c.Provider,
 		"protocol":             c.Protocol,
-		"settings":             settingsArg,
+		"settings":             c.RawSettings,
 		"encrypted_connection": encryptedConnection,
 		"created_at":           c.CreatedAt,
 		"updated_at":           c.UpdatedAt,
@@ -613,14 +620,9 @@ WHERE
 		return fmt.Errorf("cannot encrypt connection: %w", err)
 	}
 
-	var settingsArg any
-	if len(c.RawSettings) > 0 {
-		settingsArg = []byte(c.RawSettings)
-	}
-
 	args := pgx.StrictNamedArgs{
 		"id":                   c.ID,
-		"settings":             settingsArg,
+		"settings":             c.RawSettings,
 		"encrypted_connection": encryptedConnection,
 		"updated_at":           c.UpdatedAt,
 	}