feat(foreman): reconcile library for sliced-workload interface drift

pkg/foreman/slicer/reconcile.go

@@ -0,0 +1,235 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package slicer reconciles a sliced-and-unioned change against the slice
+// plan's pinned shared identifiers, catching cross-slice interface drift a
+// build cannot see (a dashboard querying a metric an exporter never emits, a
+// slice using a config key a sibling spelled differently). Ported from the
+// validated slicer experiment; the deterministic check is authoritative and
+// the injected-LLM sweep is advisory.
+package slicer
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// Drift kinds.
+const (
+	// DriftPinnedMissing is emitted by PinnedCheck: a pinned identifier is
+	// absent from a slice that must define or reference it. Deterministic and
+	// authoritative.
+	DriftPinnedMissing = "pinned-missing"
+	// DriftLLMFlagged is emitted by LLMSweep: the model flagged unpinned drift.
+	// Advisory; it can be a false positive and is adjudicated by human review.
+	DriftLLMFlagged = "llm-flagged"
+)
+
+// Drift is one cross-slice inconsistency.
+type Drift struct {
+	Identifier string
+	Slice      string
+	File       string
+	Kind       string
+}
+
+// SharedIdentifier pins one exact string that crosses a slice boundary. The
+// slice named by DefinedBy and every slice in ReferencedBy must contain the
+// exact ID.
+type SharedIdentifier struct {
+	ID           string   `yaml:"id"`
+	DefinedBy    string   `yaml:"defined_by"`
+	ReferencedBy []string `yaml:"referenced_by"`
+}
+
+// Slice is one disjoint-file unit of a plan.
+type Slice struct {
+	Name  string   `yaml:"name"`
+	Files []string `yaml:"files"`
+}
+
+// SlicePlan is the planner's output: the prose contract, the pinned shared
+// identifiers, and the disjoint-file slices.
+type SlicePlan struct {
+	Contract          string             `yaml:"contract"`
+	SharedIdentifiers []SharedIdentifier `yaml:"shared_identifiers"`
+	Slices            []Slice            `yaml:"slices"`
+}
+
+// ModelCaller is the injected LLM transport: prompt in, completion out. The
+// library never constructs a client or makes a network call; the experiment
+// passes a curl-to-reviewer closure and Foreman passes its own model client.
+type ModelCaller interface {
+	Call(prompt string) (string, error)
+}
+
+// Result is the reconcile outcome. Clean is true only when there are no
+// drifts of any kind; callers apply the authoritative-vs-advisory policy
+// (pinned-missing hard-fails, llm-flagged-only is advisory) on the drift list.
+type Result struct {
+	Clean  bool
+	Drifts []Drift
+}
+
+// PinnedCheck deterministically verifies that each pinned identifier appears,
+// as a WHOLE TOKEN (not a substring), in at least one file of every slice that
+// defines or references it. Absence yields one pinned-missing Drift for that
+// (identifier, slice). Unpinned strings are ignored.
+func PinnedCheck(ids []SharedIdentifier, repoDir string, sliceFiles map[string][]string) []Drift {
+	var drifts []Drift
+	for _, si := range ids {
+		slices := append([]string{si.DefinedBy}, si.ReferencedBy...)
+		for _, sl := range slices {
+			files := sliceFiles[sl]
+			found := false
+			for _, f := range files {
+				if present(si.ID, readFile(repoDir, f)) {
+					found = true
+					break
+				}
+			}
+			if !found {
+				drifts = append(drifts, Drift{
+					Identifier: si.ID,
+					Slice:      sl,
+					File:       strings.Join(files, ","),
+					Kind:       DriftPinnedMissing,
+				})
+			}
+		}
+	}
+	return drifts
+}
+
+var fenceRE = regexp.MustCompile("(?s)```(?:yaml)?\\s*(.*?)```")
+
+const sweepPrompt = `You are reviewing code slices written independently and then unioned. ` +
+	`Flag any SEMANTIC interface drift BETWEEN slices: a name, signature, key, ` +
+	`or value one slice uses that a sibling slice expects to be different.
+
+Return ONLY a YAML list, each item {identifier: <string>, slice: <slice name>, reason: <short>}. ` +
+	`If there is no drift, return exactly: NONE
+
+Contract:
+%s
+
+Union diff:
+%s
+`
+
+// LLMSweep asks the injected model to flag semantic drift the pins did not
+// cover, returning llm-flagged Drifts. It degrades open: any transport error,
+// or an empty / "NONE" / unparseable / non-list completion, yields no drifts
+// and never fails. A model outage therefore never blocks reconciliation.
+func LLMSweep(unionDiff, contract string, model ModelCaller) []Drift {
+	// Render the prompt before the transport call so a prompt-formatting bug is
+	// a loud programmer error, not a silent degrade-open no-op.
+	prompt := fmt.Sprintf(sweepPrompt, contract, unionDiff)
+	resp, err := model.Call(prompt)
+	if err != nil {
+		return nil
+	}
+	resp = strings.TrimSpace(resp)
+	if resp == "" || strings.HasPrefix(strings.ToUpper(resp), "NONE") {
+		return nil
+	}
+	body := resp
+	if m := fenceRE.FindStringSubmatch(resp); m != nil {
+		body = m[1]
+	}
+	var items []map[string]interface{}
+	if err := yaml.Unmarshal([]byte(body), &items); err != nil {
+		return nil
+	}
+	var drifts []Drift
+	for _, it := range items {
+		id, ok := it["identifier"]
+		if !ok {
+			continue
+		}
+		slice := ""
+		if s, ok := it["slice"]; ok {
+			slice = fmt.Sprintf("%v", s)
+		}
+		drifts = append(drifts, Drift{
+			Identifier: fmt.Sprintf("%v", id),
+			Slice:      slice,
+			File:       "",
+			Kind:       DriftLLMFlagged,
+		})
+	}
+	return drifts
+}
+
+// Reconcile runs PinnedCheck then LLMSweep and reports whether the union is
+// interface-consistent. Clean is true only when neither produced a drift.
+func Reconcile(plan SlicePlan, repoDir, unionDiff string, model ModelCaller) Result {
+	sliceFiles := make(map[string][]string, len(plan.Slices))
+	for _, s := range plan.Slices {
+		sliceFiles[s.Name] = s.Files
+	}
+	drifts := PinnedCheck(plan.SharedIdentifiers, repoDir, sliceFiles)
+	drifts = append(drifts, LLMSweep(unionDiff, plan.Contract, model)...)
+	return Result{Clean: len(drifts) == 0, Drifts: drifts}
+}
+
+// readFile returns the contents of repoDir/path, or "" if it cannot be read
+// (a missing file is not present, not an error).
+func readFile(repoDir, path string) string {
+	b, err := os.ReadFile(filepath.Join(repoDir, path))
+	if err != nil {
+		return ""
+	}
+	return string(b)
+}
+
+// present reports whether id occurs in text as a whole token: a match bounded
+// on both sides by a non-identifier character (or string edge). This is NOT
+// substring containment. A pin like "rocm_smi_gpu_temp" must not be satisfied
+// by the different metric "rocm_smi_gpu_temperature" (superstring) or by a
+// "rocm_smi_gpu_temp_degC" suffix; both are distinct identifiers.
+func present(id, text string) bool {
+	if id == "" {
+		return false
+	}
+	for i := 0; ; {
+		j := strings.Index(text[i:], id)
+		if j < 0 {
+			return false
+		}
+		start := i + j
+		end := start + len(id)
+		beforeOK := start == 0 || !isIdentByte(text[start-1])
+		afterOK := end == len(text) || !isIdentByte(text[end])
+		if beforeOK && afterOK {
+			return true
+		}
+		i = start + 1
+	}
+}
+
+// isIdentByte reports whether b is an identifier character ([A-Za-z0-9_]).
+func isIdentByte(b byte) bool {
+	return b == '_' ||
+		(b >= '0' && b <= '9') ||
+		(b >= 'a' && b <= 'z') ||
+		(b >= 'A' && b <= 'Z')
+}

pkg/foreman/slicer/reconcile_test.go

@@ -0,0 +1,159 @@
+/*
+Copyright 2025.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package slicer
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"testing"
+)
+
+// writeRepo materializes files under a temp dir and returns its path.
+func writeRepo(t *testing.T, files map[string]string) string {
+	t.Helper()
+	dir := t.TempDir()
+	for path, content := range files {
+		full := filepath.Join(dir, path)
+		if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte(content), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return dir
+}
+
+// stubModel is a canned ModelCaller: no live model in unit tests.
+type stubModel struct {
+	resp string
+	err  error
+}
+
+func (s stubModel) Call(string) (string, error) { return s.resp, s.err }
+
+func TestPinnedCheck_PresentIsClean(t *testing.T) {
+	repo := writeRepo(t, map[string]string{
+		"config/monitoring/exp.yaml": "emits rocm_smi_gpu_temp_deg here",
+		"config/grafana/dash.json":   `{"expr":"rocm_smi_gpu_temp_deg"}`,
+	})
+	ids := []SharedIdentifier{{ID: "rocm_smi_gpu_temp_deg", DefinedBy: "exp", ReferencedBy: []string{"dash"}}}
+	sf := map[string][]string{"exp": {"config/monitoring/exp.yaml"}, "dash": {"config/grafana/dash.json"}}
+	if drifts := PinnedCheck(ids, repo, sf); len(drifts) != 0 {
+		t.Fatalf("want clean, got %+v", drifts)
+	}
+}
+
+func TestPinnedCheck_MissingFromDefinerFlags(t *testing.T) {
+	repo := writeRepo(t, map[string]string{
+		"config/monitoring/exp.yaml": "emits rocm_smi_temperature_edge instead",
+		"config/grafana/dash.json":   `{"expr":"rocm_smi_gpu_temp_deg"}`,
+	})
+	ids := []SharedIdentifier{{ID: "rocm_smi_gpu_temp_deg", DefinedBy: "exp", ReferencedBy: []string{"dash"}}}
+	sf := map[string][]string{"exp": {"config/monitoring/exp.yaml"}, "dash": {"config/grafana/dash.json"}}
+	drifts := PinnedCheck(ids, repo, sf)
+	if len(drifts) != 1 {
+		t.Fatalf("want 1 drift, got %+v", drifts)
+	}
+	got := drifts[0]
+	if got.Identifier != "rocm_smi_gpu_temp_deg" || got.Slice != "exp" || got.Kind != DriftPinnedMissing {
+		t.Fatalf("unexpected drift: %+v", got)
+	}
+}
+
+// A pin must match a WHOLE TOKEN, not a substring: the different metric
+// rocm_smi_gpu_temperature must NOT satisfy the pin rocm_smi_gpu_temp
+// (regression the live validation run caught).
+func TestPinnedCheck_SuperstringIsNotAMatch(t *testing.T) {
+	repo := writeRepo(t, map[string]string{"docs/g.md": "the exporter emits rocm_smi_gpu_temperature per gpu"})
+	ids := []SharedIdentifier{{ID: "rocm_smi_gpu_temp", DefinedBy: "docs"}}
+	sf := map[string][]string{"docs": {"docs/g.md"}}
+	drifts := PinnedCheck(ids, repo, sf)
+	if len(drifts) != 1 || drifts[0].Identifier != "rocm_smi_gpu_temp" {
+		t.Fatalf("want 1 drift for rocm_smi_gpu_temp, got %+v", drifts)
+	}
+}
+
+// rocm_smi_gpu_temp must not be satisfied by rocm_smi_gpu_temp_degC (a
+// different metric); the exact token IS a match.
+func TestPinnedCheck_SuffixedTokenIsNotAMatch(t *testing.T) {
+	ids := []SharedIdentifier{{ID: "rocm_smi_gpu_temp", DefinedBy: "a"}}
+	sf := map[string][]string{"a": {"a.yaml"}}
+
+	suffixed := writeRepo(t, map[string]string{"a.yaml": "rocm_smi_gpu_temp_degC only"})
+	if drifts := PinnedCheck(ids, suffixed, sf); len(drifts) != 1 {
+		t.Fatalf("suffixed: want 1 drift, got %+v", drifts)
+	}
+
+	exact := writeRepo(t, map[string]string{"a.yaml": `value rocm_smi_gpu_temp{gpu="0"} 42`})
+	if drifts := PinnedCheck(ids, exact, sf); len(drifts) != 0 {
+		t.Fatalf("exact: want clean, got %+v", drifts)
+	}
+}
+
+func TestLLMSweep_ParsesFlaggedDrift(t *testing.T) {
+	model := stubModel{resp: "```yaml\n- identifier: foo_metric\n  slice: dash\n  reason: mismatch\n```"}
+	drifts := LLMSweep("diff", "contract", model)
+	if len(drifts) != 1 {
+		t.Fatalf("want 1 drift, got %+v", drifts)
+	}
+	if drifts[0].Identifier != "foo_metric" || drifts[0].Kind != DriftLLMFlagged {
+		t.Fatalf("unexpected drift: %+v", drifts[0])
+	}
+}
+
+func TestLLMSweep_NoneIsClean(t *testing.T) {
+	if drifts := LLMSweep("d", "c", stubModel{resp: "NONE"}); len(drifts) != 0 {
+		t.Fatalf("want clean, got %+v", drifts)
+	}
+}
+
+func TestLLMSweep_MalformedIsClean(t *testing.T) {
+	if drifts := LLMSweep("d", "c", stubModel{resp: "garbage {["}); len(drifts) != 0 {
+		t.Fatalf("want clean, got %+v", drifts)
+	}
+}
+
+func TestLLMSweep_ModelErrorIsClean(t *testing.T) {
+	if drifts := LLMSweep("d", "c", stubModel{err: errors.New("unreachable")}); len(drifts) != 0 {
+		t.Fatalf("want clean, got %+v", drifts)
+	}
+}
+
+func TestReconcile_ComposesPinnedAndLLM(t *testing.T) {
+	repo := writeRepo(t, map[string]string{"a.yaml": "wrong_name"})
+	plan := SlicePlan{
+		Contract:          "c",
+		SharedIdentifiers: []SharedIdentifier{{ID: "right_name", DefinedBy: "a"}},
+		Slices:            []Slice{{Name: "a", Files: []string{"a.yaml"}}},
+	}
+	model := stubModel{resp: "- identifier: other\n  slice: a\n  reason: r"}
+	res := Reconcile(plan, repo, "diff", model)
+	if res.Clean {
+		t.Fatalf("want not clean")
+	}
+	kinds := make([]string, 0, len(res.Drifts))
+	for _, d := range res.Drifts {
+		kinds = append(kinds, d.Kind)
+	}
+	sort.Strings(kinds)
+	if len(kinds) != 2 || kinds[0] != DriftLLMFlagged || kinds[1] != DriftPinnedMissing {
+		t.Fatalf("want [llm-flagged pinned-missing], got %v", kinds)
+	}
+}