diff --git a/README.md b/README.md index 4002e06..e465029 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,10 @@ It is possible to set authentication data via: 6. [Get Deliverable Download Secrets](docs/devops.md#156-get-deliverable-download-secrets) 7. [Check if Instance Has Sealed Secret Certificate](docs/devops.md#157-check-if-instance-has-sealed-secret-certificate) 8. [Send Deployment Metadata From Instance](docs/devops.md#158-send-deployment-metadata-from-instance) + 9. [List Product Feature Sets on an Instance Plan](docs/devops.md#159-list-product-feature-sets-on-an-instance-plan) + 10. [Switch Product Feature Set on an Instance](docs/devops.md#1510-switch-product-feature-set-on-an-instance) + 11. [Version Feature Set on a Product](docs/devops.md#1511-version-feature-set-on-a-product) + 12. [Report Deployment Failures From a CD Agent](docs/devops.md#1512-report-deployment-failures-from-a-cd-agent) 16. [Probe SBOM](#16-use-case-probe-sbom) 17. [Download Artifact](#17-use-case-download-artifact) 18. [Register PullRequest with ReARM](#18-use-case-register-pullrequest-with-rearm) diff --git a/cmd/instevent.go b/cmd/instevent.go new file mode 100644 index 0000000..cec527f --- /dev/null +++ b/cmd/instevent.go @@ -0,0 +1,322 @@ +/* +The MIT License (MIT) + +Copyright (c) 2020 - 2026 Reliza Incorporated (Reliza (tm), https://reliza.io) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*/ + +package cmd + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "regexp" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + instEventDeployment string + instEventPhase string + instEventFailureClass string + instEventMessage string + instEventDetail string + instEventDetailFile string + instEventFingerprint string + instEventFeatureSet string + instEventProduct string + instEventTargetRelease string + instEventAttemptedAt string + instEventsJson string + instEventsFile string +) + +// maxDetailChars bounds what we transmit. The server clamps again at a higher +// limit; this is the agent-side bound so a runaway helm dump never leaves the +// cluster in the first place. +const maxDetailChars = 4096 + +// maxMessageChars bounds the one-line summary. +const maxMessageChars = 1024 + +// secretPatterns redact obvious credential material out of captured stderr +// BEFORE it leaves the cluster. Helm output routinely echoes rendered values, +// and values files carry tokens/passwords -- shipping raw stderr to a hosted +// backend would be a credential-exfiltration path. Redaction is deliberately +// aggressive: a lost diagnostic line is cheap, a leaked secret is not. +var secretPatterns = []*regexp.Regexp{ + // key: value / key=value where the key CONTAINS a credential word. + // The credential word is matched as a substring on purpose: helm values are + // overwhelmingly camelCase (adminPassword, registryToken, dbSecretKey), and + // an anchored \bpassword\b misses every one of them. + regexp.MustCompile(`(?i)[\w.-]*(password|passwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential)[\w.-]*\s*[:=]\s*"?[^\s",}]+"?`), + // Short/ambiguous keys, whole-word only — "pwd" and "auth" as substrings + // would maul ordinary prose ("author", "authority", "forward"). + regexp.MustCompile(`(?i)\b(pwd|auth)\b\s*[:=]\s*"?[^\s",}]+"?`), + // Authorization headers (Bearer/Basic) + regexp.MustCompile(`(?i)\bauthorization\b\s*[:=]\s*"?(bearer|basic)?\s*[A-Za-z0-9._~+/=-]+"?`), + // PEM blocks + regexp.MustCompile(`(?s)-----BEGIN [A-Z ]*PRIVATE KEY-----.*?-----END [A-Z ]*PRIVATE KEY-----`), + // JWTs + regexp.MustCompile(`\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\b`), + // credentials embedded in URLs + regexp.MustCompile(`\b([a-zA-Z][a-zA-Z0-9+.-]*://)[^\s/:@]+:[^\s/@]+@`), +} + +// volatilePatterns are stripped only when deriving a fingerprint, so the same +// underlying fault yields one stable dedup key instead of a new row per +// reconcile. They are NOT removed from the transmitted message/detail. +var volatilePatterns = []*regexp.Regexp{ + regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`), // uuids + regexp.MustCompile(`(?i)\bsha256:[0-9a-f]{8,64}\b`), // digests + regexp.MustCompile(`(?i)\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}\S*`), // timestamps + regexp.MustCompile(`\b\d+\b`), // counters, ports, epochs +} + +// redactSecrets replaces credential-looking material with a marker. +func redactSecrets(s string) string { + for _, re := range secretPatterns { + s = re.ReplaceAllString(s, "[REDACTED]") + } + return s +} + +// truncateChars clamps to max runes, marking that it happened so an operator +// knows the tail is missing rather than assuming a short error. +func truncateChars(s string, max int) string { + r := []rune(s) + if len(r) <= max { + return s + } + return string(r[:max]) + "\n…[truncated]" +} + +// sanitizeDetail is the single path everything captured from a subprocess must +// go through before transmit. +func sanitizeDetail(s string) string { + return truncateChars(redactSecrets(s), maxDetailChars) +} + +// deriveFailureFingerprint produces a stable dedup key for a fault. Volatile tokens +// are normalised out so a message that embeds a uuid or timestamp does not +// create a fresh row on every reconcile. +func deriveFailureFingerprint(deploymentName, phase, failureClass, message string) string { + // Normalise BEFORE lowercasing: several volatile patterns are anchored on + // case-bearing syntax (the T in an RFC3339 timestamp), and lowercasing + // first silently defeated them — which let timestamps through and made a + // persistent failure insert a fresh row on every reconcile. + norm := message + for _, re := range volatilePatterns { + norm = re.ReplaceAllString(norm, "*") + } + norm = strings.ToLower(norm) + norm = strings.Join(strings.Fields(norm), " ") + sum := sha256.Sum256([]byte(deploymentName + "|" + phase + "|" + failureClass + "|" + norm)) + return hex.EncodeToString(sum[:16]) +} + +// buildInstEvent assembles one event from flags, applying redaction, +// truncation and fingerprint derivation. +func buildInstEvent() (map[string]interface{}, error) { + if instEventDeployment == "" { + return nil, fmt.Errorf("--deployment is required") + } + detail := instEventDetail + if instEventDetailFile != "" { + b, err := os.ReadFile(instEventDetailFile) + if err != nil { + return nil, fmt.Errorf("cannot read --detailfile: %w", err) + } + detail = string(b) + } + phase := strings.ToUpper(instEventPhase) + if phase == "" { + phase = "UNKNOWN" + } + failureClass := strings.ToUpper(strings.ReplaceAll(instEventFailureClass, "-", "_")) + if failureClass == "" { + failureClass = "UNKNOWN" + } + message := truncateChars(redactSecrets(instEventMessage), maxMessageChars) + fingerprint := instEventFingerprint + if fingerprint == "" { + fingerprint = deriveFailureFingerprint(instEventDeployment, phase, failureClass, instEventMessage) + } + attemptedAt := instEventAttemptedAt + if attemptedAt == "" { + attemptedAt = time.Now().UTC().Format(time.RFC3339) + } + + e := map[string]interface{}{ + "deploymentName": instEventDeployment, + "phase": phase, + "failureClass": failureClass, + "fingerprint": fingerprint, + "attemptedAt": attemptedAt, + } + if message != "" { + e["message"] = message + } + if detail != "" { + e["detail"] = sanitizeDetail(detail) + } + if namespace != "" { + e["namespace"] = namespace + } + if senderId != "" { + e["senderId"] = senderId + } + if instEventFeatureSet != "" { + e["featureSet"] = instEventFeatureSet + } + if instEventProduct != "" { + e["product"] = instEventProduct + } + if instEventTargetRelease != "" { + e["targetRelease"] = instEventTargetRelease + } + return e, nil +} + +// sanitizeEventBatch applies the same redaction/truncation guarantees to a +// caller-supplied JSON batch, so the batch path cannot bypass them. +func sanitizeEventBatch(raw []byte) ([]map[string]interface{}, error) { + var events []map[string]interface{} + if err := json.Unmarshal(raw, &events); err != nil { + return nil, fmt.Errorf("events must be a JSON array of objects: %w", err) + } + for i, e := range events { + if name, _ := e["deploymentName"].(string); name == "" { + return nil, fmt.Errorf("event %d is missing deploymentName", i) + } + if d, ok := e["detail"].(string); ok { + e["detail"] = sanitizeDetail(d) + } + if m, ok := e["message"].(string); ok { + e["message"] = truncateChars(redactSecrets(m), maxMessageChars) + } + if p, ok := e["phase"].(string); ok { + e["phase"] = strings.ToUpper(p) + } else { + e["phase"] = "UNKNOWN" + } + if fc, ok := e["failureClass"].(string); ok { + e["failureClass"] = strings.ToUpper(strings.ReplaceAll(fc, "-", "_")) + } else { + e["failureClass"] = "UNKNOWN" + } + if fp, _ := e["fingerprint"].(string); fp == "" { + msg, _ := e["message"].(string) + e["fingerprint"] = deriveFailureFingerprint( + e["deploymentName"].(string), e["phase"].(string), e["failureClass"].(string), msg) + } + if at, _ := e["attemptedAt"].(string); at == "" { + e["attemptedAt"] = time.Now().UTC().Format(time.RFC3339) + } + } + return events, nil +} + +func init() { + instEventCmd.PersistentFlags().StringVar(&instEventDeployment, "deployment", "", "Deployment name that failed (required unless --events/--eventsfile is used)") + instEventCmd.PersistentFlags().StringVar(&namespace, "namespace", "", "Namespace of the deployment (recommended; required when using a CLUSTER-scoped api key)") + instEventCmd.PersistentFlags().StringVar(&senderId, "sender", "", "Unique sender within a single namespace (optional)") + instEventCmd.PersistentFlags().StringVar(&instEventPhase, "phase", "", "Reconcile phase: VALUES_MERGE, TAG_REPLACE, SECRETS, HELM_INSTALL, HELM_UNINSTALL, WATCHER_INSTALL, BACKUP (optional, default UNKNOWN)") + instEventCmd.PersistentFlags().StringVar(&instEventFailureClass, "failureclass", "", "Failure class: RBAC_FORBIDDEN, CHART_NOT_FOUND, TIMEOUT, IMAGE_PULL, VALUES_INVALID, PRECONDITION_MISSING (optional, default UNKNOWN)") + instEventCmd.PersistentFlags().StringVar(&instEventMessage, "message", "", "One-line failure summary (optional)") + instEventCmd.PersistentFlags().StringVar(&instEventDetail, "detail", "", "Failure detail, e.g. captured stderr (optional; redacted and truncated before transmit)") + instEventCmd.PersistentFlags().StringVar(&instEventDetailFile, "detailfile", "", "Path to a file with failure detail (optional; redacted and truncated before transmit)") + instEventCmd.PersistentFlags().StringVar(&instEventFingerprint, "fingerprint", "", "Stable dedup key for this fault (optional, derived from deployment+phase+class+normalized message when omitted)") + instEventCmd.PersistentFlags().StringVar(&instEventFeatureSet, "featureset", "", "Feature set UUID (optional)") + instEventCmd.PersistentFlags().StringVar(&instEventProduct, "product", "", "Product UUID (optional)") + instEventCmd.PersistentFlags().StringVar(&instEventTargetRelease, "targetrelease", "", "Target release UUID (optional)") + instEventCmd.PersistentFlags().StringVar(&instEventAttemptedAt, "attemptedat", "", "RFC3339 time of the failed attempt (optional, defaults to now)") + instEventCmd.PersistentFlags().StringVar(&instEventsJson, "events", "", "JSON array of event objects for batch reporting (optional, mutually exclusive with single-event flags)") + instEventCmd.PersistentFlags().StringVar(&instEventsFile, "eventsfile", "", "Path to a file with the JSON array of event objects (optional)") + + devopsCmd.AddCommand(instEventCmd) +} + +var instEventCmd = &cobra.Command{ + Use: "instevent", + Short: "Reports deployment failures from a CD agent to ReARM", + Long: `Reports deployment failure events for an instance, surfaced in the ReARM Instance view. + +Events are deduplicated server-side on (namespace, deploymentName, fingerprint): +a failure that repeats on every reconcile updates one record's last-seen time and +occurrence count rather than creating new rows. Supply a stable --fingerprint when +the agent can compute one; otherwise it is derived from the deployment, phase, +failure class and a normalized message. + +Failure detail is redacted (credential-looking material) and truncated before it +leaves the cluster -- helm output can echo rendered values. + +Batch usage (preferred for a reconcile loop): + rearm devops instevent --eventsfile ./events.json + +Single event: + rearm devops instevent --deployment traefik --namespace traefik \ + --phase HELM_INSTALL --failureclass RBAC_FORBIDDEN \ + --message "UPGRADE FAILED: could not get clusterroles" --detailfile ./helm-stderr.txt`, + Run: func(cmd *cobra.Command, args []string) { + if debug == "true" { + fmt.Println("Using ReARM at", rearmUri) + } + + var events []map[string]interface{} + var err error + switch { + case instEventsJson != "": + events, err = sanitizeEventBatch([]byte(instEventsJson)) + case instEventsFile != "": + var raw []byte + raw, err = os.ReadFile(instEventsFile) + if err == nil { + events, err = sanitizeEventBatch(raw) + } + default: + var e map[string]interface{} + e, err = buildInstEvent() + if err == nil { + events = []map[string]interface{}{e} + } + } + if err != nil { + fmt.Println("Error preparing deployment events:", err) + os.Exit(1) + } + if len(events) == 0 { + if debug == "true" { + fmt.Println("No deployment events to send") + } + return + } + + if debug == "true" { + fmt.Println(events) + } + + query := ` + mutation ($events: [InstanceDeploymentEventInput!]!) { + instanceDeploymentEventsProgrammatic(events:$events) + } + ` + variables := map[string]interface{}{"events": events} + fmt.Println(sendRequest(query, variables, "instanceDeploymentEventsProgrammatic")) + }, +} diff --git a/cmd/instevent_test.go b/cmd/instevent_test.go new file mode 100644 index 0000000..858dbdf --- /dev/null +++ b/cmd/instevent_test.go @@ -0,0 +1,183 @@ +package cmd + +import ( + "strings" + "testing" +) + +// Redaction is the security boundary of this command: captured helm stderr can +// echo rendered values, and this agent runs inside the customer's cluster while +// the backend may be hosted. A miss here is credential exfiltration, so these +// cases are deliberately blunt. +func TestRedactSecretsRemovesCredentialMaterial(t *testing.T) { + cases := []struct { + name string + in string + }{ + {"password assignment", `Error rendering: password=hunter2trustno1`}, + {"password colon yaml", ` adminPassword: "s3cr3t-value"`}, + {"api key", `apiKey = ak_live_9f8e7d6c5b4a3210`}, + {"api_key underscore", `api_key: "abcd1234efgh5678"`}, + {"token", `token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345`}, + {"client secret", `client_secret=zzzz-yyyy-xxxx`}, + {"authorization bearer", `Authorization: Bearer abc.def.ghi123`}, + {"url credentials", `failed to pull https://someuser:somepass@registry.example.com/chart`}, + {"jwt", `token was eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U`}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got := redactSecrets(c.in) + if !strings.Contains(got, "[REDACTED]") { + t.Fatalf("expected redaction marker in %q", got) + } + for _, leak := range []string{"hunter2trustno1", "s3cr3t-value", "ak_live_9f8e7d6c5b4a3210", + "abcd1234efgh5678", "ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", "zzzz-yyyy-xxxx", + "abc.def.ghi123", "somepass", "dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"} { + if strings.Contains(got, leak) { + t.Fatalf("secret %q survived redaction: %q", leak, got) + } + } + }) + } +} + +func TestRedactSecretsRemovesPemPrivateKeyBlock(t *testing.T) { + in := "before\n-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA1234\nabcd\n-----END RSA PRIVATE KEY-----\nafter" + got := redactSecrets(in) + if strings.Contains(got, "MIIEowIBAAKCAQEA1234") { + t.Fatalf("PEM body survived redaction: %q", got) + } + // Surrounding diagnostic context must survive — the point is a usable error. + if !strings.Contains(got, "before") || !strings.Contains(got, "after") { + t.Fatalf("redaction destroyed surrounding context: %q", got) + } +} + +func TestRedactSecretsKeepsOrdinaryDiagnostics(t *testing.T) { + // The real traefik failure that motivated this feature must survive intact. + in := `UPGRADE FAILED: could not get information about the resource: clusterroles.rbac.authorization.k8s.io "traefik-traefik" is forbidden: User "system:serviceaccount:rearm-cd:rearm-cd" cannot get resource "clusterroles" in API group "rbac.authorization.k8s.io" at the cluster scope` + got := redactSecrets(in) + if got != in { + t.Fatalf("ordinary RBAC diagnostic was altered:\n got: %q", got) + } +} + +func TestSanitizeDetailTruncatesAndMarks(t *testing.T) { + got := sanitizeDetail(strings.Repeat("x", maxDetailChars*3)) + if len([]rune(got)) <= maxDetailChars { + t.Fatalf("expected truncation marker to be appended, got len %d", len([]rune(got))) + } + if !strings.Contains(got, "[truncated]") { + t.Fatal("truncation must be visible so an operator knows the tail is missing") + } + if len([]rune(got)) > maxDetailChars+32 { + t.Fatalf("truncated payload too long: %d", len([]rune(got))) + } +} + +// Dedup only works if the same fault yields the same fingerprint across +// reconciles — otherwise a persistent failure inserts a row every 15s. +func TestDeriveFingerprintStableAcrossVolatileTokens(t *testing.T) { + a := deriveFailureFingerprint("traefik", "HELM_INSTALL", "RBAC_FORBIDDEN", + `UPGRADE FAILED at 2026-07-25T10:15:03Z for release 4f2c9a11-3f2e-4a55-8b21-0c1d2e3f4a5b attempt 17`) + b := deriveFailureFingerprint("traefik", "HELM_INSTALL", "RBAC_FORBIDDEN", + `UPGRADE FAILED at 2026-07-25T10:15:18Z for release 9c8b7a66-1d2e-4f33-9a44-5b6c7d8e9f00 attempt 18`) + if a != b { + t.Fatalf("fingerprint drifted on volatile tokens: %s vs %s", a, b) + } +} + +func TestDeriveFingerprintDistinguishesRealDifferences(t *testing.T) { + base := deriveFailureFingerprint("traefik", "HELM_INSTALL", "RBAC_FORBIDDEN", "cannot get clusterroles") + cases := map[string]string{ + "different deployment": deriveFailureFingerprint("rearm-ui", "HELM_INSTALL", "RBAC_FORBIDDEN", "cannot get clusterroles"), + "different phase": deriveFailureFingerprint("traefik", "VALUES_MERGE", "RBAC_FORBIDDEN", "cannot get clusterroles"), + "different failure class": deriveFailureFingerprint("traefik", "HELM_INSTALL", "TIMEOUT", "cannot get clusterroles"), + "different message": deriveFailureFingerprint("traefik", "HELM_INSTALL", "RBAC_FORBIDDEN", "chart not found"), + } + for name, fp := range cases { + if fp == base { + t.Fatalf("%s collided with base fingerprint", name) + } + } +} + +// The batch path must not be a way around redaction/truncation. +func TestSanitizeEventBatchAppliesRedactionAndDefaults(t *testing.T) { + raw := []byte(`[{"deploymentName":"traefik","namespace":"traefik","detail":"password=leakme123","message":"boom"}]`) + events, err := sanitizeEventBatch(raw) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(events) != 1 { + t.Fatalf("expected 1 event, got %d", len(events)) + } + e := events[0] + if strings.Contains(e["detail"].(string), "leakme123") { + t.Fatalf("batch path bypassed redaction: %v", e["detail"]) + } + if e["phase"] != "UNKNOWN" || e["failureClass"] != "UNKNOWN" { + t.Fatalf("expected UNKNOWN defaults, got %v/%v", e["phase"], e["failureClass"]) + } + if fp, _ := e["fingerprint"].(string); fp == "" { + t.Fatal("fingerprint must be derived when omitted, else dedup breaks") + } + if at, _ := e["attemptedAt"].(string); at == "" { + t.Fatal("attemptedAt must default to now") + } +} + +func TestSanitizeEventBatchRejectsMalformedInput(t *testing.T) { + if _, err := sanitizeEventBatch([]byte(`{"deploymentName":"x"}`)); err == nil { + t.Fatal("expected error for non-array batch") + } + if _, err := sanitizeEventBatch([]byte(`[{"namespace":"traefik"}]`)); err == nil { + t.Fatal("expected error for event without deploymentName") + } +} + +func TestSanitizeEventBatchNormalizesFailureClassSeparators(t *testing.T) { + events, err := sanitizeEventBatch([]byte(`[{"deploymentName":"t","failureClass":"rbac-forbidden","phase":"helm_install"}]`)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if events[0]["failureClass"] != "RBAC_FORBIDDEN" { + t.Fatalf("expected RBAC_FORBIDDEN, got %v", events[0]["failureClass"]) + } + if events[0]["phase"] != "HELM_INSTALL" { + t.Fatalf("expected HELM_INSTALL, got %v", events[0]["phase"]) + } +} + +// Regression: helm values are overwhelmingly camelCase, so an anchored +// \bpassword\b misses adminPassword/registryToken/dbSecretKey entirely. This +// was a real leak in the first cut of the redactor. +func TestRedactSecretsHandlesCamelCaseAndPrefixedKeys(t *testing.T) { + cases := []struct{ in, leak string }{ + {` adminPassword: "s3cr3t-value"`, "s3cr3t-value"}, + {`registryToken: ghp_zzzzzzzzzzzzzzzzzzzz`, "ghp_zzzzzzzzzzzzzzzzzzzz"}, + {`dbSecretKey = topsecretvalue`, "topsecretvalue"}, + {`global.imagePullSecret: "regcred-xyz"`, "regcred-xyz"}, + {`MY_APP_PASSWORD=letmein999`, "letmein999"}, + } + for _, c := range cases { + got := redactSecrets(c.in) + if strings.Contains(got, c.leak) { + t.Fatalf("camelCase/prefixed secret survived redaction: %q -> %q", c.in, got) + } + } +} + +// Redaction must not eat ordinary Kubernetes diagnostics that merely contain +// the substring "auth" (rbac.authorization.k8s.io) or "author". +func TestRedactSecretsDoesNotMaulAuthorizationDiagnostics(t *testing.T) { + for _, in := range []string{ + `clusterroles.rbac.authorization.k8s.io "traefik-traefik" is forbidden`, + `chart author: Reliza`, + `forwarding request to authority service`, + } { + if got := redactSecrets(in); got != in { + t.Fatalf("diagnostic was mangled:\n in: %q\nout: %q", in, got) + } + } +} diff --git a/docs/devops.md b/docs/devops.md index 6475016..c908ee4 100644 --- a/docs/devops.md +++ b/docs/devops.md @@ -440,4 +440,46 @@ When multiple overrides are supplied with different branch names, the new featur - Every override component must already be a dependency of the product's BASE feature set — overrides are pure branch tweaks, not new dependencies. - A feature set with the chosen new name must not already exist on the product. -**Output:** JSON of the newly-created `Branch` (the new feature set), including its UUID for use as the input to a subsequent `switchfeatureset` call. \ No newline at end of file +**Output:** JSON of the newly-created `Branch` (the new feature set), including its UUID for use as the input to a subsequent `switchfeatureset` call. +## 15.12 Report Deployment Failures From a CD Agent + +Reports deployment failure events for an instance. Failures are surfaced in the ReARM Instance view, so a broken deploy is visible in ReARM instead of only in the CD agent's pod logs. + +Batch form (what a reconcile loop should use — one call per iteration, not one per failure): + +```bash +rearm devops instevent -i $APIKEYID -k $APIKEY --eventsfile ./events.json +``` + +Single-event form: + +```bash +rearm devops instevent -i $APIKEYID -k $APIKEY \ + --deployment traefik \ + --namespace traefik \ + --phase HELM_INSTALL \ + --failureclass RBAC_FORBIDDEN \ + --message "UPGRADE FAILED: could not get clusterroles" \ + --detailfile ./helm-stderr.txt +``` + +**Flags:** +- **--deployment** - name of the deployment that failed (required unless `--events`/`--eventsfile` is used). +- **--namespace** - namespace of the deployment. Required when authenticating with a CLUSTER-scoped API key, since that key covers many namespaces and the namespace is what resolves the owning instance. +- **--sender** - unique sender within a single namespace (optional). +- **--phase** - reconcile stage: `VALUES_MERGE`, `TAG_REPLACE`, `SECRETS`, `HELM_INSTALL`, `HELM_UNINSTALL`, `WATCHER_INSTALL`, `BACKUP` (optional, defaults to `UNKNOWN`). +- **--failureclass** - cause: `RBAC_FORBIDDEN`, `CHART_NOT_FOUND`, `TIMEOUT`, `IMAGE_PULL`, `VALUES_INVALID`, `PRECONDITION_MISSING` (optional, defaults to `UNKNOWN`). +- **--message** - one-line failure summary (optional). +- **--detail** / **--detailfile** - failure detail such as captured stderr, inline or from a file (optional). +- **--fingerprint** - stable dedup key for this fault (optional, derived when omitted). +- **--featureset**, **--product**, **--targetrelease** - UUIDs for context (optional). +- **--attemptedat** - RFC3339 time of the failed attempt (optional, defaults to now). +- **--events** / **--eventsfile** - JSON array of event objects for batch reporting, inline or from a file. Each object uses the same field names as the flags above (`deploymentName`, `namespace`, `phase`, `failureClass`, `message`, `detail`, `fingerprint`, `featureSet`, `product`, `targetRelease`, `attemptedAt`). + +**Deduplication:** ReARM keys failures on `(namespace, deploymentName, fingerprint)`. A failure that repeats on every reconcile updates one record's last-seen timestamp and occurrence count rather than creating new records. Supply `--fingerprint` when the agent can compute a stable one; otherwise it is derived from the deployment, phase, failure class and a normalized message (uuids, digests, timestamps and bare numbers are normalized out, so a message embedding volatile tokens still dedupes). + +**Redaction:** `--detail` and `--message` are scrubbed of credential-looking material (assignments to password/secret/token/apikey-style keys including camelCase variants, `Authorization` headers, PEM private-key blocks, JWTs, and credentials embedded in URLs) and truncated before anything leaves the cluster. Helm output can echo rendered values, so raw stderr is never transmitted as-is. Redaction is intentionally aggressive — a lost diagnostic line is cheaper than a leaked credential. + +**Resolution:** there is no explicit "resolved" event. ReARM infers resolution from silence — a failure that stops being re-reported is treated as fixed and ages out. Report a live failure on every reconcile for as long as it persists. + +**Output:** JSON with `status` and the count of `accepted` events.