feat: add HAR capture and runtime help

har/har.go

@@ -12,6 +12,13 @@ const defaultMaxBodySize = 64 * 1024 // 64 KB
 // -P http.har.maxBodySize=0 to capture full bodies with no cap.
 const MaxBodySizeProperty = "http.har.maxBodySize"
 
+// SensitiveProperty is the -P/properties key that disables redaction. By
+// default credentials in headers, bodies and query strings are masked, so a
+// HAR file is safe to share but cannot be replayed. Set -P http.har.sensitive=true
+// to capture them verbatim — the resulting file holds live secrets and is
+// written with 0600.
+const SensitiveProperty = "http.har.sensitive"
+
 // HARConfig controls what the HAR middleware captures and how it redacts.
 type HARConfig struct {
 	// MaxBodySize is the maximum number of bytes captured per body.
@@ -33,6 +40,11 @@ type HARConfig struct {
 	// identifiers (e.g. session ids, national-id fields) that the default
 	// heuristics don't recognise.
 	RedactedBodyKeys []string
+
+	// CaptureSensitive records credentials verbatim instead of masking them,
+	// so the archive can be replayed against the live API. Honours
+	// SensitiveProperty; off by default.
+	CaptureSensitive bool
 }
 
 // DefaultConfig returns a HARConfig with sensible defaults. The per-body
@@ -43,6 +55,7 @@ func DefaultConfig() HARConfig {
 	return HARConfig{
 		MaxBodySize:         int64(properties.Int(defaultMaxBodySize, MaxBodySizeProperty)),
 		CaptureContentTypes: []string{"application/json", "application/x-www-form-urlencoded"},
+		CaptureSensitive:    properties.On(false, SensitiveProperty),
 	}
 }
 

har/level.go

@@ -0,0 +1,49 @@
+package har
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Level selects what a HAR collector captures. Borrowed from
+// duty/connection/common.go's Debug/Trace split: at Metadata only headers,
+// query strings and timings are recorded (no bodies, so no body re-read cost);
+// at Full the standard collector middleware captures bodies too.
+type Level int
+
+const (
+	Disabled Level = iota
+	Metadata
+	Full
+)
+
+func (l Level) String() string {
+	switch l {
+	case Metadata:
+		return "metadata"
+	case Full:
+		return "full"
+	default:
+		return "disabled"
+	}
+}
+
+// ParseLevel maps a property value onto a Level. "debug"/"trace" are accepted
+// as synonyms for metadata/full, matching the log.level.*.har vocabulary duty
+// and commons-db use. An empty string yields def; anything unrecognised is an
+// error, so a typo turns into a startup failure rather than silently capturing
+// the wrong thing.
+func ParseLevel(value string, def Level) (Level, error) {
+	switch strings.ToLower(strings.TrimSpace(value)) {
+	case "":
+		return def, nil
+	case "disabled", "off", "none":
+		return Disabled, nil
+	case "metadata", "debug":
+		return Metadata, nil
+	case "full", "trace", "bodies":
+		return Full, nil
+	default:
+		return def, fmt.Errorf("invalid HAR level %q: expected metadata, full or disabled", value)
+	}
+}

har/metadata.go

@@ -0,0 +1,63 @@
+package har
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/flanksource/commons/http/middlewares"
+)
+
+// NewMetadataMiddleware captures method, URL, sanitized headers, query string,
+// status and timings — no request or response bodies. Body sizes use -1 per the
+// HAR spec ("size unknown"). Use it when you want a HAR file for traffic
+// analysis without paying the body-buffering cost.
+//
+// Ported from duty/connection/common.go's metadataHARMiddleware, which
+// commons/http and commons-db each carried their own copy of.
+func NewMetadataMiddleware(cfg HARConfig, handler func(*Entry)) middlewares.Middleware {
+	if handler == nil {
+		return func(next http.RoundTripper) http.RoundTripper {
+			return next
+		}
+	}
+	return func(next http.RoundTripper) http.RoundTripper {
+		return middlewares.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
+			entry := &Entry{
+				StartedDateTime: time.Now().UTC().Format(time.RFC3339),
+				Request: Request{
+					Method:      req.Method,
+					URL:         harURL(req.URL, cfg),
+					HTTPVersion: httpVersion(req.Proto),
+					Cookies:     []Cookie{},
+					Headers:     harHeaders(req.Header, cfg),
+					QueryString: harQueryString(req.URL.Query(), cfg),
+					HeadersSize: -1,
+					BodySize:    -1,
+				},
+			}
+
+			waitStart := time.Now()
+			resp, err := next.RoundTrip(req)
+			waitMs := float64(time.Since(waitStart).Microseconds()) / 1000.0
+
+			entry.Timings = Timings{Wait: waitMs}
+			entry.Time = waitMs
+			entry.Response = Response{
+				Cookies:     []Cookie{},
+				Headers:     []Header{},
+				Content:     Content{Size: -1},
+				HeadersSize: -1,
+				BodySize:    -1,
+			}
+			if resp != nil {
+				entry.Response.Status = resp.StatusCode
+				entry.Response.StatusText = resp.Status
+				entry.Response.HTTPVersion = httpVersion(resp.Proto)
+				entry.Response.Headers = harHeaders(resp.Header, cfg)
+			}
+
+			handler(entry)
+			return resp, err
+		})
+	}
+}

har/middleware.go

@@ -64,11 +64,11 @@ func CaptureRedirect(req *http.Request, resp *http.Response, cfg HARConfig) *Ent
 func buildRequest(req *http.Request, cfg HARConfig) Request {
 	har := Request{
 		Method:      req.Method,
-		URL:         redactURL(req.URL, cfg.RedactedBodyKeys),
+		URL:         harURL(req.URL, cfg),
 		HTTPVersion: httpVersion(req.Proto),
 		Cookies:     []Cookie{},
-		Headers:     toHARHeaders(logger.SanitizeHeaders(req.Header, cfg.RedactedHeaders...)),
-		QueryString: toQueryString(req.URL.Query(), cfg.RedactedBodyKeys),
+		Headers:     harHeaders(req.Header, cfg),
+		QueryString: harQueryString(req.URL.Query(), cfg),
 		HeadersSize: -1,
 		BodySize:    -1,
 	}
@@ -80,7 +80,7 @@ func buildRequest(req *http.Request, cfg HARConfig) Request {
 		har.BodySize = int64(len(body.raw))
 		har.PostData = &PostData{
 			MimeType: ct,
-			Text:     redactBody(body.text, ct, cfg.RedactedBodyKeys),
+			Text:     harBody(body.text, ct, cfg),
 		}
 	}
 
@@ -93,7 +93,7 @@ func buildResponse(resp *http.Response, cfg HARConfig) Response {
 		StatusText:  resp.Status,
 		HTTPVersion: httpVersion(resp.Proto),
 		Cookies:     []Cookie{},
-		Headers:     toHARHeaders(logger.SanitizeHeaders(resp.Header, cfg.RedactedHeaders...)),
+		Headers:     harHeaders(resp.Header, cfg),
 		RedirectURL: "",
 		HeadersSize: -1,
 		BodySize:    -1,
@@ -107,7 +107,7 @@ func buildResponse(resp *http.Response, cfg HARConfig) Response {
 		har.Content = Content{
 			Size:      body.totalSize,
 			MimeType:  ct,
-			Text:      redactBody(body.text, ct, cfg.RedactedBodyKeys),
+			Text:      harBody(body.text, ct, cfg),
 			Truncated: body.truncated,
 		}
 	}
@@ -154,6 +154,48 @@ func shouldCapture(contentType string, allowed []string) bool {
 	return false
 }
 
+// harHeaders, harURL, harQueryString and harBody are the single gate through
+// which every captured value passes. cfg.CaptureSensitive (-Phttp.har.sensitive)
+// bypasses redaction so the archive can be replayed against the live API; by
+// default credentials are masked with logger.PrintableSecret.
+func harHeaders(headers http.Header, cfg HARConfig) []Header {
+	if cfg.CaptureSensitive {
+		return toHARHeaders(headers)
+	}
+	return toHARHeaders(logger.SanitizeHeaders(headers, cfg.RedactedHeaders...))
+}
+
+func harURL(u *url.URL, cfg HARConfig) string {
+	if u == nil {
+		return ""
+	}
+	if cfg.CaptureSensitive {
+		return u.String()
+	}
+	return redactURL(u, cfg.RedactedBodyKeys)
+}
+
+func harQueryString(query url.Values, cfg HARConfig) []QueryString {
+	qs := make([]QueryString, 0, len(query))
+	for k, vs := range query {
+		redact := !cfg.CaptureSensitive && isRedactedKey(k, cfg.RedactedBodyKeys)
+		for _, v := range vs {
+			if redact {
+				v = logger.PrintableSecret(v)
+			}
+			qs = append(qs, QueryString{Name: k, Value: v})
+		}
+	}
+	return qs
+}
+
+func harBody(text, contentType string, cfg HARConfig) string {
+	if cfg.CaptureSensitive {
+		return text
+	}
+	return redactBody(text, contentType, cfg.RedactedBodyKeys)
+}
+
 func redactBody(text, contentType string, extraKeys []string) string {
 	ct := strings.ToLower(strings.Split(contentType, ";")[0])
 	ct = strings.TrimSpace(ct)
@@ -282,20 +324,6 @@ func toHARHeaders(h http.Header) []Header {
 	return headers
 }
 
-func toQueryString(q url.Values, extraKeys []string) []QueryString {
-	qs := make([]QueryString, 0, len(q))
-	for k, vs := range q {
-		redact := isRedactedKey(k, extraKeys)
-		for _, v := range vs {
-			if redact {
-				v = logger.PrintableSecret(v)
-			}
-			qs = append(qs, QueryString{Name: k, Value: v})
-		}
-	}
-	return qs
-}
-
 func httpVersion(proto string) string {
 	if proto == "" {
 		return "HTTP/1.1"

har/middleware_test.go

@@ -88,6 +88,55 @@ func TestHAR_AuthorizationHeaderRedacted(t *testing.T) {
 	}
 }
 
+// TestHAR_CaptureSensitiveKeepsCredentials pins the -Phttp.har.sensitive escape
+// hatch at the config layer: with it set, the archive is replayable because
+// every value is verbatim, including a query parameter the default heuristics
+// would otherwise mask.
+func TestHAR_CaptureSensitiveKeepsCredentials(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.WriteHeader(204)
+	}))
+	defer srv.Close()
+
+	const (
+		secret   = "Bearer supersecret"
+		apiKey   = "sk-live-1234567890"
+		jsonBody = `{"api_key":"sk-live-1234567890"}`
+	)
+	cfg := har.DefaultConfig()
+	cfg.CaptureSensitive = true
+
+	entry := captureOne(t, cfg, srv, http.MethodPost, "/?token="+apiKey,
+		strings.NewReader(jsonBody), map[string]string{
+			"Authorization": secret,
+			"Content-Type":  "application/json",
+		})
+
+	if got := headerValue(entry.Request.Headers, "Authorization"); got != secret {
+		t.Errorf("Authorization = %q, want the verbatim value %q", got, secret)
+	}
+	if !strings.Contains(entry.Request.URL, apiKey) {
+		t.Errorf("URL %q dropped the query credential", entry.Request.URL)
+	}
+	if entry.Request.PostData == nil || entry.Request.PostData.Text != jsonBody {
+		t.Errorf("request body was redacted: %+v", entry.Request.PostData)
+	}
+	for _, q := range entry.Request.QueryString {
+		if q.Name == "token" && q.Value != apiKey {
+			t.Errorf("query string token = %q, want %q", q.Value, apiKey)
+		}
+	}
+}
+
+func headerValue(headers []har.Header, name string) string {
+	for _, h := range headers {
+		if strings.EqualFold(h.Name, name) {
+			return h.Value
+		}
+	}
+	return ""
+}
+
 func TestHAR_CookieHeaderRedacted(t *testing.T) {
 	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.Header().Set("Set-Cookie", "session=abc123; Path=/")