diff --git a/docker-compose.yml b/docker-compose.yml index cba62ac154d..18ad6c06db9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ services: - .:/boulder:cached - ./.gocache:/root/.cache/go-build:cached - ./test/certs/.softhsm-tokens/:/var/lib/softhsm/tokens/:cached + - pkimetal-socket:/var/run/pkimetal networks: bouldernet: ipv4_address: 10.77.77.77 @@ -145,8 +146,10 @@ services: bpkimetal: image: ghcr.io/pkimetal/pkimetal:v1.41.0 - networks: - - bouldernet + volumes: + - pkimetal-socket:/var/run/pkimetal + - ./test/pkimetal-config.yaml:/config/config.yaml:ro + network_mode: none bvitess: # The `letsencrypt/boulder-vtcomboserver:latest` tag is automatically built @@ -181,6 +184,17 @@ services: aliases: - boulder-vitess +volumes: + # Shared between bpkimetal (which listens on a unix socket here) and any + # boulder container that needs to reach pkimetal. Owned by uid 1001 so + # the default pkimetal user in the container can create the socket. + pkimetal-socket: + driver: local + driver_opts: + type: tmpfs + device: tmpfs + o: "uid=1001,gid=1001,mode=0755" + networks: # This network represents the data-center internal network. It is used for # boulder services and their infrastructure, such as consul, mariadb, and diff --git a/linter/lints/rfc/lint_cert_via_pkimetal.go b/linter/lints/rfc/lint_cert_via_pkimetal.go index 31fc08d8135..ee337b736ed 100644 --- a/linter/lints/rfc/lint_cert_via_pkimetal.go +++ b/linter/lints/rfc/lint_cert_via_pkimetal.go @@ -1,121 +1,15 @@ package rfc import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "slices" - "strings" - "time" - "github.com/zmap/zcrypto/x509" "github.com/zmap/zlint/v3/lint" "github.com/zmap/zlint/v3/util" -) - -// PKIMetalConfig and its execute method provide a shared basis for linting -// both certs and CRLs using PKIMetal. -type PKIMetalConfig struct { - Addr string `toml:"addr" comment:"The address where a pkilint REST API can be reached."` - Severity string `toml:"severity" comment:"The minimum severity of findings to report (meta, debug, info, notice, warning, error, bug, or fatal)."` - Timeout time.Duration `toml:"timeout" comment:"How long, in nanoseconds, to wait before giving up."` - IgnoreLints []string `toml:"ignore_lints" comment:"The unique Validator:Code IDs of lint findings which should be ignored."` -} - -func (pkim *PKIMetalConfig) execute(endpoint string, der []byte) (*lint.LintResult, error) { - timeout := pkim.Timeout - if timeout == 0 { - timeout = 100 * time.Millisecond - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - apiURL, err := url.JoinPath(pkim.Addr, endpoint) - if err != nil { - return nil, fmt.Errorf("constructing pkimetal url: %w", err) - } - - // reqForm matches PKIMetal's documented form-urlencoded request format. It - // does not include the "profile" field, as its default value ("autodetect") - // is good for our purposes. - // https://github.com/pkimetal/pkimetal/blob/578ac224a7ca3775af51b47fce16c95753d9ac8d/doc/openapi.yaml#L179-L194 - reqForm := url.Values{} - reqForm.Set("b64input", base64.StdEncoding.EncodeToString(der)) - reqForm.Set("severity", pkim.Severity) - reqForm.Set("format", "json") - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, strings.NewReader(reqForm.Encode())) - if err != nil { - return nil, fmt.Errorf("creating pkimetal request: %w", err) - } - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - req.Header.Add("Accept", "application/json") - - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, fmt.Errorf("making POST request to pkimetal API: %s (timeout %s)", err, timeout) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("got status %d (%s) from pkimetal API", resp.StatusCode, resp.Status) - } - resJSON, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("reading response from pkimetal API: %s", err) - } - - // finding matches the repeated portion of PKIMetal's documented JSON response. - // https://github.com/pkimetal/pkimetal/blob/578ac224a7ca3775af51b47fce16c95753d9ac8d/doc/openapi.yaml#L201-L221 - type finding struct { - Linter string `json:"linter"` - Finding string `json:"finding"` - Severity string `json:"severity"` - Code string `json:"code"` - Field string `json:"field"` - } - - var res []finding - err = json.Unmarshal(resJSON, &res) - if err != nil { - return nil, fmt.Errorf("parsing response from pkimetal API: %s", err) - } - - var findings []string - for _, finding := range res { - var id string - if finding.Code != "" { - id = fmt.Sprintf("%s:%s", finding.Linter, finding.Code) - } else { - id = fmt.Sprintf("%s:%s", finding.Linter, strings.ReplaceAll(strings.ToLower(finding.Finding), " ", "_")) - } - if slices.Contains(pkim.IgnoreLints, id) { - continue - } - desc := fmt.Sprintf("%s from %s: %s", finding.Severity, id, finding.Finding) - findings = append(findings, desc) - } - - if len(findings) != 0 { - // Group the findings by severity, for human readers. - slices.Sort(findings) - return &lint.LintResult{ - Status: lint.Error, - Details: fmt.Sprintf("got %d lint findings from pkimetal API: %s", len(findings), strings.Join(findings, "; ")), - }, nil - } - - return &lint.LintResult{Status: lint.Pass}, nil -} + "github.com/letsencrypt/boulder/linter/pkimetal" +) type certViaPKIMetal struct { - PKIMetalConfig + pkimetal.Client } func init() { @@ -136,17 +30,17 @@ func NewCertViaPKIMetal() lint.CertificateLintInterface { } func (l *certViaPKIMetal) Configure() any { - return l + return &l.Config } func (l *certViaPKIMetal) CheckApplies(c *x509.Certificate) bool { // This lint applies to all certificates issued by Boulder, as long as it has - // been configured with an address to reach out to. If not, skip it. - return l.Addr != "" + // been configured with a socket to reach out to. If not, skip it. + return l.Enabled() } func (l *certViaPKIMetal) Execute(c *x509.Certificate) *lint.LintResult { - res, err := l.execute("lintcert", c.Raw) + res, err := l.Client.Execute("lintcert", c.Raw) if err != nil { return &lint.LintResult{ Status: lint.Error, diff --git a/linter/lints/rfc/lint_crl_via_pkimetal.go b/linter/lints/rfc/lint_crl_via_pkimetal.go index c927eebe525..67b7f0350d6 100644 --- a/linter/lints/rfc/lint_crl_via_pkimetal.go +++ b/linter/lints/rfc/lint_crl_via_pkimetal.go @@ -4,10 +4,12 @@ import ( "github.com/zmap/zcrypto/x509" "github.com/zmap/zlint/v3/lint" "github.com/zmap/zlint/v3/util" + + "github.com/letsencrypt/boulder/linter/pkimetal" ) type crlViaPKIMetal struct { - PKIMetalConfig + pkimetal.Client } func init() { @@ -28,17 +30,17 @@ func NewCrlViaPKIMetal() lint.RevocationListLintInterface { } func (l *crlViaPKIMetal) Configure() any { - return l + return &l.Config } func (l *crlViaPKIMetal) CheckApplies(c *x509.RevocationList) bool { // This lint applies to all CRLs issued by Boulder, as long as it has - // been configured with an address to reach out to. If not, skip it. - return l.Addr != "" + // been configured with a socket to reach out to. If not, skip it. + return l.Enabled() } func (l *crlViaPKIMetal) Execute(c *x509.RevocationList) *lint.LintResult { - res, err := l.execute("lintcrl", c.Raw) + res, err := l.Client.Execute("lintcrl", c.Raw) if err != nil { return &lint.LintResult{ Status: lint.Error, diff --git a/linter/pkimetal/client.go b/linter/pkimetal/client.go new file mode 100644 index 00000000000..4637e6a56f2 --- /dev/null +++ b/linter/pkimetal/client.go @@ -0,0 +1,146 @@ +package pkimetal + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "slices" + "strings" + "sync" + "time" + + "github.com/zmap/zlint/v3/lint" +) + +// Config holds configuration for linting both certs and CRLs using PKIMetal. +// Zlint will deserialize toml here. +type Config struct { + Socket string `toml:"socket" comment:"Path to a unix socket where pkimetal is listening."` + Severity string `toml:"severity" comment:"The minimum severity of findings to report (meta, debug, info, notice, warning, error, bug, or fatal)."` + Timeout time.Duration `toml:"timeout" comment:"How long, in nanoseconds, to wait before giving up."` + IgnoreLints []string `toml:"ignore_lints" comment:"The unique Validator:Code IDs of lint findings which should be ignored."` +} + +type Client struct { + Config + + clientOnce sync.Once + httpClient *http.Client +} + +// Enabled returns true if the client has a socket configured. +func (pkim *Client) Enabled() bool { + return pkim != nil && pkim.Socket != "" +} + +// Execute linting in pkimetal. +func (pkim *Client) Execute(endpoint string, der []byte) (*lint.LintResult, error) { + timeout := pkim.Timeout + if timeout == 0 { + timeout = 100 * time.Millisecond + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // Host is ignored by our unix-socket transport, so any valid base works. + apiURL, err := url.JoinPath("http://pkimetal", endpoint) + if err != nil { + return nil, fmt.Errorf("constructing pkimetal url: %w", err) + } + + // reqForm matches PKIMetal's documented form-urlencoded request format. It + // does not include the "profile" field, as its default value ("autodetect") + // is good for our purposes. + // https://github.com/pkimetal/pkimetal/blob/578ac224a7ca3775af51b47fce16c95753d9ac8d/doc/openapi.yaml#L179-L194 + reqForm := url.Values{} + reqForm.Set("b64input", base64.StdEncoding.EncodeToString(der)) + reqForm.Set("severity", pkim.Severity) + reqForm.Set("format", "json") + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, strings.NewReader(reqForm.Encode())) + if err != nil { + return nil, fmt.Errorf("creating pkimetal request: %w", err) + } + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Accept", "application/json") + + resp, err := pkim.getHTTPClient().Do(req) + if err != nil { + return nil, fmt.Errorf("making POST request to pkimetal API: %s (timeout %s)", err, timeout) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("got status %d (%s) from pkimetal API", resp.StatusCode, resp.Status) + } + + resJSON, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading response from pkimetal API: %s", err) + } + + // finding matches the repeated portion of PKIMetal's documented JSON response. + // https://github.com/pkimetal/pkimetal/blob/578ac224a7ca3775af51b47fce16c95753d9ac8d/doc/openapi.yaml#L201-L221 + type finding struct { + Linter string `json:"linter"` + Finding string `json:"finding"` + Severity string `json:"severity"` + Code string `json:"code"` + Field string `json:"field"` + } + + var res []finding + err = json.Unmarshal(resJSON, &res) + if err != nil { + return nil, fmt.Errorf("parsing response from pkimetal API: %s", err) + } + + var findings []string + for _, finding := range res { + var id string + if finding.Code != "" { + id = fmt.Sprintf("%s:%s", finding.Linter, finding.Code) + } else { + id = fmt.Sprintf("%s:%s", finding.Linter, strings.ReplaceAll(strings.ToLower(finding.Finding), " ", "_")) + } + if slices.Contains(pkim.IgnoreLints, id) { + continue + } + desc := fmt.Sprintf("%s from %s: %s", finding.Severity, id, finding.Finding) + findings = append(findings, desc) + } + + if len(findings) != 0 { + // Group the findings by severity, for human readers. + slices.Sort(findings) + return &lint.LintResult{ + Status: lint.Error, + Details: fmt.Sprintf("got %d lint findings from pkimetal API: %s", len(findings), strings.Join(findings, "; ")), + }, nil + } + + return &lint.LintResult{Status: lint.Pass}, nil +} + +func (pkim *Client) getHTTPClient() *http.Client { + // Create an http client on first use, as there's not a great place to do this setup ahead of time. + pkim.clientOnce.Do(func() { + socket := pkim.Socket + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + } + pkim.httpClient = &http.Client{ + Transport: transport, + } + }) + return pkim.httpClient +} diff --git a/test/config-next/zlint.toml b/test/config-next/zlint.toml index e359cb58fa8..e624c4a394b 100644 --- a/test/config-next/zlint.toml +++ b/test/config-next/zlint.toml @@ -1,5 +1,5 @@ [e_pkimetal_lint_cabf_serverauth_cert] -addr = "http://bpkimetal:8080" +socket = "/var/run/pkimetal/pkimetal.sock" severity = "notice" timeout = 2000000000 # 2 seconds ignore_lints = [ @@ -25,7 +25,7 @@ ignore_lints = [ ] [e_pkimetal_lint_cabf_serverauth_crl] -addr = "http://bpkimetal:8080" +socket = "/var/run/pkimetal/pkimetal.sock" severity = "notice" timeout = 2000000000 # 2 seconds ignore_lints = [] diff --git a/test/config/zlint.toml b/test/config/zlint.toml index 3c9709a97e6..325da361547 100644 --- a/test/config/zlint.toml +++ b/test/config/zlint.toml @@ -1,5 +1,5 @@ [e_pkimetal_lint_cabf_serverauth_cert] -addr = "http://bpkimetal:8080" +socket = "/var/run/pkimetal/pkimetal.sock" severity = "notice" timeout = 2000000000 # 2 seconds ignore_lints = [ @@ -21,7 +21,7 @@ ignore_lints = [ ] [e_pkimetal_lint_cabf_serverauth_crl] -addr = "http://bpkimetal:8080" +socket = "/var/run/pkimetal/pkimetal.sock" severity = "notice" timeout = 2000000000 # 2 seconds ignore_lints = [] diff --git a/test/entrypoint.sh b/test/entrypoint.sh index f24758e2afe..f772341b4ca 100755 --- a/test/entrypoint.sh +++ b/test/entrypoint.sh @@ -46,8 +46,8 @@ configure_database_endpoints ./test/wait-for-it.sh boulder-mariadb 3306 ./test/wait-for-it.sh boulder-proxysql 6033 -# make sure we can reach pkilint -./test/wait-for-it.sh bpkimetal 8080 +# make sure pkimetal's unix socket is ready +./test/wait-for-socket.sh /var/run/pkimetal/pkimetal.sock if [[ $# -eq 0 ]]; then exec python3 ./start.py diff --git a/test/pkimetal-config.yaml b/test/pkimetal-config.yaml new file mode 100644 index 00000000000..c662c38feb1 --- /dev/null +++ b/test/pkimetal-config.yaml @@ -0,0 +1,4 @@ +server: + # Disable the TCP webserver and only listen on a unix socket + webserverPort: 0 + webserverPath: /var/run/pkimetal/pkimetal.sock diff --git a/test/wait-for-socket.sh b/test/wait-for-socket.sh new file mode 100755 index 00000000000..7f6422f2cde --- /dev/null +++ b/test/wait-for-socket.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e -u + +socket="${1}" +max_tries=40 + +for n in $(seq 1 "${max_tries}"); do + if [ -S "${socket}" ]; then + echo "Socket ${socket} is ready" + exit 0 + fi + echo "$(date) - still waiting for socket ${socket}" + sleep 1 +done + +echo "timed out waiting for socket ${socket}" +exit 1