diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e725f7d3..6f71ff765 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ * DNS via proxy improvements, also IPv6 support for proxy * Client simulation runs in wide mode which is even better readable * Added --reqheader to support custom headers in HTTP requests +* `--phone-out` checks the HSTS preload list on https://hstspreload.org/ * Deprecating --fast and --ssl-native (warning only but still av) * Compatible to GNU grep >=3.8, bash 5.x * Don't use external pwd command anymore diff --git a/CREDITS.md b/CREDITS.md index b90181083..8f47cf008 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -55,6 +55,7 @@ Full contribution, see git log. - maximum certificate lifespan of 398 days - ssl renegotiation amount variable - custom http request headers + - HSTS preload list lookup * Frank Breedijk - Detection of insecure redirects diff --git a/doc/testssl.1.md b/doc/testssl.1.md index 7e22d5da9..db58e8e8c 100644 --- a/doc/testssl.1.md +++ b/doc/testssl.1.md @@ -152,7 +152,7 @@ The same can be achieved by setting the environment variable `WARNINGS`. `--ids-friendly` is a switch which may help to get a scan finished which otherwise would be blocked by a server side IDS. This switch skips tests for the following vulnerabilities: Heartbleed, CCS Injection, Ticketbleed and ROBOT. The environment variable OFFENSIVE set to false will achieve the same result. Please be advised that as an alternative or as a general approach you can try to apply evasion techniques by changing the variables USLEEP_SND and / or USLEEP_REC and maybe MAX_WAITSOCK. -`--phone-out` Checking for revoked certificates via CRL and OCSP is not done per default. This switch instructs testssl.sh to query external -- in a sense of the current run -- URIs. By using this switch you acknowledge that the check might have privacy issues, a download of several megabytes (CRL file) may happen and there may be network connectivity problems while contacting the endpoint which testssl.sh doesn't handle. PHONE_OUT is the environment variable for this which needs to be set to true if you want this. +`--phone-out` Checking for revoked certificates via CRL and OCSP, as well as the HSTS preload list status via hstspreload.org, is not done per default. This switch instructs testssl.sh to query external -- in a sense of the current run -- URIs. By using this switch you acknowledge that the check might have privacy issues, a download of several megabytes (CRL file) may happen and there may be network connectivity problems while contacting the endpoint which testssl.sh doesn't handle. PHONE_OUT is the environment variable for this which needs to be set to true if you want this. `--add-ca ` enables you to add your own CA(s) in PEM format for trust chain checks. `CAfile` can be a directory containing files with a \.pem extension, a single file or multiple files as a comma separated list of root CAs. Internally they will be added during runtime to all CA stores. This is (only) useful for internal hosts whose certificates are issued by internal CAs. Alternatively ADDTL_CA_FILES is the environment variable for this. @@ -213,6 +213,7 @@ Also for multiple server certificates are being checked for as well as for the c `-h, --header, --headers` if the service is HTTP (either by detection or by enforcing via `--assume-http`. It tests several HTTP headers like * HTTP Strict Transport Security (HSTS) + - HSTS preload list status (when `--phone-out` supplied) * HTTP Public Key Pinning (HPKP) * Server banner * HTTP date+time diff --git a/t/53_hsts_preload.t b/t/53_hsts_preload.t new file mode 100644 index 000000000..08f2be469 --- /dev/null +++ b/t/53_hsts_preload.t @@ -0,0 +1,52 @@ +#!/usr/bin/env perl + +# Check the HSTS preload list status against the hstspreload.org API (needs --phone-out). +# github.com is on the preload list, example.com is not. +# +# We don't use a full run, only the HTTP header section. + +use strict; +use Test::More; + +my $tests = 0; +my $prg="./testssl.sh"; +my $csv="tmp.csv"; +my $cat_csv=""; +my $check2run="-q --color 0 --phone-out --ip=one --headers --csvfile $csv"; +my $uri="github.com"; +my @args=""; + +die "Unable to open $prg" unless -f $prg; + +# Provide proper start conditions +unlink $csv; + +#1 run -- a domain which is on the HSTS preload list +printf "\n%s\n", "Unit test for HSTS preload list status against \"$uri\""; +@args="$prg $check2run $uri >/dev/null"; +system("@args") == 0 + or die ("FAILED: \"@args\" "); +$cat_csv=`cat $csv`; + +# github.com is on the preload list +like($cat_csv, qr/"HSTS_preloadlist".*"preloaded"/,"\"$uri\" should be on the HSTS preload list"); +$tests++; +unlink $csv; + +#2 run -- a domain which is NOT on the HSTS preload list +$uri="example.com"; +@args="$prg $check2run $uri >/dev/null"; +system("@args") == 0 + or die ("FAILED: \"@args\" "); +$cat_csv=`cat $csv`; + +# example.com is not on the preload list +like($cat_csv, qr/"HSTS_preloadlist".*"no entry"/,"\"$uri\" should not be on the HSTS preload list"); +$tests++; +unlink $csv; + +done_testing($tests); +printf "\n"; + + +# vim:ts=5:sw=5:expandtab diff --git a/testssl.sh b/testssl.sh index 526605bdc..0428b007c 100755 --- a/testssl.sh +++ b/testssl.sh @@ -2262,6 +2262,74 @@ check_revocation_ocsp() { fi } +# Checks a domain against the hstspreload.org HSTS preload list API (requires --phone-out). +# arg1: domain to check +# arg2: JSON key to check (e.g. status, bulk, preloadedDomain). Empty: only (re)fetch the response. +# arg3: value the key is expected to have (without surrounding quotes; quoting is handled here) +# Return values: +# 0 - request made, nothing compared (no key supplied) +# 1 - API request failed (connection error) +# 10 - key matched the expected value +# 20 - key present but value did not match +# 21 - key not found in the response +check_hsts_preloadlist_match() { + local domain="$1" + local key="$2" + local value="$3" + local response="" + local tmpfile="$TEMPDIR/$NODE.hsts-preloadlist.txt" + local uri_api_status="https://hstspreload.org/api/v2/status?domain=$domain" + + "$PHONE_OUT" || return 0 + + # Only query the API once per host, then reuse the cached response + if [[ ! -f "$tmpfile" ]]; then + http_get "$uri_api_status" "$tmpfile" || return 1 + fi + response="$(<"$tmpfile")" + + # Without a key we only (re)fetched the response + [[ -z "$key" ]] && return 0 + + # The key must be present, otherwise the API may have changed + [[ "$response" == *"\"$key\""* ]] || { debugme echo "HSTS preloadlist key unrecognized: $key"; return 21; } + + # String values are quoted in the JSON, booleans are not, so accept either form + [[ "$response" == *"\"$key\": \"$value\""* || "$response" == *"\"$key\": $value"* ]] && return 10 + return 20 +} + +# Returns the value of a known key from the hstspreload.org preload list API. +# Depends on check_hsts_preloadlist_match(). +# arg1: domain to check +# arg2: key to resolve (status or bulk) +# Echoes the matched value and returns 0, or returns 1 if no known value matched. +check_hsts_preloadlist_value() { + local domain="$1" + local key="$2" + local -a values=() + local value + local value_ret="" + + [[ -z "$key" ]] && return 1 + + # Only test against known values instead of echoing the API response back, + # so no untrusted input is reflected. + case "$key" in + status) values=("unknown" "pending" "rejected" "preloaded") ;; + bulk) values=("true" "false") ;; + *) return 1 ;; + esac + + for value in "${values[@]}"; do + check_hsts_preloadlist_match "$domain" "$key" "$value" + [[ $? -eq 10 ]] && value_ret="$value" && break + done + + [[ -n "$value_ret" ]] && echo "$value_ret" && return 0 + return 1 +} + # waits maxsleep 1/10 seconds (arg2) until process with arg1 (pid) will be killed # # return values @@ -2918,6 +2986,8 @@ run_hsts() { local hsts_age_days local spaces=" " local jsonID="HSTS" + local json_postfix="" + local preloadmarked preloadsame preloadbulk preloadcombined="" if [[ ! -s $HEADERFILE ]]; then run_http_header "$1" || return 1 @@ -2971,18 +3041,74 @@ run_hsts() { fi if preload "$TMPFILE"; then fileout "${jsonID}_preload" "OK" "domain IS marked for preloading" + preloadmarked=true else fileout "${jsonID}_preload" "INFO" "domain is NOT marked for preloading" - #FIXME: To be checked against preloading lists, - # e.g. https://dxr.mozilla.org/mozilla-central/source/security/manager/boot/src/nsSTSPreloadList.inc - # https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json + preloadmarked=false fi else pr_svrty_low "not offered" fileout "$jsonID" "LOW" "not offered" + preloadmarked=false fi outln + # Check the domain against the hstspreload.org HSTS preload list (requires --phone-out). + # Run this regardless of the served header: a domain may still be listed after the header + # was removed, or be rejected because the served header does not meet the requirements. + if "$PHONE_OUT"; then + json_postfix="_preloadlist" + pr_bold " HSTS preload list " + + # If the domain itself is the preloaded entry, it may be fine that the header omits 'preload' + check_hsts_preloadlist_match "$NODE" "preloadedDomain" "$NODE" + [[ $? -eq 10 ]] && preloadsame=true || preloadsame=false + + # bulk=true: added via the submission form; false: manual addition or a subdomain + check_hsts_preloadlist_match "$NODE" "bulk" "true" + [[ $? -eq 10 ]] && preloadbulk=true || preloadbulk=false + + # Combine the three booleans for a compact lookup, e.g. marked+same+bulk -> "111" + [[ $preloadmarked == true ]] && preloadcombined="${preloadcombined}1" || preloadcombined="${preloadcombined}0" + [[ $preloadsame == true ]] && preloadcombined="${preloadcombined}1" || preloadcombined="${preloadcombined}0" + [[ $preloadbulk == true ]] && preloadcombined="${preloadcombined}1" || preloadcombined="${preloadcombined}0" + debugme echo "Temporary lookupvariable: $preloadcombined" + + # Determine and show the outcome + case "$(check_hsts_preloadlist_value "$NODE" "status")" in + "unknown") # Not found in the HSTS preload list + case "$preloadcombined" in + "000" | "001" | "010" | "011") outln "no entry"; fileout "${jsonID}${json_postfix}" "INFO" "no entry" ;; + "100" | "101" | "110" | "111") pr_svrty_low "no entry"; outln " -- submit to HSTS preload list"; fileout "${jsonID}${json_postfix}" "LOW" "no entry" ;; + esac + ;; + "pending") # Currently in the HSTS pending list + case "$preloadcombined" in + "000" | "001" | "010" | "100" | "101" | "110" | "111") outln "pending"; fileout "${jsonID}${json_postfix}" "INFO" "pending" ;; + "011") pr_svrty_medium "pending"; outln " -- addition going to fail, add header"; fileout "${jsonID}${json_postfix}" "MEDIUM" "pending" ;; + esac + ;; + "rejected") # Entry is considered rejected by the HSTS list + case "$preloadcombined" in + "000" | "001" | "010" | "011") outln "rejected"; fileout "${jsonID}${json_postfix}" "INFO" "rejected" ;; + "100" | "101" | "110" | "111") pr_svrty_medium "rejected"; outln " -- check other requirements"; fileout "${jsonID}${json_postfix}" "MEDIUM" "rejected" ;; + esac + ;; + "preloaded") # Marked as 'preload' in the HSTS preload list + case "$preloadcombined" in + "000" | "001") prln_svrty_good "preloaded"; fileout "${jsonID}${json_postfix}" "OK" "preloaded" ;; + "010") outln "preloaded -- manual addition detected"; fileout "${jsonID}${json_postfix}" "INFO" "preloaded" ;; + "011") pr_svrty_medium "preloaded"; outln " -- list may remove entry, add header"; fileout "${jsonID}${json_postfix}" "MEDIUM" "preloaded" ;; + "100" | "101" | "110" | "111") prln_svrty_best "preloaded"; fileout "${jsonID}${json_postfix}" "OK" "preloaded" ;; + esac + ;; + *) # Empty: the hstspreload.org API was unreachable or returned an unexpected response + prln_warning "not checked (HSTS preload list lookup failed)" + fileout "${jsonID}${json_postfix}" "WARN" "HSTS preload list could not be checked" + ;; + esac + fi + tmpfile_handle ${FUNCNAME[0]}.txt return 0 } @@ -21708,7 +21834,7 @@ tuning / connect options (most also can be preset via environment variables): --sneaky leave less traces in target logs: user agent, referer --user-agent set a custom user agent instead of the standard user agent --ids-friendly skips a few vulnerability checks which may cause IDSs to block the scanning IP - --phone-out allow to contact external servers for CRL download and querying OCSP responder + --phone-out allow to contact external servers for CRL download, querying OCSP responder and the HSTS preload list --add-ca path to with *.pem or a comma separated list of CA files to include in trust check --mtls path to file in PEM format containing unencrypted certificate key (beta) --basicauth provide HTTP basic auth information