From e0c0a6658f5dbcdc7dd753135b8d8e514fcd6d1f Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Sat, 30 May 2026 17:40:34 +0200 Subject: [PATCH 1/9] Provide HTTPS RR functionality This is a fresh start for #2484 as the PR wasn't ready yet for 3.2 by the time it was released. And it continues #2866 which was kind of messed up by accident. The info for the HTTPS RR shows up in the very beginning, i.e. in `service_detection()`. All keys are listed now in bold, values in a regular font. `get_https_rrecord()` was introduced by copying and modifying `get_caa_rr_record()`. There's a similar obstacle as with CAA RRs: older binaries show the resource records binary encoded. Thus a new set of global vars is introduced HAS_*_HTTPS which check whether the binaries support decoding the RR directly. As of now raw decoding doesn't work completely. Todo: - Add logic in QUIC - if RR is detected and not QUIC is possible - add time for QUIC detection when RR is retrieved - show full HTTPS RR record, at least when having a new DNS client - coninue with raw decoding, if possible (otherwise problematic for MacOS) - shorten the comments in `get_https_rrecord()` - man page - when ASSUME_HTTP is set and no services was detected: this needs to be handled - The placement of the output should be reconsidered and/or cached when multiple IPs belong to a FQDN --- .github/workflows/codespell.yml | 4 +- CHANGELOG.md | 1 + t/baseline_data/default_testssl.csvfile | 1 + testssl.sh | 272 ++++++++++++++++++++++-- 4 files changed, 256 insertions(+), 22 deletions(-) diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 18ff48d6e..c0177b58d 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,5 +12,5 @@ jobs: - uses: actions/checkout@v6 - uses: codespell-project/actions-codespell@master with: - skip: ca_hashes.txt,tls_data.txt,*.pem,OPENSSL-LICENSE.txt,CREDITS.md,openssl.cnf,fedora-dirk-ipv6.diff,testssl.1 - ignore_words_list: borken,gost,ciph,ba,bloc,isnt,chello,fo,alle,anull + skip: ca_hashes.txt,tls_data.txt,*.pem,OPENSSL-LICENSE.txt,CREDITS.md,openssl.cnf,testssl.1 + ignore_words_list: borken,gost,ciph,ba,bloc,isnt,chello,fo,alle,anull,expt diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e725f7d3..d98a7b584 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Bump SSLlabs rating guide to 2009r * Check for Opossum vulnerability * Enable IPv6 automagically, i.e. if target via IPv6 is reachable just (also) scan it +* Detect and show DNS HTTPS RR (RFC 9460) * Provide an FAQ ### Features implemented / improvements in 3.2 diff --git a/t/baseline_data/default_testssl.csvfile b/t/baseline_data/default_testssl.csvfile index f43bbb912..a37854e0c 100644 --- a/t/baseline_data/default_testssl.csvfile +++ b/t/baseline_data/default_testssl.csvfile @@ -1,5 +1,6 @@ "id","fqdn/ip","port","severity","finding","cve","cwe" "engine_problem","/","443","WARN","No engine or GOST support via engine with your ./bin/openssl.Linux.x86_64","","" +"DNS_HTTPS_rrecord","testssl.sh/81.169.166.184","443","OK","1 . alpn='h2'","","" "service","testssl.sh/81.169.235.32","443","INFO","HTTP","","" "pre_128cipher","testssl.sh/81.169.235.32","443","INFO","No 128 cipher limit bug","","" "SSLv2","testssl.sh/81.169.235.32","443","OK","not offered","","" diff --git a/testssl.sh b/testssl.sh index 18c741a96..df23a0772 100755 --- a/testssl.sh +++ b/testssl.sh @@ -387,6 +387,11 @@ HAS_IDN2=false HAS_AVAHIRESOLVE=false HAS_DSCACHEUTIL=false HAS_DIG_NOIDNOUT=false +HAS_DIG_HTTPS=false # *_HTTPS: whether the binaries support HTTPS RR directly +HAS_DRILL_HTTPS=false +HAS_HOST_HTTPS=false +HAS_NSLOOKUP_HTTPS=false + HAS_XXD=false OSSL_CIPHERS_S="" @@ -2501,6 +2506,7 @@ s_client_options() { # determines whether the port has an HTTP service running or not (plain TLS, no STARTTLS) # arg1 could be the protocol determined as "working". IIS6 needs that. +# sets global $SERVICE # service_detection() { local -i was_killed @@ -2545,24 +2551,29 @@ service_detection() { debugme head -50 $TMPFILE | sed -e '//,$d' -e '//,$d' -e '/ trying HTTP checks" SERVICE=HTTP fileout "${jsonID}" "DEBUG" "Couldn't determine service -- ASSUME_HTTP set" elif [[ "$CLIENT_AUTH" == required ]] && [[ -z $MTLS ]]; then - out " certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" - echo "certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" >$TMPFILE + out " certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" | tee $TMPFILE fileout "${jsonID}" "INFO" "certificate-based authentication without providing client certificate and private key => skipping all HTTP checks" else out " Couldn't determine what's running on port $PORT" @@ -9396,6 +9407,7 @@ certificate_info() { local first=true local badocsp=1 local len_cert_serial=0 + local avoid_complaints="^(1\.1\.1\.1|1\.0\.0\.1|8\.8\.8\.8|8\.8\.4\.4|9\.9\.9\.9)$" if [[ $number_of_certificates -gt 1 ]]; then [[ $certificate_number -eq 1 ]] && outln @@ -10273,9 +10285,9 @@ certificate_info() { out "$indent"; pr_bold " DNS CAA RR"; out " (experimental) " jsonID="DNS_CAArecord" - if is_ipv4addr "$NODE" || is_ipv6addr "$NODE"; then - out "not checked (IP address scan -- no domain to query)" - fileout "${jsonID}${json_postfix}" "INFO" "not checked (IP address scan)" + if [[ ! $tmp =~ [a-zA-Z] ]] && [[ ! $tmp =~ $avoid_complaints ]]; then + out "not checked: IP address scan, no domain to query" + fileout "${jsonID}${json_postfix}" "INFO" "not checked IP address scan, no domain to query" else caa_node="$NODE" caa="" @@ -17732,7 +17744,7 @@ run_ticketbleed() { local hint="" [[ -n "$STARTTLS" ]] && return 0 - pr_bold " Ticketbleed"; out " ($cve), experiment. " + pr_bold " Ticketbleed"; out " ($cve), experimental " if [[ "$SERVICE" != HTTP ]] && [[ "$CLIENT_AUTH" != required ]]; then outln "(applicable only for HTTP service)" @@ -22353,6 +22365,8 @@ get_local_a() { # check_resolver_bins() { local saved_openssl_conf="$OPENSSL_CONF" + local testhost=localhost + local str="" OPENSSL_CONF="" # see https://github.com/testssl/testssl.sh/issues/134 type -p dig &> /dev/null && HAS_DIG=true @@ -22377,12 +22391,36 @@ check_resolver_bins() { HAS_DIG_NOIDNOUT=true fi fi + + # Pre-checking the following for HTTPS RR, see get_https_rrecord() + if "$HAS_DIG"; then + str=$(dig +short $testhost HTTPS) + if [[ -z "$str" ]] && [[ ! "$str" =~ 127.0.0.1 ]] ; then + HAS_DIG_HTTPS=true + fi + elif "$HAS_DRILL"; then + if drill $testhost HTTPS | grep -Eq 'IN.*HTTPS'; then + HAS_DRILL_HTTPS=true + fi + elif "$HAS_HOST"; then + host -t HTTPS $testhost 2>&1 | grep -q 'invalid type' + if [[ $? -ne 0 ]]; then + HAS_HOST_HTTPS=true + fi + elif "$HAS_NSLOOKUP"; then + nslookup -type=HTTPS $testhost | grep -q 'unknown query type' + if [[ $? -ne 0 ]]; then + HAS_NSLOOKUP_HTTPS=true + fi + fi + OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/testssl/testssl.sh/issues/134 return 0 } # arg1: a host name. Returned will be 0-n IPv4 addresses # watch out: $1 can also be a cname! --> all checked +# get_a_record() { local ip4="" local saved_openssl_conf="$OPENSSL_CONF" @@ -22436,6 +22474,7 @@ get_a_record() { # arg1: a host name. Returned will be 0-n IPv6 addresses # watch out: $1 can also be a cname! --> all checked +# get_aaaa_record() { local ip6="" local saved_openssl_conf="$OPENSSL_CONF" @@ -22485,9 +22524,12 @@ get_aaaa_record() { echo "$ip6" } + # RFC6844: DNS Certification Authority Authorization (CAA) Resource Record # arg1: domain to check for -get_caa_rr_record() { +#FIXME: should be refactored, see get_https_rrecord() +# +get_caa_rrecord() { local raw_caa="" local hash len line local -i len_caa_property @@ -22515,12 +22557,16 @@ get_caa_rr_record() { raw_caa="$(drill $1 type257 | awk '/'"^${1}"'.*CAA/ { print $5,$6,$7 }')" elif "$HAS_HOST"; then raw_caa="$(host -t type257 $1)" - if grep -Ewvq "has no CAA|has no TYPE257" <<< "$raw_caa"; then - raw_caa="$(sed -e 's/^.*has CAA record //' -e 's/^.*has TYPE257 record //' <<< "$raw_caa")" + if [[ "$raw_caa" =~ "has no CAA|has no TYPE257" ]]; then + raw_caa="" + else + raw_caa="${raw_caa/$1 has CAA record /}" + raw_caa="${raw_caa/$1 has TYPE257 record /}" fi elif "$HAS_NSLOOKUP"; then raw_caa="$(strip_lf "$(nslookup -type=type257 $1 | grep -w rdata_257)")" if [[ -n "$raw_caa" ]]; then + #FIXME: modernize here or see HTTPS RR raw_caa="$(sed 's/^.*rdata_257 = //' <<< "$raw_caa")" fi else @@ -22563,11 +22609,171 @@ get_caa_rr_record() { return 1 fi -# to do: +#TODO: # 4: check whether $1 is a CNAME and take this return 0 } + +# Service Binding and Parameter Specification via the DNS (SVCB and HTTPS Resource Records). +# https://www.rfc-editor.org/rfc/rfc9460.html +# arg1: domain to check for +# returns: string for record +# +get_https_rrecord() { + local raw_https="" + local hash="" len line="" + local len_alpnID="" + local alpnID="" + local alpnID_wire="" + local saved_openssl_conf="$OPENSSL_CONF" + local all_https="" + local noidnout="" + local svc_priority="" + + [[ -n "$NODNS" ]] && return 2 # if minimum DNS lookup was instructed, leave here + "$HAS_DIG_NOIDNOUT" && noidnout="+noidnout" + + # There's a) the possibility to query HTTPS RR records directly like "dig +short HTTPS dev.testssl.sh", + # "drill HTTPS FQDN" or "nslookup -type=HTTPS FQDN". This works for newer binaries only, unfortunately. + # On top of that b) there's also an extended format which e.g. cloudflare uses: + # $ host -t type65 testssl.net + # testssl.net has TYPE65 record \# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041A70020002057F87361C7B5A3B8CD3C028892690D35 2863623DAD4E03D33B231A4C3C8BB02B0004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A + # $ host -t HTTPS testssl.net + # testssl.net has HTTPS record 1 . alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 ech=AEX+DQBBpwAgACBX+HNhx7WjuM08AoiSaQ01KGNiPa1OA9M7IxpMPIuwKwAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a + # ECH is the encrypted client hello --> for esni (https://datatracker.ietf.org/doc/draft-ietf-tls-esni/) + # Nice description: https://www.netmeister.org/blog/https-rrs.html + + # Thus we try first whether we can query the HTTPS records directly as this gives us that already + # in clear text and also we can avoid to parse the encoded format. We'll do that as a fallback but + # at this moment we're trying to scrape only the values alpn from it, if they come first. + + OPENSSL_CONF="" + if "$HAS_DIG_HTTPS"; then + text_httpsrr=$(dig +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null) + elif "$HAS_DRILL_HTTPS"; then + text_httpsrr=$(drill -Q HTTPS $1 2>/dev/null) + elif "$HAS_HOST_HTTPS"; then + text_httpsrr=$(host -t HTTPS $1 2>/dev/null) + text_httpsrr=${text_httpsrr#*record } + elif "$HAS_NSLOOKUP_HTTPS"; then # from 4th field onwards \/ + text_httpsrr=$(nslookup -type=HTTPS $1 | awk '/'"^${1}"'.*rdata_65// { print substr($0,index($0,$4)) }') + fi + + if [[ -n "$text_httpsrr" ]]; then + safe_echo "$text_httpsrr" + return 0 + fi + + # Now we need to try parsing the raw output + # Format probably: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) + + # If there's a type65 record there are 2x3 output formats, mostly depending on age of distribution + # -- roughly that's the difference between text and binary format -- and the type of DNS client + + # for host: + # 1) 'google.com has HTTPS record 1 . alpn="h2,h3" ' + # 2) 'google.com has TYPE65 record \# 13 0001000001000602683202683 ' + + # for drill and dig it's like + #1) google.com. 18665 IN TYPE65 \# 13 00010000010006026832026833 + #2) google.com. 18301 IN HTTPS 1 . alpn="h2,h3" + + # nslookup: + # 1) dev.testssl.sh rdata_65 = 1 . alpn="h2" + # 2) dev.testssl.sh rdata_65 = \# 10 00010000010003026832 + + if "$HAS_DIG"; then + raw_https="$(dig $DIG_R +short +search +timeout=3 +tries=3 $noidnout type65 "$1" 2>/dev/null)" + # empty if there's no such record + elif "$HAS_DRILL"; then + raw_https="$(drill $1 type65 | grep -v '^;;' | awk '/'"^${1}"'.*TYPE65/ { print substr($0,index($0,$5)) }' )" # from 5th field onwards + # empty if there's no such record + elif "$HAS_HOST"; then + raw_https="$(host -t type65 $1)" + if [[ "$raw_https" =~ "has no HTTPS|has no TYPE65" ]]; then + raw_https="" + else + raw_https="${raw_https/$1 has HTTPS record /}" + raw_https="${raw_https/$1 has TYPE65 record /}" + fi + elif "$HAS_NSLOOKUP"; then + raw_https="$(strip_lf "$(nslookup -type=type65 $1 | awk '/'"^${1}"'.*rdata_65/ { print substr($0,index($0,$4)) }' )")" + # empty if there's no such record + else + return 1 + # No dig, drill, host, or nslookup --> complaint was elsewhere already + fi + OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/drwetter/testssl.sh/issues/134 + +# dig +short HTTPS dev.testssl.sh / dig +short type65 dev.testssl.sh +# 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 +# +# 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 +# alpn| L h 2 443 2a010238... L=len +# +# ----------------- +# testssl.net (split over a couple of lines) +# +# 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 +# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D +# alpn| L h 3 L h 2 |IPv4#1||IPv4#2| + +# ech=AEX+DQBBzgAgACBQGA9EFbz+PkJAXSXtcqJluxLlhxIgzhJ+GhTtRd4nJQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a +# 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A +# | cloudflare-ech.com | IPv6#1 #IPv6#2 + + +# Now comes the last straw, decoding og the stream. It only works for short entries like 1. alpn=h2,h3 + if [[ -z "$raw_https" ]]; then + return 1 + elif [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then + while read hash len line ;do + # \# 10 00010000010003026832 + # \# 36 000100000100030268320003000201BB000600102A01023842816755 10000000000B1337 + [[ $DEBUG -eq 1 ]] && echo "$hash $len $line" + if [[ "${line:0:4}" == 0001 ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.4.3 + svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, 0 is alias + if [[ $svc_priority == 1 ]]; then + # mock text representation + svc_priority="$svc_priority . " + alpnID="${alpnID}${svc_priority}" + fi + if [[ ${line:8:2} == 01 ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=1 + alpnID="${alpnID}alpn=\"" # double quote for clear text + else + continue # If the 1st element is not alpn, next iteration of loop will fail. + fi # Should we care as SvcParamKey!=alpn doesn't seems not very common? + len_alpnID=${line:12:2} # length of alpn entries (1st?) + alpnID_wire=${line:16:4} # value of first entry + alpnID=${alpnID}$(hex2ascii $alpnID_wire) +# from here it only works for one simple entry (like h2 or h2,h3) + if [[ "$len_alpnID" != "03" ]]; then # 06 would be another entry e.g. h3, quote the rhs! + alpnID_wire=${line:22:4} #FIXME: we can't cope with three entries yet + alpnID="${alpnID},$(hex2ascii $alpnID_wire)" + fi + [[ ${line:8:2} == 01 ]] && alpnID="${alpnID}\"" # if alpn add trailing double quote + +# best is to check len and then stop now + if [[ $len -eq 10 ]]; then + [[ $DEBUG -eq 1 ]] && echo "end 10" + break + fi + +# len_alpnID=$((len_alpnID*2)) # =>word! Now get name from 4th and value from 4th+len position... +# alpnID="$(hex2ascii ${line:4:$len_alpnID})" +# alpnID_wire="$(hex2ascii "${line:$((4+len_alpnID)):100}")" + else + out "please report unknown HTTPS RR $line with flag @ $NODE" + return 7 + fi + done <<< "$raw_https" + safe_echo "$alpnID" + fi + return 0 +} + + # arg1: domain to check for. Returned will be the MX record as a string get_mx_record() { local mx="" @@ -23392,6 +23598,33 @@ determine_optimal_proto() { } +dns_https_rr () { + local jsonID="DNS_HTTPS_rrecord" + local https_rr="" + local indent="" + + out "$indent"; pr_bold " DNS HTTPS RR"; out " (expt.): " + if [[ -n "$NODNS" ]]; then + out "(instructed to minimize/skip DNS queries)" + fileout "${jsonID}" "INFO" "check skipped as instructed" + elif "$DNS_VIA_PROXY"; then + out "(instructed to use the proxy for DNS only)" + fileout "${jsonID}" "INFO" "check skipped as instructed (proxy)" + else + https_rr="$(get_https_rrecord $NODE)" + if [[ -n "$https_rr" ]]; then + pr_svrty_good "yes" ; out ": " + prln_italic "$(out_row_aligned_max_width "$https_rr" "$indent " $TERM_WIDTH)" + fileout "${jsonID}" "OK" "$https_rr" + else + outln "--" + fileout "${jsonID}" "INFO" " no resource record found" + fi + fi +} + + + # Check messages which needed to be processed. I.e. those which would have destroyed the nice # screen output and thus havve been postponed. This is just an idea and is only used once # but can be extended in the future. An array might be more handy @@ -23442,7 +23675,7 @@ determine_service() { fi GET_REQ11="GET $URL_PATH HTTP/1.1\r\nHost: $NODE\r\nUser-Agent: $ua\r\n${basicauth_header}${reqheader}Accept-Encoding: identity\r\nAccept: */*\r\nConnection: Close\r\n\r\n" determine_optimal_proto - # returns always 0: + # returns always 0 and sets $SERVICE service_detection $OPTIMAL_PROTO check_msg else # STARTTLS @@ -23515,7 +23748,7 @@ determine_service() { determine_optimal_sockets_params determine_optimal_proto "$1" - out " Service set:$CORRECT_SPACES STARTTLS via " + pr_bold " Service set"; out ":$CORRECT_SPACES STARTTLS via " out "$(toupper "$protocol")" [[ "$protocol" == mysql ]] && out " (experimental)" fileout "service" "INFO" "$protocol" @@ -23529,7 +23762,6 @@ determine_service() { # It comes handy later also for STARTTLS injection to define this global. When we do banner grabbing # or replace service_detection() we might not need that anymore SERVICE=$protocol - fi tmpfile_handle ${FUNCNAME[0]}.txt @@ -23595,7 +23827,7 @@ display_rdns_etc() { outln "$PROXYIP:$PROXYPORT " fi if [[ $(count_words "$IPADDRs2SHOW") -gt 1 ]]; then - out " Further IP addresses: $CORRECT_SPACES" + pr_bold " Further IP addresses"; out ": $CORRECT_SPACES" for ip in $IPADDRs2SHOW; do if [[ "$ip" == $NODEIP ]] || [[ "[$ip]" == $NODEIP ]]; then continue @@ -23616,11 +23848,11 @@ display_rdns_etc() { outln " A record via: $CORRECT_SPACES supplied IP \"$CMDLINE_IP\"" fi fi + pr_bold " rDNS " + out "$(printf "%-19s" "($nodeip):")" if [[ "$rDNS" =~ instructed ]]; then - out "$(printf " %-23s " "rDNS ($nodeip):")" out "$rDNS" elif [[ -n "$rDNS" ]]; then - out "$(printf " %-23s " "rDNS ($nodeip):")" out "$(out_row_aligned_max_width "$rDNS" " $CORRECT_SPACES" $TERM_WIDTH)" fi } From e365ccf03ff879bbd5396788eea7440a3515134e Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Sun, 31 May 2026 19:59:01 +0200 Subject: [PATCH 2/9] try to squash the baseline comparison check --- t/baseline_data/default_testssl.csvfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/baseline_data/default_testssl.csvfile b/t/baseline_data/default_testssl.csvfile index a37854e0c..155ccf798 100644 --- a/t/baseline_data/default_testssl.csvfile +++ b/t/baseline_data/default_testssl.csvfile @@ -1,6 +1,6 @@ "id","fqdn/ip","port","severity","finding","cve","cwe" "engine_problem","/","443","WARN","No engine or GOST support via engine with your ./bin/openssl.Linux.x86_64","","" -"DNS_HTTPS_rrecord","testssl.sh/81.169.166.184","443","OK","1 . alpn='h2'","","" +"DNS_HTTPS_rrecord","testssl.sh/81.169.235.32","443","OK","1 . alpn='h2'","","" "service","testssl.sh/81.169.235.32","443","INFO","HTTP","","" "pre_128cipher","testssl.sh/81.169.235.32","443","INFO","No 128 cipher limit bug","","" "SSLv2","testssl.sh/81.169.235.32","443","OK","not offered","","" From ba7d9604a90d78d348adec8278e86b876dfb9cbc Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Mon, 1 Jun 2026 10:14:25 +0200 Subject: [PATCH 3/9] Getting from github runner under MacOS as there is an inexplicable difference between a real Mac which passes the run and the one in github -"DNS_HTTPS_rrecord","testssl.sh/81.169.235.32","443","OK","81.169.235.32","","" +"DNS_HTTPS_rrecord","testssl.sh/81.169.235.32","443","OK","1 . alpn='h2'","","" The first line comes from the runner --- .github/workflows/unit_tests_macos.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/unit_tests_macos.yml b/.github/workflows/unit_tests_macos.yml index 0cf273d74..8d6835660 100644 --- a/.github/workflows/unit_tests_macos.yml +++ b/.github/workflows/unit_tests_macos.yml @@ -45,6 +45,8 @@ jobs: printf "%s\n" "----------" bash --version printf "%s\n" "----------" + echo $PATH + printf "%s\n" "----------" - name: Install perl modules run: | From 84bd9dd1a3681c2200722ee2c14da2c2f2093bcb Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Mon, 1 Jun 2026 16:14:29 +0200 Subject: [PATCH 4/9] Updatesr get_https_rrecord() - quote vars (hoping it'll resolve the Mac runner issue) - make sure CNAMEs are properly parsed - end get_https_rrecord() earlier when there's no record but DNS binaries are "HTTPS record aware" - while loop was redundant - better comments Elsewhere: make sure get_https_rrecord is called with a trailing dot for the NODE --- testssl.sh | 149 +++++++++++++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 73 deletions(-) diff --git a/testssl.sh b/testssl.sh index df23a0772..60d8a4d61 100755 --- a/testssl.sh +++ b/testssl.sh @@ -2359,7 +2359,7 @@ hex2binary() { # convert 414243 into ABC hex2ascii() { - hex2binary $1 + hex2binary "$1" } # arg1: text string @@ -22622,7 +22622,8 @@ get_caa_rrecord() { # get_https_rrecord() { local raw_https="" - local hash="" len line="" + local hash="" line="" + local len # better fix as integer? local len_alpnID="" local alpnID="" local alpnID_wire="" @@ -22634,42 +22635,36 @@ get_https_rrecord() { [[ -n "$NODNS" ]] && return 2 # if minimum DNS lookup was instructed, leave here "$HAS_DIG_NOIDNOUT" && noidnout="+noidnout" - # There's a) the possibility to query HTTPS RR records directly like "dig +short HTTPS dev.testssl.sh", - # "drill HTTPS FQDN" or "nslookup -type=HTTPS FQDN". This works for newer binaries only, unfortunately. - # On top of that b) there's also an extended format which e.g. cloudflare uses: - # $ host -t type65 testssl.net - # testssl.net has TYPE65 record \# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041A70020002057F87361C7B5A3B8CD3C028892690D35 2863623DAD4E03D33B231A4C3C8BB02B0004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A - # $ host -t HTTPS testssl.net - # testssl.net has HTTPS record 1 . alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 ech=AEX+DQBBpwAgACBX+HNhx7WjuM08AoiSaQ01KGNiPa1OA9M7IxpMPIuwKwAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a - # ECH is the encrypted client hello --> for esni (https://datatracker.ietf.org/doc/draft-ietf-tls-esni/) - # Nice description: https://www.netmeister.org/blog/https-rrs.html - - # Thus we try first whether we can query the HTTPS records directly as this gives us that already - # in clear text and also we can avoid to parse the encoded format. We'll do that as a fallback but - # at this moment we're trying to scrape only the values alpn from it, if they come first. + # There's the possibility to query HTTPS RR records directly like "dig +short HTTPS dev.testssl.sh", + # "drill HTTPS FQDN" or "nslookup -type=HTTPS FQDN". This works for new binaries only. Thus we try first + # whether we can query the HTTPS records directly as this gives us that already everything we want in + # in clear text and also we can avoid to parse the encoded formats. + # "tail -1" and the awk commands make sure we use the right lines when we encounter a CNAME OPENSSL_CONF="" if "$HAS_DIG_HTTPS"; then - text_httpsrr=$(dig +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null) + text_httpsrr="$(dig +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null | tail -1)" elif "$HAS_DRILL_HTTPS"; then - text_httpsrr=$(drill -Q HTTPS $1 2>/dev/null) + text_httpsrr="$(drill -Q HTTPS "$1" 2>/dev/null | tail -1)" elif "$HAS_HOST_HTTPS"; then - text_httpsrr=$(host -t HTTPS $1 2>/dev/null) - text_httpsrr=${text_httpsrr#*record } - elif "$HAS_NSLOOKUP_HTTPS"; then # from 4th field onwards \/ - text_httpsrr=$(nslookup -type=HTTPS $1 | awk '/'"^${1}"'.*rdata_65// { print substr($0,index($0,$4)) }') + text_httpsrr="$(host -t HTTPS "$1" 2>/dev/null | awk -F'HTTP service bindings ' '/HTTP service bindings /{print $2}')" + elif "$HAS_NSLOOKUP_HTTPS"; then + text_httpsrr="$(nslookup -type=HTTPS "$1" | awk -F'rdata_65 = ' '/rdata_65 =/{print $2}' )" fi if [[ -n "$text_httpsrr" ]]; then safe_echo "$text_httpsrr" + OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/drwetter/testssl.sh/issues/134 + return 0 + elif "$HAS_DIG_HTTPS" || "$HAS_DRILL_HTTPS" || "$HAS_HOST_HTTPS" || "$HAS_NSLOOKUP_HTTPS"; then + # no record despite binaries are "HTTPS record aware" + OPENSSL_CONF="$saved_openssl_conf" return 0 fi - # Now we need to try parsing the raw output - # Format probably: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) - - # If there's a type65 record there are 2x3 output formats, mostly depending on age of distribution - # -- roughly that's the difference between text and binary format -- and the type of DNS client + # As we didn't succeed yet, we need to try parsing the raw output. First is to get the TYPE65 record + # as text. These days (2026) it's not that common anymore. Mac is the party pooper as it normally returns + # a hex stream only --in 2026. Here's how output of old+ancient client DNS binaries may look like with TYPE65 # for host: # 1) 'google.com has HTTPS record 1 . alpn="h2,h3" ' @@ -22687,10 +22682,10 @@ get_https_rrecord() { raw_https="$(dig $DIG_R +short +search +timeout=3 +tries=3 $noidnout type65 "$1" 2>/dev/null)" # empty if there's no such record elif "$HAS_DRILL"; then - raw_https="$(drill $1 type65 | grep -v '^;;' | awk '/'"^${1}"'.*TYPE65/ { print substr($0,index($0,$5)) }' )" # from 5th field onwards + raw_https="$(drill "$1" type65 | grep -v '^;;' | awk '/'"^${1}"'.*TYPE65/ { print substr($0,index($0,$5)) }' )" # from 5th field onwards # empty if there's no such record elif "$HAS_HOST"; then - raw_https="$(host -t type65 $1)" + raw_https="$(host -t type65 "$1")" if [[ "$raw_https" =~ "has no HTTPS|has no TYPE65" ]]; then raw_https="" else @@ -22698,76 +22693,82 @@ get_https_rrecord() { raw_https="${raw_https/$1 has TYPE65 record /}" fi elif "$HAS_NSLOOKUP"; then - raw_https="$(strip_lf "$(nslookup -type=type65 $1 | awk '/'"^${1}"'.*rdata_65/ { print substr($0,index($0,$4)) }' )")" + raw_https="$(strip_lf "$(nslookup -type=type65 "$1" | awk '/'"^${1}"'.*rdata_65/ { print substr($0,index($0,$4)) }' )")" # empty if there's no such record else return 1 # No dig, drill, host, or nslookup --> complaint was elsewhere already fi - OPENSSL_CONF="$saved_openssl_conf" # see https://github.com/drwetter/testssl.sh/issues/134 + OPENSSL_CONF="$saved_openssl_conf" # We're done now with openssl, see https://github.com/drwetter/testssl.sh/issues/134 -# dig +short HTTPS dev.testssl.sh / dig +short type65 dev.testssl.sh -# 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 -# -# 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 -# alpn| L h 2 443 2a010238... L=len -# -# ----------------- -# testssl.net (split over a couple of lines) -# -# 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 -# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D -# alpn| L h 3 L h 2 |IPv4#1||IPv4#2| - -# ech=AEX+DQBBzgAgACBQGA9EFbz+PkJAXSXtcqJluxLlhxIgzhJ+GhTtRd4nJQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a -# 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A -# | cloudflare-ech.com | IPv6#1 #IPv6#2 + # Now comes the third, more tricky part and the last straw --> parsing the hex stream which was returned if it was returned. + # Format is like: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) + # dev.testssl.sh: + # 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 + # + # 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 + # TL alpn| L h 2 443 2a010238... L=len of alpn entries, TL=total length of the following by, excluding spaces + # + # ----------------- + # testssl.net (here hown over a couple of lines): + # 1 . alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 ech=AEX+DQBBpwAgACBX+HNhx7WjuM08AoiSaQ01KGNiPa1OA9M7IxpMPIuwKwAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a + # ECH is the encrypted client hello --> for esni (https://datatracker.ietf.org/doc/draft-ietf-tls-esni/) + # Nice description: https://www.netmeister.org/blog/https-rrs.html + # + # 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 + # 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D + # TL alpn| L h 3 L h 2 |IPv4#1||IPv4#2| + # + # ech=AEX+DQBBzgAgACBQGA9EFbz+PkJAXSXtcqJluxLlhxIgzhJ+GhTtRd4nJQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a + # 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A + # | cloudflare-ech.com | IPv6#1 #IPv6#2 -# Now comes the last straw, decoding og the stream. It only works for short entries like 1. alpn=h2,h3 + # Be aware that all variables are strings here! Therefore we use double quotes so that e.g. 03 won't become 3 + # For now the following only works for short entries like 1. alpn=h2,h3 if [[ -z "$raw_https" ]]; then return 1 elif [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then - while read hash len line ;do + # signals that we're on the right track with type65 interpretation + read hash len line <<< "$raw_https" # \# 10 00010000010003026832 # \# 36 000100000100030268320003000201BB000600102A01023842816755 10000000000B1337 [[ $DEBUG -eq 1 ]] && echo "$hash $len $line" + if [[ "${line:0:4}" == 0001 ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.4.3 svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, 0 is alias - if [[ $svc_priority == 1 ]]; then + if [[ $svc_priority == "1" ]]; then # mock text representation - svc_priority="$svc_priority . " + svc_priority+=" . " alpnID="${alpnID}${svc_priority}" fi - if [[ ${line:8:2} == 01 ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=1 - alpnID="${alpnID}alpn=\"" # double quote for clear text + if [[ ${line:8:2} == "01" ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=1 + alpnID+="\"" # double quote for clear text else - continue # If the 1st element is not alpn, next iteration of loop will fail. + continue # If the 1st element is not alpn, next iteration of loop will fail for now fi # Should we care as SvcParamKey!=alpn doesn't seems not very common? - len_alpnID=${line:12:2} # length of alpn entries (1st?) - alpnID_wire=${line:16:4} # value of first entry - alpnID=${alpnID}$(hex2ascii $alpnID_wire) -# from here it only works for one simple entry (like h2 or h2,h3) - if [[ "$len_alpnID" != "03" ]]; then # 06 would be another entry e.g. h3, quote the rhs! - alpnID_wire=${line:22:4} #FIXME: we can't cope with three entries yet - alpnID="${alpnID},$(hex2ascii $alpnID_wire)" + len_alpnID="${line:12:2}" # length of alpn entries, e.g. 03 or 06 + alpnID_wire="${line:16:4}" # value of first entry + alpnID="${alpnID}$(hex2ascii "$alpnID_wire")" + localPTR=20 + if [[ "$len_alpnID" == "06" ]]; then # 06 would be another entry e.g. h3, quote the rhs! + alpnID_wire="${line:22:4}" #FIXME: we can't cope with three entries yet + alpnID="${alpnID},$(hex2ascii "$alpnID_wire")" + localPTR=26 fi - [[ ${line:8:2} == 01 ]] && alpnID="${alpnID}\"" # if alpn add trailing double quote + [[ ${line:8:2} == "01" ]] && alpnID+="\"" # if alpn and we're done add trailing double quote -# best is to check len and then stop now - if [[ $len -eq 10 ]]; then - [[ $DEBUG -eq 1 ]] && echo "end 10" - break - fi + # done if localPTR / 2 = len or localPTR not empty + echo "key: ${line:localPTR:4}" + # key=1: alpn + # key=3: port + # key=4: IPv4 + # key=6: IPv6 -# len_alpnID=$((len_alpnID*2)) # =>word! Now get name from 4th and value from 4th+len position... -# alpnID="$(hex2ascii ${line:4:$len_alpnID})" -# alpnID_wire="$(hex2ascii "${line:$((4+len_alpnID)):100}")" else out "please report unknown HTTPS RR $line with flag @ $NODE" return 7 fi - done <<< "$raw_https" safe_echo "$alpnID" fi return 0 @@ -23602,6 +23603,7 @@ dns_https_rr () { local jsonID="DNS_HTTPS_rrecord" local https_rr="" local indent="" + local https_rr_node="$NODE" out "$indent"; pr_bold " DNS HTTPS RR"; out " (expt.): " if [[ -n "$NODNS" ]]; then @@ -23611,9 +23613,11 @@ dns_https_rr () { out "(instructed to use the proxy for DNS only)" fileout "${jsonID}" "INFO" "check skipped as instructed (proxy)" else - https_rr="$(get_https_rrecord $NODE)" + # append a dot if there was none + [[ $https_rr_node =~ '.'$ ]] || https_rr_node+="." + https_rr="$(get_https_rrecord $https_rr_node)" if [[ -n "$https_rr" ]]; then - pr_svrty_good "yes" ; out ": " + pr_svrty_good "yes" ; out ": " prln_italic "$(out_row_aligned_max_width "$https_rr" "$indent " $TERM_WIDTH)" fileout "${jsonID}" "OK" "$https_rr" else @@ -23624,7 +23628,6 @@ dns_https_rr () { } - # Check messages which needed to be processed. I.e. those which would have destroyed the nice # screen output and thus havve been postponed. This is just an idea and is only used once # but can be extended in the future. An array might be more handy From a92cd8f702972379f00c5f2d2489b2cde7a937ee Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Mon, 1 Jun 2026 16:56:47 +0200 Subject: [PATCH 5/9] fix shellcheck complaint --- testssl.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/testssl.sh b/testssl.sh index 60d8a4d61..2476b3251 100755 --- a/testssl.sh +++ b/testssl.sh @@ -22744,9 +22744,7 @@ get_https_rrecord() { fi if [[ ${line:8:2} == "01" ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=1 alpnID+="\"" # double quote for clear text - else - continue # If the 1st element is not alpn, next iteration of loop will fail for now - fi # Should we care as SvcParamKey!=alpn doesn't seems not very common? + fi len_alpnID="${line:12:2}" # length of alpn entries, e.g. 03 or 06 alpnID_wire="${line:16:4}" # value of first entry alpnID="${alpnID}$(hex2ascii "$alpnID_wire")" From 51ba8327a89d1b4f255203a3ade65e3c8b39ebb7 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 2 Jun 2026 19:04:17 +0200 Subject: [PATCH 6/9] introduce subfunctions decode_* First implemented and tested working is decode_https_rr_alpn(). Also we use the svk params in a case statement to decipher the hexstream better. The hexstream ($line) has now no blanks anymore. They seem to be arbitrary. Variables need to be declared in get_https_rrecord() . --- testssl.sh | 75 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/testssl.sh b/testssl.sh index 2476b3251..4c52bbdac 100755 --- a/testssl.sh +++ b/testssl.sh @@ -22726,6 +22726,9 @@ get_https_rrecord() { # Be aware that all variables are strings here! Therefore we use double quotes so that e.g. 03 won't become 3 # For now the following only works for short entries like 1. alpn=h2,h3 + +local text="" + if [[ -z "$raw_https" ]]; then return 1 elif [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then @@ -22733,45 +22736,69 @@ get_https_rrecord() { read hash len line <<< "$raw_https" # \# 10 00010000010003026832 # \# 36 000100000100030268320003000201BB000600102A01023842816755 10000000000B1337 - [[ $DEBUG -eq 1 ]] && echo "$hash $len $line" + len=${len// /} # remove spaces if [[ "${line:0:4}" == 0001 ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.4.3 svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, 0 is alias if [[ $svc_priority == "1" ]]; then # mock text representation - svc_priority+=" . " - alpnID="${alpnID}${svc_priority}" - fi - if [[ ${line:8:2} == "01" ]]; then # Then comes SvcParamKeys, see rfc 14.3.2 which should be alpn=1 - alpnID+="\"" # double quote for clear text - fi - len_alpnID="${line:12:2}" # length of alpn entries, e.g. 03 or 06 - alpnID_wire="${line:16:4}" # value of first entry - alpnID="${alpnID}$(hex2ascii "$alpnID_wire")" - localPTR=20 - if [[ "$len_alpnID" == "06" ]]; then # 06 would be another entry e.g. h3, quote the rhs! - alpnID_wire="${line:22:4}" #FIXME: we can't cope with three entries yet - alpnID="${alpnID},$(hex2ascii "$alpnID_wire")" - localPTR=26 + svc_priority+=" . " #FIXME? needs more testing + text="${text}${svc_priority}" fi - [[ ${line:8:2} == "01" ]] && alpnID+="\"" # if alpn and we're done add trailing double quote - - # done if localPTR / 2 = len or localPTR not empty - echo "key: ${line:localPTR:4}" - # key=1: alpn - # key=3: port - # key=4: IPv4 - # key=6: IPv6 + len_entry=${line:12:2} + len_entry=$(( ((10#$len_entry)) * 2 )) # make sure we count in the right system + entry=${line:14:$len_entry} + # Service Parameter Keys https://www.rfc-editor.org/info/rfc9460/#name-initial-contents + case ${line:8:2} in + 00) # "mandatory" + ;; + 01) # "alpn" + text+=$(decode_https_rr_alpn $entry) ;; + 02) # "no-default-alpn" + ;; + 03) # "port" + ;; + 04) # "ipv4hint" + ;; + 05) # "ech" + ;; + 06) # "ipv6hint" + ;; + esac else out "please report unknown HTTPS RR $line with flag @ $NODE" return 7 fi - safe_echo "$alpnID" + safe_echo "$text" + [[ $DEBUG -eq 1 ]] && echo "$hash $len $line" >&2 + [[ $DEBUG -eq 1 ]] && echo "key: ${line:localPTR:4}" >&2 fi return 0 } +decode_https_rr_alpn() { + local entry="$1" + local -i len="${#entry}" + local -i ptr=0 + local alpn_wire="" str="" + local alpn_len="" + + ptr=0 + while (( ptr < len )); do + [[ -n "$alpn_str" ]] && alpn_str+="," # add a comma in the >=2 round + alpn_len=${entry:$ptr:2} + alpn_len=$(( ((10#$alpn_len)) * 2 )) # conversion, make sure it's the right format + + ptr=$((ptr + 2)) # len field is always 2 bytes + alpn_wire=${entry:$ptr:$alpn_len} + str=$(hex2ascii $alpn_wire) + ptr=$((ptr + alpn_len)) + alpn_str+="$str" + done + safe_echo "alpn=\"$alpn_str\"" +} + # arg1: domain to check for. Returned will be the MX record as a string get_mx_record() { From 37135fa75251db6b66417d470e6f2b617aa13a07 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 9 Jun 2026 22:14:45 +0200 Subject: [PATCH 7/9] Save work - dig needs to be called with $DIG_R - basic parsing for alpn on Mac should be fine now - case statement filled with moste of the functions - port function tested + added, but not called yet - ipv4hint function tested + added but not called yet - ipv6hint function tested + added but not called yet. Doesn't do compression of ipv6 address yet - stub functions dohpath+ech --- testssl.sh | 142 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 125 insertions(+), 17 deletions(-) diff --git a/testssl.sh b/testssl.sh index 556dd3e70..125ceeca7 100755 --- a/testssl.sh +++ b/testssl.sh @@ -22393,8 +22393,10 @@ check_resolver_bins() { # Pre-checking the following for HTTPS RR, see get_https_rrecord() if "$HAS_DIG"; then - str=$(dig +short $testhost HTTPS) - if [[ -z "$str" ]] && [[ ! "$str" =~ 127.0.0.1 ]] ; then + str=$(dig $DIG_R +short $testhost HTTPS) + if [[ -z "$str" ]] && [[ ! "$str" =~ 127.0.0.1 ]] && \ + # MacOS runners are problematic otherwise: + dig $DIG_R +nocomments $testhost HTTPS | grep -q 'IN.*HTTPS'; then HAS_DIG_HTTPS=true fi elif "$HAS_DRILL"; then @@ -22642,7 +22644,7 @@ get_https_rrecord() { # "tail -1" and the awk commands make sure we use the right lines when we encounter a CNAME OPENSSL_CONF="" if "$HAS_DIG_HTTPS"; then - text_httpsrr="$(dig +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null | tail -1)" + text_httpsrr="$(dig $DIG_R +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null | tail -1)" elif "$HAS_DRILL_HTTPS"; then text_httpsrr="$(drill -Q HTTPS "$1" 2>/dev/null | tail -1)" elif "$HAS_HOST_HTTPS"; then @@ -22707,7 +22709,7 @@ get_https_rrecord() { # 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 # # 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 - # TL alpn| L h 2 443 2a010238... L=len of alpn entries, TL=total length of the following by, excluding spaces + # TL alpn| L h 2 |=2 round alpn_len=${entry:$ptr:2} alpn_len=$(( ((10#$alpn_len)) * 2 )) # conversion, make sure it's the right format - ptr=$((ptr + 2)) # len field is always 2 bytes + ptr=$((ptr + 2)) # len field is always 2 bytes alpn_wire=${entry:$ptr:$alpn_len} str=$(hex2ascii $alpn_wire) ptr=$((ptr + alpn_len)) @@ -22798,6 +22803,109 @@ decode_https_rr_alpn() { safe_echo "alpn=\"$alpn_str\"" } +# key 3 — port: single u16 override port +# +decode_https_rr_port() { + local entry="$1" + local -i len="${#entry}" + local -i ptr=2 + local port_wire="" str="" + + # we assume it's one port only and it starts at $ptr and is $len-$ptr long + port_wire=${entry:$ptr:$((len - ptr))} + str=$((16#$port_wire)) # hex2dec + port_str+="$str" + safe_echo "port=\"$port_str\"" +} + +# key 4 — ipv4hint: one or more 4-byte IPv4 addresse +# +decode_https_rr_ipv4() { + local entry="$1" + local -i len="${#entry}" + local -i ptr=2 + local ipv4_wire="" str="" + # we currently don't need that: + # local nr_ips="${1:0:2}" + + while (( ptr < len )); do + ipv4_wire=${entry:$ptr:2} + str=$((16#$ipv4_wire)) # hex2dec + ipv4_str+="$str" + + # if the end is not reached yet + # after address 2,4,6, 10,12,14, ... we need a dot + # after address 18, 16, ... we need a comma + + if [[ $len -ne $((ptr + 2)) ]]; then + if [[ $((ptr % 8 )) -eq 0 ]] ; then + ipv4_str+="," + else + ipv4_str+="." + fi + fi + ptr=$((ptr + 2)) # two bytes per octet + done + safe_echo "ipv4hint=\"$ipv4_str\"" +} + + +# key 5 — ech: opaque ECHConfigList blob, show as truncated hex +# +decode_https_rr_ech() { + echo +} + + +# key 6 — ipv6hint: one or more 16-byte IPv6 addresses +#FIXME: doesn't do IPv6 compression yet +decode_https_rr_ipv6() { + local entry="$1" + local -i len="${#entry}" + local -i ptr=2 # we start at pos 2 + local ipv6_wire="" str="" + # local nr_ips="${1:0:2}" + + while (( ptr < len )); do + ipv6_wire=${entry:$ptr:4} + ipv6_str+="$ipv6_wire" + + # We have 8 octets filled with zero if needed --> 32 chars + + if [[ $len -ne $((ptr + 4)) ]]; then + if [[ $((ptr % 30 )) -eq 0 ]] ; then # we have two bytes pointer 30+2=32 + ipv6_str+="," + else + ipv6_str+=":" + fi + fi + ptr=$((ptr + 4)) # two byte per octett + done + + ipv6_str="$(tolower "$ipv6_str")" + + safe_echo "ipv6hint=\"$ipv6_str\"" +} + + +# key 7 — dohpath: UTF-8 URI template for DNS-over-HTTPS +#FIXME --> to test! +# +decode_dohpath() { + local entry="$1" + local -i len="${#entry}" + # local len=$1 + local path="" + local -i i + + for (( i = 0; i < len; i++ )); do + path+=$(printf "\\$(printf '%03o' "${PARAM_VALUE_BYTES[$i]}")") + done + safe_echo "$path" +} + + + # arg1: domain to check for. Returned will be the MX record as a string get_mx_record() { From 913bf1406dc40fdab72836db2f7bb1d087678ac3 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 16 Jun 2026 11:00:14 +0200 Subject: [PATCH 8/9] Save work - parsing output from old dig versions (Mac) works for almost every svc_key - for old dig versions: double lined RR work (but output is not nice yet) - cleaned up comments - separate function https_rr_raw_parser() - commented output from claude.ai for ech for later interpretation - get_mx_record() has a warning when get_https_rrecord returned != 0 --- testssl.sh | 254 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 148 insertions(+), 106 deletions(-) diff --git a/testssl.sh b/testssl.sh index 125ceeca7..db2844bfa 100755 --- a/testssl.sh +++ b/testssl.sh @@ -22623,15 +22623,10 @@ get_caa_rrecord() { # get_https_rrecord() { local raw_https="" - local hash="" line="" - local len # better fix as integer? - local len_alpnID="" - local alpnID="" - local alpnID_wire="" + local line="" local saved_openssl_conf="$OPENSSL_CONF" local all_https="" local noidnout="" - local svc_priority="" [[ -n "$NODNS" ]] && return 2 # if minimum DNS lookup was instructed, leave here "$HAS_DIG_NOIDNOUT" && noidnout="+noidnout" @@ -22642,6 +22637,8 @@ get_https_rrecord() { # in clear text and also we can avoid to parse the encoded formats. # "tail -1" and the awk commands make sure we use the right lines when we encounter a CNAME + #FIXME: likely causes a problem with mulitline RR + OPENSSL_CONF="" if "$HAS_DIG_HTTPS"; then text_httpsrr="$(dig $DIG_R +short +search +timeout=3 +tries=3 $noidnout HTTPS "$1" 2>/dev/null | tail -1)" @@ -22702,80 +22699,100 @@ get_https_rrecord() { fi OPENSSL_CONF="$saved_openssl_conf" # We're done now with openssl, see https://github.com/drwetter/testssl.sh/issues/134 - # Now comes the third, more tricky part and the last straw --> parsing the hex stream which was returned if it was returned. - # Format is like: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) + if [[ -z "$raw_https" ]]; then + return 1 + fi - # dev.testssl.sh: - # 1 . alpn="h2" port=443 ipv6hint=2a01:238:4308:a920:1000:0:b:1337 - # - # 36 000100000100030268320003000201BB000600102A0102384308A920 10000000000B1337 - # TL alpn| L h 2 | for esni (https://datatracker.ietf.org/doc/draft-ietf-tls-esni/) - # Nice description: https://www.netmeister.org/blog/https-rrs.html - # - # 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 - # 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D - # TL alpn| L h 3 L h 2 |IPv4#1||IPv4#2| - # - # ech=AEX+DQBBzgAgACBQGA9EFbz+PkJAXSXtcqJluxLlhxIgzhJ+GhTtRd4nJQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a - # 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A - # | cloudflare-ech.com | IPv6#1 #IPv6#2 + # Now comes the third, tricky part (old dig for Macs e.g.) --> parsing the hex stream which was returned if it was returned. + # https_rr_raw_parser() takes care of that. Format is like: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) - # Be aware that all variables are strings here! Therefore we use double quotes so that e.g. 03 won't become 3 - # For now the following only works for short entries like 1. alpn=h2,h3 + local -i i=0 + while IFS= read -r line; do + https_rr_raw_parser "$line" || return 1 + ((i++)) + # in rare cases there can be two lines (sodiao.cc) or more, #FIXME: output formatting is wrong + [[ "$raw_https" == *$'\n'* ]] && [[ $i -ge 1 ]] && outln + done <<< "$raw_https" +} -local text="" - if [[ -z "$raw_https" ]]; then - return 1 - elif [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then - # signals that we're on the right track with type65 interpretation +https_rr_raw_parser () { + local raw_https="$1" + local hash="" line="" + local len="" len_next_entry="" + local svc_priority="" svc_key="" + local text="" + local -i ptr=0 + local first=true + + if [[ "$raw_https" =~ \#\ [0-9][0-9] ]]; then # check we're on the right track with type65 interpretation read hash len line <<< "$raw_https" - # \# 10 00010000010003026832 - # \# 36 000100000100030268320003000201BB000600102A01023842816755 10000000000B1337 - - len=${len// /} # remove spaces - if [[ "${line:0:4}" == 0001 ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.4.3 - svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, 0 is alias - if [[ $svc_priority == "1" ]]; then - # mock text representation - svc_priority+=" . " #FIXME? needs more testing - text="${text}${svc_priority}" + + # testssl.sh \# 10 00010000010003026832 --> 1. alpn="h2" + # dev.testssl.sh \# 36 000100000100030268320003000201BB000600102A01023842816755 10000000000B1337 ----> 1. alpn="h2" port=443 ipv6hint=2a01:238:4281:6755:1000:0:b:1337 + # google.com \# 13 00010000010006026832026833 --> 1. alpn="h2,h3" + # b-cdn.net \# 27 0001000001000C02683208687474702F312E3100040004A996F722 --> alpn="h2,http/1.1" ipv4hint=169,150.247.34 + # testssl.net \# 136 00010000010006026833026832000400086815229AAC43CDE7000500 470045FE0D0041F3002000202BD0935ED66980C1862F2570C0D6014D 733A7CFAAEA5E4DD9CA43D4C24199E330004000100010012636C6F75 64666C6172652D6563682E636F6D0000000600202606470030310000 00000000AC43CDE72606470030360000000000006815229A + # --> 1. alpn="h3,h2" ipv4hint=104.21.34.154,172.67.205.231 ech=AEX+DQBB1gAgACDasOut8j3EAZ6Rc04Wy0Vm+fj/SiHZWUZIeH3bRtoyAQAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= ipv6hint=2606:4700:3031::ac43:cde7,2606:4700:3036::6815:229a + # more @ https://github.com/yzzhn/imc2024dnshttps + + line=${line// /} # remove spaces + if [[ $((len * 2)) -ne ${#line} ]]; then # again a consistency check + echo "inconsistent length for type65 hex stream parsing" + return 1 + fi + if [[ "${line:0:4}" =~ ^(0001|0002)$ ]]; then # marker to proceed, belongs to SvcPriority, see rfc9460, 2.4.3 + svc_priority=$(printf "%0d" "$((10#${line:2:2}))") # 1 is most often, 2 is possible, 0 is alias (to be tested) + if [[ $svc_priority =~ ^(1|2)$ ]]; then + # mock text representation + svc_priority+=" . " #FIXME: what about 0? + text="${text}${svc_priority}" + ptr=6 # This is at the start + fi + while (( ptr < ${#line} )); do + if "$first"; then + first=false + else + text+=" " fi - len_entry=${line:12:2} - len_entry=$(( ((10#$len_entry)) * 2 )) # make sure we count in the right system - entry=${line:14:$len_entry} + ptr=$(( ptr + 2 )) + svc_key=${line:$ptr:2} + ptr=$(( ptr + 4 )) + + len_next_entry=${line:$ptr:2} + len_next_entry=$((16#${len_next_entry})) # it's a hex number + len_next_entry=$((len_next_entry * 2 )) + ptr=$(( ptr + 2 )) + entry=${line:$ptr:$len_next_entry} + + debugme echo "-- $svc_key : $entry ($len_next_entry) --" # Service Parameter Keys https://www.rfc-editor.org/info/rfc9460/#name-initial-contents - case ${line:8:2} in + case $svc_key in 00) # = "mandatory", skipping that ;; - 01) # = "alpn" - text+=$(decode_https_rr_alpn $entry) ;; - 02) # = "no-default-alpn", skipping that + 01) text+=$(decode_https_rr_alpn $entry) ;; - 03) # = "port" - text+=$(decode_https_rr_port $entry) ;; - 04) # = "ipv4hint" - text+=$(decode_https_rr_ipv4hint $entry) ;; - 05) # = "ech" - text+=$(decode_https_rr_ech $entry) ;; - 06) # = "ipv6hint" - text+=$(decode_https_rr_ipv6hint $entry) ;; - 07) # = "dohpath" - text+=$(decode_https_rr_dohpath $entry) ;; - esac - else - out "please report unknown HTTPS RR $line with flag @ $NODE" - return 7 - fi - safe_echo "$text" - [[ $DEBUG -eq 1 ]] && echo "$hash $len $line" >&2 - [[ $DEBUG -eq 1 ]] && echo "key: ${line:localPTR:4}" >&2 + 02) text+="no-default-alpn" + ;; + 03) text+=$(decode_https_rr_port $entry) + ;; + 04) text+=$(decode_https_rr_ipv4 $entry) + ;; + 05) text+=$(decode_https_rr_ech $entry) + ;; + 06) text+=$(decode_https_rr_ipv6 $entry) + ;; + 07) text+=$(decode_https_rr_dohpath $entry) + ;; + esac + ptr=$((10#${#entry} + ptr )) + done + else + safe_echo "please report unknown HTTPS RR $line from $NODE" + return 1 + fi + safe_echo "$text" fi return 0 } @@ -22792,7 +22809,7 @@ decode_https_rr_alpn() { while (( ptr < len )); do [[ -n "$alpn_str" ]] && alpn_str+="," # add a comma in the >=2 round alpn_len=${entry:$ptr:2} - alpn_len=$(( ((10#$alpn_len)) * 2 )) # conversion, make sure it's the right format + alpn_len=$(( ((10#$alpn_len)) * 2 )) # also make sure it's a number ptr=$((ptr + 2)) # len field is always 2 bytes alpn_wire=${entry:$ptr:$alpn_len} @@ -22803,30 +22820,27 @@ decode_https_rr_alpn() { safe_echo "alpn=\"$alpn_str\"" } -# key 3 — port: single u16 override port +# key 3 — port: single one # decode_https_rr_port() { local entry="$1" local -i len="${#entry}" - local -i ptr=2 local port_wire="" str="" - # we assume it's one port only and it starts at $ptr and is $len-$ptr long - port_wire=${entry:$ptr:$((len - ptr))} - str=$((16#$port_wire)) # hex2dec + # we assume it's one port only and it starts at $ptr and is $len long + port_wire=${entry:0:$len} # we start @ pos=0 and assume, it's one port only, otherwise we need to extend this, see ipv6 func e.g. + str=$((16#$port_wire)) # hex2dec. Works too: printf "%d\n" "0x$port_wire" port_str+="$str" - safe_echo "port=\"$port_str\"" + safe_echo "port=${port_str}" } -# key 4 — ipv4hint: one or more 4-byte IPv4 addresse +# key 4 — ipv4hint: one or more 4-byte IPv4 addresses # decode_https_rr_ipv4() { local entry="$1" local -i len="${#entry}" - local -i ptr=2 + local -i ptr=0 # we start @ pos=0 local ipv4_wire="" str="" - # we currently don't need that: - # local nr_ips="${1:0:2}" while (( ptr < len )); do ipv4_wire=${entry:$ptr:2} @@ -22838,7 +22852,7 @@ decode_https_rr_ipv4() { # after address 18, 16, ... we need a comma if [[ $len -ne $((ptr + 2)) ]]; then - if [[ $((ptr % 8 )) -eq 0 ]] ; then + if [[ $(( ((ptr + 2 )) % 8 )) -eq 0 ]] ; then ipv4_str+="," else ipv4_str+="." @@ -22846,67 +22860,92 @@ decode_https_rr_ipv4() { fi ptr=$((ptr + 2)) # two bytes per octet done - safe_echo "ipv4hint=\"$ipv4_str\"" + safe_echo "ipv4hint=${ipv4_str}" } -# key 5 — ech: opaque ECHConfigList blob, show as truncated hex +# key 5 — encrypted client hello: pub key and more # decode_https_rr_ech() { - echo + # cloudflare-ech.com (base64 format conversion between the two): + # text format: AEX+DQBB+QAgACD4885ZLoES1IllBXr15/nI6vPXjTcxfiM02O8nxfZgXwAEAAEAAQASY2xvdWRmbGFyZS1lY2guY29tAAA= + # wire format: 0045FE0D0041F900200020F8F3CE592E8112D48965057AF5E7F9C8EAF3D78D37317E2334D8EF27C5F6605F0004000100010012636C6F7564666C6172652D6563682E636F6D0000 + +# interpretation from claude.ai, to be double checked: +# 00 45 ECHConfigList.length = 0x0045 = 69 +# FE 0D ECHConfig.version = 0xfe0d (ECH draft-13) +# 00 41 ECHConfig.length = 0x0041 = 65 +# F9 config_id = 0xF9 (249) +# 00 20 kem_id = 0x0020 = DHKEM(X25519, HKDF-SHA256) +# 00 20 public_key_len = 32 +# F8 F3 CE 59 2E 81 12 D4 +# 89 65 05 7A F5 E7 F9 C8 +# EA F3 D7 8D 37 31 7E 23 +# 34 D8 EF 27 C5 F6 60 5F public_key (32 bytes, X25519 pubkey) +# 00 04 cipher_suites_len = 4 +# 00 01 00 01 one suite: KDF=0x0001 (HKDF-SHA256), AEAD=0x0001 (AES-128-GCM) +# 00 maximum_name_length = 0 +# 12 public_name_len = 18 +# 63 6C 6F 75 64 66 6C 61 +# 72 65 2D 65 63 68 2E 63 +# 6F 6D public_name = "cloudflare-ech.com" +# 00 00 extensions_len = 0 + + # for now we just encode the wire format to the base64 format + safe_echo "ech=$(hex2ascii "$1" | $OPENSSL base64 -A 2>/dev/null)" } - # key 6 — ipv6hint: one or more 16-byte IPv6 addresses -#FIXME: doesn't do IPv6 compression yet +# decode_https_rr_ipv6() { local entry="$1" local -i len="${#entry}" - local -i ptr=2 # we start at pos 2 local ipv6_wire="" str="" - # local nr_ips="${1:0:2}" + local -i ptr=0 # we start @ pos=0 while (( ptr < len )); do - ipv6_wire=${entry:$ptr:4} + ipv6_wire=${entry:$ptr:4} # we have 8 hextets, length 4, filled with zero if needed --> 32 chars ipv6_str+="$ipv6_wire" - # We have 8 octets filled with zero if needed --> 32 chars - if [[ $len -ne $((ptr + 4)) ]]; then - if [[ $((ptr % 30 )) -eq 0 ]] ; then # we have two bytes pointer 30+2=32 + if [[ $(( ((ptr + 4)) % 32 )) -eq 0 ]]; then # we have two bytes pointer 30+2=32 ipv6_str+="," else ipv6_str+=":" fi fi - ptr=$((ptr + 4)) # two byte per octett + ptr=$((ptr + 4)) # two byte per hextets done ipv6_str="$(tolower "$ipv6_str")" - safe_echo "ipv6hint=\"$ipv6_str\"" -} + # poor man's compression, max 5 zero hextets + ipv6_str=${ipv6_str//:0000:0000:0000:0000:0000:/::} + ipv6_str=${ipv6_str//:0000:0000:0000:0000:/::} + ipv6_str=${ipv6_str//:0000:0000:0000:/::} + ipv6_str=${ipv6_str//:0000:0000:/::} + ipv6_str=${ipv6_str//:0000:/::} + + # strip up to 3 leading zeros in a hextet + ipv6_str=${ipv6_str//:0/:} + ipv6_str=${ipv6_str//:0/:} + ipv6_str=${ipv6_str//:0/:} + safe_echo "ipv6hint=${ipv6_str}" +} # key 7 — dohpath: UTF-8 URI template for DNS-over-HTTPS -#FIXME --> to test! +#FIXME: likely doesn't work, not tested # decode_dohpath() { local entry="$1" local -i len="${#entry}" - # local len=$1 - local path="" - local -i i + local path=$( hex2ascii "$1" ) - for (( i = 0; i < len; i++ )); do - path+=$(printf "\\$(printf '%03o' "${PARAM_VALUE_BYTES[$i]}")") - done - safe_echo "$path" + safe_echo "$path (please report this @ github)" } - - # arg1: domain to check for. Returned will be the MX record as a string get_mx_record() { local mx="" @@ -23748,7 +23787,10 @@ dns_https_rr () { # append a dot if there was none [[ $https_rr_node =~ '.'$ ]] || https_rr_node+="." https_rr="$(get_https_rrecord $https_rr_node)" - if [[ -n "$https_rr" ]]; then + if [[ $? -ne 0 ]]; then + prln_warning "$https_rr" + fileout "${jsonID}" "WARN" "$https_rr" + elif [[ -n "$https_rr" ]]; then pr_svrty_good "yes" ; out ": " prln_italic "$(out_row_aligned_max_width "$https_rr" "$indent " $TERM_WIDTH)" fileout "${jsonID}" "OK" "$https_rr" From 1f9e61afbcaa05c5323f351f71f86e42faede4a7 Mon Sep 17 00:00:00 2001 From: Dirk Wetter Date: Tue, 16 Jun 2026 13:15:11 +0200 Subject: [PATCH 9/9] Fix CI runner for Mac --- testssl.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/testssl.sh b/testssl.sh index db2844bfa..e9029776c 100755 --- a/testssl.sh +++ b/testssl.sh @@ -22707,12 +22707,16 @@ get_https_rrecord() { # https_rr_raw_parser() takes care of that. Format is like: https://www.rfc-editor.org/rfc/rfc3597 (plus updates) local -i i=0 + local -i nr_lines=$(grep -c '^' <<< "$raw_https") + # In rare cases there can be two lines (sodiao.cc) or more while IFS= read -r line; do https_rr_raw_parser "$line" || return 1 + [[ $nr_lines -eq 1 ]] && break # return here for a one liner, otherwise next time we hit return 1 ((i++)) - # in rare cases there can be two lines (sodiao.cc) or more, #FIXME: output formatting is wrong - [[ "$raw_https" == *$'\n'* ]] && [[ $i -ge 1 ]] && outln + [[ $i -eq $nr_lines ]] && break # we hit the last line + [[ $i -ge 1 ]] && out " / " # hack: two lines are merged into one output line and separated by "/" done <<< "$raw_https" + return 0 }