waf: allow to drop request for which body is unreadable

config_example.conf

@@ -39,4 +39,6 @@ APPSEC_CONNECT_TIMEOUT=
 APPSEC_SEND_TIMEOUT=
 APPSEC_PROCESS_TIMEOUT=
 ALWAYS_SEND_TO_APPSEC=false
+# When true, drop requests whose body cannot be read (e.g. HTTP/2 or HTTP/3 requests without a content-length header) instead of forwarding them to AppSec without a body.
+APPSEC_DROP_UNREADABLE_BODY=false
 SSL_VERIFY=true

lib/crowdsec.lua

@@ -34,6 +34,13 @@ local APPSEC_TRANSFER_ENCODING_HEADER = "x-crowdsec-appsec-transfer-encoding"
 local REMEDIATION_API_KEY_HEADER = 'x-api-key'
 local METRICS_PERIOD = 900
 
+local METHODS_WITH_BODY = {
+  POST = true,
+  PUT = true,
+  PATCH = true,
+  DELETE = true,
+}
+
 --- only for debug purpose
 --- called only from within the nginx configuration file in the CI
 function csmod.debug_metrics()
@@ -218,6 +225,12 @@ function csmod.init(configFile, userAgent)
     runtime.conf["ALWAYS_SEND_TO_APPSEC"] = true
   end
 
+  if runtime.conf["APPSEC_DROP_UNREADABLE_BODY"] == "true" then
+    runtime.conf["APPSEC_DROP_UNREADABLE_BODY"] = true
+  else
+    runtime.conf["APPSEC_DROP_UNREADABLE_BODY"] = false
+  end
+
   runtime.conf["APPSEC_ENABLED"] = false
 
   if runtime.conf["APPSEC_URL"] ~= "" then
@@ -343,7 +356,7 @@ local function get_body()
   -- do not even try to read the body if there's no content-length as the LUA API will throw an error
   if ngx.req.http_version() >= 2 and ngx.var.http_content_length == nil then
     ngx.log(ngx.DEBUG, "No content-length header in request")
-    return nil
+    return nil, METHODS_WITH_BODY[ngx.var.request_method] == true
   end
   ngx.req.read_body()
   local body = ngx.req.get_body_data()
@@ -357,7 +370,7 @@ local function get_body()
       end
     end
   end
-  return body
+  return body, false
 end
 
 function csmod.GetCaptchaBackendKey()
@@ -599,7 +612,11 @@ function csmod.AppSecCheck(ip)
 
   local method = "GET"
 
-  local body = get_body()
+  local body, unreadable_body = get_body()
+  if unreadable_body and runtime.conf["APPSEC_DROP_UNREADABLE_BODY"] then
+    ngx.log(ngx.WARN, "Dropping request because body is unreadable and APPSEC_DROP_UNREADABLE_BODY is enabled")
+    return false, runtime.conf["FALLBACK_REMEDIATION"], ngx.HTTP_FORBIDDEN, nil
+  end
   if body ~= nil then
     if #body > 0 then
       method = "POST"

lib/plugins/crowdsec/config.lua

@@ -1,6 +1,6 @@
 local config = {}
 
-local valid_params = {'ENABLED', 'ENABLE_INTERNAL', 'API_URL', 'API_KEY', 'BOUNCING_ON_TYPE', 'MODE', 'SECRET_KEY', 'SITE_KEY', 'BAN_TEMPLATE_PATH' ,'CAPTCHA_TEMPLATE_PATH', 'REDIRECT_LOCATION', 'RET_CODE', 'CAPTCHA_RET_CODE', 'EXCLUDE_LOCATION', 'FALLBACK_REMEDIATION', 'CAPTCHA_PROVIDER', 'APPSEC_URL', 'APPSEC_FAILURE_ACTION', 'ALWAYS_SEND_TO_APPSEC', 'SSL_VERIFY', 'USE_TLS_AUTH', 'TLS_CLIENT_CERT', 'TLS_CLIENT_KEY'}
+local valid_params = {'ENABLED', 'ENABLE_INTERNAL', 'API_URL', 'API_KEY', 'BOUNCING_ON_TYPE', 'MODE', 'SECRET_KEY', 'SITE_KEY', 'BAN_TEMPLATE_PATH' ,'CAPTCHA_TEMPLATE_PATH', 'REDIRECT_LOCATION', 'RET_CODE', 'CAPTCHA_RET_CODE', 'EXCLUDE_LOCATION', 'FALLBACK_REMEDIATION', 'CAPTCHA_PROVIDER', 'APPSEC_URL', 'APPSEC_FAILURE_ACTION', 'ALWAYS_SEND_TO_APPSEC', 'APPSEC_DROP_UNREADABLE_BODY', 'SSL_VERIFY', 'USE_TLS_AUTH', 'TLS_CLIENT_CERT', 'TLS_CLIENT_KEY'}
 local valid_int_params = {'CACHE_EXPIRATION', 'CACHE_SIZE', 'REQUEST_TIMEOUT', 'UPDATE_FREQUENCY', 'CAPTCHA_EXPIRATION', 'APPSEC_CONNECT_TIMEOUT', 'APPSEC_SEND_TIMEOUT', 'APPSEC_PROCESS_TIMEOUT', 'STREAM_REQUEST_TIMEOUT'}
 -- CACHE_SIZE is not used in the code, but as is was valid parameter for the configuration file, not removing it now
 local valid_bouncing_on_type_values = {'ban', 'captcha', 'all'}
@@ -26,6 +26,7 @@ local default_values = {
     ['APPSEC_FAILURE_ACTION'] = "passthrough",
     ['SSL_VERIFY'] = "true",
     ['ALWAYS_SEND_TO_APPSEC'] = "false",
+    ['APPSEC_DROP_UNREADABLE_BODY'] = "false",
     ['CAPTCHA_RET_CODE'] = 0,
     ['CACHE_EXPIRATION'] = 1,
     ['USE_TLS_AUTH'] = "false",
@@ -112,6 +113,11 @@ function config.loadConfig(file, default)
                     ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'false' instead")
                     value = "false"
                 end
+            elseif key == "APPSEC_DROP_UNREADABLE_BODY" then
+                if not has_value(valid_truefalse_values, value) then
+                    ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'false' instead")
+                    value = "false"
+                end
             elseif key == "MODE" then
                 if not has_value({'stream', 'live'}, value) then
                 ngx.log(ngx.ERR, "unsupported value '" .. value .. "' for variable '" .. key .. "'. Using default value 'stream' instead")

t/18_appsec_drop_unreadable_body.t

@@ -0,0 +1,77 @@
+use Test::Nginx::Socket 'no_plan';
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: APPSEC_DROP_UNREADABLE_BODY=true bans a POST whose body is unreadable
+
+--- main_config
+load_module /usr/share/nginx/modules/ndk_http_module.so;
+load_module /usr/share/nginx/modules/ngx_http_lua_module.so;
+
+--- http_config
+
+lua_package_path './lib/?.lua;;';
+lua_shared_dict crowdsec_cache 50m;
+lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+
+init_by_lua_block
+{
+        cs = require "crowdsec"
+        local ok, err = cs.init("./t/conf_t/18_appsec_drop_unreadable_body_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8")
+        if ok == nil then
+                ngx.log(ngx.ERR, "[Crowdsec] " .. err)
+                error()
+        end
+        ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done")
+}
+
+access_by_lua_block {
+        local cs = require "crowdsec"
+        -- Simulate an HTTP/2+ request so the bouncer treats the missing
+        -- content-length as an unreadable body.
+        ngx.req.http_version = function() return 2.0 end
+        cs.Allow(ngx.var.remote_addr)
+}
+
+server {
+    listen 8081;
+
+       location = /v1/decisions {
+            content_by_lua_block {
+                ngx.print('null')
+            }
+       }
+}
+
+server {
+    listen 7422;
+
+       location / {
+            content_by_lua_block {
+                ngx.log(ngx.ERR, "[appsec mock] should not be reached when dropping unreadable body")
+                ngx.status = 200
+                ngx.print('{"action":"allow"}')
+            }
+       }
+}
+
+
+--- config
+
+
+location = /t {
+    set_real_ip_from 127.0.0.1;
+    real_ip_header   X-Forwarded-For;
+    real_ip_recursive on;
+    content_by_lua_block {
+        ngx.say("Hello, world")
+    }
+}
+
+--- raw_request eval
+"POST /t HTTP/1.1\r\nHost: localhost\r\nX-Forwarded-For: 1.1.1.2\r\nConnection: close\r\n\r\n"
+
+--- error_code: 403

t/19_appsec_drop_unreadable_body_get.t

@@ -0,0 +1,83 @@
+use Test::Nginx::Socket 'no_plan';
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: APPSEC_DROP_UNREADABLE_BODY=true does not drop GET requests (no body expected)
+
+--- main_config
+load_module /usr/share/nginx/modules/ndk_http_module.so;
+load_module /usr/share/nginx/modules/ngx_http_lua_module.so;
+
+--- http_config
+
+lua_package_path './lib/?.lua;;';
+lua_shared_dict crowdsec_cache 50m;
+lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+
+init_by_lua_block
+{
+        cs = require "crowdsec"
+        local ok, err = cs.init("./t/conf_t/18_appsec_drop_unreadable_body_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8")
+        if ok == nil then
+                ngx.log(ngx.ERR, "[Crowdsec] " .. err)
+                error()
+        end
+        ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done")
+}
+
+access_by_lua_block {
+        local cs = require "crowdsec"
+        -- Simulate an HTTP/2+ request so the missing content-length would
+        -- normally be flagged as unreadable; since the method is GET, the
+        -- request must still go through.
+        ngx.req.http_version = function() return 2.0 end
+        cs.Allow(ngx.var.remote_addr)
+}
+
+server {
+    listen 8081;
+
+       location = /v1/decisions {
+            content_by_lua_block {
+                ngx.print('null')
+            }
+       }
+}
+
+server {
+    listen 7422;
+
+       location / {
+            content_by_lua_block {
+                ngx.status = 200
+                ngx.print('{"action":"allow"}')
+            }
+       }
+}
+
+
+--- config
+
+
+location = /t {
+    set_real_ip_from 127.0.0.1;
+    real_ip_header   X-Forwarded-For;
+    real_ip_recursive on;
+    content_by_lua_block {
+        ngx.say("Hello, world")
+    }
+}
+
+--- more_headers
+X-Forwarded-For: 1.1.1.2
+
+--- request
+GET /t
+
+--- response_body
+Hello, world
+
+--- error_code: 200

t/20_appsec_keep_unreadable_body.t

@@ -0,0 +1,80 @@
+use Test::Nginx::Socket 'no_plan';
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: APPSEC_DROP_UNREADABLE_BODY=false (default) lets a POST with unreadable body reach AppSec
+
+--- main_config
+load_module /usr/share/nginx/modules/ndk_http_module.so;
+load_module /usr/share/nginx/modules/ngx_http_lua_module.so;
+
+--- http_config
+
+lua_package_path './lib/?.lua;;';
+lua_shared_dict crowdsec_cache 50m;
+lua_ssl_trusted_certificate /etc/ssl/certs/ca-certificates.crt;
+
+
+init_by_lua_block
+{
+        cs = require "crowdsec"
+        local ok, err = cs.init("./t/conf_t/19_appsec_keep_unreadable_body_crowdsec_nginx_bouncer.conf", "crowdsec-nginx-bouncer/v1.0.8")
+        if ok == nil then
+                ngx.log(ngx.ERR, "[Crowdsec] " .. err)
+                error()
+        end
+        ngx.log(ngx.ALERT, "[Crowdsec] Initialisation done")
+}
+
+access_by_lua_block {
+        local cs = require "crowdsec"
+        -- Simulate an HTTP/2+ request with no content-length to make the
+        -- bouncer treat the body as unreadable; the request must still be
+        -- forwarded since the option is disabled.
+        ngx.req.http_version = function() return 2.0 end
+        cs.Allow(ngx.var.remote_addr)
+}
+
+server {
+    listen 8081;
+
+       location = /v1/decisions {
+            content_by_lua_block {
+                ngx.print('null')
+            }
+       }
+}
+
+server {
+    listen 7422;
+
+       location / {
+            content_by_lua_block {
+                ngx.status = 200
+                ngx.print('{"action":"allow"}')
+            }
+       }
+}
+
+
+--- config
+
+
+location = /t {
+    set_real_ip_from 127.0.0.1;
+    real_ip_header   X-Forwarded-For;
+    real_ip_recursive on;
+    content_by_lua_block {
+        ngx.say("Hello, world")
+    }
+}
+
+--- raw_request eval
+"POST /t HTTP/1.1\r\nHost: localhost\r\nX-Forwarded-For: 1.1.1.2\r\nConnection: close\r\n\r\n"
+
+--- response_body
+Hello, world
+
+--- error_code: 200

t/conf_t/18_appsec_drop_unreadable_body_crowdsec_nginx_bouncer.conf

@@ -0,0 +1,21 @@
+APPSEC_URL=http://127.0.0.1:7422
+APPSEC_DROP_UNREADABLE_BODY=true
+ENABLED=true
+API_URL=http://127.0.0.1:8081
+API_KEY=test_key
+CACHE_EXPIRATION=1
+BOUNCING_ON_TYPE=all
+FALLBACK_REMEDIATION=ban
+REQUEST_TIMEOUT=3000
+UPDATE_FREQUENCY=10
+MODE=live
+EXCLUDE_LOCATION=
+BAN_TEMPLATE_PATH=./ban
+REDIRECT_LOCATION=
+RET_CODE=
+CAPTCHA_PROVIDER=
+SECRET_KEY=
+SITE_KEY=
+CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html
+CAPTCHA_EXPIRATION=3600
+APPSEC_FAILURE_ACTION=passthrough

t/conf_t/19_appsec_keep_unreadable_body_crowdsec_nginx_bouncer.conf

@@ -0,0 +1,21 @@
+APPSEC_URL=http://127.0.0.1:7422
+APPSEC_DROP_UNREADABLE_BODY=false
+ENABLED=true
+API_URL=http://127.0.0.1:8081
+API_KEY=test_key
+CACHE_EXPIRATION=1
+BOUNCING_ON_TYPE=all
+FALLBACK_REMEDIATION=ban
+REQUEST_TIMEOUT=3000
+UPDATE_FREQUENCY=10
+MODE=live
+EXCLUDE_LOCATION=
+BAN_TEMPLATE_PATH=./ban
+REDIRECT_LOCATION=
+RET_CODE=
+CAPTCHA_PROVIDER=
+SECRET_KEY=
+SITE_KEY=
+CAPTCHA_TEMPLATE_PATH=/var/lib/crowdsec/lua/templates/captcha.html
+CAPTCHA_EXPIRATION=3600
+APPSEC_FAILURE_ACTION=passthrough