diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4585bd..cf5fba4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,20 +9,28 @@ on: jobs: build: strategy: + fail-fast: false matrix: - op_version: - - "1.25.3.1" - - "1.27.1.2" + runtime: + - name: apisix-runtime + script_url: "https://raw.githubusercontent.com/api7/apisix-build-tools/master/build-apisix-runtime.sh" + script_name: "build-apisix-runtime.sh" + exclude_tests: "" + - name: api7ee-runtime + script_url: "https://raw.githubusercontent.com/api7/apisix-build-tools/release/api7ee-runtime/build-api7ee-runtime.sh" + script_name: "build-api7ee-runtime.sh" + # Tests requiring nginx features not available in OR 1.21 (nginx 1.21) + exclude_tests: "t/pipe.t t/upstream_pass_trailers.t" runs-on: "ubuntu-latest" + name: build (${{ matrix.runtime.name }}) env: - OPENRESTY_VERSION: ${{ matrix.op_version }} OPENRESTY_PREFIX: "/usr/local/openresty" steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up build environment run: | @@ -43,13 +51,20 @@ jobs: - name: Install run: | - wget https://raw.githubusercontent.com/api7/apisix-build-tools/master/build-apisix-base.sh - chmod +x build-apisix-base.sh - OR_PREFIX=$OPENRESTY_PREFIX CC="gcc -fsanitize=address -fdiagnostics-color=always -Wno-unused-but-set-variable -Wno-unused-parameter" \ - cc_opt="-Werror" ./build-apisix-base.sh latest - + wget ${{ matrix.runtime.script_url }} + chmod +x ${{ matrix.runtime.script_name }} + # Fix permission issue: EE script does rm without sudo after sudo make install + sed -i 's|rm -rf "$OPENSSL_PREFIX"/share|sudo rm -rf "$OPENSSL_PREFIX"/share|' ${{ matrix.runtime.script_name }} + OR_PREFIX=$OPENRESTY_PREFIX CC="gcc -fsanitize=address -fdiagnostics-color=always -Werror -Wno-unused-but-set-variable -Wno-unused-parameter" \ + ./${{ matrix.runtime.script_name }} latest - name: Script run: | export PATH=$OPENRESTY_PREFIX/nginx/sbin:$PATH - prove -I. -Itest-nginx/lib -r t/ + TEST_FILES=$(find t/ -name '*.t' -type f | sort) + if [ -n "${{ matrix.runtime.exclude_tests }}" ]; then + for f in ${{ matrix.runtime.exclude_tests }}; do + TEST_FILES=$(echo "$TEST_FILES" | grep -v "^${f}$") + done + fi + echo "$TEST_FILES" | xargs prove -I. -Itest-nginx/lib diff --git a/lib/resty/apisix/upstream.lua b/lib/resty/apisix/upstream.lua index 11bd462..7d624d0 100644 --- a/lib/resty/apisix/upstream.lua +++ b/lib/resty/apisix/upstream.lua @@ -18,6 +18,12 @@ ngx_int_t ngx_http_apisix_upstream_set_ssl_trusted_store(ngx_http_request_t *r, int ngx_http_apisix_upstream_set_ssl_verify(ngx_http_request_t *r, int verify); ngx_int_t ngx_http_apisix_set_upstream_pass_trailers(ngx_http_request_t *r, int on); + +ngx_int_t ngx_http_apisix_push_upstream_state(ngx_http_request_t *r, + const unsigned char *addr, size_t addr_len, ngx_int_t status, + ngx_int_t connect_time_ms, ngx_int_t header_time_ms); +ngx_int_t ngx_http_apisix_update_upstream_state(ngx_http_request_t *r, + ngx_int_t response_time_ms, intptr_t response_length); ]]) local _M = {} @@ -124,4 +130,55 @@ function _M.set_pass_trailers(on) end +function _M.push_upstream_state(opts) + if type(opts) ~= "table" then + return nil, "opts must be a table" + end + + local r = get_request() + if not r then + return nil, "no request found" + end + + local addr = opts.addr + if addr ~= nil and type(addr) ~= "string" then + return nil, "addr must be a string" + end + local addr_len = addr and #addr or 0 + local ret = C.ngx_http_apisix_push_upstream_state( + r, + addr, addr_len, + opts.status or 0, + opts.connect_time or -1, + opts.header_time or -1) + if ret == NGX_ERROR then + return nil, "error while pushing upstream state" + end + + return true +end + + +function _M.update_upstream_state(opts) + if type(opts) ~= "table" then + return nil, "opts must be a table" + end + + local r = get_request() + if not r then + return nil, "no request found" + end + + local ret = C.ngx_http_apisix_update_upstream_state( + r, + opts.response_time or -1, + opts.response_length or -1) + if ret == NGX_ERROR then + return nil, "error while updating upstream state" + end + + return true +end + + return _M diff --git a/src/ngx_http_apisix_module.c b/src/ngx_http_apisix_module.c index 7338fbc..3c17fb5 100644 --- a/src/ngx_http_apisix_module.c +++ b/src/ngx_http_apisix_module.c @@ -1080,3 +1080,88 @@ ngx_http_apisix_is_upstream_pass_trailers(ngx_http_request_t *r) return 1; } + + +ngx_int_t +ngx_http_apisix_push_upstream_state(ngx_http_request_t *r, + const u_char *addr, size_t addr_len, ngx_int_t status, + ngx_msec_int_t connect_time_ms, ngx_msec_int_t header_time_ms) +{ + ngx_http_upstream_state_t *state; + ngx_str_t *peer; + u_char *p; + + if (r->upstream_states == NULL) { + r->upstream_states = ngx_array_create(r->pool, 1, + sizeof(ngx_http_upstream_state_t)); + if (r->upstream_states == NULL) { + return NGX_ERROR; + } + } + + state = ngx_array_push(r->upstream_states); + if (state == NULL) { + return NGX_ERROR; + } + + ngx_memzero(state, sizeof(ngx_http_upstream_state_t)); + + /* unset timings render as "-" in logs, not "0.000" */ + state->response_time = (ngx_msec_t) -1; + state->connect_time = (ngx_msec_t) -1; + state->header_time = (ngx_msec_t) -1; + + if (addr != NULL && addr_len > 0) { + peer = ngx_palloc(r->pool, sizeof(ngx_str_t)); + if (peer == NULL) { + return NGX_ERROR; + } + + p = ngx_pnalloc(r->pool, addr_len); + if (p == NULL) { + return NGX_ERROR; + } + + ngx_memcpy(p, addr, addr_len); + peer->data = p; + peer->len = addr_len; + state->peer = peer; + } + + state->status = status; + + if (connect_time_ms >= 0) { + state->connect_time = (ngx_msec_t) connect_time_ms; + } + + if (header_time_ms >= 0) { + state->header_time = (ngx_msec_t) header_time_ms; + } + + return NGX_OK; +} + + +ngx_int_t +ngx_http_apisix_update_upstream_state(ngx_http_request_t *r, + ngx_msec_int_t response_time_ms, off_t response_length) +{ + ngx_http_upstream_state_t *state; + + if (r->upstream_states == NULL || r->upstream_states->nelts == 0) { + return NGX_ERROR; + } + + state = (ngx_http_upstream_state_t *) r->upstream_states->elts + + (r->upstream_states->nelts - 1); + + if (response_time_ms >= 0) { + state->response_time = (ngx_msec_t) response_time_ms; + } + + if (response_length >= 0) { + state->response_length = response_length; + } + + return NGX_OK; +} diff --git a/src/ngx_http_apisix_module.h b/src/ngx_http_apisix_module.h index 37b8345..1d2ed36 100644 --- a/src/ngx_http_apisix_module.h +++ b/src/ngx_http_apisix_module.h @@ -71,4 +71,10 @@ char * ngx_http_apisix_error_log_request_id(ngx_conf_t *cf, ngx_command_t *cmd, ngx_int_t ngx_http_apisix_set_upstream_pass_trailers(ngx_http_request_t *r, int on); ngx_int_t ngx_http_apisix_is_upstream_pass_trailers(ngx_http_request_t *r); + +ngx_int_t ngx_http_apisix_push_upstream_state(ngx_http_request_t *r, + const u_char *addr, size_t addr_len, ngx_int_t status, + ngx_msec_int_t connect_time_ms, ngx_msec_int_t header_time_ms); +ngx_int_t ngx_http_apisix_update_upstream_state(ngx_http_request_t *r, + ngx_msec_int_t response_time_ms, off_t response_length); #endif /* _NGX_HTTP_APISIX_H_INCLUDED_ */ diff --git a/t/upstream_state.t b/t/upstream_state.t new file mode 100644 index 0000000..7a216d0 --- /dev/null +++ b/t/upstream_state.t @@ -0,0 +1,272 @@ +use t::APISIX_NGINX 'no_plan'; + +repeat_each(2); + +run_tests(); + +__DATA__ + +=== TEST 1: push upstream state - status and addr via ngx.var +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + local ok, err = upstream.push_upstream_state({ + addr = "1.2.3.4:8080", + status = 200, + }) + if not ok then + ngx.say("push failed: ", err) + return + end + ngx.say("status: ", ngx.var.upstream_status) + ngx.say("addr: ", ngx.var.upstream_addr) + } + } +--- response_body +status: 200 +addr: 1.2.3.4:8080 + + + +=== TEST 2: push + update upstream state - all timing fields +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + local ok, err = upstream.push_upstream_state({ + addr = "10.0.0.1:443", + status = 200, + connect_time = 50, + header_time = 120, + }) + if not ok then + ngx.say("push failed: ", err) + return + end + + local ok, err = upstream.update_upstream_state({ + response_time = 1500, + response_length = 4096, + }) + if not ok then + ngx.say("update failed: ", err) + return + end + + ngx.say("status: ", ngx.var.upstream_status) + ngx.say("connect_time: ", ngx.var.upstream_connect_time) + ngx.say("header_time: ", ngx.var.upstream_header_time) + ngx.say("response_time: ", ngx.var.upstream_response_time) + ngx.say("response_length: ", ngx.var.upstream_response_length) + } + } +--- response_body +status: 200 +connect_time: 0.050 +header_time: 0.120 +response_time: 1.500 +response_length: 4096 + + + +=== TEST 3: push upstream state - unset timings render as dash +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + local ok, err = upstream.push_upstream_state({ + addr = "10.0.0.1:443", + status = 502, + }) + if not ok then + ngx.say("push failed: ", err) + return + end + ngx.say("status: ", ngx.var.upstream_status) + ngx.say("connect_time: ", ngx.var.upstream_connect_time) + ngx.say("header_time: ", ngx.var.upstream_header_time) + ngx.say("response_time: ", ngx.var.upstream_response_time) + ngx.say("response_length: ", ngx.var.upstream_response_length) + } + } +--- response_body +status: 502 +connect_time: - +header_time: - +response_time: - +response_length: 0 + + + +=== TEST 4: update upstream state without push fails +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + local ok, err = upstream.update_upstream_state({ + response_time = 100, + }) + if not ok then + ngx.say("expected error: ", err) + return + end + ngx.say("should not reach here") + } + } +--- response_body +expected error: error while updating upstream state + + + +=== TEST 5: multiple push calls (retry scenario) +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + + -- first attempt fails + upstream.push_upstream_state({ + addr = "10.0.0.1:443", + status = 502, + connect_time = 30, + }) + upstream.update_upstream_state({ + response_time = 50, + }) + + -- second attempt succeeds + upstream.push_upstream_state({ + addr = "10.0.0.2:443", + status = 200, + connect_time = 20, + header_time = 80, + }) + upstream.update_upstream_state({ + response_time = 500, + response_length = 2048, + }) + + ngx.say("status: ", ngx.var.upstream_status) + ngx.say("addr: ", ngx.var.upstream_addr) + } + } +--- response_body +status: 502, 200 +addr: 10.0.0.1:443, 10.0.0.2:443 + + + +=== TEST 6: push upstream state with no addr +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + local ok, err = upstream.push_upstream_state({ + status = 200, + }) + if not ok then + ngx.say("push failed: ", err) + return + end + ngx.say("status: ", ngx.var.upstream_status) + } + } +--- response_body +status: 200 + + + +=== TEST 7: push and update upstream state verified via ngx.var (integration check) +--- config + location /t { + content_by_lua_block { + local upstream = require("resty.apisix.upstream") + upstream.push_upstream_state({ + addr = "10.0.0.1:443", + status = 200, + connect_time = 50, + header_time = 120, + }) + upstream.update_upstream_state({ + response_time = 1500, + response_length = 8192, + }) + + ngx.say("status: ", ngx.var.upstream_status) + ngx.say("addr: ", ngx.var.upstream_addr) + ngx.say("response_time: ", ngx.var.upstream_response_time) + ngx.say("header_time: ", ngx.var.upstream_header_time) + ngx.say("connect_time: ", ngx.var.upstream_connect_time) + ngx.say("response_length: ", ngx.var.upstream_response_length) + } + } +--- response_body +status: 200 +addr: 10.0.0.1:443 +response_time: 1.500 +header_time: 0.120 +connect_time: 0.050 +response_length: 8192 + + + +=== TEST 8: access log with upstream state from cosocket request +--- http_config + log_format upstream_log '$upstream_status $upstream_addr $upstream_response_time $upstream_header_time $upstream_connect_time $upstream_response_length'; + server { + listen 10888; + location / { + content_by_lua_block { + ngx.say("hello from upstream") + } + } + } +--- config + access_log logs/access.log upstream_log; + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + + local t0 = ngx.now() + local ok, err = httpc:connect("127.0.0.1", 10888) + if not ok then + ngx.say("connect failed: ", err) + return + end + local connect_time = math.floor((ngx.now() - t0) * 1000) + + local res, err = httpc:request({ + method = "GET", + path = "/", + }) + if not res then + ngx.say("request failed: ", err) + return + end + local header_time = math.floor((ngx.now() - t0) * 1000) + + local body = res:read_body() + local response_time = math.floor((ngx.now() - t0) * 1000) + + local upstream = require("resty.apisix.upstream") + upstream.push_upstream_state({ + addr = "127.0.0.1:10888", + status = res.status, + connect_time = connect_time, + header_time = header_time, + }) + upstream.update_upstream_state({ + response_time = response_time, + response_length = #body, + }) + + httpc:set_keepalive() + ngx.say("done") + } + } +--- response_body +done +--- access_log eval +qr/200 127\.0\.0\.1:10888 \d+\.\d{3} \d+\.\d{3} \d+\.\d{3} 20\n/