diff --git a/build-apisix-runtime.sh b/build-apisix-runtime.sh index 92f340878..bbf86f613 100755 --- a/build-apisix-runtime.sh +++ b/build-apisix-runtime.sh @@ -21,10 +21,14 @@ ld_opt=${ld_opt:-"-L$zlib_prefix/lib -L$pcre_prefix/lib -L$OPENSSL_PREFIX/lib -W # dependencies for building openresty OPENSSL_VERSION=${OPENSSL_VERSION:-"3.4.1"} -OPENRESTY_VERSION="1.27.1.2" +OPENRESTY_VERSION=${OPENRESTY_VERSION:-"1.29.2.4"} ngx_multi_upstream_module_ver="1.3.2" +# TODO: remove this local patch after ngx_multi_upstream_module releases OpenResty 1.29.2 support. +ngx_multi_upstream_module_patch="patches/ngx_multi_upstream_module/nginx-1.29.2.patch" mod_dubbo_ver="1.0.2" -apisix_nginx_module_ver="1.19.4" +apisix_nginx_module_ver=${apisix_nginx_module_ver:-"openresty-1.29.2.4-patches"} +# TODO: switch back to an apisix-nginx-module release tag after the 1.29.2.4 patches are released. +apisix_nginx_module_commit=${apisix_nginx_module_commit-"36c6de78d74dd0f093e9a25910875762f6f56da6"} wasm_nginx_module_ver="0.7.0" lua_var_nginx_module_ver="v0.5.3" lua_resty_events_ver="0.2.0" @@ -52,7 +56,7 @@ install_openssl_3(){ --with-zlib-lib=$zlib_prefix/lib \ --with-zlib-include=$zlib_prefix/include make -j $(nproc) LD_LIBRARY_PATH= CC="gcc" - sudo make install + sudo make install_sw install_ssldirs if [ -f "$OPENSSL_CONF_PATH" ]; then sudo cp "$OPENSSL_CONF_PATH" "$OPENSSL_PREFIX"/ssl/openssl.cnf fi @@ -69,6 +73,7 @@ if ([ $# -gt 0 ] && [ "$1" == "latest" ]) || [ "$runtime_version" == "0.0.0" ]; fi prev_workdir="$PWD" +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo=$(basename "$prev_workdir") workdir=$(mktemp -d) cd "$workdir" || exit 1 @@ -94,6 +99,12 @@ else https://github.com/api7/ngx_multi_upstream_module.git \ ngx_multi_upstream_module-${ngx_multi_upstream_module_ver} fi +if [[ "$OPENRESTY_VERSION" == 1.29.2.* ]]; then + cp "$script_dir/$ngx_multi_upstream_module_patch" \ + ngx_multi_upstream_module-${ngx_multi_upstream_module_ver}/nginx-1.29.2.patch + patch -d ngx_multi_upstream_module-${ngx_multi_upstream_module_ver} -p0 \ + < "$script_dir/patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch" +fi if [ "$repo" == mod_dubbo ]; then cp -r "$prev_workdir" ./mod_dubbo-${mod_dubbo_ver} @@ -104,12 +115,24 @@ else fi if [ "$repo" == apisix-nginx-module ]; then + apisix_nginx_module_cloned=0 cp -r "$prev_workdir" ./apisix-nginx-module-${apisix_nginx_module_ver} else - git clone --depth=1 -b $apisix_nginx_module_ver \ + apisix_nginx_module_cloned=1 + apisix_nginx_module_clone_ref="$apisix_nginx_module_ver" + if [ -n "$apisix_nginx_module_commit" ]; then + apisix_nginx_module_clone_ref="main" + fi + git clone --depth=1 -b $apisix_nginx_module_clone_ref \ https://github.com/api7/apisix-nginx-module.git \ apisix-nginx-module-${apisix_nginx_module_ver} fi +if [ -n "$apisix_nginx_module_commit" ] && [ "$apisix_nginx_module_cloned" = 1 ]; then + git -C apisix-nginx-module-${apisix_nginx_module_ver} fetch --depth=1 \ + origin "$apisix_nginx_module_commit" + git -C apisix-nginx-module-${apisix_nginx_module_ver} checkout \ + "$apisix_nginx_module_commit" +fi if [ "$repo" == wasm-nginx-module ]; then cp -r "$prev_workdir" ./wasm-nginx-module-${wasm_nginx_module_ver} diff --git a/patches/ngx_multi_upstream_module/nginx-1.29.2.patch b/patches/ngx_multi_upstream_module/nginx-1.29.2.patch new file mode 100644 index 000000000..af4970e18 --- /dev/null +++ b/patches/ngx_multi_upstream_module/nginx-1.29.2.patch @@ -0,0 +1,262 @@ +diff --git src/core/ngx_connection.h src/core/ngx_connection.h +index 84dd804..21e7c4e 100644 +--- src/core/ngx_connection.h ++++ src/core/ngx_connection.h +@@ -200,6 +200,9 @@ struct ngx_connection_s { + #if (NGX_THREADS || NGX_COMPAT) + ngx_thread_task_t *sendfile_task; + #endif ++#if (T_NGX_MULTI_UPSTREAM) ++ void *multi_c; ++#endif + }; + + +diff --git src/http/ngx_http_request.h src/http/ngx_http_request.h +index ad11f14..da51f19 100644 +--- src/http/ngx_http_request.h ++++ src/http/ngx_http_request.h +@@ -604,6 +604,13 @@ struct ngx_http_request_s { + + unsigned http_minor:16; + unsigned http_major:16; ++ ++#if (T_NGX_MULTI_UPSTREAM) ++ ngx_queue_t *multi_item; ++ ngx_queue_t *backend_r; ++ ngx_queue_t waiting_queue; ++ ngx_flag_t waiting; ++#endif + }; + + +diff --git src/http/ngx_http_upstream.c src/http/ngx_http_upstream.c +index 6ff78fc..cf19c07 100644 +--- src/http/ngx_http_upstream.c ++++ src/http/ngx_http_upstream.c +@@ -9,6 +9,10 @@ + #include + #include + ++#if (T_NGX_MULTI_UPSTREAM) ++#include ++#endif ++ + + #if (NGX_HTTP_CACHE) + static ngx_int_t ngx_http_upstream_cache(ngx_http_request_t *r, +@@ -1560,6 +1564,12 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r, + } + + ++#if (T_NGX_MULTI_UPSTREAM) ++ ++#include "ngx_http_multi_upstream.c" ++ ++#endif /* T_NGX_MULTI_UPSTREAM */ ++ + static void + ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) + { +@@ -1641,6 +1651,49 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u) + + c = u->peer.connection; + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi) { ++ if (!(u->multi_mode & NGX_MULTI_UPS_SUPPORT_MULTI)) { ++ ngx_http_multi_upstream_finalize_request(c, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ngx_log_error(NGX_LOG_ERR, c->log, 0, ++ "multi: upstream configured multi, but handler no support"); ++ return; ++ } ++ ++ if (rc == NGX_AGAIN) { //first real connect ++ c->read->handler = ngx_http_multi_upstream_connect_handler; ++ c->write->handler = ngx_http_multi_upstream_connect_handler; ++ ngx_add_timer(c->write, u->conf->connect_timeout); ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, ++ "multi: connect new to backend %p", c); ++ } else if (rc == NGX_DONE) { //use exist connection ++ if (ngx_multi_connected(c)) { ++ ngx_log_error(NGX_LOG_INFO, c->log, 0, "multi: connect reuse %p", c); ++ ++ ngx_http_multi_upstream_init_request(c, r); ++ ngx_http_multi_upstream_process(c, 1); ++ } else { ++ ngx_log_error(NGX_LOG_ERR, c->log, 0, "multi: connect reuse unfinished %p", c); ++ } ++ } else { ++ ngx_http_multi_upstream_finalize_request(c, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ngx_log_error(NGX_LOG_ERR, c->log, 0, ++ "multi: connect return %i error", rc); ++ } ++ ++ return; ++ } else if ((u->multi_mode & NGX_MULTI_UPS_NEED_MULTI) == NGX_MULTI_UPS_NEED_MULTI) { ++ ngx_http_upstream_finalize_request(r, u, ++ NGX_HTTP_INTERNAL_SERVER_ERROR); ++ ngx_log_error(NGX_LOG_ERR, c->log, 0, ++ "multi: need multi, but upstream not support, " ++ "maybe need configuration 'multi' in upstream"); ++ return; ++ } ++#endif ++ + c->requests++; + + c->data = r; +@@ -1875,6 +1926,13 @@ ngx_http_upstream_ssl_handshake(ngx_http_request_t *r, ngx_http_upstream_t *u, + u->output.sendfile = 0; + } + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi) { ++ ngx_http_multi_upstream_connect_init(c); ++ return; ++ } ++#endif ++ + c->write->handler = ngx_http_upstream_handler; + c->read->handler = ngx_http_upstream_handler; + +@@ -2202,6 +2260,14 @@ ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u, + ngx_post_event(c->read, &ngx_posted_events); + } + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi && r->connection != u->peer.connection && !r->waiting) { ++ ngx_multi_connection_t *multi_c = ngx_get_multi_connection(c); ++ ngx_queue_insert_tail(&multi_c->waiting_list, &r->waiting_queue); ++ r->waiting = 1; ++ } ++#endif ++ + return; + } + +@@ -2243,6 +2309,12 @@ ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u, + ngx_add_timer(c->read, u->read_timeout); + + if (c->read->ready) { ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi) { ++ ngx_http_multi_upstream_read_handler(c); ++ return; ++ } ++#endif + ngx_http_upstream_process_header(r, u); + return; + } +@@ -2280,6 +2352,11 @@ ngx_http_upstream_send_request_body(ngx_http_request_t *r, + u->request_body_blocked = 1; + + } else { ++#if T_NGX_MULTI_UPSTREAM ++ if (u->multi && rc == NGX_OK) { ++ ngx_multi_clean_leak(u->peer.connection); ++ } ++#endif + u->request_body_blocked = 0; + } + +@@ -3880,6 +3957,13 @@ ngx_http_upstream_process_non_buffered_downstream(ngx_http_request_t *r) + return; + } + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi) { ++ ngx_http_multi_upstream_process_non_buffered_request(r); ++ return; ++ } ++#endif ++ + ngx_http_upstream_process_non_buffered_request(r, 1); + } + +@@ -4568,6 +4652,16 @@ ngx_http_upstream_next(ngx_http_request_t *r, ngx_http_upstream_t *u, + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http next upstream, %xi", ft_type); + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi && ngx_http_multi_connection_fake(r)) { ++ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0, ++ "multi: http next upstream fake_r %p", r); ++ ++ ngx_http_multi_upstream_next(r->connection, ft_type); ++ return; ++ } ++#endif ++ + if (u->peer.sockaddr) { + + if (u->peer.connection) { +@@ -4738,6 +4832,16 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r, + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "finalize http upstream request: %i", rc); + ++#if (T_NGX_MULTI_UPSTREAM) ++ if (u->multi && ngx_http_multi_connection_fake(r)) { ++ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, ++ "http finalize upstream fake_r %p", r); ++ ++ ngx_http_multi_upstream_finalize_request(r->connection, rc); ++ return; ++ } ++#endif ++ + if (u->cleanup == NULL) { + /* the request was already finalized */ + ngx_http_finalize_request(r, NGX_DONE); +diff --git src/http/ngx_http_upstream.h src/http/ngx_http_upstream.h +index 8f6317c..0eba977 100644 +--- src/http/ngx_http_upstream.h ++++ src/http/ngx_http_upstream.h +@@ -418,6 +418,14 @@ struct ngx_http_upstream_s { + unsigned request_body_sent:1; + unsigned request_body_blocked:1; + unsigned header_sent:1; ++ ++#if (T_NGX_MULTI_UPSTREAM) ++ unsigned multi:1; ++ void *multi_init; ++ ngx_pool_t *send_pool; ++ ngx_flag_t multi_mode; ++#endif ++ + }; + + +diff --git src/stream/ngx_stream.h src/stream/ngx_stream.h +index 2819029..375ad3b 100644 +--- src/stream/ngx_stream.h ++++ src/stream/ngx_stream.h +@@ -296,6 +296,13 @@ struct ngx_stream_session_s { + unsigned health_check:1; + + unsigned limit_conn_status:2; ++ ++#if (T_NGX_MULTI_UPSTREAM) ++ ngx_queue_t *multi_item; ++ ngx_queue_t *backend_r; ++ ngx_queue_t waiting_queue; ++ ngx_flag_t waiting; ++#endif + }; + + +diff --git src/stream/ngx_stream_upstream.h src/stream/ngx_stream_upstream.h +index 2f989ac..e154eed 100644 +--- src/stream/ngx_stream_upstream.h ++++ src/stream/ngx_stream_upstream.h +@@ -148,6 +148,10 @@ typedef struct { + unsigned connected:1; + unsigned proxy_protocol:2; + unsigned half_closed:1; ++ ++#if (T_NGX_MULTI_UPSTREAM) ++ unsigned multi:1; ++#endif + } ngx_stream_upstream_t; diff --git a/patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch b/patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch new file mode 100644 index 000000000..ed2193d36 --- /dev/null +++ b/patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch @@ -0,0 +1,14 @@ +diff --git patch.sh patch.sh +index e3a764a..ea09ad7 100755 +--- patch.sh ++++ patch.sh +@@ -34,6 +34,9 @@ elif [[ "$1" == *openresty-1.25.3.* ]]; then + elif [[ "$1" == *openresty-1.27.1.* ]]; then + patch="$PWD/nginx-1.27.1.patch" + dir="$1/bundle/nginx-1.27.1" ++elif [[ "$1" == *openresty-1.29.2.* ]]; then ++ patch="$PWD/nginx-1.29.2.patch" ++ dir="$1/bundle/nginx-1.29.2" + else + err "can't detect OpenResty version" + exit 1