Skip to content
31 changes: 27 additions & 4 deletions build-apisix-runtime.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,14 @@ ld_opt=${ld_opt:-"-L$zlib_prefix/lib -L$pcre_prefix/lib -L$OPENSSL_PREFIX/lib -W

# dependencies for building openresty
OPENSSL_VERSION=${OPENSSL_VERSION:-"3.4.1"}
OPENRESTY_VERSION="1.27.1.2"
OPENRESTY_VERSION=${OPENRESTY_VERSION:-"1.29.2.4"}
ngx_multi_upstream_module_ver="1.3.2"
# TODO: remove this local patch after ngx_multi_upstream_module releases OpenResty 1.29.2 support.
ngx_multi_upstream_module_patch="patches/ngx_multi_upstream_module/nginx-1.29.2.patch"
mod_dubbo_ver="1.0.2"
Comment thread
jarvis9443 marked this conversation as resolved.
Comment thread
jarvis9443 marked this conversation as resolved.
apisix_nginx_module_ver="1.19.4"
apisix_nginx_module_ver=${apisix_nginx_module_ver:-"openresty-1.29.2.4-patches"}
# TODO: switch back to an apisix-nginx-module release tag after the 1.29.2.4 patches are released.
apisix_nginx_module_commit=${apisix_nginx_module_commit-"36c6de78d74dd0f093e9a25910875762f6f56da6"}
wasm_nginx_module_ver="0.7.0"
lua_var_nginx_module_ver="v0.5.3"
lua_resty_events_ver="0.2.0"
Expand Down Expand Up @@ -52,7 +56,7 @@ install_openssl_3(){
--with-zlib-lib=$zlib_prefix/lib \
--with-zlib-include=$zlib_prefix/include
make -j $(nproc) LD_LIBRARY_PATH= CC="gcc"
sudo make install
sudo make install_sw install_ssldirs
if [ -f "$OPENSSL_CONF_PATH" ]; then
sudo cp "$OPENSSL_CONF_PATH" "$OPENSSL_PREFIX"/ssl/openssl.cnf
fi
Expand All @@ -69,6 +73,7 @@ if ([ $# -gt 0 ] && [ "$1" == "latest" ]) || [ "$runtime_version" == "0.0.0" ];
fi

prev_workdir="$PWD"
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
repo=$(basename "$prev_workdir")
workdir=$(mktemp -d)
cd "$workdir" || exit 1
Expand All @@ -94,6 +99,12 @@ else
https://github.com/api7/ngx_multi_upstream_module.git \
ngx_multi_upstream_module-${ngx_multi_upstream_module_ver}
fi
if [[ "$OPENRESTY_VERSION" == 1.29.2.* ]]; then
cp "$script_dir/$ngx_multi_upstream_module_patch" \
ngx_multi_upstream_module-${ngx_multi_upstream_module_ver}/nginx-1.29.2.patch
patch -d ngx_multi_upstream_module-${ngx_multi_upstream_module_ver} -p0 \
< "$script_dir/patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch"
fi

if [ "$repo" == mod_dubbo ]; then
cp -r "$prev_workdir" ./mod_dubbo-${mod_dubbo_ver}
Expand All @@ -104,12 +115,24 @@ else
fi

if [ "$repo" == apisix-nginx-module ]; then
apisix_nginx_module_cloned=0
cp -r "$prev_workdir" ./apisix-nginx-module-${apisix_nginx_module_ver}
else
git clone --depth=1 -b $apisix_nginx_module_ver \
apisix_nginx_module_cloned=1
apisix_nginx_module_clone_ref="$apisix_nginx_module_ver"
if [ -n "$apisix_nginx_module_commit" ]; then
apisix_nginx_module_clone_ref="main"
fi
git clone --depth=1 -b $apisix_nginx_module_clone_ref \
https://github.com/api7/apisix-nginx-module.git \
apisix-nginx-module-${apisix_nginx_module_ver}
fi
if [ -n "$apisix_nginx_module_commit" ] && [ "$apisix_nginx_module_cloned" = 1 ]; then
git -C apisix-nginx-module-${apisix_nginx_module_ver} fetch --depth=1 \
origin "$apisix_nginx_module_commit"
git -C apisix-nginx-module-${apisix_nginx_module_ver} checkout \
"$apisix_nginx_module_commit"
fi

if [ "$repo" == wasm-nginx-module ]; then
cp -r "$prev_workdir" ./wasm-nginx-module-${wasm_nginx_module_ver}
Expand Down
262 changes: 262 additions & 0 deletions patches/ngx_multi_upstream_module/nginx-1.29.2.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,262 @@
diff --git src/core/ngx_connection.h src/core/ngx_connection.h
index 84dd804..21e7c4e 100644
--- src/core/ngx_connection.h
+++ src/core/ngx_connection.h
@@ -200,6 +200,9 @@ struct ngx_connection_s {
#if (NGX_THREADS || NGX_COMPAT)
ngx_thread_task_t *sendfile_task;
#endif
+#if (T_NGX_MULTI_UPSTREAM)
+ void *multi_c;
+#endif
};


diff --git src/http/ngx_http_request.h src/http/ngx_http_request.h
index ad11f14..da51f19 100644
--- src/http/ngx_http_request.h
+++ src/http/ngx_http_request.h
@@ -604,6 +604,13 @@ struct ngx_http_request_s {

unsigned http_minor:16;
unsigned http_major:16;
+
+#if (T_NGX_MULTI_UPSTREAM)
+ ngx_queue_t *multi_item;
+ ngx_queue_t *backend_r;
+ ngx_queue_t waiting_queue;
+ ngx_flag_t waiting;
+#endif
};


diff --git src/http/ngx_http_upstream.c src/http/ngx_http_upstream.c
index 6ff78fc..cf19c07 100644
--- src/http/ngx_http_upstream.c
+++ src/http/ngx_http_upstream.c
@@ -9,6 +9,10 @@
#include <ngx_core.h>
#include <ngx_http.h>

+#if (T_NGX_MULTI_UPSTREAM)
+#include <ngx_http_multi_upstream_module.h>
+#endif
+

#if (NGX_HTTP_CACHE)
static ngx_int_t ngx_http_upstream_cache(ngx_http_request_t *r,
@@ -1560,6 +1564,12 @@ ngx_http_upstream_check_broken_connection(ngx_http_request_t *r,
}


+#if (T_NGX_MULTI_UPSTREAM)
+
+#include "ngx_http_multi_upstream.c"
+
+#endif /* T_NGX_MULTI_UPSTREAM */
+
static void
ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)
{
@@ -1641,6 +1651,49 @@ ngx_http_upstream_connect(ngx_http_request_t *r, ngx_http_upstream_t *u)

c = u->peer.connection;

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi) {
+ if (!(u->multi_mode & NGX_MULTI_UPS_SUPPORT_MULTI)) {
+ ngx_http_multi_upstream_finalize_request(c,
+ NGX_HTTP_INTERNAL_SERVER_ERROR);
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "multi: upstream configured multi, but handler no support");
+ return;
+ }
+
+ if (rc == NGX_AGAIN) { //first real connect
+ c->read->handler = ngx_http_multi_upstream_connect_handler;
+ c->write->handler = ngx_http_multi_upstream_connect_handler;
+ ngx_add_timer(c->write, u->conf->connect_timeout);
+ ngx_log_error(NGX_LOG_INFO, c->log, 0,
+ "multi: connect new to backend %p", c);
+ } else if (rc == NGX_DONE) { //use exist connection
+ if (ngx_multi_connected(c)) {
+ ngx_log_error(NGX_LOG_INFO, c->log, 0, "multi: connect reuse %p", c);
+
+ ngx_http_multi_upstream_init_request(c, r);
+ ngx_http_multi_upstream_process(c, 1);
+ } else {
+ ngx_log_error(NGX_LOG_ERR, c->log, 0, "multi: connect reuse unfinished %p", c);
+ }
+ } else {
+ ngx_http_multi_upstream_finalize_request(c,
+ NGX_HTTP_INTERNAL_SERVER_ERROR);
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "multi: connect return %i error", rc);
+ }
+
+ return;
Comment thread
jarvis9443 marked this conversation as resolved.
+ } else if ((u->multi_mode & NGX_MULTI_UPS_NEED_MULTI) == NGX_MULTI_UPS_NEED_MULTI) {
+ ngx_http_upstream_finalize_request(r, u,
+ NGX_HTTP_INTERNAL_SERVER_ERROR);
+ ngx_log_error(NGX_LOG_ERR, c->log, 0,
+ "multi: need multi, but upstream not support, "
+ "maybe need configuration 'multi' in upstream");
+ return;
+ }
+#endif
+
c->requests++;

c->data = r;
@@ -1875,6 +1926,13 @@ ngx_http_upstream_ssl_handshake(ngx_http_request_t *r, ngx_http_upstream_t *u,
u->output.sendfile = 0;
}

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi) {
+ ngx_http_multi_upstream_connect_init(c);
+ return;
+ }
+#endif
+
c->write->handler = ngx_http_upstream_handler;
c->read->handler = ngx_http_upstream_handler;

@@ -2202,6 +2260,14 @@ ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u,
ngx_post_event(c->read, &ngx_posted_events);
}

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi && r->connection != u->peer.connection && !r->waiting) {
+ ngx_multi_connection_t *multi_c = ngx_get_multi_connection(c);
+ ngx_queue_insert_tail(&multi_c->waiting_list, &r->waiting_queue);
+ r->waiting = 1;
+ }
+#endif
+
return;
}

@@ -2243,6 +2309,12 @@ ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u,
ngx_add_timer(c->read, u->read_timeout);

if (c->read->ready) {
+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi) {
+ ngx_http_multi_upstream_read_handler(c);
+ return;
+ }
+#endif
ngx_http_upstream_process_header(r, u);
return;
}
@@ -2280,6 +2352,11 @@ ngx_http_upstream_send_request_body(ngx_http_request_t *r,
u->request_body_blocked = 1;

} else {
+#if T_NGX_MULTI_UPSTREAM
+ if (u->multi && rc == NGX_OK) {
+ ngx_multi_clean_leak(u->peer.connection);
+ }
+#endif
Comment on lines +156 to +161
u->request_body_blocked = 0;
}

@@ -3880,6 +3957,13 @@ ngx_http_upstream_process_non_buffered_downstream(ngx_http_request_t *r)
return;
}

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi) {
+ ngx_http_multi_upstream_process_non_buffered_request(r);
+ return;
+ }
+#endif
+
ngx_http_upstream_process_non_buffered_request(r, 1);
}

@@ -4568,6 +4652,16 @@ ngx_http_upstream_next(ngx_http_request_t *r, ngx_http_upstream_t *u,
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http next upstream, %xi", ft_type);

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi && ngx_http_multi_connection_fake(r)) {
+ ngx_log_error(NGX_LOG_INFO, r->connection->log, 0,
+ "multi: http next upstream fake_r %p", r);
+
+ ngx_http_multi_upstream_next(r->connection, ft_type);
+ return;
+ }
+#endif
+
if (u->peer.sockaddr) {

if (u->peer.connection) {
@@ -4738,6 +4832,16 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r,
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"finalize http upstream request: %i", rc);

+#if (T_NGX_MULTI_UPSTREAM)
+ if (u->multi && ngx_http_multi_connection_fake(r)) {
+ ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+ "http finalize upstream fake_r %p", r);
+
+ ngx_http_multi_upstream_finalize_request(r->connection, rc);
+ return;
+ }
+#endif
+
if (u->cleanup == NULL) {
/* the request was already finalized */
ngx_http_finalize_request(r, NGX_DONE);
diff --git src/http/ngx_http_upstream.h src/http/ngx_http_upstream.h
index 8f6317c..0eba977 100644
--- src/http/ngx_http_upstream.h
+++ src/http/ngx_http_upstream.h
@@ -418,6 +418,14 @@ struct ngx_http_upstream_s {
unsigned request_body_sent:1;
unsigned request_body_blocked:1;
unsigned header_sent:1;
+
+#if (T_NGX_MULTI_UPSTREAM)
+ unsigned multi:1;
+ void *multi_init;
+ ngx_pool_t *send_pool;
+ ngx_flag_t multi_mode;
+#endif
+
};


diff --git src/stream/ngx_stream.h src/stream/ngx_stream.h
index 2819029..375ad3b 100644
--- src/stream/ngx_stream.h
+++ src/stream/ngx_stream.h
@@ -296,6 +296,13 @@ struct ngx_stream_session_s {
unsigned health_check:1;

unsigned limit_conn_status:2;
+
+#if (T_NGX_MULTI_UPSTREAM)
+ ngx_queue_t *multi_item;
+ ngx_queue_t *backend_r;
+ ngx_queue_t waiting_queue;
+ ngx_flag_t waiting;
+#endif
};


diff --git src/stream/ngx_stream_upstream.h src/stream/ngx_stream_upstream.h
index 2f989ac..e154eed 100644
--- src/stream/ngx_stream_upstream.h
+++ src/stream/ngx_stream_upstream.h
@@ -148,6 +148,10 @@ typedef struct {
unsigned connected:1;
unsigned proxy_protocol:2;
unsigned half_closed:1;
+
+#if (T_NGX_MULTI_UPSTREAM)
+ unsigned multi:1;
+#endif
} ngx_stream_upstream_t;
14 changes: 14 additions & 0 deletions patches/ngx_multi_upstream_module/patch-sh-openresty-1.29.2.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
diff --git patch.sh patch.sh
index e3a764a..ea09ad7 100755
--- patch.sh
+++ patch.sh
@@ -34,6 +34,9 @@ elif [[ "$1" == *openresty-1.25.3.* ]]; then
elif [[ "$1" == *openresty-1.27.1.* ]]; then
patch="$PWD/nginx-1.27.1.patch"
dir="$1/bundle/nginx-1.27.1"
+elif [[ "$1" == *openresty-1.29.2.* ]]; then
+ patch="$PWD/nginx-1.29.2.patch"
+ dir="$1/bundle/nginx-1.29.2"
else
err "can't detect OpenResty version"
exit 1
Loading