Use mirrored images for all Docker image references

.github/workflows/system-tests.yml

@@ -19,7 +19,7 @@ jobs:
       #   - https://github.com/DataDog/system-tests/blob/main/utils/scripts/load-binary.sh
       NGINX_VERSION: 1.29.5
       WAF: ON
-      BASE_IMAGE: alpine:3.20.3
+      MIRROR_REGISTRY: ""
     steps:
       - name: checkout
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0

.gitlab/build-and-test-fast.yml

@@ -10,7 +10,7 @@ include:
 
 format:
   extends: .build-and-test-fast
-  image: registry.ddbuild.io/ci/nginx-datadog/dd-trace-cpp-ci
+  image: $CI_REGISTRY/dd-trace-cpp-ci
   tags: ["arch:amd64"]
   script:
     - pip install -r requirements.txt
@@ -19,10 +19,10 @@ format:
 
 shellcheck:
   extends: .build-and-test-fast
-  image: registry.ddbuild.io/ci/nginx-datadog/nginx_musl_toolchain
+  image: $CI_REGISTRY/nginx_musl_toolchain
   tags: ["arch:amd64"]
   script:
-    - find bin/ test/ example/ -type f -executable | xargs shellcheck --exclude SC1071,SC1091,SC2317
+    - find bin/ test/ example/ -type f -executable -not -name '*.py' | xargs shellcheck --exclude SC1071,SC1091,SC2317
 
 build-nginx-fast:
   extends:

.gitlab/common.yml

@@ -1,5 +1,9 @@
+variables:
+  CI_REGISTRY: "registry.ddbuild.io/ci/nginx-datadog"
+  TESTS_IMAGES_REGISTRY: "$CI_REGISTRY/mirror"
+
 .build:
-  image: registry.ddbuild.io/ci/nginx-datadog/nginx_musl_toolchain
+  image: $CI_REGISTRY/nginx_musl_toolchain
   tags: ["arch:$ARCH"]
   variables:
     GIT_SUBMODULE_STRATEGY: recursive
@@ -53,13 +57,13 @@
       - artifacts-ingress/$ARCH/$INGRESS_NGINX_VERSION/
 
 .test:
-  image: registry.ddbuild.io/ci/nginx-datadog/test
+  image: $CI_REGISTRY/test
   tags: ["docker-in-docker:$ARCH"]
 
 .test-nginx:
   extends: .test
   script:
-    - python3 test/bin/run.py --image $BASE_IMAGE --module-path artifacts/$ARCH/$NGINX_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
+    - python3 test/bin/run.py --image $TESTS_IMAGES_REGISTRY/$BASE_IMAGE --module-path artifacts/$ARCH/$NGINX_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
   artifacts:
     paths:
       - test/logs/test.log
@@ -69,7 +73,7 @@
   variables:
     RUM: "ON"
   script:
-    - python3 test/bin/run.py --image $BASE_IMAGE --module-path artifacts-rum/$ARCH/$NGINX_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
+    - python3 test/bin/run.py --image $TESTS_IMAGES_REGISTRY/$BASE_IMAGE --module-path artifacts-rum/$ARCH/$NGINX_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
   artifacts:
     paths:
       - test/logs/test.log
@@ -79,15 +83,15 @@
   variables:
     NGINX_FLAVOR: ingress-nginx
   script:
-    - python3 test/bin/run.py --image registry.k8s.io/ingress-nginx/controller:v$INGRESS_NGINX_VERSION --module-path artifacts-ingress/$ARCH/$INGRESS_NGINX_VERSION/ngx_http_datadog_module.so -- --verbose --failfast
+    - python3 test/bin/run.py --image $TESTS_IMAGES_REGISTRY/registry.k8s.io/ingress-nginx/controller:v$INGRESS_NGINX_VERSION --module-path artifacts-ingress/$ARCH/$INGRESS_NGINX_VERSION/ngx_http_datadog_module.so -- --verbose --failfast
   artifacts:
     paths:
       - test/logs/test.log
 
 .test-openresty:
   extends: .test
   script:
-    - python3 test/bin/run.py --image openresty/openresty:$RESTY_VERSION-alpine --module-path artifacts-openresty/$ARCH/$RESTY_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
+    - python3 test/bin/run.py --image $TESTS_IMAGES_REGISTRY/openresty/openresty:$RESTY_VERSION-alpine --module-path artifacts-openresty/$ARCH/$RESTY_VERSION/$WAF/ngx_http_datadog_module.so -- --verbose --failfast
   artifacts:
     paths:
       - test/logs/test.log

Makefile

@@ -52,7 +52,7 @@ endif
 
 .PHONY: build-local-musl-toolchain
 build-local-musl-toolchain:
-	docker build --progress=plain --platform $(DOCKER_PLATFORM) --build-arg ARCH=$(ARCH) $(if $(BASE_IMAGE), --build-arg BASE_IMAGE=$(BASE_IMAGE),) -t $(BUILD_IMAGE) build_env
+	docker build --progress=plain --platform $(DOCKER_PLATFORM) --build-arg ARCH=$(ARCH) $(if $(filter environment command,$(origin MIRROR_REGISTRY)),--build-arg MIRROR_REGISTRY=$(MIRROR_REGISTRY),) -t $(BUILD_IMAGE) build_env
 
 .PHONY: build-local-uwsgi-test-image
 build-local-uwsgi-test-image:

build_env/Dockerfile

@@ -1,7 +1,8 @@
 # Dockerfile for registry.ddbuild.io/ci/nginx-datadog/nginx_musl_toolchain
 
-ARG BASE_IMAGE=registry.ddbuild.io/images/mirror/library/alpine:3.20.3
-FROM ${BASE_IMAGE} AS sysroot
+ARG MIRROR_REGISTRY=registry.ddbuild.io/ci/nginx-datadog/mirror/
+FROM ${MIRROR_REGISTRY}ghcr.io/astral-sh/uv:0.9.28 AS uv
+FROM ${MIRROR_REGISTRY}alpine:3.20.3 AS sysroot
 
 ARG LLVM_VERSION=17.0.6
 ARG ARCH
@@ -85,7 +86,7 @@ RUN cargo install --locked cbindgen --version 0.26.0 && \
 RUN apk add --no-cache shellcheck
 
 # UV and Python packages for tests
-COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
+COPY --from=uv /uv /usr/local/bin/uv
 COPY pyproject.toml .
 RUN uv sync
 ENV PATH="/.venv/bin:$PATH"

example/tracing/services/fastcgi/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace node-fastcgi
 
 COPY ./fastcgi.js /opt/app/fastcgi.js

example/tracing/services/grpc/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace @grpc/grpc-js @grpc/proto-loader grpc-node-server-reflection yarn
 
 COPY ./grpc.js ./grpc.proto /opt/app/

example/tracing/services/http/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace
 
 COPY ./http.js /opt/app/http.js

injection/ingress-nginx/docker-compose.yaml

@@ -13,7 +13,7 @@ services:
 
   # testagent is used to collect data from the library to validate.
   testagent:
-    image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
+    image: registry.ddbuild.io/ci/nginx-datadog/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0
     ports:
       - "8126:8126"
 
@@ -22,7 +22,7 @@ services:
     depends_on:
       - lib_inject
       - testagent
-    image: nginx:1.25.3
+    image: registry.ddbuild.io/ci/nginx-datadog/mirror/nginx:1.25.3
     environment:
       - DD_TRACE_AGENT_URL=http://testagent:8126
       - DD_AGENT_HOST=testagent

test/Dockerfile

@@ -1,8 +1,10 @@
-FROM python:3.14-alpine
+ARG MIRROR_REGISTRY=registry.ddbuild.io/ci/nginx-datadog/mirror/
+FROM ${MIRROR_REGISTRY}ghcr.io/astral-sh/uv:0.9.28 AS uv
+FROM ${MIRROR_REGISTRY}python:3.14-alpine
 
 RUN apk --no-cache add bash docker-cli-compose
 
-COPY --from=ghcr.io/astral-sh/uv:0.9.28 /uv /usr/local/bin/uv
+COPY --from=uv /uv /usr/local/bin/uv
 COPY pyproject.toml .
 RUN uv sync
 ENV PATH="/.venv/bin:$PATH"

test/bin/run.py

@@ -13,6 +13,7 @@
 import argparse
 import sys
 import os
+import time
 
 DRY_RUN = False
 PROJECT_DIR = os.path.dirname(
@@ -25,6 +26,26 @@ def run_cmd(cmd: str, *args, **kwargs) -> None:
         subprocess.run(cmd, shell=True, check=True, *args, **kwargs)
 
 
+def run_cmd_with_retries(cmd: str, retries: int = 3, **kwargs) -> None:
+    start = time.monotonic()
+    for attempt in range(1, retries + 1):
+        try:
+            run_cmd(cmd, **kwargs)
+            return
+        except subprocess.CalledProcessError:
+            elapsed = time.monotonic() - start
+            banner = '!' * 60
+            print(f"\n{banner}")
+            print(
+                f"  RETRY: Attempt {attempt}/{retries} failed after {elapsed:.1f}s"
+            )
+            if attempt == retries:
+                print(f"  BUILD FAILED after {retries} attempts")
+                print(f"{banner}\n")
+                raise
+            print(f"{banner}\n")
+
+
 def validate_file(arg):
     if not os.path.exists(arg):
         raise argparse.ArgumentTypeError(f"file {arg} does not exist")
@@ -53,7 +74,7 @@ def main() -> int:
     shutil.copy(src=args.module_path,
                 dst=os.path.join(PROJECT_DIR, "test", "services", "nginx"))
 
-    run_cmd(
+    run_cmd_with_retries(
         f"docker compose build --build-arg BASE_IMAGE={args.image} --parallel --progress=plain",
         cwd=os.path.join(PROJECT_DIR, "test"),
     )

test/services/agent/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install massagepack
 
 COPY ./agent.js /opt/app/agent.js

test/services/client/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine/curl-http3
+FROM registry.ddbuild.io/ci/nginx-datadog/mirror/alpine/curl-http3
 
 COPY install-tools.sh /tmp/install-tools.sh
 USER 0

test/services/fastcgi/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace node-fastcgi
 
 COPY ./fastcgi.js /opt/app/fastcgi.js

test/services/grpc/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace @grpc/grpc-js @grpc/proto-loader grpc-node-server-reflection yarn
 
 COPY ./grpc.js ./grpc.proto /opt/app/

test/services/http/Dockerfile

@@ -7,6 +7,7 @@ RUN mkdir /opt/app
 WORKDIR /opt/app
 
 RUN apk update && apk add nodejs npm
+RUN npm config set fetch-retries 5
 RUN npm install dd-trace
 RUN npm install ws
 

test/services/nginx/Dockerfile

@@ -15,7 +15,7 @@ COPY ./nginx.conf /datadog-tests/nginx.conf
 
 # the uwsgi_params are not at the same location of openresty and nginx containers
 ARG BASE_IMAGE
-RUN if echo "${BASE_IMAGE}" | grep -q '^openresty'; then \
+RUN if echo "${BASE_IMAGE}" | grep -q 'openresty'; then \
         ln -s /usr/local/openresty/nginx/logs/nginx.pid /run/nginx.pid; \
         ln -s /usr/local/openresty/nginx/conf/uwsgi_params /etc/nginx/uwsgi_params; \
         # used to activate openresty test