Add supported_versions.yml and CI generator

.gitlab-ci.yml

@@ -14,7 +14,9 @@ default:
 
 include:
   - local: ".gitlab/build-and-test-all.yml"
+  - local: ".gitlab/build-and-test-common.yml"
   - local: ".gitlab/build-and-test-fast.yml"
+  - local: ".gitlab/checks.yml"
 
 macrobenchmarks:
   stage: benchmarks

.gitlab/build-and-test-all.yml

@@ -1,22 +1,5 @@
-include:
-  - local: ".gitlab/common.yml"
-
-# We split build and test all in two stages because GitLab is not able to handle dependencies graph
-# with more than 100 jobs!
-
-.build-and-test-all:
-  rules:
-    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/
-      when: always
-    - when: never
-
-.build-all:
-  extends: .build-and-test-all
-  stage: build-all
-
-.test-all:
-  extends: .build-and-test-all
-  stage: test-all
+# This file is generated by bin/generate_gitlab_ci.py from supported_versions.yml.
+# Do not edit manually. Run: uv run bin/generate_gitlab_ci.py
 
 build-nginx-all:
   extends:
@@ -221,14 +204,6 @@ test-nginx-all:
         BASE_IMAGE: ["nginx:1.28.2", "nginx:1.28.2-alpine"]
         NGINX_VERSION: ["1.28.2"]
         WAF: ["ON", "OFF"]
-
-# GitLab is not able to handle a matrix with more than 200 jobs!
-test-nginx-all-bis:
-  extends:
-    - .test-all
-    - .test-nginx
-  parallel:
-    matrix:
       - ARCH: ["amd64", "arm64"]
         BASE_IMAGE: ["nginx:1.29.0", "nginx:1.29.0-alpine"]
         NGINX_VERSION: ["1.29.0"]
@@ -249,6 +224,13 @@ test-nginx-all-bis:
         BASE_IMAGE: ["nginx:1.29.4", "nginx:1.29.4-alpine"]
         NGINX_VERSION: ["1.29.4"]
         WAF: ["ON", "OFF"]
+
+test-nginx-all-2:
+  extends:
+    - .test-all
+    - .test-nginx
+  parallel:
+    matrix:
       - ARCH: ["amd64", "arm64"]
         BASE_IMAGE: ["nginx:1.29.5", "nginx:1.29.5-alpine"]
         NGINX_VERSION: ["1.29.5"]

.gitlab/build-and-test-common.yml

@@ -0,0 +1,23 @@
+include:
+  - local: ".gitlab/common.yml"
+
+.build-and-test-fast:
+  stage: build-and-test-fast
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: never
+    - when: always
+
+.build-and-test-all:
+  rules:
+    - if: $CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+/
+      when: always
+    - when: never
+
+.build-all:
+  extends: .build-and-test-all
+  stage: build-all
+
+.test-all:
+  extends: .build-and-test-all
+  stage: test-all

.gitlab/build-and-test-fast.yml

@@ -1,48 +1,5 @@
-include:
-  - local: ".gitlab/common.yml"
-
-.build-and-test-fast:
-  stage: build-and-test-fast
-  rules:
-    - if: $CI_COMMIT_TAG
-      when: never
-    - when: always
-
-build-formatter-image:
-  extends: .build-and-test-fast
-  image: registry.ddbuild.io/images/nydus:v2.3.8-dd
-  tags: ["docker-in-docker:amd64"]
-  script:
-    - |
-      HASH=$(cat Dockerfile.formatter | sha256sum | cut -d ' ' -f 1)
-    - echo "FORMATTER_IMAGE_TAG=$CI_REGISTRY/formatter:$HASH" >> formatter-image.env
-    - |
-      if docker buildx imagetools inspect "$CI_REGISTRY/formatter:$HASH" > /dev/null 2>&1; then
-        echo "Image $CI_REGISTRY/formatter:$HASH already exists. Skipping build."
-      else
-        docker build --progress=plain --platform linux/amd64 -t $CI_REGISTRY/formatter:$HASH -f Dockerfile.formatter .
-        docker push $CI_REGISTRY/formatter:$HASH
-        nydus-convert $CI_REGISTRY/formatter:$HASH $CI_REGISTRY/formatter:$HASH
-      fi
-  artifacts:
-    reports:
-      dotenv: formatter-image.env
-
-lint:
-  extends: .build-and-test-fast
-  needs:
-    - build-formatter-image
-  image: $FORMATTER_IMAGE_TAG
-  tags: ["arch:amd64"]
-  script:
-    - make lint
-
-shellcheck:
-  extends: .build-and-test-fast
-  image: $CI_REGISTRY/nginx_musl_toolchain
-  tags: ["arch:amd64"]
-  script:
-    - find bin/ test/ example/ -type f -executable -not -name '*.py' | xargs shellcheck --exclude SC1071,SC1091,SC2317
+# This file is generated by bin/generate_gitlab_ci.py from supported_versions.yml.
+# Do not edit manually. Run: uv run bin/generate_gitlab_ci.py
 
 build-nginx-fast:
   extends:
@@ -51,7 +8,6 @@ build-nginx-fast:
   parallel:
     matrix:
       - ARCH: ["amd64", "arm64"]
-        # The version used by system-tests must be one of the ones below. See https://github.com/DataDog/system-tests/pull/6113.
         NGINX_VERSION:
           - "1.24.0"
           - "1.25.5"
@@ -199,16 +155,3 @@ test-nginx-rum-fast:
         BASE_IMAGE: ["nginx:1.29.6"]
         NGINX_VERSION: ["1.29.6"]
         WAF: ["OFF"]
-
-coverage:
-  extends:
-    - .build-and-test-fast
-    - .build
-  variables:
-    ARCH: "amd64"
-    BASE_IMAGE: "nginx:1.26.0"
-    NGINX_VERSION: "1.26.0"
-    WAF: "ON"
-  tags: ["docker-in-docker:$ARCH"]
-  script:
-    - make coverage

.gitlab/checks.yml

@@ -0,0 +1,55 @@
+build-formatter-image:
+  extends: .build-and-test-fast
+  image: registry.ddbuild.io/images/nydus:v2.3.8-dd
+  tags: ["docker-in-docker:amd64"]
+  script:
+    - |
+      HASH=$(cat Dockerfile.formatter | sha256sum | cut -d ' ' -f 1)
+    - echo "FORMATTER_IMAGE_TAG=$CI_REGISTRY/formatter:$HASH" >> formatter-image.env
+    - |
+      if docker buildx imagetools inspect "$CI_REGISTRY/formatter:$HASH" > /dev/null 2>&1; then
+        echo "Image $CI_REGISTRY/formatter:$HASH already exists. Skipping build."
+      else
+        docker build --progress=plain --platform linux/amd64 -t $CI_REGISTRY/formatter:$HASH -f Dockerfile.formatter .
+        docker push $CI_REGISTRY/formatter:$HASH
+        nydus-convert $CI_REGISTRY/formatter:$HASH $CI_REGISTRY/formatter:$HASH
+      fi
+  artifacts:
+    reports:
+      dotenv: formatter-image.env
+
+lint:
+  extends: .build-and-test-fast
+  needs:
+    - build-formatter-image
+  image: $FORMATTER_IMAGE_TAG
+  tags: ["arch:amd64"]
+  script:
+    - make lint
+
+shellcheck:
+  extends: .build-and-test-fast
+  image: $CI_REGISTRY/nginx_musl_toolchain
+  tags: ["arch:amd64"]
+  script:
+    - find bin/ test/ example/ -type f -executable -not -name '*.py' | xargs shellcheck --exclude SC1071,SC1091,SC2317
+
+check-generated-ci:
+  extends: .build-and-test-fast
+  image: registry.ddbuild.io/ci/nginx-datadog/formatter:102793200
+  tags: ["arch:amd64"]
+  script:
+    - uv run bin/generate_gitlab_ci.py --check
+
+coverage:
+  extends:
+    - .build-and-test-fast
+    - .build
+  variables:
+    ARCH: "amd64"
+    BASE_IMAGE: "nginx:1.26.0"
+    NGINX_VERSION: "1.26.0"
+    WAF: "ON"
+  tags: ["docker-in-docker:$ARCH"]
+  script:
+    - make coverage