Build images with deps versions

.github/workflows/ci-build-binary-artifacts.yaml

@@ -20,8 +20,6 @@
 name: CI - Build binary artifacts
 on:
   push:
-    branches:
-      - '*'
     tags:
       - '*'
 
@@ -43,8 +41,21 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/deb
+          load: true
+          tags: build-deb-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build Debian packages
-        run: pkg/deb/docker-build-deb-x86_64.sh
+        run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -67,8 +78,22 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/deb
+          load: true
+          tags: build-deb-arm64:latest
+          build-args: PLATFORM=aarch64
+          platforms: linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build Debian packages
-        run: pkg/deb/docker-build-deb-arm64.sh
+        run: pkg/deb/docker-build-deb-arm64.sh build-deb-arm64:latest
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -88,8 +113,21 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/rpm
+          load: true
+          tags: build-rpm-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build RPM packages
-        run: pkg/rpm/docker-build-rpm-x86_64.sh
+        run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
@@ -112,11 +150,97 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/rpm
+          load: true
+          tags: build-rpm-arm64:latest
+          build-args: PLATFORM=aarch64
+          platforms: linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build RPM packages
-        run: pkg/rpm/docker-build-rpm-arm64.sh
+        run: pkg/rpm/docker-build-rpm-arm64.sh build-rpm-arm64:latest
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:
           name: rpm-arm64
           path: pkg/rpm/RPMS
+
+  apk-packaging-x86_64:
+    name: Build Alpine Linux APK Package - x86_64
+    runs-on: ubuntu-20.04
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Package Pulsar source
+        run: build-support/generate-source-archive.sh
+
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/apk
+          load: true
+          tags: build-apk-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build APK packages
+        run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: apk-x86_64
+          path: pkg/apk/build/x86_64
+
+  apk-packaging-arm64:
+    name: Build Alpine Linux APK Package - arm64
+    runs-on: ubuntu-20.04
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Package Pulsar source
+        run: build-support/generate-source-archive.sh
+
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/apk
+          load: true
+          tags: build-apk-arm64:latest
+          build-args: PLATFORM=aarch64
+          platforms: linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build APK packages
+        run: pkg/apk/docker-build-apk-arm64.sh build-apk-arm64:latest
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: apk-arm64
+          path: pkg/apk/build/aarch64

.github/workflows/ci-pr-validation.yaml

@@ -22,9 +22,6 @@ on:
   pull_request:
     branches:
       - main
-  push:
-    branches:
-      - main
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -209,10 +206,22 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/deb
+          load: true
+          tags: build-deb-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build Debian packages
-        run: |
-          echo "Build Debian packages"
-          BUILD_IMAGE=1 pkg/deb/docker-build-deb-x86_64.sh
+        run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest
+
 
   rpm-packaging:
     name: Build RPM Package
@@ -227,25 +236,48 @@ jobs:
       - name: Package Pulsar source
         run: build-support/generate-source-archive.sh
 
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/rpm
+          load: true
+          tags: build-rpm-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
       - name: Build RPM packages
-        run: |
-          echo "Build RPM packages"
-          BUILD_IMAGE=1 pkg/rpm/docker-build-rpm-x86_64.sh
-
-#  apk-packaging:
-#    name: Build Alpine Linux APK Package
-#    runs-on: ubuntu-20.04
-#    needs: unit-tests
-#    timeout-minutes: 120
-#
-#    steps:
-#      - name: checkout
-#        uses: actions/checkout@v2
-#
-#      - name: Package Pulsar source
-#        run: build-support/generate-source-archive.sh
-#
-#      - name: Build APK packages
-#        run: |
-#          echo "Build APK packages"
-#          BUILD_IMAGE=1 pkg/apk/docker-build-apk.sh
+        run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest
+
+
+  apk-packaging:
+    name: Build Alpine Linux APK Package
+    runs-on: ubuntu-20.04
+    needs: unit-tests
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Package Pulsar source
+        run: build-support/generate-source-archive.sh
+
+      - uses: docker/setup-buildx-action@v2
+      - run: build-support/copy-deps-versionfile.sh
+
+      - name: Build dependencies Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./pkg/apk
+          load: true
+          tags: build-apk-x86_64:latest
+          build-args: PLATFORM=x86_64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Build APK packages
+        run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest

build-support/copy-deps-versionfile.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e -x
+
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+for dir in apk deb rpm; do
+  mkdir -p pkg/$dir/.build
+  cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build
+  cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build
+done

build-support/dep-version.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+import yaml, sys
+
+deps = yaml.safe_load(open('dependencies.yaml'))
+print(deps[sys.argv[1]])

dependencies.yaml

@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+boost : 1.80.0
+cmake: 3.24.2
+protobuf: 3.20.0
+zlib: 1.2.12
+zstd: 1.5.2
+snappy: 1.1.9
+openssl: 1.1.1q
+curl: 7.85.0

pkg/apk/.gitignore

@@ -1,5 +1,3 @@
 pkg
-perf
-examples
-lib
-src
+build
+.build

pkg/apk/APKBUILD

@@ -27,7 +27,7 @@ url="https://pulsar.apache.org"
 arch="all"
 license="https://www.apache.org/licenses/LICENSE-2.0.txt"
 depends=""
-makedepends="cmake"
+makedepends=""
 install=""
 subpackages="$pkgname-dev"
 source=""
@@ -48,7 +48,7 @@ build() {
 		-DBUILD_TESTS=OFF \
                 -DLINK_STATIC=ON  \
 		${CMAKE_CROSSOPTS} ${ROOT_DIR}
-	make
+	make -j6
 }
 
 package() {