From f5c11c07175378d5e83a30723779ab60ab6ce058 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:16:24 -0700 Subject: [PATCH 01/12] Fixed building Alpine Linux APK packaged for x86_64 and arm64 --- .../workflows/ci-build-binary-artifacts.yaml | 44 +++++++++ .github/workflows/ci-pr-validation.yaml | 42 ++++----- pkg/apk/APKBUILD | 2 +- pkg/apk/Dockerfile | 89 +++++++++++++++++++ pkg/apk/create-images.sh | 32 +++++++ ...build-apk.sh => docker-build-apk-arm64.sh} | 11 ++- pkg/apk/docker-build-apk-x86_64.sh | 32 +++++++ 7 files changed, 223 insertions(+), 29 deletions(-) create mode 100644 pkg/apk/Dockerfile create mode 100755 pkg/apk/create-images.sh rename pkg/apk/{docker-build-apk.sh => docker-build-apk-arm64.sh} (82%) create mode 100755 pkg/apk/docker-build-apk-x86_64.sh diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 61b1fe46..42e38c72 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -120,3 +120,47 @@ jobs: with: name: rpm-arm64 path: pkg/rpm/RPMS + + apk-packaging-x86_64: + name: Build Alpine Linux APK Package - x86_64 + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-x86_64.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: apk-x86_64 + path: pkg/apk/pkg/x86_64 + + apk-packaging-arm64: + name: Build Alpine Linux APK Package - arm64 + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-arm64.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: apk-arm64 + path: pkg/apk/pkg/aarch64 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 321a4a84..7dcaef42 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -210,9 +210,7 @@ jobs: run: build-support/generate-source-archive.sh - name: Build Debian packages - run: | - echo "Build Debian packages" - BUILD_IMAGE=1 pkg/deb/docker-build-deb-x86_64.sh + run: pkg/deb/docker-build-deb-x86_64.sh rpm-packaging: name: Build RPM Package @@ -228,24 +226,20 @@ jobs: run: build-support/generate-source-archive.sh - name: Build RPM packages - run: | - echo "Build RPM packages" - BUILD_IMAGE=1 pkg/rpm/docker-build-rpm-x86_64.sh - -# apk-packaging: -# name: Build Alpine Linux APK Package -# runs-on: ubuntu-20.04 -# needs: unit-tests -# timeout-minutes: 120 -# -# steps: -# - name: checkout -# uses: actions/checkout@v2 -# -# - name: Package Pulsar source -# run: build-support/generate-source-archive.sh -# -# - name: Build APK packages -# run: | -# echo "Build APK packages" -# BUILD_IMAGE=1 pkg/apk/docker-build-apk.sh \ No newline at end of file + run: pkg/rpm/docker-build-rpm-x86_64.sh + + apk-packaging: + name: Build Alpine Linux APK Package + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-x86_64.sh \ No newline at end of file diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index e3ae244f..e86cabaf 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -48,7 +48,7 @@ build() { -DBUILD_TESTS=OFF \ -DLINK_STATIC=ON \ ${CMAKE_CROSSOPTS} ${ROOT_DIR} - make + make -j6 } package() { diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile new file mode 100644 index 00000000..b2880659 --- /dev/null +++ b/pkg/apk/Dockerfile @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM alpine:3.16 + +ARG PLATFORM + +# perl is required to install OpenSSL +RUN apk add \ + abuild \ + bash \ + curl \ + g++ \ + make \ + python3 \ + cmake \ + boost-dev \ + perl \ + sudo + +# Download and copile protoubf +RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ + tar xfz protobuf-cpp-3.20.0.tar.gz && \ + cd protobuf-3.20.0/ && \ + CXXFLAGS=-fPIC ./configure && \ + make -j8 && make install && \ + rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + +# ZLib +RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ + tar xfz v1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ + CFLAGS="-fPIC -O3" ./configure && \ + make && make install && \ + rm -rf /v1.2.12.tar.gz /zlib-1.2.12 + +# Zstandard +RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz && \ + tar xfz zstd-1.5.2.tar.gz && \ + cd zstd-1.5.2 && \ + CFLAGS="-fPIC -O3" make -j8 && \ + make install && \ + rm -rf /zstd-1.5.2 /zstd-1.5.2.tar.gz + +# Snappy +RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ + tar xfz snappy-1.1.3.tar.gz && \ + cd snappy-1.1.3 && \ + CXXFLAGS="-fPIC -O3" ./configure && \ + make && make install && \ + rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz + +RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ + tar xfz OpenSSL_1_1_1n.tar.gz && \ + cd openssl-OpenSSL_1_1_1n/ && \ + ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ + make -j8 && make install && \ + rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + +ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: + +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ + tar xfz curl-7.61.0.tar.gz && \ + cd curl-7.61.0 && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ + make -j8 && make install && \ + rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ + +RUN adduser pulsar -D +RUN addgroup pulsar abuild diff --git a/pkg/apk/create-images.sh b/pkg/apk/create-images.sh new file mode 100755 index 00000000..4c0612a0 --- /dev/null +++ b/pkg/apk/create-images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR/pkg/apk + +# ARM +IMAGE=apachepulsar/pulsar-build:alpine-3.16-arm64 +docker build --platform arm64 -t $IMAGE . --build-arg PLATFORM=aarch64 + +# X86_64 +IMAGE=apachepulsar/pulsar-build:alpine-3.16-x86_64 +docker build --platform x86_64 -t $IMAGE . --build-arg PLATFORM=x86_64 diff --git a/pkg/apk/docker-build-apk.sh b/pkg/apk/docker-build-apk-arm64.sh similarity index 82% rename from pkg/apk/docker-build-apk.sh rename to pkg/apk/docker-build-apk-arm64.sh index c400c7d7..cc47ca73 100755 --- a/pkg/apk/docker-build-apk.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -18,12 +18,15 @@ # under the License. # -set -e +set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE=apachepulsar/pulsar-build:alpine-3.11 -docker pull $IMAGE +IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-arm64 -docker run -i -v $ROOT_DIR:/pulsar-client-cpp $IMAGE \ +#docker pull $IMAGE_NAME +docker run -v $ROOT_DIR:/pulsar-client-cpp \ + --env PLATFORM=arm64 \ + $IMAGE_NAME \ /pulsar-client-cpp/pkg/apk/build-apk.sh + diff --git a/pkg/apk/docker-build-apk-x86_64.sh b/pkg/apk/docker-build-apk-x86_64.sh new file mode 100755 index 00000000..833a75ec --- /dev/null +++ b/pkg/apk/docker-build-apk-x86_64.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) + +IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-x86_64 + +#docker pull $IMAGE_NAME +docker run -v $ROOT_DIR:/pulsar-client-cpp \ + --env PLATFORM=x86_64 \ + $IMAGE_NAME \ + /pulsar-client-cpp/pkg/apk/build-apk.sh + From ef72eeec9fcf342fd7439c5663d50e1dd291caca Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:27:11 -0700 Subject: [PATCH 02/12] Removed job depedency --- .github/workflows/ci-build-binary-artifacts.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 42e38c72..0250741b 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -124,7 +124,6 @@ jobs: apk-packaging-x86_64: name: Build Alpine Linux APK Package - x86_64 runs-on: ubuntu-20.04 - needs: unit-tests timeout-minutes: 120 steps: @@ -146,7 +145,6 @@ jobs: apk-packaging-arm64: name: Build Alpine Linux APK Package - arm64 runs-on: ubuntu-20.04 - needs: unit-tests timeout-minutes: 120 steps: From 0ee8e623cc49fe7cb850f39b2ebc53bd1eb6c868 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:57:20 -0700 Subject: [PATCH 03/12] Fixed permissions --- .github/workflows/ci-build-binary-artifacts.yaml | 2 -- .github/workflows/ci-pr-validation.yaml | 3 --- pkg/apk/build-apk.sh | 5 ++--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 0250741b..98659175 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -20,8 +20,6 @@ name: CI - Build binary artifacts on: push: - branches: - - '*' tags: - '*' diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 7dcaef42..804a937d 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -22,9 +22,6 @@ on: pull_request: branches: - main - push: - branches: - - main concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index b31a1311..c81a2e94 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -41,9 +41,8 @@ chmod 755 ~ chmod 755 ~/.abuild chmod 644 ~/.abuild/* -mkdir -p /root/packages -chmod 777 /root/packages +mkdir -p pkg +chmod 777 pkg sudo -E -u pulsar abuild -r -mv /root/packages/pkg . From dfefe9898d894c4d90e98f1d0a8371022dce1f81 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 10:31:30 -0700 Subject: [PATCH 04/12] Fixed directory ownership --- pkg/apk/build-apk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index c81a2e94..cd6460c5 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -43,6 +43,6 @@ chmod 644 ~/.abuild/* mkdir -p pkg chmod 777 pkg +chgrp pulsar pkg sudo -E -u pulsar abuild -r - From f9960b626fc9fb50ffabd83abc129c17c66d1c18 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 3 Oct 2022 18:53:53 -0700 Subject: [PATCH 05/12] Fixed building APK as root --- pkg/apk/APKBUILD | 2 +- pkg/apk/Dockerfile | 1 + pkg/apk/build-apk.sh | 10 +--------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index e86cabaf..3c3cdc70 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -27,7 +27,7 @@ url="https://pulsar.apache.org" arch="all" license="https://www.apache.org/licenses/LICENSE-2.0.txt" depends="" -makedepends="cmake" +makedepends="" install="" subpackages="$pkgname-dev" source="" diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index b2880659..d03b1f1f 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -23,6 +23,7 @@ ARG PLATFORM # perl is required to install OpenSSL RUN apk add \ + build-base \ abuild \ bash \ curl \ diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index cd6460c5..a60f6a8e 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -37,12 +37,4 @@ export VERSION=`echo $POM_VERSION | sed -E 's/\-[a-zA-Z]+//'` echo "VERSION: $VERSION" abuild-keygen -a -i -n -chmod 755 ~ -chmod 755 ~/.abuild -chmod 644 ~/.abuild/* - -mkdir -p pkg -chmod 777 pkg -chgrp pulsar pkg - -sudo -E -u pulsar abuild -r +abuild -F -c -r From 3197a539c04808f189ee27ee9d2aaf0665ee264a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 4 Oct 2022 15:29:45 -0500 Subject: [PATCH 06/12] Use qemu to build arm64 package --- .github/workflows/ci-build-binary-artifacts.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 98659175..09197e0f 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -149,6 +149,9 @@ jobs: - name: checkout uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Package Pulsar source run: build-support/generate-source-archive.sh From bb608450b7ef80abd12dadc25e1e52a5eb18a295 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 4 Oct 2022 16:35:34 -0500 Subject: [PATCH 07/12] fixed copying the APK package to artifacts --- .github/workflows/ci-build-binary-artifacts.yaml | 4 ++-- pkg/apk/.gitignore | 5 +---- pkg/apk/build-apk.sh | 2 ++ 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 09197e0f..c4e6543b 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -138,7 +138,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: apk-x86_64 - path: pkg/apk/pkg/x86_64 + path: pkg/apk/build/x86_64 apk-packaging-arm64: name: Build Alpine Linux APK Package - arm64 @@ -162,4 +162,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: apk-arm64 - path: pkg/apk/pkg/aarch64 + path: pkg/apk/build/aarch64 diff --git a/pkg/apk/.gitignore b/pkg/apk/.gitignore index 842209a0..a3661865 100644 --- a/pkg/apk/.gitignore +++ b/pkg/apk/.gitignore @@ -1,5 +1,2 @@ pkg -perf -examples -lib -src +build \ No newline at end of file diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index a60f6a8e..66cb7633 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -38,3 +38,5 @@ echo "VERSION: $VERSION" abuild-keygen -a -i -n abuild -F -c -r + +cp -r /root/packages/pkg ./build From 42464d4a1841617a2d0b7bb99d4219374b6cfe76 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 4 Oct 2022 22:36:19 -0500 Subject: [PATCH 08/12] Build images with fixed dependencies --- .../workflows/ci-build-binary-artifacts.yaml | 87 ++++++++++++- .github/workflows/ci-pr-validation.yaml | 44 ++++++- build-support/copy-deps-versionfile.sh | 29 +++++ build-support/dep-version.py | 24 ++++ dependencies.yaml | 10 ++ pkg/apk/.gitignore | 3 +- pkg/apk/Dockerfile | 90 ++++++++------ pkg/apk/create-images.sh | 2 + pkg/deb/.gitignore | 1 + pkg/deb/Dockerfile | 114 +++++++++++------- pkg/deb/create-images.sh | 2 + pkg/deb/docker-build-deb-arm64.sh | 2 +- pkg/deb/docker-build-deb-x86_64.sh | 2 +- pkg/rpm/.gitignore | 3 +- pkg/rpm/Dockerfile | 107 +++++++++------- pkg/rpm/create-images.sh | 2 + 16 files changed, 388 insertions(+), 134 deletions(-) create mode 100755 build-support/copy-deps-versionfile.sh create mode 100755 build-support/dep-version.py create mode 100644 dependencies.yaml diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index c4e6543b..3bd6f1ea 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -41,8 +41,20 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/deb + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh + run: pkg/deb/docker-build-deb-x86_64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -65,8 +77,21 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/deb + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=aarch64 + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build Debian packages - run: pkg/deb/docker-build-deb-arm64.sh + run: pkg/deb/docker-build-deb-arm64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -86,8 +111,20 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/rpm + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh + run: pkg/rpm/docker-build-rpm-x86_64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -110,8 +147,21 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/rpm + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=aarch64 + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-arm64.sh + run: pkg/rpm/docker-build-rpm-arm64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -131,8 +181,20 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/apk + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh + run: pkg/apk/docker-build-apk-x86_64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -155,8 +217,21 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/apk + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=aarch64 + platforms: linux/arm64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build APK packages - run: pkg/apk/docker-build-apk-arm64.sh + run: pkg/apk/docker-build-apk-arm64.sh ${{ env.TEST_TAG }} - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 804a937d..fa42a3c5 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -206,8 +206,21 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/deb + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh + run: pkg/deb/docker-build-deb-x86_64.sh ${{ env.TEST_TAG }} + rpm-packaging: name: Build RPM Package @@ -222,8 +235,21 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/rpm + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh + run: pkg/rpm/docker-build-rpm-x86_64.sh ${{ env.TEST_TAG }} + apk-packaging: name: Build Alpine Linux APK Package @@ -238,5 +264,17 @@ jobs: - name: Package Pulsar source run: build-support/generate-source-archive.sh + - uses: docker/setup-buildx-action@v2 + - run: build-support/copy-deps-versionfile.sh + + - name: Build dependencies Docker image + uses: docker/build-push-action@v3 + with: + context: ./pkg/apk + tags: ${{ env.TEST_TAG }} + build-args: PLATFORM=x86_64 + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh \ No newline at end of file + run: pkg/apk/docker-build-apk-x86_64.sh ${{ env.TEST_TAG }} \ No newline at end of file diff --git a/build-support/copy-deps-versionfile.sh b/build-support/copy-deps-versionfile.sh new file mode 100755 index 00000000..8ecc8c00 --- /dev/null +++ b/build-support/copy-deps-versionfile.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) + +for dir in apk deb rpm; do + mkdir -p pkg/$dir/.build + cp $ROOT_DIR/dependencies.yaml pkg/$dir/.build + cp $ROOT_DIR/build-support/dep-version.py pkg/$dir/.build +done diff --git a/build-support/dep-version.py b/build-support/dep-version.py new file mode 100755 index 00000000..2421d428 --- /dev/null +++ b/build-support/dep-version.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +import yaml, sys + +deps = yaml.safe_load(open('dependencies.yaml')) +print(deps[sys.argv[1]]) diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 00000000..458bfa95 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,10 @@ + + +boost : 1.80.0 +cmake: 3.24.2 +protobuf: 3.20.0 +zlib: 1.2.12 +zstd: 1.5.2 +snappy: 1.1.9 +openssl: 1.1.1q +curl: 7.85.0 diff --git a/pkg/apk/.gitignore b/pkg/apk/.gitignore index a3661865..c3a14d58 100644 --- a/pkg/apk/.gitignore +++ b/pkg/apk/.gitignore @@ -1,2 +1,3 @@ pkg -build \ No newline at end of file +build +.build \ No newline at end of file diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index d03b1f1f..7b596827 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -24,67 +24,89 @@ ARG PLATFORM # perl is required to install OpenSSL RUN apk add \ build-base \ + linux-headers \ abuild \ bash \ curl \ g++ \ make \ - python3 \ cmake \ - boost-dev \ + python3 \ + py3-pip \ perl \ sudo -# Download and copile protoubf -RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ - tar xfz protobuf-cpp-3.20.0.tar.gz && \ - cd protobuf-3.20.0/ && \ +RUN pip3 install pyyaml + +ADD .build/dependencies.yaml / +ADD .build/dep-version.py /usr/local/bin + +# Download and compile boost +RUN BOOST_VERSION=$(dep-version.py boost) && \ + BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + ./bootstrap.sh --with-libraries=regex && \ + ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ + rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + +# Download and compile protobuf +RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION}/ && \ CXXFLAGS=-fPIC ./configure && \ make -j8 && make install && \ - rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} # ZLib -RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ - tar xfz v1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN ZLIB_VERSION=$(dep-version.py zlib) && \ + curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ + tar xfz v${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ CFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /v1.2.12.tar.gz /zlib-1.2.12 + make -j8 && make install && \ + rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} # Zstandard -RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz && \ - tar xfz zstd-1.5.2.tar.gz && \ - cd zstd-1.5.2 && \ +RUN ZSTD_VERSION=$(dep-version.py zstd) && \ + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ + tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ + cd zstd-${ZSTD_VERSION} && \ CFLAGS="-fPIC -O3" make -j8 && \ make install && \ - rm -rf /zstd-1.5.2 /zstd-1.5.2.tar.gz + rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz # Snappy -RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ - tar xfz snappy-1.1.3.tar.gz && \ - cd snappy-1.1.3 && \ - CXXFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz +RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ + tar xfz ${SNAPPY_VERSION}.tar.gz && \ + cd snappy-${SNAPPY_VERSION} && \ + CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ + make -j8 && make install && \ + rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz -RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ - tar xfz OpenSSL_1_1_1n.tar.gz && \ - cd openssl-OpenSSL_1_1_1n/ && \ +RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ + OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ make -j8 && make install && \ - rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ # LibCurl -RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ - tar xfz curl-7.61.0.tar.gz && \ - cd curl-7.61.0 && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ +RUN CURL_VERSION=$(dep-version.py curl) && \ + CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ + tar xfz curl-${CURL_VERSION}.tar.gz && \ + cd curl-${CURL_VERSION} && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ make -j8 && make install && \ - rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ -RUN adduser pulsar -D -RUN addgroup pulsar abuild diff --git a/pkg/apk/create-images.sh b/pkg/apk/create-images.sh index 4c0612a0..eb8a89f2 100755 --- a/pkg/apk/create-images.sh +++ b/pkg/apk/create-images.sh @@ -23,6 +23,8 @@ set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/pkg/apk +$ROOT_DIR/build-support/copy-deps-versionfile.sh + # ARM IMAGE=apachepulsar/pulsar-build:alpine-3.16-arm64 docker build --platform arm64 -t $IMAGE . --build-arg PLATFORM=aarch64 diff --git a/pkg/deb/.gitignore b/pkg/deb/.gitignore index 89620f31..a71cdd76 100644 --- a/pkg/deb/.gitignore +++ b/pkg/deb/.gitignore @@ -1 +1,2 @@ BUILD +.build \ No newline at end of file diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index 839f05a0..da5de23b 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -25,69 +25,93 @@ ARG PLATFORM # perl is required to install OpenSSL RUN apt-get update -y && \ - apt-get install -y curl g++ make perl dpkg-dev python3 + apt-get install -y \ + curl \ + g++ \ + make \ + perl \ + dpkg-dev \ + python3 \ + python3-pip + +RUN pip3 install pyyaml + +ADD .build/dependencies.yaml / +ADD .build/dep-version.py /usr/local/bin # Download and compile boost -RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz && \ - tar xfz boost_1_79_0.tar.gz && \ - cd boost_1_79_0 && \ - ./bootstrap.sh && \ - ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_1_79_0.tar.gz /boost_1_79_0 - -RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-${PLATFORM}.tar.gz && \ - tar xfz cmake-3.24.0-linux-${PLATFORM}.tar.gz && \ - cp cmake-3.24.0-linux-${PLATFORM}/bin/* /usr/bin/ && \ - cp -r cmake-3.24.0-linux-${PLATFORM}/share/cmake-3.24 /usr/share/ && \ - rm -rf cmake-3.24.0-linux-${PLATFORM} cmake-3.24.0-linux-${PLATFORM}.tar.gz - -# Download and copile protoubf -RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ - tar xfz protobuf-cpp-3.20.0.tar.gz && \ - cd protobuf-3.20.0/ && \ +RUN BOOST_VERSION=$(dep-version.py boost) && \ + BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + cd boost_${BOOST_VERSION_UNDESRSCORE} && \ + ./bootstrap.sh --with-libraries=regex && \ + ./b2 -d0 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ + rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} + +RUN CMAKE_VERSION=$(dep-version.py cmake) && \ + curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ + tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ + cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \ + cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \ + rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz + +# Download and compile protobuf +RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION}/ && \ CXXFLAGS=-fPIC ./configure && \ make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} # ZLib -RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ - tar xfz v1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN ZLIB_VERSION=$(dep-version.py zlib) && \ + curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ + tar xfz v${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ CFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /v1.2.12.tar.gz /zlib-1.2.12 + make -j8 && make install && \ + rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} # Zstandard -RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ - tar xfz zstd-1.3.7.tar.gz && \ - cd zstd-1.3.7 && \ +RUN ZSTD_VERSION=$(dep-version.py zstd) && \ + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ + tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ + cd zstd-${ZSTD_VERSION} && \ CFLAGS="-fPIC -O3" make -j8 && \ make install && \ - rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz + rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz # Snappy -RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ - tar xfz snappy-1.1.3.tar.gz && \ - cd snappy-1.1.3 && \ - CXXFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz - -RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ - tar xfz OpenSSL_1_1_1n.tar.gz && \ - cd openssl-OpenSSL_1_1_1n/ && \ +RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ + tar xfz ${SNAPPY_VERSION}.tar.gz && \ + cd snappy-${SNAPPY_VERSION} && \ + CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ + make -j8 && make install && \ + rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz + +RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ + OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ make -j8 && make install && \ - rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ # LibCurl -RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ - tar xfz curl-7.61.0.tar.gz && \ - cd curl-7.61.0 && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ +RUN CURL_VERSION=$(dep-version.py curl) && \ + CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ + tar xfz curl-${CURL_VERSION}.tar.gz && \ + cd curl-${CURL_VERSION} && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ make -j8 && make install && \ - rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} + -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ diff --git a/pkg/deb/create-images.sh b/pkg/deb/create-images.sh index ad853574..14b64b54 100755 --- a/pkg/deb/create-images.sh +++ b/pkg/deb/create-images.sh @@ -23,6 +23,8 @@ set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/pkg/deb +$ROOT_DIR/build-support/copy-deps-versionfile.sh + # ARM IMAGE=apachepulsar/pulsar-build:debian-9-2.11-arm64 docker build --platform arm64 -t $IMAGE . --build-arg PLATFORM=aarch64 diff --git a/pkg/deb/docker-build-deb-arm64.sh b/pkg/deb/docker-build-deb-arm64.sh index 5a814b27..44b29ee8 100755 --- a/pkg/deb/docker-build-deb-arm64.sh +++ b/pkg/deb/docker-build-deb-arm64.sh @@ -22,7 +22,7 @@ set -ex ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:debian-9-2.11-arm64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64} docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ diff --git a/pkg/deb/docker-build-deb-x86_64.sh b/pkg/deb/docker-build-deb-x86_64.sh index 5a958a62..ded463f0 100755 --- a/pkg/deb/docker-build-deb-x86_64.sh +++ b/pkg/deb/docker-build-deb-x86_64.sh @@ -22,7 +22,7 @@ set -ex ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:debian-9-2.11-x86_64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64} docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ diff --git a/pkg/rpm/.gitignore b/pkg/rpm/.gitignore index b77cada7..f825e316 100644 --- a/pkg/rpm/.gitignore +++ b/pkg/rpm/.gitignore @@ -2,4 +2,5 @@ BUILD BUILDROOT SOURCES RPMS -SRPMS \ No newline at end of file +SRPMS +.build \ No newline at end of file diff --git a/pkg/rpm/Dockerfile b/pkg/rpm/Dockerfile index e4b809b9..905f3783 100644 --- a/pkg/rpm/Dockerfile +++ b/pkg/rpm/Dockerfile @@ -24,71 +24,94 @@ FROM centos:7 ARG PLATFORM RUN yum update -y && \ - yum install -y gcc-c++ make rpm-build which \ - createrepo libstdc++-static python3 + yum install -y \ + gcc-c++ \ + make \ + rpm-build \ + which \ + createrepo \ + libstdc++-static \ + python3 + +RUN pip3 install pyyaml + +ADD .build/dependencies.yaml / +ADD .build/dep-version.py /usr/local/bin # Download and compile boost # GCC 4.8.2 implementation of std::regex is buggy, so we install boost::regex here -RUN curl -O -L https://boostorg.jfrog.io/artifactory/main/release/1.79.0/source/boost_1_79_0.tar.gz && \ - tar xfz boost_1_79_0.tar.gz && \ - cd boost_1_79_0 && \ +RUN BOOST_VERSION=$(dep-version.py boost) && \ + echo "BOOST VERSION: '${BOOST_VERSION}'" && \ + BOOST_VERSION_UNDESRSCORE=$(echo $BOOST_VERSION | sed 's/\./_/g') && \ + curl -O -L https://boostorg.jfrog.io/artifactory/main/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + tar xfz boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz && \ + cd boost_${BOOST_VERSION_UNDESRSCORE} && \ ./bootstrap.sh --with-libraries=regex && \ ./b2 address-model=64 cxxflags=-fPIC link=static threading=multi variant=release install && \ - rm -rf /boost_1_79_0.tar.gz /boost_1_79_0 + rm -rf /boost_${BOOST_VERSION_UNDESRSCORE}.tar.gz /boost_${BOOST_VERSION_UNDESRSCORE} -RUN curl -O -L https://github.com/Kitware/CMake/releases/download/v3.24.0/cmake-3.24.0-linux-${PLATFORM}.tar.gz && \ - tar xfz cmake-3.24.0-linux-${PLATFORM}.tar.gz && \ - cp cmake-3.24.0-linux-${PLATFORM}/bin/* /usr/bin/ && \ - cp -r cmake-3.24.0-linux-${PLATFORM}/share/cmake-3.24 /usr/share/ && \ - rm -rf cmake-3.24.0-linux-${PLATFORM} cmake-3.24.0-linux-${PLATFORM}.tar.gz +RUN CMAKE_VERSION=$(dep-version.py cmake) && \ + curl -O -L https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ + tar xfz cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz && \ + cp cmake-${CMAKE_VERSION}-linux-${PLATFORM}/bin/* /usr/bin/ && \ + cp -r cmake-${CMAKE_VERSION}-linux-${PLATFORM}/share/cmake-* /usr/share/ && \ + rm -rf cmake-${CMAKE_VERSION}-linux-${PLATFORM} cmake-${CMAKE_VERSION}-linux-${PLATFORM}.tar.gz # Download and copile protoubf -RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ - tar xfz protobuf-cpp-3.20.0.tar.gz && \ - cd protobuf-3.20.0/ && \ +RUN PROTOBUF_VERSION=$(dep-version.py protobuf) && \ + curl -O -L https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + tar xfz protobuf-cpp-${PROTOBUF_VERSION}.tar.gz && \ + cd protobuf-${PROTOBUF_VERSION}/ && \ CXXFLAGS=-fPIC ./configure && \ make -j8 && make install && ldconfig && \ - rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + rm -rf /protobuf-cpp-${PROTOBUF_VERSION}.tar.gz /protobuf-${PROTOBUF_VERSION} # ZLib -RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ - tar xfz v1.2.12.tar.gz && \ - cd zlib-1.2.12 && \ +RUN ZLIB_VERSION=$(dep-version.py zlib) && \ + curl -O -L https://github.com/madler/zlib/archive/v${ZLIB_VERSION}.tar.gz && \ + tar xfz v${ZLIB_VERSION}.tar.gz && \ + cd zlib-${ZLIB_VERSION} && \ CFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /v1.2.12.tar.gz /zlib-1.2.12 + make -j8 && make install && \ + rm -rf /v${ZLIB_VERSION}.tar.gz /zlib-${ZLIB_VERSION} # Zstandard -RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.3.7/zstd-1.3.7.tar.gz && \ - tar xfz zstd-1.3.7.tar.gz && \ - cd zstd-1.3.7 && \ +RUN ZSTD_VERSION=$(dep-version.py zstd) && \ + curl -O -L https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz && \ + tar xfz zstd-${ZSTD_VERSION}.tar.gz && \ + cd zstd-${ZSTD_VERSION} && \ CFLAGS="-fPIC -O3" make -j8 && \ make install && \ - rm -rf /zstd-1.3.7 /zstd-1.3.7.tar.gz + rm -rf /zstd-${ZSTD_VERSION} /zstd-${ZSTD_VERSION}.tar.gz # Snappy -RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ - tar xfz snappy-1.1.3.tar.gz && \ - cd snappy-1.1.3 && \ - CXXFLAGS="-fPIC -O3" ./configure && \ - make && make install && \ - rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz - -RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ - tar xfz OpenSSL_1_1_1n.tar.gz && \ - cd openssl-OpenSSL_1_1_1n/ && \ +RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ + curl -O -L https://github.com/google/snappy/archive/refs/tags/${SNAPPY_VERSION}.tar.gz && \ + tar xfz ${SNAPPY_VERSION}.tar.gz && \ + cd snappy-${SNAPPY_VERSION} && \ + CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ + make -j8 && make install && \ + rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz + +RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ + OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + tar xfz OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz && \ + cd openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE}/ && \ ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ make -j8 && make install && \ - rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + rm -rf /OpenSSL_${OPENSSL_VERSION_UNDERSCORE}.tar.gz /openssl-OpenSSL_${OPENSSL_VERSION_UNDERSCORE} ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ # LibCurl -RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ - tar xfz curl-7.61.0.tar.gz && \ - cd curl-7.61.0 && \ - CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ - make && make install && \ - rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 +RUN CURL_VERSION=$(dep-version.py curl) && \ + CURL_VERSION_UNDERSCORE=$(echo $CURL_VERSION | sed 's/\./_/g') && \ + curl -O -L https://github.com/curl/curl/releases/download/curl-${CURL_VERSION_UNDERSCORE}/curl-${CURL_VERSION}.tar.gz && \ + tar xfz curl-${CURL_VERSION}.tar.gz && \ + cd curl-${CURL_VERSION} && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ --without-zstd && \ + make -j8 && make install && \ + rm -rf /curl-${CURL_VERSION}.tar.gz /curl-${CURL_VERSION} -ENV OPENSSL_ROOT_DIR /usr/local/ssl/ diff --git a/pkg/rpm/create-images.sh b/pkg/rpm/create-images.sh index 9bfb3ee8..0e57d357 100755 --- a/pkg/rpm/create-images.sh +++ b/pkg/rpm/create-images.sh @@ -23,6 +23,8 @@ set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) cd $ROOT_DIR/pkg/rpm +$ROOT_DIR/build-support/copy-deps-versionfile.sh + # ARM IMAGE=apachepulsar/pulsar-build:centos-7-2.11-arm64 docker build --platform arm64 -t $IMAGE . --build-arg PLATFORM=aarch64 From 77d97053d4f1c4d9d0672507bc70094239e2f925 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 7 Oct 2022 06:39:19 -0700 Subject: [PATCH 09/12] Apply suggestions from code review Co-authored-by: Yunze Xu --- pkg/apk/Dockerfile | 2 +- pkg/deb/Dockerfile | 2 +- pkg/rpm/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index 7b596827..2a3d3a03 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -85,7 +85,7 @@ RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ cd snappy-${SNAPPY_VERSION} && \ CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz + rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ diff --git a/pkg/deb/Dockerfile b/pkg/deb/Dockerfile index da5de23b..82297629 100644 --- a/pkg/deb/Dockerfile +++ b/pkg/deb/Dockerfile @@ -90,7 +90,7 @@ RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ cd snappy-${SNAPPY_VERSION} && \ CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz + rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ diff --git a/pkg/rpm/Dockerfile b/pkg/rpm/Dockerfile index 905f3783..119ca9d9 100644 --- a/pkg/rpm/Dockerfile +++ b/pkg/rpm/Dockerfile @@ -91,7 +91,7 @@ RUN SNAPPY_VERSION=$(dep-version.py snappy) && \ cd snappy-${SNAPPY_VERSION} && \ CXXFLAGS="-fPIC -O3" cmake . -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF && \ make -j8 && make install && \ - rm -rf /snappy-${SNAPPY_VERSION} /snappy-${SNAPPY_VERSION}.tar.gz + rm -rf /snappy-${SNAPPY_VERSION} /${SNAPPY_VERSION}.tar.gz RUN OPENSSL_VERSION=$(dep-version.py openssl) && \ OPENSSL_VERSION_UNDERSCORE=$(echo $OPENSSL_VERSION | sed 's/\./_/g') && \ From 2e09e07028402e7bd1f8a8431113635c78574177 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 7 Oct 2022 09:07:08 -0700 Subject: [PATCH 10/12] Fixed docker tag names --- .../workflows/ci-build-binary-artifacts.yaml | 24 +++++++++---------- .github/workflows/ci-pr-validation.yaml | 12 +++++----- pkg/apk/docker-build-apk-arm64.sh | 1 - pkg/apk/docker-build-apk-x86_64.sh | 1 - pkg/deb/docker-build-deb-arm64.sh | 1 - pkg/deb/docker-build-deb-x86_64.sh | 1 - pkg/rpm/docker-build-rpm-arm64.sh | 2 -- pkg/rpm/docker-build-rpm-x86_64.sh | 2 -- 8 files changed, 18 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 3bd6f1ea..b1d4903a 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -48,13 +48,13 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb - tags: ${{ env.TEST_TAG }} + tags: build-deb-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh ${{ env.TEST_TAG }} + run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -84,14 +84,14 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb - tags: ${{ env.TEST_TAG }} + tags: build-deb-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build Debian packages - run: pkg/deb/docker-build-deb-arm64.sh ${{ env.TEST_TAG }} + run: pkg/deb/docker-build-deb-arm64.sh build-deb-arm64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -118,13 +118,13 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm - tags: ${{ env.TEST_TAG }} + tags: build-rpm-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh ${{ env.TEST_TAG }} + run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -154,14 +154,14 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm - tags: ${{ env.TEST_TAG }} + tags: build-rpm-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-arm64.sh ${{ env.TEST_TAG }} + run: pkg/rpm/docker-build-rpm-arm64.sh build-rpm-arm64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -188,13 +188,13 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk - tags: ${{ env.TEST_TAG }} + tags: build-apk-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh ${{ env.TEST_TAG }} + run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 @@ -224,14 +224,14 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk - tags: ${{ env.TEST_TAG }} + tags: build-apk-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build APK packages - run: pkg/apk/docker-build-apk-arm64.sh ${{ env.TEST_TAG }} + run: pkg/apk/docker-build-apk-arm64.sh build-apk-arm64:latest - name: Upload artifacts uses: actions/upload-artifact@v3 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index fa42a3c5..cff3469f 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -213,13 +213,13 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb - tags: ${{ env.TEST_TAG }} + tags: build-deb-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build Debian packages - run: pkg/deb/docker-build-deb-x86_64.sh ${{ env.TEST_TAG }} + run: pkg/deb/docker-build-deb-x86_64.sh build-deb-x86_64:latest rpm-packaging: @@ -242,13 +242,13 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm - tags: ${{ env.TEST_TAG }} + tags: build-rpm-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build RPM packages - run: pkg/rpm/docker-build-rpm-x86_64.sh ${{ env.TEST_TAG }} + run: pkg/rpm/docker-build-rpm-x86_64.sh build-rpm-x86_64:latest apk-packaging: @@ -271,10 +271,10 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk - tags: ${{ env.TEST_TAG }} + tags: build-apk-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha cache-to: type=gha,mode=max - name: Build APK packages - run: pkg/apk/docker-build-apk-x86_64.sh ${{ env.TEST_TAG }} \ No newline at end of file + run: pkg/apk/docker-build-apk-x86_64.sh build-apk-x86_64:latest \ No newline at end of file diff --git a/pkg/apk/docker-build-apk-arm64.sh b/pkg/apk/docker-build-apk-arm64.sh index cc47ca73..e62369da 100755 --- a/pkg/apk/docker-build-apk-arm64.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -24,7 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-arm64 -#docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=arm64 \ $IMAGE_NAME \ diff --git a/pkg/apk/docker-build-apk-x86_64.sh b/pkg/apk/docker-build-apk-x86_64.sh index 833a75ec..a5bea003 100755 --- a/pkg/apk/docker-build-apk-x86_64.sh +++ b/pkg/apk/docker-build-apk-x86_64.sh @@ -24,7 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-x86_64 -#docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=x86_64 \ $IMAGE_NAME \ diff --git a/pkg/deb/docker-build-deb-arm64.sh b/pkg/deb/docker-build-deb-arm64.sh index 44b29ee8..f8c397c3 100755 --- a/pkg/deb/docker-build-deb-arm64.sh +++ b/pkg/deb/docker-build-deb-arm64.sh @@ -24,7 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-arm64} -docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=arm64 \ $IMAGE_NAME \ diff --git a/pkg/deb/docker-build-deb-x86_64.sh b/pkg/deb/docker-build-deb-x86_64.sh index ded463f0..fecf6a1e 100755 --- a/pkg/deb/docker-build-deb-x86_64.sh +++ b/pkg/deb/docker-build-deb-x86_64.sh @@ -24,7 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=${1:-apachepulsar/pulsar-build:debian-9-2.11-x86_64} -docker pull $IMAGE_NAME docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=amd64 \ $IMAGE_NAME \ diff --git a/pkg/rpm/docker-build-rpm-arm64.sh b/pkg/rpm/docker-build-rpm-arm64.sh index 07d05d6f..41ae840c 100755 --- a/pkg/rpm/docker-build-rpm-arm64.sh +++ b/pkg/rpm/docker-build-rpm-arm64.sh @@ -24,8 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=apachepulsar/pulsar-build:centos-7-2.11-arm64 -docker pull $IMAGE_NAME - docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=aarch64 \ $IMAGE_NAME \ diff --git a/pkg/rpm/docker-build-rpm-x86_64.sh b/pkg/rpm/docker-build-rpm-x86_64.sh index 69707c18..73667aa9 100755 --- a/pkg/rpm/docker-build-rpm-x86_64.sh +++ b/pkg/rpm/docker-build-rpm-x86_64.sh @@ -24,8 +24,6 @@ ROOT_DIR=$(git rev-parse --show-toplevel) IMAGE_NAME=apachepulsar/pulsar-build:centos-7-2.11-x86_64 -docker pull $IMAGE_NAME - docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=x86_64 \ $IMAGE_NAME \ From fc3cf045771a000b874969d244835ba15ba72ad4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 7 Oct 2022 09:30:30 -0700 Subject: [PATCH 11/12] Added load: true --- .github/workflows/ci-build-binary-artifacts.yaml | 6 ++++++ .github/workflows/ci-pr-validation.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index b1d4903a..dd482313 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -48,6 +48,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb + load: true tags: build-deb-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha @@ -84,6 +85,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb + load: true tags: build-deb-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 @@ -118,6 +120,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm + load: true tags: build-rpm-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha @@ -154,6 +157,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm + load: true tags: build-rpm-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 @@ -188,6 +192,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk + load: true tags: build-apk-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha @@ -224,6 +229,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk + load: true tags: build-apk-arm64:latest build-args: PLATFORM=aarch64 platforms: linux/arm64 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index cff3469f..0277d938 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -213,6 +213,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/deb + load: true tags: build-deb-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha @@ -242,6 +243,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/rpm + load: true tags: build-rpm-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha @@ -271,6 +273,7 @@ jobs: uses: docker/build-push-action@v3 with: context: ./pkg/apk + load: true tags: build-apk-x86_64:latest build-args: PLATFORM=x86_64 cache-from: type=gha From 29bd563268474ed0a03491dc2f2dfff839c7f7e7 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Fri, 7 Oct 2022 09:59:00 -0700 Subject: [PATCH 12/12] Fixed overriding image name --- dependencies.yaml | 19 ++++++++++++++++++- pkg/apk/docker-build-apk-arm64.sh | 2 +- pkg/apk/docker-build-apk-x86_64.sh | 2 +- pkg/rpm/docker-build-rpm-arm64.sh | 2 +- pkg/rpm/docker-build-rpm-x86_64.sh | 2 +- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 458bfa95..bc2c531b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -1,4 +1,21 @@ - +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# boost : 1.80.0 cmake: 3.24.2 diff --git a/pkg/apk/docker-build-apk-arm64.sh b/pkg/apk/docker-build-apk-arm64.sh index e62369da..75631523 100755 --- a/pkg/apk/docker-build-apk-arm64.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -22,7 +22,7 @@ set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-arm64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=arm64 \ diff --git a/pkg/apk/docker-build-apk-x86_64.sh b/pkg/apk/docker-build-apk-x86_64.sh index a5bea003..985d87d1 100755 --- a/pkg/apk/docker-build-apk-x86_64.sh +++ b/pkg/apk/docker-build-apk-x86_64.sh @@ -22,7 +22,7 @@ set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-x86_64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:alpine-3.16-x86_64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=x86_64 \ diff --git a/pkg/rpm/docker-build-rpm-arm64.sh b/pkg/rpm/docker-build-rpm-arm64.sh index 41ae840c..13ca8d51 100755 --- a/pkg/rpm/docker-build-rpm-arm64.sh +++ b/pkg/rpm/docker-build-rpm-arm64.sh @@ -22,7 +22,7 @@ set -ex ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:centos-7-2.11-arm64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:centos-7-2.11-arm64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=aarch64 \ diff --git a/pkg/rpm/docker-build-rpm-x86_64.sh b/pkg/rpm/docker-build-rpm-x86_64.sh index 73667aa9..c78dc7b4 100755 --- a/pkg/rpm/docker-build-rpm-x86_64.sh +++ b/pkg/rpm/docker-build-rpm-x86_64.sh @@ -22,7 +22,7 @@ set -ex ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE_NAME=apachepulsar/pulsar-build:centos-7-2.11-x86_64 +IMAGE_NAME=${1:-apachepulsar/pulsar-build:centos-7-2.11-x86_64} docker run -v $ROOT_DIR:/pulsar-client-cpp \ --env PLATFORM=x86_64 \