From f5c11c07175378d5e83a30723779ab60ab6ce058 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:16:24 -0700 Subject: [PATCH 1/7] Fixed building Alpine Linux APK packaged for x86_64 and arm64 --- .../workflows/ci-build-binary-artifacts.yaml | 44 +++++++++ .github/workflows/ci-pr-validation.yaml | 42 ++++----- pkg/apk/APKBUILD | 2 +- pkg/apk/Dockerfile | 89 +++++++++++++++++++ pkg/apk/create-images.sh | 32 +++++++ ...build-apk.sh => docker-build-apk-arm64.sh} | 11 ++- pkg/apk/docker-build-apk-x86_64.sh | 32 +++++++ 7 files changed, 223 insertions(+), 29 deletions(-) create mode 100644 pkg/apk/Dockerfile create mode 100755 pkg/apk/create-images.sh rename pkg/apk/{docker-build-apk.sh => docker-build-apk-arm64.sh} (82%) create mode 100755 pkg/apk/docker-build-apk-x86_64.sh diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 61b1fe46..42e38c72 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -120,3 +120,47 @@ jobs: with: name: rpm-arm64 path: pkg/rpm/RPMS + + apk-packaging-x86_64: + name: Build Alpine Linux APK Package - x86_64 + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-x86_64.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: apk-x86_64 + path: pkg/apk/pkg/x86_64 + + apk-packaging-arm64: + name: Build Alpine Linux APK Package - arm64 + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-arm64.sh + + - name: Upload artifacts + uses: actions/upload-artifact@v3 + with: + name: apk-arm64 + path: pkg/apk/pkg/aarch64 diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 321a4a84..7dcaef42 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -210,9 +210,7 @@ jobs: run: build-support/generate-source-archive.sh - name: Build Debian packages - run: | - echo "Build Debian packages" - BUILD_IMAGE=1 pkg/deb/docker-build-deb-x86_64.sh + run: pkg/deb/docker-build-deb-x86_64.sh rpm-packaging: name: Build RPM Package @@ -228,24 +226,20 @@ jobs: run: build-support/generate-source-archive.sh - name: Build RPM packages - run: | - echo "Build RPM packages" - BUILD_IMAGE=1 pkg/rpm/docker-build-rpm-x86_64.sh - -# apk-packaging: -# name: Build Alpine Linux APK Package -# runs-on: ubuntu-20.04 -# needs: unit-tests -# timeout-minutes: 120 -# -# steps: -# - name: checkout -# uses: actions/checkout@v2 -# -# - name: Package Pulsar source -# run: build-support/generate-source-archive.sh -# -# - name: Build APK packages -# run: | -# echo "Build APK packages" -# BUILD_IMAGE=1 pkg/apk/docker-build-apk.sh \ No newline at end of file + run: pkg/rpm/docker-build-rpm-x86_64.sh + + apk-packaging: + name: Build Alpine Linux APK Package + runs-on: ubuntu-20.04 + needs: unit-tests + timeout-minutes: 120 + + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Package Pulsar source + run: build-support/generate-source-archive.sh + + - name: Build APK packages + run: pkg/apk/docker-build-apk-x86_64.sh \ No newline at end of file diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index e3ae244f..e86cabaf 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -48,7 +48,7 @@ build() { -DBUILD_TESTS=OFF \ -DLINK_STATIC=ON \ ${CMAKE_CROSSOPTS} ${ROOT_DIR} - make + make -j6 } package() { diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile new file mode 100644 index 00000000..b2880659 --- /dev/null +++ b/pkg/apk/Dockerfile @@ -0,0 +1,89 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +FROM alpine:3.16 + +ARG PLATFORM + +# perl is required to install OpenSSL +RUN apk add \ + abuild \ + bash \ + curl \ + g++ \ + make \ + python3 \ + cmake \ + boost-dev \ + perl \ + sudo + +# Download and copile protoubf +RUN curl -O -L https://github.com/google/protobuf/releases/download/v3.20.0/protobuf-cpp-3.20.0.tar.gz && \ + tar xfz protobuf-cpp-3.20.0.tar.gz && \ + cd protobuf-3.20.0/ && \ + CXXFLAGS=-fPIC ./configure && \ + make -j8 && make install && \ + rm -rf /protobuf-cpp-3.20.0.tar.gz /protobuf-3.20.0 + +# ZLib +RUN curl -O -L https://github.com/madler/zlib/archive/v1.2.12.tar.gz && \ + tar xfz v1.2.12.tar.gz && \ + cd zlib-1.2.12 && \ + CFLAGS="-fPIC -O3" ./configure && \ + make && make install && \ + rm -rf /v1.2.12.tar.gz /zlib-1.2.12 + +# Zstandard +RUN curl -O -L https://github.com/facebook/zstd/releases/download/v1.5.2/zstd-1.5.2.tar.gz && \ + tar xfz zstd-1.5.2.tar.gz && \ + cd zstd-1.5.2 && \ + CFLAGS="-fPIC -O3" make -j8 && \ + make install && \ + rm -rf /zstd-1.5.2 /zstd-1.5.2.tar.gz + +# Snappy +RUN curl -O -L https://github.com/google/snappy/releases/download/1.1.3/snappy-1.1.3.tar.gz && \ + tar xfz snappy-1.1.3.tar.gz && \ + cd snappy-1.1.3 && \ + CXXFLAGS="-fPIC -O3" ./configure && \ + make && make install && \ + rm -rf /snappy-1.1.3 /snappy-1.1.3.tar.gz + +RUN curl -O -L https://github.com/openssl/openssl/archive/OpenSSL_1_1_1n.tar.gz && \ + tar xfz OpenSSL_1_1_1n.tar.gz && \ + cd openssl-OpenSSL_1_1_1n/ && \ + ./Configure -fPIC --prefix=/usr/local/ssl/ linux-${PLATFORM} && \ + make -j8 && make install && \ + rm -rf /OpenSSL_1_1_1n.tar.gz /openssl-OpenSSL_1_1_1n + +ENV LD_LIBRARY_PATH /usr/local/ssl/lib/: + +# LibCurl +RUN curl -O -L https://github.com/curl/curl/releases/download/curl-7_61_0/curl-7.61.0.tar.gz && \ + tar xfz curl-7.61.0.tar.gz && \ + cd curl-7.61.0 && \ + CFLAGS=-fPIC ./configure --with-ssl=/usr/local/ssl/ && \ + make -j8 && make install && \ + rm -rf /curl-7.61.0.tar.gz /curl-7.61.0 + +ENV OPENSSL_ROOT_DIR /usr/local/ssl/ + +RUN adduser pulsar -D +RUN addgroup pulsar abuild diff --git a/pkg/apk/create-images.sh b/pkg/apk/create-images.sh new file mode 100755 index 00000000..4c0612a0 --- /dev/null +++ b/pkg/apk/create-images.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) +cd $ROOT_DIR/pkg/apk + +# ARM +IMAGE=apachepulsar/pulsar-build:alpine-3.16-arm64 +docker build --platform arm64 -t $IMAGE . --build-arg PLATFORM=aarch64 + +# X86_64 +IMAGE=apachepulsar/pulsar-build:alpine-3.16-x86_64 +docker build --platform x86_64 -t $IMAGE . --build-arg PLATFORM=x86_64 diff --git a/pkg/apk/docker-build-apk.sh b/pkg/apk/docker-build-apk-arm64.sh similarity index 82% rename from pkg/apk/docker-build-apk.sh rename to pkg/apk/docker-build-apk-arm64.sh index c400c7d7..cc47ca73 100755 --- a/pkg/apk/docker-build-apk.sh +++ b/pkg/apk/docker-build-apk-arm64.sh @@ -18,12 +18,15 @@ # under the License. # -set -e +set -e -x ROOT_DIR=$(git rev-parse --show-toplevel) -IMAGE=apachepulsar/pulsar-build:alpine-3.11 -docker pull $IMAGE +IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-arm64 -docker run -i -v $ROOT_DIR:/pulsar-client-cpp $IMAGE \ +#docker pull $IMAGE_NAME +docker run -v $ROOT_DIR:/pulsar-client-cpp \ + --env PLATFORM=arm64 \ + $IMAGE_NAME \ /pulsar-client-cpp/pkg/apk/build-apk.sh + diff --git a/pkg/apk/docker-build-apk-x86_64.sh b/pkg/apk/docker-build-apk-x86_64.sh new file mode 100755 index 00000000..833a75ec --- /dev/null +++ b/pkg/apk/docker-build-apk-x86_64.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -e -x + +ROOT_DIR=$(git rev-parse --show-toplevel) + +IMAGE_NAME=apachepulsar/pulsar-build:alpine-3.16-x86_64 + +#docker pull $IMAGE_NAME +docker run -v $ROOT_DIR:/pulsar-client-cpp \ + --env PLATFORM=x86_64 \ + $IMAGE_NAME \ + /pulsar-client-cpp/pkg/apk/build-apk.sh + From ef72eeec9fcf342fd7439c5663d50e1dd291caca Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:27:11 -0700 Subject: [PATCH 2/7] Removed job depedency --- .github/workflows/ci-build-binary-artifacts.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 42e38c72..0250741b 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -124,7 +124,6 @@ jobs: apk-packaging-x86_64: name: Build Alpine Linux APK Package - x86_64 runs-on: ubuntu-20.04 - needs: unit-tests timeout-minutes: 120 steps: @@ -146,7 +145,6 @@ jobs: apk-packaging-arm64: name: Build Alpine Linux APK Package - arm64 runs-on: ubuntu-20.04 - needs: unit-tests timeout-minutes: 120 steps: From 0ee8e623cc49fe7cb850f39b2ebc53bd1eb6c868 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 09:57:20 -0700 Subject: [PATCH 3/7] Fixed permissions --- .github/workflows/ci-build-binary-artifacts.yaml | 2 -- .github/workflows/ci-pr-validation.yaml | 3 --- pkg/apk/build-apk.sh | 5 ++--- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 0250741b..98659175 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -20,8 +20,6 @@ name: CI - Build binary artifacts on: push: - branches: - - '*' tags: - '*' diff --git a/.github/workflows/ci-pr-validation.yaml b/.github/workflows/ci-pr-validation.yaml index 7dcaef42..804a937d 100644 --- a/.github/workflows/ci-pr-validation.yaml +++ b/.github/workflows/ci-pr-validation.yaml @@ -22,9 +22,6 @@ on: pull_request: branches: - main - push: - branches: - - main concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index b31a1311..c81a2e94 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -41,9 +41,8 @@ chmod 755 ~ chmod 755 ~/.abuild chmod 644 ~/.abuild/* -mkdir -p /root/packages -chmod 777 /root/packages +mkdir -p pkg +chmod 777 pkg sudo -E -u pulsar abuild -r -mv /root/packages/pkg . From dfefe9898d894c4d90e98f1d0a8371022dce1f81 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Sun, 2 Oct 2022 10:31:30 -0700 Subject: [PATCH 4/7] Fixed directory ownership --- pkg/apk/build-apk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index c81a2e94..cd6460c5 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -43,6 +43,6 @@ chmod 644 ~/.abuild/* mkdir -p pkg chmod 777 pkg +chgrp pulsar pkg sudo -E -u pulsar abuild -r - From f9960b626fc9fb50ffabd83abc129c17c66d1c18 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 3 Oct 2022 18:53:53 -0700 Subject: [PATCH 5/7] Fixed building APK as root --- pkg/apk/APKBUILD | 2 +- pkg/apk/Dockerfile | 1 + pkg/apk/build-apk.sh | 10 +--------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/apk/APKBUILD b/pkg/apk/APKBUILD index e86cabaf..3c3cdc70 100644 --- a/pkg/apk/APKBUILD +++ b/pkg/apk/APKBUILD @@ -27,7 +27,7 @@ url="https://pulsar.apache.org" arch="all" license="https://www.apache.org/licenses/LICENSE-2.0.txt" depends="" -makedepends="cmake" +makedepends="" install="" subpackages="$pkgname-dev" source="" diff --git a/pkg/apk/Dockerfile b/pkg/apk/Dockerfile index b2880659..d03b1f1f 100644 --- a/pkg/apk/Dockerfile +++ b/pkg/apk/Dockerfile @@ -23,6 +23,7 @@ ARG PLATFORM # perl is required to install OpenSSL RUN apk add \ + build-base \ abuild \ bash \ curl \ diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index cd6460c5..a60f6a8e 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -37,12 +37,4 @@ export VERSION=`echo $POM_VERSION | sed -E 's/\-[a-zA-Z]+//'` echo "VERSION: $VERSION" abuild-keygen -a -i -n -chmod 755 ~ -chmod 755 ~/.abuild -chmod 644 ~/.abuild/* - -mkdir -p pkg -chmod 777 pkg -chgrp pulsar pkg - -sudo -E -u pulsar abuild -r +abuild -F -c -r From 3197a539c04808f189ee27ee9d2aaf0665ee264a Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 4 Oct 2022 15:29:45 -0500 Subject: [PATCH 6/7] Use qemu to build arm64 package --- .github/workflows/ci-build-binary-artifacts.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 98659175..09197e0f 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -149,6 +149,9 @@ jobs: - name: checkout uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Package Pulsar source run: build-support/generate-source-archive.sh From bb608450b7ef80abd12dadc25e1e52a5eb18a295 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Tue, 4 Oct 2022 16:35:34 -0500 Subject: [PATCH 7/7] fixed copying the APK package to artifacts --- .github/workflows/ci-build-binary-artifacts.yaml | 4 ++-- pkg/apk/.gitignore | 5 +---- pkg/apk/build-apk.sh | 2 ++ 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-build-binary-artifacts.yaml b/.github/workflows/ci-build-binary-artifacts.yaml index 09197e0f..c4e6543b 100644 --- a/.github/workflows/ci-build-binary-artifacts.yaml +++ b/.github/workflows/ci-build-binary-artifacts.yaml @@ -138,7 +138,7 @@ jobs: uses: actions/upload-artifact@v3 with: name: apk-x86_64 - path: pkg/apk/pkg/x86_64 + path: pkg/apk/build/x86_64 apk-packaging-arm64: name: Build Alpine Linux APK Package - arm64 @@ -162,4 +162,4 @@ jobs: uses: actions/upload-artifact@v3 with: name: apk-arm64 - path: pkg/apk/pkg/aarch64 + path: pkg/apk/build/aarch64 diff --git a/pkg/apk/.gitignore b/pkg/apk/.gitignore index 842209a0..a3661865 100644 --- a/pkg/apk/.gitignore +++ b/pkg/apk/.gitignore @@ -1,5 +1,2 @@ pkg -perf -examples -lib -src +build \ No newline at end of file diff --git a/pkg/apk/build-apk.sh b/pkg/apk/build-apk.sh index a60f6a8e..66cb7633 100755 --- a/pkg/apk/build-apk.sh +++ b/pkg/apk/build-apk.sh @@ -38,3 +38,5 @@ echo "VERSION: $VERSION" abuild-keygen -a -i -n abuild -F -c -r + +cp -r /root/packages/pkg ./build