Skip to content

Create and upload release candidate artifacts #6

Create and upload release candidate artifacts

Create and upload release candidate artifacts #6

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
name: Create and upload release candidate artifacts
on:
workflow_dispatch:
inputs:
tag:
description: 'Existing Git tag (e.g., v1.1.0-incubating-rc1)'
required: true
type: string
rc_number:
description: 'Release candidate number for artifacts (e.g., 1 for RC1, 2 for RC2)'
required: true
type: string
default: '1'
image_registry:
description: 'Container image registry prefix (e.g., ghcr.io/apache, docker.io/apache)'
required: false
type: string
default: 'ghcr.io/apache'
jobs:
create-rc:
runs-on: ubuntu-latest
outputs:
version: ${{ steps.vars.outputs.version }}
rc_num: ${{ steps.vars.outputs.rc_num }}
tag_name: ${{ steps.vars.outputs.tag_name }}
rc_dir: ${{ steps.vars.outputs.rc_dir }}
commit_hash: ${{ steps.vars.outputs.commit_hash }}
src_tarball: ${{ steps.vars.outputs.src_tarball }}
compose_tarball: ${{ steps.vars.outputs.compose_tarball }}
helm_tarball: ${{ steps.vars.outputs.helm_tarball }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0 # Full history for proper tagging
- name: Validate tag exists
run: |
TAG_NAME="${{ github.event.inputs.tag }}"
# Check if tag exists
if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then
echo "Error: Tag '$TAG_NAME' does not exist"
echo "Available tags:"
git tag -l | tail -10
exit 1
fi
echo "✓ Tag validation passed: $TAG_NAME"
- name: Set up variables
id: vars
run: |
TAG_NAME="${{ github.event.inputs.tag }}"
RC_NUM="${{ github.event.inputs.rc_number }}"
IMAGE_REGISTRY="${{ github.event.inputs.image_registry }}"
# Parse version from tag (format: v1.1.0-incubating or v1.1.0-incubating-rcN)
# Both formats are accepted, but we use the input rc_number for artifacts
if [[ "$TAG_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+-incubating)(-rc[0-9]+)?$ ]]; then
VERSION="${BASH_REMATCH[1]}"
else
echo "Error: Tag must be in format vX.Y.Z-incubating or vX.Y.Z-incubating-rcN (e.g., v1.1.0-incubating-rc1)"
exit 1
fi
COMMIT_HASH=$(git rev-parse "$TAG_NAME")
COMMIT_SHORT=$(git rev-parse "$TAG_NAME" | cut -c1-9)
RC_DIR="${VERSION}-RC${RC_NUM}"
SRC_TARBALL="apache-texera-${VERSION}-src.tar.gz"
COMPOSE_TARBALL="apache-texera-${VERSION}-docker-compose.tar.gz"
HELM_TARBALL="apache-texera-${VERSION}-helm.tgz"
echo "version=$VERSION" >> $GITHUB_OUTPUT
echo "rc_num=$RC_NUM" >> $GITHUB_OUTPUT
echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT
echo "rc_dir=$RC_DIR" >> $GITHUB_OUTPUT
echo "commit_hash=$COMMIT_HASH" >> $GITHUB_OUTPUT
echo "commit_short=$COMMIT_SHORT" >> $GITHUB_OUTPUT
echo "image_registry=$IMAGE_REGISTRY" >> $GITHUB_OUTPUT
echo "src_tarball=$SRC_TARBALL" >> $GITHUB_OUTPUT
echo "compose_tarball=$COMPOSE_TARBALL" >> $GITHUB_OUTPUT
echo "helm_tarball=$HELM_TARBALL" >> $GITHUB_OUTPUT
echo "Release Candidate: $TAG_NAME"
echo "Version: $VERSION"
echo "RC Number: $RC_NUM"
echo "Commit: $COMMIT_HASH ($COMMIT_SHORT)"
echo "Image Registry: $IMAGE_REGISTRY"
echo "Staging directory: dist/dev/incubator/texera/$RC_DIR"
- name: Create source tarball
run: |
TAG_NAME="${{ steps.vars.outputs.tag_name }}"
SRC_TARBALL="${{ steps.vars.outputs.src_tarball }}"
VERSION="${{ steps.vars.outputs.version }}"
TEMP_DIR=$(mktemp -d)
# Export the git repository at the tag
git archive --format=tar --prefix="apache-texera-${VERSION}-src/" "$TAG_NAME" | tar -x -C "$TEMP_DIR"
# Create tarball
cd "$TEMP_DIR"
tar -czf "$GITHUB_WORKSPACE/$SRC_TARBALL" "apache-texera-${VERSION}-src"
cd "$GITHUB_WORKSPACE"
# Verify tarball was created
if [[ ! -f "$SRC_TARBALL" ]]; then
echo "Error: Source tarball was not created"
exit 1
fi
# Show tarball info
ls -lh "$SRC_TARBALL"
echo "✓ Created source tarball: $SRC_TARBALL"
- name: Create Docker Compose deployment bundle
run: |
VERSION="${{ steps.vars.outputs.version }}"
TAG_NAME="${{ steps.vars.outputs.tag_name }}"
IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}"
COMMIT_SHORT="${{ steps.vars.outputs.commit_short }}"
COMPOSE_TARBALL="${{ steps.vars.outputs.compose_tarball }}"
TEMP_DIR=$(mktemp -d)
BUNDLE_DIR="$TEMP_DIR/apache-texera-${VERSION}-docker-compose"
mkdir -p "$BUNDLE_DIR"
# Export the single-node directory from the tagged source
mkdir -p "$TEMP_DIR/_raw"
git archive --format=tar "$TAG_NAME" -- bin/single-node/ sql/ | tar -x -C "$TEMP_DIR/_raw"
# Copy deployment files
cp "$TEMP_DIR/_raw/bin/single-node/docker-compose.yml" "$BUNDLE_DIR/"
cp "$TEMP_DIR/_raw/bin/single-node/nginx.conf" "$BUNDLE_DIR/"
cp -r "$TEMP_DIR/_raw/sql" "$BUNDLE_DIR/"
# Patch the SQL mount path for the self-contained bundle layout
# In the repo it's ../../sql (relative to bin/single-node/), in the bundle it's ./sql
sed -i 's|../../sql|./sql|g' "$BUNDLE_DIR/docker-compose.yml"
# Generate a release-pinned .env file with the version tag
# Start from the source .env and ensure IMAGE_REGISTRY and IMAGE_TAG are set
cp "$TEMP_DIR/_raw/bin/single-node/.env" "$BUNDLE_DIR/.env"
# Replace if line exists, otherwise append
if grep -q '^IMAGE_REGISTRY=' "$BUNDLE_DIR/.env"; then
sed -i "s|^IMAGE_REGISTRY=.*|IMAGE_REGISTRY=${IMAGE_REGISTRY}|" "$BUNDLE_DIR/.env"
else
echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}" >> "$BUNDLE_DIR/.env"
fi
if grep -q '^IMAGE_TAG=' "$BUNDLE_DIR/.env"; then
sed -i "s|^IMAGE_TAG=.*|IMAGE_TAG=${COMMIT_SHORT}|" "$BUNDLE_DIR/.env"
else
echo "IMAGE_TAG=${COMMIT_SHORT}" >> "$BUNDLE_DIR/.env"
fi
# Include the README from the repo
cp "$TEMP_DIR/_raw/bin/single-node/README.md" "$BUNDLE_DIR/"
# Create tarball
cd "$TEMP_DIR"
tar -czf "$GITHUB_WORKSPACE/$COMPOSE_TARBALL" "apache-texera-${VERSION}-docker-compose"
cd "$GITHUB_WORKSPACE"
ls -lh "$COMPOSE_TARBALL"
echo "✓ Created Docker Compose bundle: $COMPOSE_TARBALL"
- name: Create Helm chart package
run: |
VERSION="${{ steps.vars.outputs.version }}"
TAG_NAME="${{ steps.vars.outputs.tag_name }}"
COMMIT_SHORT="${{ steps.vars.outputs.commit_short }}"
IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}"
HELM_TARBALL="${{ steps.vars.outputs.helm_tarball }}"
# Install Helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# Export the Helm chart from the tagged source
# Include sql/ because bin/k8s/files/ has symlinks to ../../../sql/
TEMP_DIR=$(mktemp -d)
git archive --format=tar "$TAG_NAME" -- bin/k8s/ sql/ | tar -x -C "$TEMP_DIR"
CHART_DIR="$TEMP_DIR/bin/k8s"
# Resolve symlinks in the chart so it packages as a self-contained artifact
find "$CHART_DIR" -type l | while read -r link; do
target=$(readlink -f "$link")
rm "$link"
cp "$target" "$link"
done
# Update Chart.yaml with release version
sed -i "s/^version:.*/version: ${VERSION}/" "$CHART_DIR/Chart.yaml"
sed -i "s/^appVersion:.*/appVersion: \"${VERSION}\"/" "$CHART_DIR/Chart.yaml"
# Update values.yaml with release image registry and tag
sed -i "s|imageRegistry:.*|imageRegistry: ${IMAGE_REGISTRY}|" "$CHART_DIR/values.yaml"
sed -i "s|imageTag:.*|imageTag: ${COMMIT_SHORT}|" "$CHART_DIR/values.yaml"
# Download chart dependencies declared in Chart.yaml
# These tarballs are .gitignored so they're not in git archive
helm dependency build "$CHART_DIR"
# Package the Helm chart
helm package "$CHART_DIR" \
--version "$VERSION" \
--app-version "$VERSION" \
--destination "$GITHUB_WORKSPACE"
# Rename to our expected artifact name
HELM_PKG=$(ls "$GITHUB_WORKSPACE"/texera-helm-*.tgz 2>/dev/null | head -1)
if [[ -n "$HELM_PKG" && "$HELM_PKG" != "$GITHUB_WORKSPACE/$HELM_TARBALL" ]]; then
mv "$HELM_PKG" "$GITHUB_WORKSPACE/$HELM_TARBALL"
fi
ls -lh "$GITHUB_WORKSPACE/$HELM_TARBALL"
echo "✓ Created Helm chart package: $HELM_TARBALL"
- name: Import GPG key
run: |
echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --batch --import
# List imported keys
gpg --list-secret-keys
echo "✓ GPG key imported successfully"
- name: Sign and checksum all artifacts
run: |
for artifact in \
"${{ steps.vars.outputs.src_tarball }}" \
"${{ steps.vars.outputs.compose_tarball }}" \
"${{ steps.vars.outputs.helm_tarball }}"; do
# GPG signature
echo "${{ secrets.GPG_PASSPHRASE }}" | gpg --batch --yes --pinentry-mode loopback --passphrase-fd 0 \
--armor --detach-sign --output "${artifact}.asc" "$artifact"
gpg --verify "${artifact}.asc" "$artifact"
echo "✓ Signed: ${artifact}"
# SHA512 checksum
sha512sum "$artifact" > "${artifact}.sha512"
echo "✓ Checksum: ${artifact}.sha512"
done
- name: Generate vote email template
id: vote_email
run: |
VERSION="${{ steps.vars.outputs.version }}"
RC_NUM="${{ steps.vars.outputs.rc_num }}"
TAG_NAME="${{ steps.vars.outputs.tag_name }}"
RC_DIR="${{ steps.vars.outputs.rc_dir }}"
COMMIT_HASH="${{ steps.vars.outputs.commit_hash }}"
IMAGE_REGISTRY="${{ steps.vars.outputs.image_registry }}"
# Get GPG key ID from the imported key
GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep 'sec' | head -n1 | awk '{print $2}' | cut -d'/' -f2)
GPG_EMAIL=$(gpg --list-secret-keys | grep 'uid' | head -n1 | grep -oP '[\w\.-]+@[\w\.-]+')
# Copy template from repository
cp .github/release/vote-email-template.md vote-email.txt
# Substitute variables in the template
sed -i "s|\${VERSION}|${VERSION}|g" vote-email.txt
sed -i "s|\${RC_NUM}|${RC_NUM}|g" vote-email.txt
sed -i "s|\${RC_DIR}|${RC_DIR}|g" vote-email.txt
sed -i "s|\${TAG_NAME}|${TAG_NAME}|g" vote-email.txt
sed -i "s|\${COMMIT_HASH}|${COMMIT_HASH}|g" vote-email.txt
sed -i "s|\${GPG_KEY_ID}|${GPG_KEY_ID}|g" vote-email.txt
sed -i "s|\${GPG_EMAIL}|${GPG_EMAIL}|g" vote-email.txt
sed -i "s|\${IMAGE_REGISTRY}|${IMAGE_REGISTRY}|g" vote-email.txt
echo "✓ Vote email template generated!"
- name: Upload RC artifacts
uses: actions/upload-artifact@v4
with:
name: rc-artifacts
path: |
${{ steps.vars.outputs.src_tarball }}
${{ steps.vars.outputs.src_tarball }}.asc
${{ steps.vars.outputs.src_tarball }}.sha512
${{ steps.vars.outputs.compose_tarball }}
${{ steps.vars.outputs.compose_tarball }}.asc
${{ steps.vars.outputs.compose_tarball }}.sha512
${{ steps.vars.outputs.helm_tarball }}
${{ steps.vars.outputs.helm_tarball }}.asc
${{ steps.vars.outputs.helm_tarball }}.sha512
vote-email.txt
retention-days: 7
upload-rc:
runs-on: ubuntu-latest
needs: create-rc
steps:
- name: Download RC artifacts
uses: actions/download-artifact@v4
with:
name: rc-artifacts
- name: Verify downloaded artifacts
run: |
SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}"
COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}"
HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}"
echo "Verifying downloaded artifacts..."
ls -lh
for artifact in "$SRC_TARBALL" "$COMPOSE_TARBALL" "$HELM_TARBALL"; do
if [[ ! -f "$artifact" ]] || [[ ! -f "${artifact}.asc" ]] || [[ ! -f "${artifact}.sha512" ]]; then
echo "Error: Missing artifact or signature/checksum for: $artifact"
exit 1
fi
done
echo "✓ All artifacts downloaded successfully"
- name: Install SVN
run: |
sudo apt-get update
sudo apt-get install -y subversion
svn --version
- name: Checkout SVN dev directory
run: |
RC_DIR="${{ needs.create-rc.outputs.rc_dir }}"
# Checkout the dev directory with depth=empty (lightweight)
svn co --depth=empty https://dist.apache.org/repos/dist/dev/incubator/texera svn-texera \
--username "${{ secrets.SVN_USERNAME }}" \
--password "${{ secrets.SVN_PASSWORD }}" \
--no-auth-cache
cd svn-texera
# Check if RC directory already exists on the remote
SVN_BASE="https://dist.apache.org/repos/dist/dev/incubator/texera"
if svn info "$SVN_BASE/$RC_DIR" >/dev/null 2>&1; then
# Directory exists remotely — update (checkout) it into the working copy
svn update --depth=infinity "$RC_DIR" \
--username "${{ secrets.SVN_USERNAME }}" \
--password "${{ secrets.SVN_PASSWORD }}" \
--no-auth-cache || true
# If update didn't bring it down (empty parent checkout), do a sparse checkout
if [[ ! -d "$RC_DIR" ]]; then
svn update --set-depth=infinity "$RC_DIR" \
--username "${{ secrets.SVN_USERNAME }}" \
--password "${{ secrets.SVN_PASSWORD }}" \
--no-auth-cache
fi
echo "✓ RC directory already exists remotely, checked out: $RC_DIR"
else
# Directory doesn't exist remotely — create and add it
mkdir -p "$RC_DIR"
svn add "$RC_DIR"
echo "✓ Created new RC directory: $RC_DIR"
fi
- name: Stage artifacts to SVN
run: |
SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}"
COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}"
HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}"
RC_DIR="${{ needs.create-rc.outputs.rc_dir }}"
cd svn-texera/"$RC_DIR"
# Copy all artifacts
for artifact in "$SRC_TARBALL" "$COMPOSE_TARBALL" "$HELM_TARBALL"; do
cp "$GITHUB_WORKSPACE/$artifact" .
cp "$GITHUB_WORKSPACE/${artifact}.asc" .
cp "$GITHUB_WORKSPACE/${artifact}.sha512" .
done
# Add files to SVN
svn add * --force
# Check status
svn status
echo "✓ Staged all artifacts to SVN"
- name: Commit artifacts to dist/dev
run: |
VERSION="${{ needs.create-rc.outputs.version }}"
RC_NUM="${{ needs.create-rc.outputs.rc_num }}"
RC_DIR="${{ needs.create-rc.outputs.rc_dir }}"
cd svn-texera
# Commit with descriptive message
svn commit -m "Add Apache Texera ${VERSION} RC${RC_NUM} artifacts (source + docker-compose + helm)" \
--username "${{ secrets.SVN_USERNAME }}" \
--password "${{ secrets.SVN_PASSWORD }}" \
--no-auth-cache
echo "✓ Committed artifacts to dist/dev/incubator/texera/$RC_DIR"
- name: Generate release summary
run: |
VERSION="${{ needs.create-rc.outputs.version }}"
RC_NUM="${{ needs.create-rc.outputs.rc_num }}"
TAG_NAME="${{ needs.create-rc.outputs.tag_name }}"
RC_DIR="${{ needs.create-rc.outputs.rc_dir }}"
COMMIT_HASH="${{ needs.create-rc.outputs.commit_hash }}"
SRC_TARBALL="${{ needs.create-rc.outputs.src_tarball }}"
COMPOSE_TARBALL="${{ needs.create-rc.outputs.compose_tarball }}"
HELM_TARBALL="${{ needs.create-rc.outputs.helm_tarball }}"
echo "## Release Candidate Created Successfully!" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Release Information" >> $GITHUB_STEP_SUMMARY
echo "- **Version:** ${VERSION}" >> $GITHUB_STEP_SUMMARY
echo "- **RC Number:** RC${RC_NUM}" >> $GITHUB_STEP_SUMMARY
echo "- **Git Tag:** \`${TAG_NAME}\`" >> $GITHUB_STEP_SUMMARY
echo "- **Commit:** \`${COMMIT_HASH}\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Artifacts Location" >> $GITHUB_STEP_SUMMARY
echo "**Staging Directory:** https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Artifacts Created" >> $GITHUB_STEP_SUMMARY
echo "| Artifact | Description |" >> $GITHUB_STEP_SUMMARY
echo "|----------|-------------|" >> $GITHUB_STEP_SUMMARY
echo "| \`${SRC_TARBALL}\` | Source code |" >> $GITHUB_STEP_SUMMARY
echo "| \`${COMPOSE_TARBALL}\` | Docker Compose deployment bundle |" >> $GITHUB_STEP_SUMMARY
echo "| \`${HELM_TARBALL}\` | Helm chart for Kubernetes deployment |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Each artifact has a corresponding \`.asc\` (GPG signature) and \`.sha512\` (checksum) file." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
echo "1. Build and push container images using the \`Build and push images\` workflow with tag \`${VERSION}\`" >> $GITHUB_STEP_SUMMARY
echo "2. Verify the artifacts at the staging directory" >> $GITHUB_STEP_SUMMARY
echo "3. Send [VOTE] email to dev@texera.apache.org" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "### Verification" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
echo "# Import KEYS and verify signatures" >> $GITHUB_STEP_SUMMARY
echo "gpg --import KEYS" >> $GITHUB_STEP_SUMMARY
echo "gpg --verify ${SRC_TARBALL}.asc ${SRC_TARBALL}" >> $GITHUB_STEP_SUMMARY
echo "gpg --verify ${COMPOSE_TARBALL}.asc ${COMPOSE_TARBALL}" >> $GITHUB_STEP_SUMMARY
echo "gpg --verify ${HELM_TARBALL}.asc ${HELM_TARBALL}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "# Verify SHA512 checksums" >> $GITHUB_STEP_SUMMARY
echo "sha512sum -c ${SRC_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY
echo "sha512sum -c ${COMPOSE_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY
echo "sha512sum -c ${HELM_TARBALL}.sha512" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "**KEYS file:** https://downloads.apache.org/incubator/texera/KEYS" >> $GITHUB_STEP_SUMMARY
echo "✓ Release candidate workflow completed successfully!"
- name: Display vote email template
run: |
echo "## Vote Email Template" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "Copy the content below to send to dev@texera.apache.org:" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
cat "$GITHUB_WORKSPACE/vote-email.txt" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY