diff --git a/CHANGES_MAY_2026.md b/CHANGES_MAY_2026.md new file mode 100644 index 000000000..f89178436 --- /dev/null +++ b/CHANGES_MAY_2026.md @@ -0,0 +1,55 @@ +# Documentation of Recent Changes (May 2026) + +This document summarizes the significant changes made to the `zopen` tools recently, focusing on RPM generation, Pulp integration, and dependency management improvements. + +## 1. `zopen-pax2rpm` Refactoring and Improvements + +The `zopen-pax2rpm` tool, used for generating RPM spec files and building RPMs from pax archives, has undergone a major overhaul: + +- **Portability:** Switched shebang from `#!/bin/bash` to `#!/bin/sh`. +- **Consistency:** Integrated `setupMyself` to align with other `zopen` scripts and source `common.sh`. +- **Platform Support:** + - Improved architecture detection for z/OS, correctly mapping numeric machine types to `s390x`. + - Added `%define __os_install_post %{nil}` to spec files on z/OS to prevent broken post-install processing (like stripping). +- **Robust Parsing:** + - Improved filename parsing to handle `.zos` suffixes and avoid bash-specific syntax. + - Enhanced pax archive analysis with higher entry limits (up to 5000) and more robust directory structure detection. +- **Improved Build Environment:** + - Better handling of `~/.rpmmacros` and `_topdir`. + - Uses `--define "_topdir ..."` in `rpmbuild` commands for better isolation. +- **User Interface:** + - Added `--pkg-version` and `--version` options. + - Improved error messages and dependency checking. + +## 2. Pulp Integration for RPM Publishing + +Support for the Pulp repository manager has been added to the publishing workflow: + +- **`zopen-publish`**: + - New options: `--pulp` (enable Pulp upload) and `--pulp-repo ` (specify target repository). + - Automatic detection of RPM files in common build locations (`rpmbuild/RPMS` or the same directory as the pax file). + - New `uploadToPulp` function to handle the interaction with the Pulp CLI. +- **`cicd/publish.groovy`**: + - Updated Jenkins pipeline to automatically push RPMs to the appropriate Pulp repository (`zopen-stable` or `zopen-dev`) during the release process. + +## 3. Binary-Only Dependency Support (`:bin`) + +A new mechanism for handling binary-only dependencies has been introduced to `zopen-build`: + +- **Syntax:** Dependencies can now be specified with a `:bin` suffix (e.g., `git:bin`). +- **Behavior:** `:bin` dependencies are intended for tools that are only needed for execution (relying on `bin/` and `share/`), rather than for linking or compiling against. +- **`zopen-build` Changes:** + - Refactored environment sourcing (`setDepsEnv`) to handle `:bin` dependencies by temporarily unsetting `ZOPEN_IN_ZOPEN_BUILD`. + - Added `normalizeDeps` logic to prefer full packages if both full and `:bin` versions of the same package are requested. +- **Environment Protection:** + - Modified `.env` generation to wrap `PKG_CONFIG_PATH` updates in a check for `ZOPEN_IN_ZOPEN_BUILD`. This prevents build-time environment variables from leaking into the runtime environment of `:bin` dependencies. + +## 4. Shared Library Updates (`include/common.sh`) + +New utility functions were added to `common.sh` to support the new dependency logic: +- `parseDeps`: Now correctly handles the `:bin` suffix and extracts it as a property. +- `normalizeDeps`: Deduplicates and prioritizes dependencies, ensuring a consistent environment. + +## 5. Automated Reports + +- **`docs/upstreamstatus.md`**: The Upstream Patch Status Report has been updated with the latest data on project patches and historical trends. diff --git a/bin/zopen-build b/bin/zopen-build index 00a0dd858..4dcc7816f 100755 --- a/bin/zopen-build +++ b/bin/zopen-build @@ -393,6 +393,7 @@ Option: with the .rej extension. -g, --get-source get the source and apply patch without building. -gp, --generate-pax generate a pax.Z file based on the install contents. + -gr, --generate-rpm generate an RPM package from the pax archive. -h, --help, -? print this information. --no-set-active do not change the pinned version. --no-install-deps do not install project's runtime dependencies. @@ -437,6 +438,7 @@ processOptions() buildEnvFile="./buildenv" getSourceOnly=false generatePax=false + generateRPM=false setActive=true signPax=false forcePatchApply=false @@ -538,11 +540,16 @@ processOptions() "-gp" | "--generate-pax") generatePax=true ;; + "-gr" | "--generate-rpm") + generateRPM=true + generatePax=true + ;; "-s" | "--shell") startShell=true ;; "-sp" | "--sign-pax") signPax=true + generatePax=true ;; *) printError "Unknown option ${1} specified" @@ -2446,7 +2453,25 @@ install() if ! runAndLog "${ZOPEN_PAX_CMD}"; then printError "Could not generate pax \"${paxFileName}\"" fi + fi + if ${generateRPM}; then + if [ -f "${paxFileName}" ]; then + printHeader "Generating RPM from ${ZOPEN_INSTALL_DIR}" + rpm_deps=$(echo "${ZOPEN_RUNTIME_DEPS}" | xargs -n1 | sort -u | xargs) + cmd="PATH=\"${ZOPEN_ROOTFS}/usr/local/bin:${PATH}\" \"${MYDIR}/zopen-pax2rpm\" \"${paxFileName}\" --summary \"${ZOPEN_NAME} package\" --build --buildroot \"${ZOPEN_ROOT}/rpmbuild\"" + if [ -n "${rpm_deps}" ]; then + cmd="${cmd} --requires \"${rpm_deps}\"" + fi + if ! runAndLog "${cmd}"; then + printError "Could not generate RPM from \"${paxFileName}\"" + fi + else + printError "Pax file ${paxFileName} not found. Ensure --generate-pax is also used." + fi + fi + + if ${generatePax}; then #TODO: Hack so that we can use coreutils md5sum without impacting builds ZOPEN_DEPS="${ZOPEN_DEPS} coreutils jq" if [ "${signPax}" = "true" ] && ( [ -z "${ZOPEN_GPG_SECRET_KEY_FILE}" ] || [ -z "${ZOPEN_GPG_SECRET_KEY_PASSPHRASE_FILE}" ] || [ -z "${ZOPEN_GPG_PUBLIC_KEY_FILE}" ] || [ ! -r "${ZOPEN_GPG_SECRET_KEY_FILE}" ] || [ ! -r "${ZOPEN_GPG_SECRET_KEY_PASSPHRASE_FILE}" ] || [ ! -r "${ZOPEN_GPG_PUBLIC_KEY_FILE}" ] ); then diff --git a/bin/zopen-pax2rpm b/bin/zopen-pax2rpm new file mode 100755 index 000000000..d4a016057 --- /dev/null +++ b/bin/zopen-pax2rpm @@ -0,0 +1,929 @@ +#!/bin/sh +# +# RPM generation from the PAX generated file utility for zopen +# + +# +# All zopen-* scripts MUST start with this code to maintain consistency. +# +setupMyself() +{ + ME=$(basename $0) + MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" + INCDIR="${MYDIR}/../include" + if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + echo "Internal Error. Unable to find common.sh file to source." >&2 + exit 8 + fi + . "${INCDIR}/common.sh" +} +setupMyself + +# Default values +RELEASE="1" +LICENSE="Proprietary" +BUILD_ARCH=$(rpm --eval "%{_target_cpu}" 2>/dev/null || uname -m 2>/dev/null || echo "s390x") + +# On z/OS, uname -m returns numeric machine types (e.g., 8561, 3931). +# RPM expects 's390x' for the build architecture. +if [ "$(uname -s)" = "OS/390" ]; then + case "${BUILD_ARCH}" in + [0-9]*) BUILD_ARCH="s390x" ;; + esac +fi + +# Get current user safely for packager info +CURRENT_USER="${USER:-$(whoami 2>/dev/null || echo "zopen-community")}" +PACKAGER_NAME="${CURRENT_USER}" +PACKAGER_EMAIL="${CURRENT_USER}@$(hostname 2>/dev/null || echo "localhost")" + +# Build flag +BUILD_RPM=false +BUILDROOT="${HOME}/rpmbuild" +VALIDATE_SPEC=false +DRY_RUN=false +VERBOSE=false + +# Function to display usage +usage() { +exit_code="${1:-1}" + cat << EOF +Usage: $0 [options] + +Generate an RPM spec file from a z/OS pax archive. + +Arguments: + pax_file Path to the pax file (e.g., /path/to/file.pax or file.pax.Z) + +Options: + --name Override package name (default: extracted from filename) + --version Override version (default: extracted from filename) + --pkg-version Override version (alternative to --version) + --release Override release number (default: 1) + --license Specify license (default: Proprietary) + --summary Package summary (required) + --description Package description (default: same as summary) + --url Project URL (default: none) + --requires Package dependencies (e.g., "oef >= 1.1.0") + --output Output spec file (default: .spec) + --build Build the RPM after generating spec file + --buildroot RPM build root directory (default: ~/rpmbuild) + --validate Validate spec file after generation (checks syntax and runs rpmlint) + --dry-run Show what would be done without actually doing it + --verbose Enable verbose debug output + --help Display this help message + --version Display tool version + +Example: + $0 /nfsmnts/bpidrivers/oefv1r1/os390/latest/HAMN110.runnable.pax.Z \\ + --summary "HAMN110 Runtime Package" \\ + --license "IBM" \\ + --url "https://www.ibm.com" + +EOF + exit "$exit_code" +} + +# Function to extract package name and version from filename +parse_filename() { + filename="$1" + basename=$(basename "$filename") + + # Remove .pax.Z or .pax extension + basename="${basename%.pax.Z}" + basename="${basename%.pax}" + + # Handle .zos suffix commonly used in zopen packages + basename="${basename%.zos}" + + # Try to extract name and version using sed since BASH_REMATCH is a bashism + # Pattern 1: NAME-VERSION (e.g., package-1.0, curl-8.10.1.20241001_214340) + if echo "$basename" | grep -qE '^(.+)-([0-9].*)$'; then + PKG_NAME=$(echo "$basename" | sed -E 's/^(.+)-([0-9].*)$/\1/') + PKG_VERSION=$(echo "$basename" | sed -E 's/^(.+)-([0-9].*)$/\2/') + # Pattern 2: NAMEVERSION.suffix (e.g., HAMN110.runnable) + elif echo "$basename" | grep -qE '^([A-Za-z]+)([0-9]+)\.(.+)$'; then + PKG_NAME=$(echo "$basename" | sed -E 's/^([A-Za-z]+)([0-9]+)\.(.+)$/\1/') + PKG_VERSION=$(echo "$basename" | sed -E 's/^([A-Za-z]+)([0-9]+)\.(.+)$/\2/') + # Pattern 3: NAMEVERSION (e.g., HAMN110) + elif echo "$basename" | grep -qE '^([A-Za-z]+)([0-9]+)$'; then + PKG_NAME=$(echo "$basename" | sed -E 's/^([A-Za-z]+)([0-9]+)$/\1/') + PKG_VERSION=$(echo "$basename" | sed -E 's/^([A-Za-z]+)([0-9]+)$/\2/') + else + PKG_NAME="$basename" + PKG_VERSION="1.0" + fi + + # Convert to lowercase and replace invalid characters (keep alphanumeric and hyphens) + PKG_NAME=$(echo "$PKG_NAME" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/^-//;s/-$//') +} + +# Function to analyze pax contents and categorize files +analyze_pax_contents() { + pax_file="$1" + temp_dir=$(mktempdir "pax2rpm") + + printInfo "# Analyzing pax file structure..." + + # Extract pax listing (limit to first 5000 entries for performance/accuracy balance) + case "$pax_file" in + *.pax.Z) + pax_listing=$(pax -vzf "$pax_file" 2>&1 | head -5000) + ;; + *) + pax_listing=$(pax -vf "$pax_file" 2>&1 | head -5000) + ;; + esac + + # Check if pax listing failed or is empty + if [ -z "$pax_listing" ]; then + printWarning "Could not analyze pax contents, using default structure" + return 0 + fi + + # Initialize variables for different file types (removed unused FILE_CATEGORIES) + has_bin=false + has_lib=false + has_include=false + has_etc=false + has_share=false + has_doc=false + has_man=false + has_scripts=false + is_os_layout=false + + # Analyze each file + while IFS= read -r line; do + # Skip empty lines and headers + [ -z "$line" ] && continue + + # Extract filename (last field in pax -v output) + filename=$(echo "$line" | awk '{print $NF}') + [ -z "$filename" ] && continue + + # Detect directory structure (handles top-level dir if present) + # Matches: bin/, ./bin/, package-ver/bin/, ./package-ver/bin/ + # Does NOT match: usr/lpp/IBM/bin/ (too deep) + case "$filename" in + bin/* | */bin/*) + has_bin=true ;; + lib/* | */lib/*) + has_lib=true ;; + include/* | */include/*) + has_include=true ;; + etc/* | */etc/*) + has_etc=true ;; + share/* | */share/*) + has_share=true ;; + doc/* | docs/* | */doc/* | */docs/*) + has_doc=true ;; + man/* | */man/*) + has_man=true ;; + scripts/* | */scripts/*) + has_scripts=true ;; + usr/lpp/* | */usr/lpp/*) + is_os_layout=true ;; + esac + done << EOF +$pax_listing +EOF + + rm -rf "$temp_dir" +} + +# Function to list contents of pax file +list_pax_contents() { + pax_file="$1" + + printHeader "# Pax file contents (first 100 entries):" + + case "$pax_file" in + *.pax.Z) + pax -vzf "$pax_file" 2>&1 | head -100 + ;; + *) + pax -vf "$pax_file" 2>&1 | head -100 + ;; + esac +} +# Function to generate install commands based on analyzed contents +# Function to generate install commands based on analyzed contents +# Function to generate install commands based on analyzed contents +generate_install_commands() { +pax_file="$1" + + cat << 'EOF' +# Install files based on detected structure + +# Detect the source layout +if [ -d usr ]; then + echo "OS layout (usr/) detected, searching for actual content root..." + # Find the deepest directory that contains bin, lib, pyz, bash, or go + # We sort by the number of slashes to find the deepest one + FOUND_PATH=$(find . -type d \( -name bin -o -name lib -o -name pyz -o -name bash -o -name go \) -print | awk -F/ '{print NF, $0}' | sort -rn | head -n 1 | cut -d' ' -f2-) + + if [ -n "$FOUND_PATH" ]; then + FOUND_PATH=${FOUND_PATH#./} + case "$FOUND_PATH" in + */pyz) + ROOT_DIR="$FOUND_PATH" + ;; + *) + ROOT_DIR=$(echo "$FOUND_PATH" | sed 's/\/\(bin\|lib\|bash\|go\)$//') + ;; + esac + else + # Fallback for OS layout: find first dir with more than one entry + ROOT_DIR="." + while [ $(ls -1 "$ROOT_DIR" 2>/dev/null | wc -l) -eq 1 ]; do + SUB_DIR=$(ls -1 "$ROOT_DIR") + if [ -d "$ROOT_DIR/$SUB_DIR" ]; then + ROOT_DIR="$ROOT_DIR/$SUB_DIR" + else + break + fi + done + fi + echo "Detected nested root: $ROOT_DIR" + mkdir -p %{buildroot}%{install_dir} + cp -RP "$ROOT_DIR"/* %{buildroot}%{install_dir}/ +elif [ $(ls -1 | grep -v "^$" | wc -l) -eq 1 ] && [ -d * ]; then + # Standard single-directory layout (e.g., go/) + SUB_DIR=$(ls -1) + echo "Standard single-directory layout detected: $SUB_DIR" + mkdir -p %{buildroot}%{install_dir} + cp -RP "$SUB_DIR"/* %{buildroot}%{install_dir}/ +else + # Flat or mixed layout + echo "Flat or mixed layout detected" + mkdir -p %{buildroot}%{install_dir} + cp -RP * %{buildroot}%{install_dir}/ +fi + +# Install root-level documentation if it hasn't been copied already +# (Matches files in the CURRENT directory, not ROOT_DIR) +for doc in README* LICENSE* COPYING* CHANGELOG* AUTHORS* INSTALL* NEWS*; do + if [ -f "$doc" ]; then + mkdir -p %{buildroot}%{install_dir}/doc + cp -RP "$doc" %{buildroot}%{install_dir}/doc/ + fi +done +EOF +} + + +generate_files_list() { + cat << 'EOF' +%defattr(-,root,root,-) +%{install_dir} +EOF +} + +# Function to generate spec file +generate_spec() { +output_file="$1" +source_file="$2" +date=$(date "+%a %b %d %Y") + + # Analyze the pax contents first (populate global DIR_STRUCTURE) + analyze_pax_contents "$source_file" + + cat > "$output_file" << EOF +# RPM Spec file generated from pax archive +# Source: $source_file +# Generated: $(date) + +%define _rpmfilename %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}-ibm-zos.rpm + +Name: ${PKG_NAME} +Version: ${PKG_VERSION} +Release: ${RELEASE}%{?dist} +Summary: ${SUMMARY} + +%define install_dir /usr/lpp/pkg/IBM/%{name}/%{version} + +License: ${LICENSE} +${URL:+URL: $URL} +Source0: $(basename "$source_file") + +BuildArch: ${BUILD_ARCH} +${REQUIRES:+Requires: $REQUIRES} + +EOF + + if [ "$(uname -s)" = "OS/390" ]; then + cat >> "$output_file" << EOF +# On z/OS, disable broken post-install processing (stripping, etc.) +%define __os_install_post %{nil} + +EOF + fi + + cat >> "$output_file" << EOF +%description +${DESCRIPTION} + +%prep +# Extract pax archive +%setup -c -T +cd %{_builddir} +mkdir -p %{name}-%{version} +cd %{name}-%{version} + +EOF + + case "$source_file" in + *.pax.Z) + cat >> "$output_file" << EOF +# Extract compressed pax archive +pax -rzf %{_sourcedir}/$(basename "$source_file") +EOF + ;; + *) + cat >> "$output_file" << EOF +# Extract pax archive +pax -rf %{_sourcedir}/$(basename "$source_file") +EOF + ;; + esac + + cat >> "$output_file" << EOF + +%build +# No build step required for pre-built pax archives + +%install +rm -rf %{buildroot} + +# Create base installation directories +mkdir -p %{buildroot}%{install_dir} + +$(generate_install_commands "$source_file") + +%files +$(generate_files_list) + +%changelog +* ${date} ${PACKAGER_NAME} <${PACKAGER_EMAIL}> - ${PKG_VERSION}-${RELEASE} +- Initial RPM package created from pax archive +- Source: $(basename "$source_file") +EOF + + echo "RPM spec file generated: $output_file" + echo "" + echo "Detected structure:" + [ "$has_bin" = "true" ] && echo " ✓ Binaries in bin/" + [ "$has_lib" = "true" ] && echo " ✓ Libraries in lib/" + [ "$has_include" = "true" ] && echo " ✓ Headers in include/" + [ "$has_etc" = "true" ] && echo " ✓ Configuration files in etc/" + [ "$has_share" = "true" ] && echo " ✓ Data files in share/" + [ "$has_doc" = "true" ] && echo " ✓ Documentation in doc/" + [ "$has_man" = "true" ] && echo " ✓ Man pages in man/" + [ "$has_scripts" = "true" ] && echo " ✓ Scripts in scripts/" + + return 0 +} + +# Function to setup rpmbuild directories +setup_rpmbuild() { +buildroot="$1" + + echo "Setting up RPM build directories in: $buildroot" + + mkdir -p "$buildroot/BUILD" "$buildroot/BUILDROOT" "$buildroot/RPMS" "$buildroot/SOURCES" "$buildroot/SPECS" "$buildroot/SRPMS" + + if [ ! -f "$HOME/.rpmmacros" ]; then + if touch "$HOME/.rpmmacros" 2>/dev/null; then + cat > "$HOME/.rpmmacros" << EOF +%_topdir $buildroot +EOF + echo "Created ~/.rpmmacros with _topdir set to $buildroot" + else + echo "Warning: could not create ~/.rpmmacros. This is normal in some CI environments." + echo "The script will use '$buildroot' by overriding it with --define \"_topdir $buildroot\"." + fi + else + # Check if _topdir is already defined and different + existing_topdir=$(grep "%_topdir" "$HOME/.rpmmacros" | awk '{print $2}') + if [ -n "$existing_topdir" ] && [ "$existing_topdir" != "$buildroot" ]; then + echo "Warning: ~/.rpmmacros exists and has _topdir set to '$existing_topdir'." + echo "The script will use '$buildroot' by overriding it with --define \"_topdir $buildroot\"." + fi + fi +} + +# Function to check for required tools +check_required_tools() { + missing_tools="" + pax_file="$1" + + # Check for pax + if ! command -v pax > /dev/null 2>&1; then + missing_tools="${missing_tools} pax" + fi + + # Check for rpmbuild if building or validating + if [ "$BUILD_RPM" = true ] || [ "$VALIDATE_SPEC" = true ]; then + if [ "$VERBOSE" = true ]; then + echo "DEBUG: Current PATH is $PATH" >&2 + fi + if ! command -v rpmbuild > /dev/null 2>&1; then + missing_tools="${missing_tools} rpmbuild" + fi + fi + + if [ -n "$missing_tools" ]; then + echo "" >&2 + printError "Missing Required Tools" + echo "The following tools are required but not found:" >&2 + echo "" >&2 + + for tool in $missing_tools; do + echo " ✗ $tool" >&2 + + case "$tool" in + pax) + echo " Purpose: Extract pax archives" >&2 + echo " Install: Usually pre-installed on z/OS" >&2 + echo " On Linux: apt-get install pax (Debian/Ubuntu/RHEL/CentOS)" >&2 + ;; + rpmbuild) + echo " Purpose: Build RPM packages and validate spec files" >&2 + echo " Install: Part of rpm-build package" >&2 + echo " On z/OS: zopen install rpm" >&2 + echo " On Linux: apt-get install rpm (Debian/Ubuntu/RHEL/CentOS)" >&2 + ;; + esac + echo "" >&2 + done + + printInfo "Please install the missing tools and try again." + return 1 + fi + + # Optional tools check (informational only) + optional_missing="" + + if [ "$VALIDATE_SPEC" = true ] && ! command -v rpmlint > /dev/null 2>&1; then + optional_missing="${optional_missing} rpmlint" + fi + + if [ -n "$optional_missing" ]; then + echo "" >&2 + echo "Note: Optional tools not found (validation will be limited):" >&2 + for tool in $optional_missing; do + echo " - $tool (for enhanced spec file quality checks)" >&2 + case "$tool" in + rpmlint) + echo " Install: On z/OS: zopen install rpmlint" >&2 + echo " On Linux: apt-get install rpmlint (Debian/Ubuntu)" >&2 + echo " yum install rpmlint (RHEL/CentOS)" >&2 + ;; + esac + done + echo "" >&2 + fi + + return 0 +} + +# Function to validate spec file syntax +validate_spec_syntax() { + spec_file="$1" + buildroot="$2" + + echo "" + echo "==========================================" + echo "Validating spec file syntax..." + echo "==========================================" + echo "" + + # Check if rpmbuild is available + if ! command -v rpmbuild > /dev/null 2>&1; then + echo "Warning: rpmbuild not found, skipping syntax validation" >&2 + return 0 + fi + + # Try to use rpmbuild -bp (prep stage) to validate syntax + # This will check syntax without actually building + # Use --nodeps to check syntax/prep logic without requiring local RPM DB to have all deps + temp_output=$(mktemp) + if rpmbuild --define "_topdir $buildroot" -bp --nodeps "$spec_file" > "$temp_output" 2>&1; then + echo "✓ Spec file syntax is valid" + rm -f "$temp_output" + return 0 + else + # Check if it's a real syntax error or just missing source files + if grep -q "No such file or directory" "$temp_output" || grep -q "does not exist" "$temp_output"; then + echo "✓ Spec file syntax appears valid (source files not present for full validation)" + rm -f "$temp_output" + return 0 + else + echo "✗ Spec file has syntax errors:" >&2 + cat "$temp_output" | head -20 + rm -f "$temp_output" + return 1 + fi + fi +} + +# Function to run rpmlint on spec file +run_rpmlint() { + spec_file="$1" + + echo "" + echo "==========================================" + echo "Running rpmlint checks..." + echo "==========================================" + echo "" + + # Check if rpmlint is available + if ! command -v rpmlint > /dev/null 2>&1; then + echo "Note: rpmlint not found, skipping quality checks" + echo "Install rpmlint for additional spec file validation" + return 0 + fi + + # Run rpmlint + rpmlint_output=$(rpmlint "$spec_file" 2>&1 || true) + rpmlint_exit=$? + + echo "$rpmlint_output" + echo "" + + # Count errors and warnings + errors=$(echo "$rpmlint_output" | grep -c "E:" || true) + warnings=$(echo "$rpmlint_output" | grep -c "W:" || true) + + if [ $errors -gt 0 ]; then + echo "⚠ Found $errors error(s) and $warnings warning(s)" + echo "Please review and fix errors before building" + return 1 + elif [ $warnings -gt 0 ]; then + echo "⚠ Found $warnings warning(s)" + echo "Warnings can often be ignored, but please review them" + return 0 + else + echo "✓ No errors or warnings found" + return 0 + fi +} + +# Function to validate generated spec file +validate_spec_file() { +spec_file="$1" +pax_file="$2" +buildroot="$3" +validation_failed=false + + echo "" + echo "==========================================" + echo "Validating generated spec file..." + echo "==========================================" + + # Setup environment for validation (rpmbuild -bp needs source) + setup_rpmbuild "$buildroot" + + # Copy pax file to SOURCES if not already there +source_name=$(basename "$pax_file") + if [ ! -f "$buildroot/SOURCES/$source_name" ] || [ "$pax_file" -nt "$buildroot/SOURCES/$source_name" ]; then + echo "Copying source file to $buildroot/SOURCES/$source_name for validation" + cp "$pax_file" "$buildroot/SOURCES/" + fi + + # Check syntax + if ! validate_spec_syntax "$spec_file" "$buildroot"; then + validation_failed=true + fi + + # Run rpmlint + if ! run_rpmlint "$spec_file"; then + validation_failed=true + fi + + if [ "$validation_failed" = true ]; then + echo "" + echo "==========================================" + echo "✗ Validation failed" + echo "==========================================" + echo "" + echo "Please review the errors above and edit the spec file:" + echo " $spec_file" + echo "" + return 1 + else + echo "" + echo "==========================================" + echo "✓ Validation passed" + echo "==========================================" + echo "" + return 0 + fi +} + +# Function to perform dry run +perform_dry_run() { +pax_file="$1" + + echo "" + echo "==========================================" + echo "DRY RUN MODE - No files will be created" + echo "==========================================" + echo "" + + echo "Would perform the following actions:" + echo "" + echo "1. Analyze pax file: $pax_file" + echo "2. Extract package information:" + echo " - Name: $PKG_NAME" + echo " - Version: $PKG_VERSION" + echo " - Release: $RELEASE" + echo "3. Generate spec file: $OUTPUT_FILE" + + if [ "$VALIDATE_SPEC" = true ]; then + echo "4. Validate spec file syntax and run rpmlint" + fi + + if [ "$BUILD_RPM" = true ]; then + echo "5. Build RPM package in: $BUILDROOT" + echo " - Copy $pax_file to $BUILDROOT/SOURCES/" + echo " - Copy spec file to $BUILDROOT/SPECS/" + echo " - Run: rpmbuild --define \"_topdir $BUILDROOT\" -ba $OUTPUT_FILE" + fi + + echo "" + echo "To execute these actions, run without --dry-run flag" + echo "" +} + +# Function to build RPM +build_rpm() { +spec_file="$1" +pax_file="$2" +buildroot="$3" + + echo "" + echo "==========================================" + echo "Building RPM package..." + echo "==========================================" + echo "" + + # Setup rpmbuild directories + setup_rpmbuild "$buildroot" + + # Copy pax file to SOURCES +source_name=$(basename "$pax_file") + echo "Copying source file to $buildroot/SOURCES/$source_name" + cp "$pax_file" "$buildroot/SOURCES/" + + # Copy spec file to SPECS + echo "Copying spec file to $buildroot/SPECS/" + cp "$spec_file" "$buildroot/SPECS/" + + # Build the RPM + echo "" + echo "Running rpmbuild --define \"_topdir $buildroot\" -ba $spec_file" + echo "" + + if rpmbuild --define "_topdir $buildroot" -ba "$spec_file"; then + echo "" + echo "==========================================" + echo "✓ RPM build completed successfully!" + echo "==========================================" + echo "" + echo "Generated packages:" + echo "" + echo "Binary RPMs:" + find "$buildroot/RPMS" -name "*.rpm" -type f 2>/dev/null | while read rpm; do + echo " - $rpm" + done + echo "" + echo "Source RPMs:" + find "$buildroot/SRPMS" -name "*.rpm" -type f 2>/dev/null | while read rpm; do + echo " - $rpm" + done + echo "" + return 0 + else + echo "" + echo "==========================================" + echo "✗ RPM build failed!" + echo "==========================================" + echo "" + echo "Please review the spec file and build output above." + echo "Common issues:" + echo " - Missing BuildRequires dependencies" + echo " - Incorrect file paths in %install or %files sections" + echo " - Pax extraction errors" + echo "" + return 1 + fi +} + +# Main script +main() { + if [ $# -eq 0 ]; then + usage + fi + + case "$1" in + --help) + usage 0 + ;; + --version) + if [ -x "${MYDIR}/zopen-version" ]; then + "${MYDIR}/zopen-version" "${ME}" + else + echo "${ME} version $(cat "${INCDIR}/zopen_version" 2>/dev/null || echo "unknown")" + fi + exit 0 + ;; + esac + + PAX_FILE="$1" + shift + + # Check if pax file exists + if [ ! -f "$PAX_FILE" ]; then + echo "Error: Pax file not found: $PAX_FILE" >&2 + exit 1 + fi + + # Parse filename to get default name and version + parse_filename "$PAX_FILE" + + # Parse command line options + while [ $# -gt 0 ]; do + case "$1" in + --name) + [ -n "$2" ] || { echo "Error: --name requires a value" >&2; usage; } + PKG_NAME="$2" + shift 2 + ;; + --version) + case "$2" in + "" | --*) + if [ -x "${MYDIR}/zopen-version" ]; then + "${MYDIR}/zopen-version" "${ME}" + else + echo "${ME} version $(cat "${INCDIR}/zopen_version" 2>/dev/null || echo "unknown")" + fi + exit 0 + ;; + *) + PKG_VERSION="$2" + shift 2 + ;; + esac + ;; + --pkg-version) + [ -n "$2" ] || { echo "Error: --pkg-version requires a value" >&2; usage; } + PKG_VERSION="$2" + shift 2 + ;; + --release) + [ -n "$2" ] || { echo "Error: --release requires a value" >&2; usage; } + RELEASE="$2" + shift 2 + ;; + --license) + [ -n "$2" ] || { echo "Error: --license requires a value" >&2; usage; } + LICENSE="$2" + shift 2 + ;; + --summary) + [ -n "$2" ] || { echo "Error: --summary requires a value" >&2; usage; } + SUMMARY="$2" + shift 2 + ;; + --description) + [ -n "$2" ] || { echo "Error: --description requires a value" >&2; usage; } + DESCRIPTION="$2" + shift 2 + ;; + --url) + [ -n "$2" ] || { echo "Error: --url requires a value" >&2; usage; } + URL="$2" + shift 2 + ;; + --output) + [ -n "$2" ] || { echo "Error: --output requires a value" >&2; usage; } + OUTPUT_FILE="$2" + shift 2 + ;; + --requires) + [ -n "$2" ] || { echo "Error: --requires requires a value" >&2; usage; } + REQUIRES="$2" + shift 2 + ;; + --build) + BUILD_RPM=true + shift + ;; + --buildroot) + [ -n "$2" ] || { echo "Error: --buildroot requires a value" >&2; usage; } + BUILDROOT="$2" + shift 2 + ;; + --validate) + VALIDATE_SPEC=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --verbose) + VERBOSE=true + set -x + shift + ;; + --help) + usage 0 + ;; + *) + echo "Error: Unknown option: $1" >&2 + usage + ;; + esac + done + + # Set defaults for required fields + if [ -z "$SUMMARY" ]; then + echo "Error: --summary is required" >&2 + exit 1 + fi + + if [ -z "$DESCRIPTION" ]; then + DESCRIPTION="$SUMMARY" + fi + + if [ -z "$OUTPUT_FILE" ]; then + OUTPUT_FILE="${PKG_NAME}.spec" + fi + + # Check for required tools + if ! check_required_tools "$PAX_FILE"; then + exit 1 + fi + + # Display extracted information + echo "Package Information:" + echo " Name: $PKG_NAME" + echo " Version: $PKG_VERSION" + echo " Release: $RELEASE" + echo " License: $LICENSE" + echo " Summary: $SUMMARY" + echo " Source: $(basename "$PAX_FILE")" + echo " Output: $OUTPUT_FILE" + echo "" + + # Handle dry-run mode + if [ "$DRY_RUN" = true ]; then + perform_dry_run "$PAX_FILE" + exit 0 + fi + + # List pax contents for reference + list_pax_contents "$PAX_FILE" || echo " (Could not list contents)" + echo "" + + # Generate spec file + generate_spec "$OUTPUT_FILE" "$PAX_FILE" + + # Validate spec file if requested + if [ "$VALIDATE_SPEC" = true ]; then + if ! validate_spec_file "$OUTPUT_FILE" "$PAX_FILE" "$BUILDROOT"; then + echo "Validation failed. Please fix the errors and try again." >&2 + exit 1 + fi + fi + + # Build RPM if requested + if [ "$BUILD_RPM" = true ]; then + if build_rpm "$OUTPUT_FILE" "$PAX_FILE" "$BUILDROOT"; then + echo "RPM package built successfully!" + else + echo "Warning: RPM build failed. Please review the spec file and try again." >&2 + exit 1 + fi + else + echo "" + echo "Next steps:" + echo " 1. Review and edit the generated spec file: $OUTPUT_FILE" + echo " 2. Adjust the %install and %files sections if needed" + if [ "$VALIDATE_SPEC" = false ]; then + echo " 3. Validate the spec: $0 $PAX_FILE --summary \"$SUMMARY\" --validate" + echo " 4. Copy the pax file to rpmbuild/SOURCES/" + echo " 5. Build the RPM: rpmbuild --define \"_topdir $BUILDROOT\" -ba $OUTPUT_FILE" + else + echo " 3. Copy the pax file to rpmbuild/SOURCES/" + echo " 4. Build the RPM: rpmbuild --define \"_topdir $BUILDROOT\" -ba $OUTPUT_FILE" + fi + echo "" + echo "Or run with --build flag to build automatically:" + echo " $0 $PAX_FILE --summary \"$SUMMARY\" --build" + fi +} + +main "$@" diff --git a/bin/zopen-publish b/bin/zopen-publish index 7fbfbc987..f8fc21dd3 100755 --- a/bin/zopen-publish +++ b/bin/zopen-publish @@ -8,7 +8,7 @@ setupMyself() { ME=$(basename "$0") MYDIR="$(cd "$(dirname "$0")" > /dev/null 2>&1 && pwd -P)" INCDIR="${MYDIR}/../include" - if ! [ -d "${INCDIR}" ] && ! [ -f "${INCDIR}/common.sh" ]; then + if ! [ -d "${INCDIR}" ] || ! [ -f "${INCDIR}/common.sh" ]; then echo "Internal Error. Unable to find common.sh file to source." >&2 exit 8 fi @@ -40,6 +40,8 @@ printSyntax() echo " GitHub Personal Access Token (required for GitHub publish, or set GITHUB_TOKEN env var)" echo " -o, --github-org ORG GitHub Organization (default: zopencommunity)" echo " -l, --latest Mark release as 'Latest' (not pre-release)." + echo " --pulp Push RPM artifacts to Pulp repository." + echo " --pulp-repo REPO Pulp repository name (default: zopen-stable or zopen-dev)" echo " -w, --whl WHL_FILE Path to a Python wheel file to publish to Pulp PyPI" echo " --pulp-url URL Pulp PyPI repository URL (e.g., http://host:8080/pypi/repo/)" echo " --pulp-user USER Pulp username (or set PULP_USER env var, default: admin)" @@ -72,6 +74,12 @@ checkDependencies() { printError "Error: jq command not found. Please install it." return 1 fi + if [ "$PUSH_TO_PULP" = true ]; then + if ! command -v pulp > /dev/null 2>&1; then + printError "Error: pulp command not found. Please install it to use --pulp option." + return 1 + fi + fi return 0 } @@ -302,9 +310,9 @@ createGitHubRelease() { fi - if [ -z "$release_id" ] || [[ "$release_id" == "null" ]]; then + if [ -z "$release_id" ] || [ "$release_id" = "null" ]; then printError "Error creating GitHub release tag: $tag.\n$error_message" - if [ -n "$api_error_message" ] && [[ "$api_error_message" != "null" ]]; then + if [ -n "$api_error_message" ] && [ "$api_error_message" != "null" ]; then printError "GitHub API Error: $api_error_message" fi printError "Full GitHub API response (for debugging):" @@ -349,9 +357,9 @@ uploadGitHubReleaseAsset() { return 1 fi - if [ -z "$asset_state" ] || [[ "$asset_state" == "null" ]] || [ "$asset_state" != "uploaded" ]; then + if [ -z "$asset_state" ] || [ "$asset_state" = "null" ] || [ "$asset_state" != "uploaded" ]; then printError "Error uploading asset: $asset_file to release ID: $release_id" - if [ -n "$api_error_message" ] && [[ "$api_error_message" != "null" ]]; then + if [ -n "$api_error_message" ] && [ "$api_error_message" != "null" ]; then printError "GitHub API Error: $api_error_message" fi printError "Full GitHub API response (for debugging):" # Print full response for debugging @@ -389,7 +397,7 @@ deleteGitHubReleaseByTag() { fi - if [ -z "$release_id" ] || [[ "$release_id" == "null" ]]; then + if [ -z "$release_id" ] || [ "$release_id" = "null" ]; then printVerbose "No existing release found for tag: $tag. Skipping deletion." return 0 # No release to delete - consider it success. fi @@ -419,7 +427,7 @@ deleteGitHubReleaseByTag() { fi - if [ -n "$delete_api_error_message" ] && [[ "$delete_api_error_message" != "null" ]]; then + if [ -n "$delete_api_error_message" ] && [ "$delete_api_error_message" != "null" ]; then printError "Error deleting GitHub release ID: $release_id for tag: $tag. GitHub API Error: $delete_api_error_message. Full GitHub API response (for debugging): $DELETE_RESPONSE" return 1 fi @@ -428,6 +436,50 @@ deleteGitHubReleaseByTag() { return 0 } +uploadToPulp() { + rpm_file="$1" + build_line="$2" + + # Default to repo.zopen.community if not specified in environment + if [ -z "$PULP_BASE_ADDR" ]; then + export PULP_BASE_ADDR="https://repo.zopen.community" + printVerbose "PULP_BASE_ADDR not set; defaulting to $PULP_BASE_ADDR" + fi + + if [ -n "$CUSTOM_PULP_REPO" ]; then + PULP_REPO="$CUSTOM_PULP_REPO" + elif [ "$build_line" = "STABLE" ]; then + PULP_REPO="zopen-stable" + else + PULP_REPO="zopen-dev" + fi + + printInfo "- Pushing $rpm_file to Pulp repository: $PULP_REPO..." + + # Perform the upload and automatic repository version update + if ! pulp rpm repository upload --name "$PULP_REPO" --file "$rpm_file"; then + printError "Failed to push RPM to Pulp repository: $PULP_REPO" + return 1 + fi + + printVerbose "Successfully pushed $rpm_file to Pulp repository: $PULP_REPO" + + # Update the distribution to point to the latest publication + DIST_NAME="$PULP_REPO" + + printInfo "- Updating Pulp distribution $DIST_NAME..." + LATEST_PUB=$(pulp rpm publication list --repository "${PULP_REPO}" --limit 1 | jq -r '.[0].pulp_href') + if [ -n "$LATEST_PUB" ] && [ "$LATEST_PUB" != "null" ]; then + if ! pulp rpm distribution update --name "${DIST_NAME}" --publication "${LATEST_PUB}" > /dev/null; then + printWarning "Warning: Failed to update Pulp distribution ${DIST_NAME}" + fi + else + printWarning "Warning: Could not find latest publication for ${PULP_REPO}" + fi + + return 0 +} + publishRelease() { PAX_BASENAME=$(basename "${PAX_FILE}") @@ -472,7 +524,7 @@ publishRelease() { if [ -z "$BUILD_LINE_OPT" ]; then BUILD_LINE=$(jq -r '.product.buildline' "$METADATA_FILE") BUILD_LINE_UPPER=$(echo "$BUILD_LINE" | tr '[:lower:]' '[:upper:]') - if [[ "$BUILD_LINE_UPPER" != "DEV" && "$BUILD_LINE_UPPER" != "STABLE" ]]; then + if [ "$BUILD_LINE_UPPER" != "DEV" ] && [ "$BUILD_LINE_UPPER" != "STABLE" ]; then printError "Error: Invalid build line in metadata.json: $BUILD_LINE. Must be 'DEV' or 'STABLE'." return 1 fi @@ -482,7 +534,7 @@ publishRelease() { else BUILD_LINE="$BUILD_LINE_OPT" BUILD_LINE_UPPER=$(echo "$BUILD_LINE" | tr '[:lower:]' '[:upper:]') - if [[ "$BUILD_LINE_UPPER" != "DEV" && "$BUILD_LINE_UPPER" != "STABLE" ]]; then + if [ "$BUILD_LINE_UPPER" != "DEV" ] && [ "$BUILD_LINE_UPPER" != "STABLE" ]; then printError "Invalid build line: $BUILD_LINE. Must be 'DEV' or 'STABLE'." printSyntax return 1 @@ -498,7 +550,7 @@ publishRelease() { DESCRIPTION="${PORT_DESCRIPTION}
" DESCRIPTION="${DESCRIPTION}Version: ${VERSION}
" URL_LINE="https://github.com/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/releases/download/${TAG}/${PAX_BASENAME}" - DESCRIPTION="${DESCRIPTION}
Command to download and install on z/OS (if you have curl) 
curl -o ${PAX_BASENAME} -L ${URL_LINE} && pax -rf ${PAX_BASENAME} && cd ${PORT_NAME} && . ./.env
" + DESCRIPTION="${DESCRIPTION}
Command to download and install on z/OS (if you have curl) 
curl -o ${PAX_BASENAME} -L ${URL_LINE} && pax -rf ${PAX_BASENAME} && cd ${PORT_NAME} && . ./setup.sh
" DESCRIPTION="${DESCRIPTION}
Or use: 
zopen install ${PORT_NAME}
" if ${FORCE_OVERWRITE} || ! deleteGitHubReleaseByTag "${TAG}"; then @@ -529,6 +581,43 @@ publishRelease() { return 1 # metadata.json upload failed fi + if [ "$PUSH_TO_PULP" = true ]; then + # Look for RPMs in rpmbuild/RPMS first (typical build location) + RPM_COUNT=0 + RPM_DIRS="rpmbuild/RPMS" + + # Check if rpmbuild/RPMS exists + if [ -d "$RPM_DIRS" ]; then + for rpm_file in "$RPM_DIRS"/*.rpm; do + if [ -f "$rpm_file" ]; then + if ! uploadToPulp "$rpm_file" "$BUILD_LINE"; then + return 1 + fi + RPM_COUNT=$((RPM_COUNT + 1)) + fi + done + fi + + # If not found there, look in the same directory as the PAX file + if [ $RPM_COUNT -eq 0 ]; then + PAX_DIR=$(dirname "$PAX_FILE") + for rpm_file in "$PAX_DIR"/*.rpm; do + if [ -f "$rpm_file" ]; then + if ! uploadToPulp "$rpm_file" "$BUILD_LINE"; then + return 1 + fi + RPM_COUNT=$((RPM_COUNT + 1)) + fi + done + fi + + if [ $RPM_COUNT -gt 0 ]; then + printInfo "- Found $RPM_COUNT RPM(s) to push to Pulp." + else + printWarning "Pulp push requested, but no RPM files were found." + fi + fi + echo "Release published successfully to https://github.com/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/releases/tag/${TAG}" return 0 diff --git a/cicd/build.groovy b/cicd/build.groovy index 8d471bc4c..fb173c7a1 100755 --- a/cicd/build.groovy +++ b/cicd/build.groovy @@ -59,7 +59,7 @@ fi git clone -b "${PORT_BRANCH}" "${PORT_GITHUB_REPO}" ${PORT_NAME} && cd ${PORT_NAME} # Always run tests and update dependencies and generate pax file -zopen build -v -b release -u -gp -sp --no-set-active $extraOptions +zopen build -v -b release -u -gr -sp --no-set-active $extraOptions # Clean the cache after build is complete zopen clean -c -v diff --git a/cicd/pipeline.jenkins b/cicd/pipeline.jenkins index 9dbe318bf..d79dda45a 100644 --- a/cicd/pipeline.jenkins +++ b/cicd/pipeline.jenkins @@ -27,7 +27,7 @@ node('linux') { stage('Build and Test') { // Build Job : https://cicd.zopen.community/view/Framework/job/Port-Build/ - buildResult = build job: 'Port-Build', propagate: false, parameters: [string(name: 'PORT_GITHUB_REPO', value: "${repo}"), + buildResult = build job: 'Port-Build-Test', propagate: false, parameters: [string(name: 'PORT_GITHUB_REPO', value: "${repo}"), string(name: 'PORT_BRANCH', value: "${branch}"), string(name: 'node', value: "${node_label}"), booleanParam(name: 'FORCE_CLANG', value: params.FORCE_CLANG), @@ -39,7 +39,7 @@ node('linux') } copyArtifacts filter: '**/test.status', fingerprintArtifacts: true, - projectName: 'Port-Build', + projectName: 'Port-Build-Test', selector: specific(buildResult.number.toString()), optional: true def testStatusFile = sh(script: 'find . -name test.status | head -n 1', returnStdout: true).trim() @@ -52,7 +52,7 @@ node('linux') stage('Promote') { if(!params.NO_PROMOTE) { // Publish Job : https://128.168.139.253:8443/view/Framework/job/Port-Publish/ - promoteResult = build job: 'Port-Publish', propagate: false, parameters: + promoteResult = build job: 'Port-Publish-Test', propagate: false, parameters: [string(name: 'BUILD_SELECTOR', value: " ${buildResult.number.toString()}"), string(name: 'PORT_GITHUB_REPO', value: "${repo}"), string(name: 'PORT_DESCRIPTION', value: "${description}"), string(name: 'BUILD_LINE', value: "${build_line}") diff --git a/cicd/publish.groovy b/cicd/publish.groovy index 9336e696a..dafbe8bb6 100755 --- a/cicd/publish.groovy +++ b/cicd/publish.groovy @@ -1,229 +1,201 @@ #!/bin/bash -# Publish Job : https://cicd.zopen.community/view/Framework/job/Port-Publish/ -# This publish job will a pax.Z artifact from the Port-Build (https://cicd.zopen.community/view/Framework/job/Port-Build/) -# Inputs: -# - PORT_GITHUB_REPO : Github repoistory to publish the artifact to e.g: https://github.com/zopencommunity/xzport.git -# - PORT_DESCRIPTION : Description of the tool that is presented in the Github release page -# - BUILD_LINE: dev or stable -# zos.pax.Z artifact is copied as input -# requires a GITHUB_TOKEN environment variable (already configured in Jenkins) -# Output: -# - pax.Z artifact is published as a Jenkins artifact -# - package is copied to /jenkins/build +set -euo pipefail + +echo "=== STARTING PUBLISH JOB ===" + +# --- REQUIRED ENV --- +: "${PORT_GITHUB_REPO:?Missing PORT_GITHUB_REPO}" +: "${PORT_DESCRIPTION:?Missing PORT_DESCRIPTION}" +: "${BUILD_NUMBER:?Missing BUILD_NUMBER}" +: "${GITHUB_TOKEN:?Missing GITHUB_TOKEN}" +: "${PULP_USERNAME:?Missing PULP_USERNAME}" +: "${PULP_PASSWORD:?Missing PULP_PASSWORD}" + +# --- STATIC CONFIG --- GITHUB_ORGANIZATION="zopencommunity" -RELEASE_NOTES_SCRIPT="tools/create_release_notes.sh" # Path to your release notes script +RELEASE_NOTES_SCRIPT="tools/create_release_notes.sh" +PULP_HOST="https://repo.zopen.community" # ✅ FIXED +BUILD_LINE=${BUILD_LINE:-DEV} + +# --- DERIVE VALUES --- RELEASE_PREFIX=$(basename "${PORT_GITHUB_REPO}") -# Used for Release/Tag name RELEASE_PREFIX=${RELEASE_PREFIX%%.*} -PORT_NAME=${RELEASE_PREFIX%%port} -# Get the REPO name +PORT_NAME=$(echo "$RELEASE_PREFIX" | sed 's/port$//') GITHUB_REPO=$RELEASE_PREFIX -# PAX file should be a copied artifact -PAX=`find . -type f -path "*install/*zos.pax.Z"` -METADATA=`find . -type f -path "*install/metadata.json"` -BUILD_STATUS=`find . -name "test.status" | xargs cat` -DEPENDENCIES=`find . -name ".runtimedeps" | xargs cat` -BUILD_DEPENDENCIES=`find . -name ".builddeps" | xargs cat` -VERSION=`find . -name "*install/.version" | xargs cat` - -if [ ! -f "$PAX" ]; then - echo "Port pax file does not exist"; - exit 1; -fi +# --- FIND FILES --- +PAX=$(find . -type f -name "*zos.pax.Z" | head -n 1) +METADATA=$(find . -type f -name "metadata.json" | head -n 1) -if [ ! -f "$METADATA" ]; then - echo "Port metadata.json file does not exist"; - exit 1; -fi +[ -f "$PAX" ] || { echo "ERROR: Missing PAX file"; exit 1; } +[ -f "$METADATA" ] || { echo "ERROR: Missing metadata.json"; exit 1; } -echo "PAX: $PAX" -echo "METADATA: $METADATA" +# --- RPM FILES --- +RPM_FILES=() +while IFS= read -r -d '' f; do + RPM_FILES+=("$f") +done < <(find rpmbuild/RPMS -type f -name "*.rpm" -print0 2>/dev/null) -if [ -z "$DEPENDENCIES" ]; then - DEPENDENCIES="No dependencies"; -fi +NUM_RPMS=${#RPM_FILES[@]} -if [ -z "$BUILD_DEPENDENCIES" ]; then - BUILD_DEPENDENCIES="No dependencies"; -fi +# --- INFO --- +BUILD_STATUS=$(find . -name "test.status" -exec cat {} + 2>/dev/null || echo "Unknown") +DEPENDENCIES=$(find . -name ".runtimedeps" -exec cat {} + 2>/dev/null || echo "None") +VERSION=$(find . -name ".version" -exec cat {} + 2>/dev/null || echo "unknown") -if [ ! -z "$VERSION" ]; then - VERSION="$VERSION "; -fi +unset http_proxy https_proxy -PAX_BASENAME=$(basename "${PAX}") -DIR_NAME=${PAX_BASENAME%%.pax.Z} -DIR_NAME=$(echo "$DIR_NAME" | sed -e "s/\.202[0-9]*_[0-9]*\.zos/.zos/g" -e "s/\.zos//g") -BUILD_ID=${BUILD_NUMBER} +PAX_BASENAME=$(basename "$PAX") +TAG="${BUILD_LINE}_${RELEASE_PREFIX}_${BUILD_NUMBER}" +NAME="${PORT_NAME} ${VERSION} (Build ${BUILD_NUMBER}) - (${BUILD_LINE})" -# Check for python dependencies -if pip3 show numpy &> /dev/null; then - echo "NumPy is already installed." +# --- RELEASE NOTES --- +if [ -f "$RELEASE_NOTES_SCRIPT" ]; then + RELEASE_NOTES=$("$RELEASE_NOTES_SCRIPT" -m release -p "$GITHUB_REPO" -t markdown || echo "No notes") else - echo "NumPy is not installed. Installing now..." - pip3 install numpy + RELEASE_NOTES="No release notes" fi -if pip3 show PyGithub &> /dev/null; then - echo "PyGithub is already installed." -else - echo "PyGithub is not installed. Installing now..." - pip3 install PyGithub -fi +echo "$RELEASE_NOTES" > CHANGELOG.md -if pip3 show matplotlib &> /dev/null; then - echo "matplotlib is already installed." -else - echo "matplotlib is not installed. Installing now..." - pip3 install matplotlib -fi +DESCRIPTION="${PORT_DESCRIPTION}
Status: ${BUILD_STATUS}
" +DESCRIPTION+="Dependencies: ${DEPENDENCIES}
" +DESCRIPTION+=$'\n\n'"$RELEASE_NOTES" -# Needed for uploading releases -unset http_proxy -unset https_proxy +# --- TOOL CHECK --- +command -v github-release >/dev/null || { echo "ERROR: github-release not installed"; exit 1; } -DESCRIPTION="${PORT_DESCRIPTION}" -DESCRIPTION="${DESCRIPTION}
Test Status: ${BUILD_STATUS}
" -DESCRIPTION="${DESCRIPTION}Runtime Dependencies: ${DEPENDENCIES}
" -DESCRIPTION="${DESCRIPTION}Build Dependencies: ${BUILD_DEPENDENCIES}
" - -if [ -z "$BUILD_LINE" ]; then - BUILD_LINE="STABLE" +# --- CREATE / RECREATE RELEASE --- +if github-release info -u "$GITHUB_ORGANIZATION" -r "$GITHUB_REPO" --tag "$TAG" -j &>/dev/null; then + github-release delete --user "$GITHUB_ORGANIZATION" --repo "$GITHUB_REPO" --tag "$TAG" fi -TAG="${BUILD_LINE}_${RELEASE_PREFIX}_${BUILD_ID}" +github-release release \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$NAME" \ + --description "$DESCRIPTION" -URL_LINE="https://github.com/${GITHUB_ORGANIZATION}/${GITHUB_REPO}/releases/download/${TAG}/$PAX_BASENAME" -DESCRIPTION="${DESCRIPTION}
Command to download and install on z/OS (if you have curl) 
curl -o ${PAX_BASENAME} -L ${URL_LINE} && pax -rf ${PAX_BASENAME} && cd $DIR_NAME && . ./.env
" -DESCRIPTION="${DESCRIPTION}
Or use: 
zopen install ${PORT_NAME}
" +# --- VALIDATE RELEASE (FIXES YOUR ERROR) --- +github-release info \ + -u "$GITHUB_ORGANIZATION" \ + -r "$GITHUB_REPO" \ + --tag "$TAG" || { + echo "ERROR: Release creation failed" + exit 1 +} -NAME="${PORT_NAME} ${VERSION}(Build ${BUILD_ID}) - ($BUILD_LINE)" +# --- UPLOAD PAX --- +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$PAX_BASENAME" \ + --file "$PAX" -# --- Generate Release Notes using create_release_notes.sh in Markdown format --- -RELEASE_NOTES_MD_CONTENT=$( - "$RELEASE_NOTES_SCRIPT" -m release -p "$GITHUB_REPO" -t markdown -) +# ========================================================= +# 🔧 PULP SETUP (AUTO INSTALL + CONFIG) +# ========================================================= -# --- Generate Release Notes using create_release_notes.sh in Text format --- -RELEASE_NOTES_TXT_CONTENT=$( # New variable for text content - "$RELEASE_NOTES_SCRIPT" -m release -p "$GITHUB_REPO" -t text -) +echo "=== Checking Pulp CLI ===" +if ! command -v pulp >/dev/null 2>&1; then + echo "Pulp CLI not found. Installing..." -if [ -z "$RELEASE_NOTES_MD_CONTENT" ]; then - echo "Error: No Markdown release notes were generated by $RELEASE_NOTES_SCRIPT." - exit 1 -else - echo "Markdown release notes generated successfully by $RELEASE_NOTES_SCRIPT" + if command -v pip3 >/dev/null 2>&1; then + python3 -m pip install --user pulp-cli || true + export PATH="$HOME/.local/bin:$PATH" + fi fi -if [ -z "$RELEASE_NOTES_TXT_CONTENT" ]; then - echo "Error: No Text release notes were generated by $RELEASE_NOTES_SCRIPT." - exit 1 +if command -v pulp >/dev/null 2>&1; then + echo "Pulp CLI available" + + pulp config create \ + --base-url "$PULP_HOST" \ + --username "$PULP_USERNAME" \ + --password "$PULP_PASSWORD" \ + --overwrite + + pulp status || { + echo "ERROR: Pulp connection failed" + exit 1 + } + + PULP_AVAILABLE=true else - echo "Text release notes generated successfully by $RELEASE_NOTES_SCRIPT" + echo "WARNING: Pulp not available, skipping RPM upload" + PULP_AVAILABLE=false fi +# ========================================================= +# 📦 RPM UPLOAD +# ========================================================= -# --- Create release notes files --- -CHANGELOG_TXT_FILE="CHANGELOG.txt" -RELEASE_NOTES_MD_FILE="CHANGELOG.md" +PULP_REPO="zopen" -echo "$RELEASE_NOTES_TXT_CONTENT" > "$CHANGELOG_TXT_FILE" # Write TEXT release notes to .txt file # Using new content variable -echo "$RELEASE_NOTES_MD_CONTENT" > "$RELEASE_NOTES_MD_FILE" # Write Markdown notes to .md file +if [ "$NUM_RPMS" -gt 0 ] && [ "$PULP_AVAILABLE" = true ]; then -echo "Release notes written to $CHANGELOG_TXT_FILE" -echo "Release notes written to $RELEASE_NOTES_MD_FILE" + echo "--- Configuring Pulp Pipeline ---" -# --- Append CHANGELOG.md content to DESCRIPTION --- -DESCRIPTION_WITH_CHANGELOG="$DESCRIPTION" # Start with the basic description -DESCRIPTION_WITH_CHANGELOG+=$'\n\n' # Add a couple of newlines to separate -DESCRIPTION_WITH_CHANGELOG+="$RELEASE_NOTES_MD_CONTENT" # Append the Markdown changelog -DESCRIPTION="$DESCRIPTION_WITH_CHANGELOG" # Update DESCRIPTION to include changelog + # 1. Enable autopublish so every upload triggers a new live version + pulp rpm repository update --name "$PULP_REPO" --autopublish + # 2. Link the distribution to the repository + pulp rpm distribution update --name "$PULP_REPO" --repository "$PULP_REPO" -exists=$(github-release info -u ${GITHUB_ORGANIZATION} -r ${GITHUB_REPO} --tag "${TAG}" -j) -if [ $? -gt 0 ]; then - echo "Creating a new tag in github" - github-release -v release --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "${NAME}" --description "${DESCRIPTION}" # Use --description - if [ $? -eq 0 ]; then - echo "Release created successfully!" - else - echo "Failed to create release." - exit 1 - fi -else - echo "Deleting and creating new tag" - github-release -v delete --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" - github-release -v release --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "${NAME}" --description "${DESCRIPTION}" # Use --description - if [ $? -eq 0 ]; then - echo "Release recreated successfully!" - else - echo "Failed to recreate release." - exit 1 - fi -fi + echo "Uploading $NUM_RPMS RPMs to Pulp repo: $PULP_REPO" + + for RPM in "${RPM_FILES[@]}"; do + RPM_NAME=$(basename "$RPM") + + # Upload to GitHub + github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$RPM_NAME" \ + --file "$RPM" -# check up to 5 times for the release to appear on github -for i in {1..5}; do - github-release info -u "${GITHUB_ORGANIZATION}" -r "${GITHUB_REPO}" --tag "${TAG}" -j - if [ $? -eq 0 ]; then - echo "Release found!" - break - else - if [ $i -eq 5 ]; then - echo "Command failed after 5 attempts. Recreating release..." - github-release -v release --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "${NAME}" --description "${DESCRIPTION}" # Use --description - if [ $? -eq 0 ]; then - echo "Release recreated successfully!" + # Upload to Pulp (with retry) + for i in 1 2 3; do + if pulp rpm repository upload --name "$PULP_REPO" --file "$RPM"; then + echo "SUCCESS: $RPM_NAME" + break else - echo "Failed to recreate release." + echo "Retry $i for $RPM_NAME" + sleep 5 + fi + + if [ $i -eq 3 ]; then + echo "ERROR: Failed to upload $RPM_NAME" exit 1 fi - else - echo "Command failed. Retrying in 10 seconds..." - sleep 10 - fi - fi -done + done + done -echo "Uploading the artifacts into github" -github-release -v upload --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "${PAX_BASENAME}" --file "${PAX}" -if [ $? -eq 0 ]; then - echo "PAX Artifact uploaded successfully!" -else - echo "Failed to upload PAX artifact!" - exit 1 -fi + echo "=== SUCCESS: Packages are being processed ===" + echo "Check the status at: ${PULP_HOST}/pulp/content/rpm/${PULP_REPO}/" -echo "Uploading metadata artifacts into github" -github-release -v upload --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "metadata.json" --file "${METADATA}" -if [ $? -eq 0 ]; then - echo "Metadata Artifact uploaded successfully!" else - echo "Failed to upload Metadata artifact!" - exit 1 + echo "Skipping RPM upload" fi -# --- Upload CHANGELOG.md as release asset --- -echo "Uploading CHANGELOG.md as release asset into github" -github-release -v upload --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "CHANGELOG.md" --file "$RELEASE_NOTES_MD_FILE" -if [ $? -eq 0 ]; then - echo "CHANGELOG.md uploaded successfully as release asset!" -else - echo "Failed to upload CHANGELOG.md as release asset!" - exit 1 -fi +# --- FINAL FILES --- +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "metadata.json" \ + --file "$METADATA" -# --- Upload CHANGELOG.txt as release asset --- -echo "Uploading CHANGELOG.txt as release asset into github" -github-release -v upload --user ${GITHUB_ORGANIZATION} --repo ${GITHUB_REPO} --tag "${TAG}" --name "CHANGELOG.txt" --file "$CHANGELOG_TXT_FILE" # Use CHANGELOG_TXT_FILE -if [ $? -eq 0 ]; then - echo "CHANGELOG.txt uploaded successfully as release asset!" -else - echo "Failed to upload CHANGELOG.txt as release asset!" - exit 1 -fi +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "CHANGELOG.md" \ + --file "CHANGELOG.md" +echo "=== SUCCESS: PUBLISH COMPLETED ===" diff --git a/cicd/publish.groovy.orig b/cicd/publish.groovy.orig new file mode 100755 index 000000000..85d3c8a64 --- /dev/null +++ b/cicd/publish.groovy.orig @@ -0,0 +1,215 @@ +#!/bin/bash +set -euo pipefail + +echo "=== STARTING PUBLISH JOB ===" + +# --- REQUIRED ENV --- +: "${PORT_GITHUB_REPO:?Missing PORT_GITHUB_REPO}" +: "${PORT_DESCRIPTION:?Missing PORT_DESCRIPTION}" +: "${BUILD_NUMBER:?Missing BUILD_NUMBER}" +: "${GITHUB_TOKEN:?Missing GITHUB_TOKEN}" +: "${PULP_USERNAME:?Missing PULP_USERNAME}" +: "${PULP_PASSWORD:?Missing PULP_PASSWORD}" + +# --- STATIC CONFIG --- +GITHUB_ORGANIZATION="zopencommunity" +RELEASE_NOTES_SCRIPT="tools/create_release_notes.sh" +PULP_HOST="https://repo.zopen.community" # ✅ FIXED + +BUILD_LINE=${BUILD_LINE:-DEV} + +# --- DERIVE VALUES --- +RELEASE_PREFIX=$(basename "${PORT_GITHUB_REPO}") +RELEASE_PREFIX=${RELEASE_PREFIX%%.*} +PORT_NAME=$(echo "$RELEASE_PREFIX" | sed 's/port$//') +GITHUB_REPO=$RELEASE_PREFIX + +# --- FIND FILES --- +PAX=$(find . -type f -name "*zos.pax.Z" | head -n 1) +METADATA=$(find . -type f -name "metadata.json" | head -n 1) + +[ -f "$PAX" ] || { echo "ERROR: Missing PAX file"; exit 1; } +[ -f "$METADATA" ] || { echo "ERROR: Missing metadata.json"; exit 1; } + +# --- RPM FILES --- +RPM_FILES=() +while IFS= read -r -d '' f; do + RPM_FILES+=("$f") +done < <(find rpmbuild/RPMS -type f -name "*.rpm" -print0 2>/dev/null) + +NUM_RPMS=${#RPM_FILES[@]} + +# --- INFO --- +BUILD_STATUS=$(find . -name "test.status" -exec cat {} + 2>/dev/null || echo "Unknown") +DEPENDENCIES=$(find . -name ".runtimedeps" -exec cat {} + 2>/dev/null || echo "None") +VERSION=$(find . -name ".version" -exec cat {} + 2>/dev/null || echo "unknown") + +unset http_proxy https_proxy + +PAX_BASENAME=$(basename "$PAX") +TAG="${BUILD_LINE}_${RELEASE_PREFIX}_${BUILD_NUMBER}" +NAME="${PORT_NAME} ${VERSION} (Build ${BUILD_NUMBER}) - (${BUILD_LINE})" + +# --- RELEASE NOTES --- +if [ -f "$RELEASE_NOTES_SCRIPT" ]; then + RELEASE_NOTES=$("$RELEASE_NOTES_SCRIPT" -m release -p "$GITHUB_REPO" -t markdown || echo "No notes") +else + RELEASE_NOTES="No release notes" +fi + +echo "$RELEASE_NOTES" > CHANGELOG.md + +DESCRIPTION="${PORT_DESCRIPTION}
Status: ${BUILD_STATUS}
" +DESCRIPTION+="Dependencies: ${DEPENDENCIES}
" +DESCRIPTION+=$'\n\n'"$RELEASE_NOTES" + +# --- TOOL CHECK --- +command -v github-release >/dev/null || { echo "ERROR: github-release not installed"; exit 1; } + +# --- CREATE / RECREATE RELEASE --- +if github-release info -u "$GITHUB_ORGANIZATION" -r "$GITHUB_REPO" --tag "$TAG" -j &>/dev/null; then + github-release delete --user "$GITHUB_ORGANIZATION" --repo "$GITHUB_REPO" --tag "$TAG" +fi + +github-release release \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$NAME" \ + --description "$DESCRIPTION" + +# --- VALIDATE RELEASE (FIXES YOUR ERROR) --- +github-release info \ + -u "$GITHUB_ORGANIZATION" \ + -r "$GITHUB_REPO" \ + --tag "$TAG" || { + echo "ERROR: Release creation failed" + exit 1 +} + +# --- UPLOAD PAX --- +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$PAX_BASENAME" \ + --file "$PAX" + +# ========================================================= +# 🔧 PULP SETUP (AUTO INSTALL + CONFIG) +# ========================================================= + +echo "=== Checking Pulp CLI ===" + +if ! command -v pulp >/dev/null 2>&1; then + echo "Pulp CLI not found. Installing..." + + if command -v pip3 >/dev/null 2>&1; then + python3 -m pip install --user pulp-cli || true + export PATH="$HOME/.local/bin:$PATH" + fi +fi + +if command -v pulp >/dev/null 2>&1; then + echo "Pulp CLI available" + + pulp config create \ + --base-url "$PULP_HOST" \ + --username "$PULP_USERNAME" \ + --password "$PULP_PASSWORD" \ + --overwrite + + pulp status || { + echo "ERROR: Pulp connection failed" + exit 1 + } + + PULP_AVAILABLE=true +else + echo "WARNING: Pulp not available, skipping RPM upload" + PULP_AVAILABLE=false +fi + +# ========================================================= +# 📦 RPM UPLOAD +# ========================================================= + +if [ "$NUM_RPMS" -gt 0 ] && [ "$PULP_AVAILABLE" = true ]; then + + if [ "$BUILD_LINE" = "STABLE" ]; then + PULP_REPO="zopen-stable" + else + PULP_REPO="zopen-dev" + fi + + echo "Uploading RPMs to Pulp repo: $PULP_REPO" + + for RPM in "${RPM_FILES[@]}"; do + RPM_NAME=$(basename "$RPM") + + # Upload to GitHub + github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "$RPM_NAME" \ + --file "$RPM" + + # Upload to Pulp (with retry) + for i in 1 2 3; do + if pulp rpm repository upload --name "$PULP_REPO" --file "$RPM"; then + echo "SUCCESS: $RPM_NAME" + break + else + echo "Retry $i for $RPM_NAME" + sleep 5 + fi + + if [ $i -eq 3 ]; then + echo "ERROR: Failed to upload $RPM_NAME" + exit 1 + fi + done + done + +<<<<<<< HEAD + # After all RPMs are uploaded, update the distribution to the latest publication + echo "Updating Pulp distribution for ${PULP_REPO}..." + + # Determine distribution name (matching the repo name in this setup) + DIST_NAME="$PULP_REPO" + + LATEST_PUB=$(pulp rpm publication list --repository "${PULP_REPO}" --limit 1 | jq -r '.[0].pulp_href') + if [ -n "$LATEST_PUB" ] && [ "$LATEST_PUB" != "null" ]; then + echo "Updating distribution ${DIST_NAME} to publication ${LATEST_PUB}" + pulp rpm distribution update --name "${DIST_NAME}" --publication "${LATEST_PUB}" + if [ $? -eq 0 ]; then + echo "Successfully updated Pulp distribution ${DIST_NAME}" + else + echo "Warning: Failed to update Pulp distribution ${DIST_NAME}" + fi + else + echo "Warning: Could not find latest publication for ${PULP_REPO}" + fi +======= +>>>>>>> b58d0e1dbd1abc524d970a83223def67877fed5d +else + echo "Skipping RPM upload" +fi + +# --- FINAL FILES --- +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "metadata.json" \ + --file "$METADATA" + +github-release upload \ + --user "$GITHUB_ORGANIZATION" \ + --repo "$GITHUB_REPO" \ + --tag "$TAG" \ + --name "CHANGELOG.md" \ + --file "CHANGELOG.md" + +echo "=== SUCCESS: PUBLISH COMPLETED ===" diff --git a/zopen-diagnostics b/zopen-diagnostics deleted file mode 100644 index e69de29bb..000000000