diff --git a/.github/workflows/export-dynamic.yaml b/.github/workflows/export-dynamic.yaml index 0e7b410..fe3e99e 100644 --- a/.github/workflows/export-dynamic.yaml +++ b/.github/workflows/export-dynamic.yaml @@ -17,15 +17,15 @@ on: type: string required: true janus-cli-version: - description: Version of the janus-idp/cli package. + description: Version of @red-hat-developer-hub/cli (legacy input name). type: string required: true cli-package: - description: Alternative CLI package to use for plugin export instead of @janus-idp/cli. + description: npm package for plugin export (@red-hat-developer-hub/cli). type: string required: false - default: "@janus-idp/cli" + default: "@red-hat-developer-hub/cli" plugins-repo: description: @@ -126,41 +126,105 @@ on: description: Optional commit ID of the last successful publishing of plugin container images type: string required: false + + force-export: + description: > + Export all plugins even when the overlay workspace is unchanged since + last-publish-commit. Use for smoke tests and manual re-publishes. + type: boolean + required: false + default: false + + skip-metadata-validation: + description: > + Skip catalog metadata validation. When false, validation runs in the compile + job (against post-override-sources trees) and a gate job fails the workflow + after OCI publish — matching main semantics (images pushed before metadata failure). + type: boolean + required: false + default: false + + export-builder-image: + description: > + UBI-based container image for the export compile job. Must match the RHDH runtime ABI + and Node major from versions.json (e.g. export-builder:ubi9-node24). + Selected in export-workspaces-as-dynamic prepare from EXPORT_BUILDER_GHCR_IMAGE env. + type: string + required: false + default: ghcr.io/redhat-developer/rhdh-plugin-export-utils/export-builder:ubi9-node24 + + cli-caller: + description: Path to pre-installed rhdh-cli inside the export-builder image + type: string + required: false + default: "" + + export-utils-repository: + description: Repository for export-utils publish scripts checkout + type: string + required: false + default: redhat-developer/rhdh-plugin-export-utils + + export-utils-ref: + description: Git ref for export-utils publish scripts checkout + type: string + required: false + default: main outputs: published-exports: - value: '${{ jobs.export.outputs.published-exports }}' + value: '${{ jobs.export-publish.outputs.published-exports }}' failed-exports: - value: '${{ jobs.export.outputs.failed-exports }}' + value: "${{ inputs.publish-container && jobs.export-publish.outputs.failed-exports != '' && jobs.export-publish.outputs.failed-exports || jobs.export-compile.outputs.failed-exports }}" metadata-validation-passed: description: Whether the metadata validation passed (true/false) - value: '${{ jobs.export.outputs.metadata-validation-passed }}' + value: '${{ jobs.export-compile.outputs.metadata-validation-passed }}' metadata-validation-errors: description: JSON array of metadata validation errors, empty array if validation passed - value: '${{ jobs.export.outputs.metadata-validation-errors }}' + value: '${{ jobs.export-compile.outputs.metadata-validation-errors }}' metadata-validation-error-count: description: Number of metadata validation errors found - value: '${{ jobs.export.outputs.metadata-validation-error-count }}' + value: '${{ jobs.export-compile.outputs.metadata-validation-error-count }}' jobs: - export: - name: Export + export-compile: + name: Export (compile) runs-on: ubuntu-latest + container: + image: ${{ inputs.export-builder-image }} + env: NODE_OPTIONS: --max-old-space-size=8192 + NPM_CONFIG_IGNORE_SCRIPTS: "true" + YARN_ENABLE_SCRIPTS: "false" + NPM_CONFIG_cache: /tmp/npm-cache + INPUTS_CLI_CALLER: ${{ inputs.cli-caller }} + # Local composite actions break in container jobs (action_path vs mount). Call scripts directly. + EXPORT_UTILS_ACTIONS: ${{ github.workspace }}/export-utils-actions defaults: run: working-directory: source-repo/${{ inputs.plugins-root }} + permissions: + contents: write + outputs: - published-exports: '${{ steps.export-dynamic.outputs.published-exports }}' + published-exports: '' failed-exports: '${{ steps.export-dynamic.outputs.failed-exports }}' - metadata-validation-passed: '${{ steps.validate-metadata.outputs.validation-passed }}' - metadata-validation-errors: '${{ steps.validate-metadata.outputs.validation-errors }}' - metadata-validation-error-count: '${{ steps.validate-metadata.outputs.validation-error-count }}' + staging-artifact-name: '${{ steps.staging-artifact.outputs.name }}' + staging-uploaded: '${{ steps.upload-staging.outcome == ''success'' }}' + metadata-validation-passed: >- + ${{ inputs.skip-metadata-validation && 'skipped' + || steps.validate-metadata.outputs.metadata-validation-passed }} + metadata-validation-errors: >- + ${{ inputs.skip-metadata-validation && '[]' + || steps.validate-metadata.outputs.metadata-validation-errors }} + metadata-validation-error-count: >- + ${{ inputs.skip-metadata-validation && '0' + || steps.validate-metadata.outputs.metadata-validation-error-count }} steps: - name: Validate Inputs @@ -171,20 +235,14 @@ jobs: with: script: | const pluginsRoot = core.getInput('plugins_root'); - if (pluginsRoot.startsWith('/') || pluginsRoot.includes('..')) { + if (pluginsRoot.startsWith('/') || pluginsRoot.includes('..')) { core.setFailed(`Invalid plugins root: ${pluginsRoot}`); } const overlayRoot = core.getInput('overlay_root'); - if (overlayRoot.startsWith('/') || overlayRoot.includes('..')) { + if (overlayRoot.startsWith('/') || overlayRoot.includes('..')) { core.setFailed(`Invalid overlay root: ${overlayRoot}`); } - - name: Setup Node.js ${{ inputs.node-version }} - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: ${{ inputs.node-version }} - registry-url: https://registry.npmjs.org/ # Needed for auth - - name: Checkout plugins repository ${{ inputs.plugins-repo }} uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 if: ${{ inputs.plugins-root == '.' }} @@ -215,11 +273,26 @@ jobs: path: overlay-repo fetch-depth: 50 - - name: Override Sources - uses: redhat-developer/rhdh-plugin-export-utils/override-sources@main + - name: Checkout export-utils composite actions + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - overlay-root: ${{ github.workspace }}/overlay-repo/${{inputs.overlay-root}} - workspace-root: source-repo/${{ inputs.plugins-root }} + repository: ${{ inputs.export-utils-repository }} + ref: ${{ inputs.export-utils-ref }} + sparse-checkout: | + override-sources + export-dynamic + validate-metadata + sparse-checkout-cone-mode: false + path: export-utils-actions + + - name: Override Sources + env: + OVERLAY_ROOT: ${{ github.workspace }}/overlay-repo/${{inputs.overlay-root}} + WORKSPACE_ROOT: ${{ github.workspace }}/source-repo/${{ inputs.plugins-root }} + SOURCE_OVERLAY_FOLDER_NAME: overlay + run: | + bash "${EXPORT_UTILS_ACTIONS}/override-sources/override-sources.sh" \ + "${OVERLAY_ROOT}" "${WORKSPACE_ROOT}" "${SOURCE_OVERLAY_FOLDER_NAME}" - name: Dump content env: @@ -246,21 +319,19 @@ jobs: exit 1 fi - - name: Install required native libraries - run: | - sudo apt-get update - sudo apt-get install build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - - - name: Enable Corepack - run: corepack enable - - name: Run yarn install + env: + NPM_CONFIG_IGNORE_SCRIPTS: "true" + YARN_ENABLE_SCRIPTS: "false" run: | yarn --version yarn install --immutable - name: Debug yarn install if: failure() + env: + NPM_CONFIG_IGNORE_SCRIPTS: "true" + YARN_ENABLE_SCRIPTS: "false" run: | cp yarn.lock yarn.lock.before @@ -269,50 +340,68 @@ jobs: echo "::endgroup::" echo "::group::Required changes in yarn.lock" - diff -u yarn.lock.before yarn.lock || true + if command -v diff >/dev/null 2>&1; then + diff -u yarn.lock.before yarn.lock || true + else + echo "diff not installed in export-builder image; yarn.lock differs from immutable install if mutable install changed lockfile" + cmp -s yarn.lock.before yarn.lock && echo "yarn.lock unchanged" || echo "yarn.lock changed" + fi echo "::endgroup::" - name: Run Typescript type checking run: yarn tsc - - name: Log in to container registry - if: ${{ inputs.publish-container }} - uses: redhat-actions/podman-login@4934294ad0449894bcd1e9f191899d7292469603 # v1.7 + - name: Export dynamic plugin packages + if: ${{ success() }} + id: export-dynamic + working-directory: source-repo/${{ inputs.plugins-root }} env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - with: - username: ${{ inputs.image-registry-user }} - password: ${{ secrets.image-registry-password }} - registry: ${{ inputs.image-repository-prefix }} + YARN_ENABLE_IMMUTABLE_INSTALLS: "false" + INPUTS_DESTINATION: ${{ github.workspace }}/dynamic-plugin-archives + INPUTS_CLI_VERSION: ${{ inputs.janus-cli-version }} + INPUTS_CLI_PACKAGE: ${{ inputs.cli-package }} + INPUTS_PLUGINS_FILE: ${{ github.workspace }}/overlay-repo/${{ inputs.overlay-root }}/plugins-list.yaml + INPUTS_APP_CONFIG_FILE_NAME: app-config.dynamic.yaml + INPUTS_SCALPRUM_CONFIG_FILE_NAME: scalprum-config.json + INPUTS_SOURCE_OVERLAY_FOLDER_NAME: overlay + INPUTS_SOURCE_PATCH_FILE_NAME: patch + INPUTS_LAST_PUBLISH_COMMIT: ${{ inputs.last-publish-commit }} + INPUTS_FORCE_EXPORT: ${{ inputs.force-export }} + INPUTS_CLI_CALLER: ${{ inputs.cli-caller }} + STAGING_ROOT: ${{ inputs.publish-container && format('{0}/export-staging', github.workspace) || '' }} + INPUTS_IMAGE_TAG_PREFIX: ${{ inputs.image-tag-prefix }} + INPUTS_PLUGINS_ROOT: ${{ github.workspace }}/source-repo/${{ inputs.plugins-root }} + run: | + bash "${EXPORT_UTILS_ACTIONS}/export-dynamic/export-dynamic.sh" || true + if [[ -f "${GITHUB_WORKSPACE}/.export-dynamic-outputs" ]]; then + cat "${GITHUB_WORKSPACE}/.export-dynamic-outputs" >> "${GITHUB_OUTPUT}" + fi + if [[ -n "${STAGING_ROOT}" ]]; then + bash "${EXPORT_UTILS_ACTIONS}/export-dynamic/create-export-staging.sh" + fi - - name: Set Plugin Image Repository Prefix + - name: Set staging artifact name if: ${{ inputs.publish-container }} + id: staging-artifact env: - IMAGE_REPOSITORY_PREFIX: ${{ inputs.image-repository-prefix }} - - id: set-image-tag-name - shell: bash + INPUT_OVERLAY_ROOT: ${{ inputs.overlay-root }} run: | - if [[ "${{ inputs.publish-container }}" ]] - then - echo "IMAGE_REPOSITORY_PREFIX=$IMAGE_REPOSITORY_PREFIX" >> $GITHUB_OUTPUT - else - echo "IMAGE_REPOSITORY_PREFIX=" >> $GITHUB_OUTPUT - fi + name="export-staging-$(echo "${INPUT_OVERLAY_ROOT}" | sed 's:/:-:g')" + echo "name=${name}" >> "$GITHUB_OUTPUT" - - name: Export dynamic plugin packages - if: ${{ success() }} - id: export-dynamic - uses: redhat-developer/rhdh-plugin-export-utils/export-dynamic@main + - name: Upload export staging for OCI publish + id: upload-staging + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: >- + ${{ inputs.publish-container && success() + && (inputs.force-export + || steps.export-dynamic.outputs.workspace-skipped-unchanged-since == 'false') }} with: - plugins-root: source-repo/${{inputs.plugins-root}} - plugins-file: ${{ github.workspace }}/overlay-repo/${{inputs.overlay-root}}/plugins-list.yaml - destination: ${{ github.workspace }}/dynamic-plugin-archives - janus-cli-version: ${{inputs.janus-cli-version}} - cli-package: ${{inputs.cli-package}} - image-repository-prefix: ${{ steps.set-image-tag-name.outputs.IMAGE_REPOSITORY_PREFIX }} - image-tag-prefix: ${{ inputs.image-tag-prefix }} - last-publish-commit: ${{ inputs.last-publish-commit }} + name: ${{ steps.staging-artifact.outputs.name }} + path: ${{ github.workspace }}/export-staging + if-no-files-found: error + retention-days: 1 + overwrite: true - name: Set artifacts name suffix id: set-artifacts-name-suffix @@ -330,7 +419,10 @@ jobs: - name: Upload exported archives to workflow artifacts uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: ${{ !inputs.publish-release-assets && success() && steps.export-dynamic.outputs.workspace-skipped-unchanged-since == 'false' }} + if: >- + ${{ !inputs.publish-release-assets && success() + && (inputs.force-export + || steps.export-dynamic.outputs.workspace-skipped-unchanged-since == 'false') }} with: name: dynamic plugin packages${{ steps.set-artifacts-name-suffix.outputs.ARTIFACTS_NAME_SUFFIX }} path: ${{ github.workspace }}/dynamic-plugin-archives @@ -353,23 +445,11 @@ jobs: overwrite: true include-hidden-files: true - - name: Log container image names - if: ${{ success() && steps.export-dynamic.outputs.published-exports != '' }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - INPUT_PUBLISHED_EXPORTS: ${{ steps.export-dynamic.outputs.published-exports }} - INPUT_OVERLAY_ROOT: ${{ inputs.overlay-root }} - with: - script: | - const publishedExports = core.getMultilineInput('published_exports'); - const overlayRoot = core.getInput('overlay_root'); - core.summary - .addHeading(`Published container images for workspace '${overlayRoot}' :`, 4) - .addList(publishedExports) - .write(); - - name: Log that the workspace has been skipped - if: ${{ success() && steps.export-dynamic.outputs.workspace-skipped-unchanged-since != 'false' }} + if: >- + ${{ success() + && steps.export-dynamic.outputs.workspace-skipped-unchanged-since != 'false' + && steps.export-dynamic.outputs.workspace-skipped-unchanged-since != '' }} uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: INPUT_UNCHANGED_SINCE: ${{ steps.export-dynamic.outputs.workspace-skipped-unchanged-since }} @@ -413,12 +493,159 @@ jobs: repository: ${{ github.repository }} - name: Validate Catalog Metadata - if: ${{ failure() || steps.export-dynamic.outputs.workspace-skipped-unchanged-since == 'false' }} + if: >- + ${{ !inputs.skip-metadata-validation + && (failure() || steps.export-dynamic.outputs.workspace-skipped-unchanged-since == 'false') }} id: validate-metadata - uses: redhat-developer/rhdh-plugin-export-utils/validate-metadata@main + continue-on-error: true + working-directory: ${{ github.workspace }}/export-utils-actions/validate-metadata + env: + INPUTS_OVERLAY_ROOT: ${{ github.workspace }}/overlay-repo/${{inputs.overlay-root}} + INPUTS_PLUGINS_ROOT: ${{ github.workspace }}/source-repo/${{inputs.plugins-root}} + INPUTS_TARGET_BACKSTAGE_VERSION: ${{ inputs.target-backstage-version }} + INPUTS_IMAGE_REPOSITORY_PREFIX: ${{ inputs.image-repository-prefix }} + INPUTS_COMPUTED_IMAGE_TAG_PREFIX: ${{ inputs.computed-image-tag-prefix }} + run: | + npm install + node validate-metadata.ts + + export-publish: + name: Publish OCI + + needs: export-compile + if: >- + ${{ inputs.publish-container + && needs.export-compile.result == 'success' + && needs.export-compile.outputs.staging-uploaded == 'true' }} + + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + attestations: write + id-token: write + + outputs: + published-exports: '${{ steps.publish-staging.outputs.published-exports }}' + failed-exports: '${{ steps.publish-staging.outputs.failed-exports }}' + + steps: + - name: Download export staging + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ needs.export-compile.outputs.staging-artifact-name }} + path: export-staging + + - name: Checkout export-utils scripts + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ inputs.export-utils-repository }} + ref: ${{ inputs.export-utils-ref }} + sparse-checkout: scripts + path: export-utils + + - name: Publish OCI images from staging + id: publish-staging + env: + EXPORT_BUILDER_IMAGE: ${{ inputs.export-builder-image }} + STAGING_ROOT: ${{ github.workspace }}/export-staging + INPUTS_IMAGE_REPOSITORY_PREFIX: ${{ inputs.image-repository-prefix }} + INPUTS_IMAGE_TAG_PREFIX: ${{ inputs.image-tag-prefix }} + INPUTS_CLI_CALLER: ${{ inputs.cli-caller }} + GITHUB_ACTOR: ${{ github.actor }} + IMAGE_REGISTRY_USER: ${{ inputs.image-registry-user != '' && inputs.image-registry-user || github.actor }} + IMAGE_REGISTRY_PASSWORD: ${{ secrets.image-registry-password }} + run: | + # GHA service containers cannot run rootless buildah (no user namespaces). + # Run the export-builder image privileged on the host Docker daemon instead. + docker run --rm --privileged \ + --user 0 \ + --entrypoint "" \ + -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" \ + -w "${GITHUB_WORKSPACE}" \ + -e "STAGING_ROOT=${STAGING_ROOT}" \ + -e "INPUTS_IMAGE_REPOSITORY_PREFIX=${INPUTS_IMAGE_REPOSITORY_PREFIX}" \ + -e "INPUTS_IMAGE_TAG_PREFIX=${INPUTS_IMAGE_TAG_PREFIX}" \ + -e "INPUTS_CLI_CALLER=${INPUTS_CLI_CALLER}" \ + -e "INPUTS_CONTAINER_BUILD_TOOL=buildah" \ + -e "INPUTS_PUSH_CONTAINER_IMAGE=true" \ + -e "GITHUB_ACTOR=${GITHUB_ACTOR}" \ + -e "GITHUB_WORKSPACE=${GITHUB_WORKSPACE}" \ + -e "IMAGE_REGISTRY_USER=${IMAGE_REGISTRY_USER}" \ + -e "IMAGE_REGISTRY_PASSWORD=${IMAGE_REGISTRY_PASSWORD}" \ + -e "BUILDAH_ISOLATION=chroot" \ + -e "STORAGE_DRIVER=vfs" \ + "${EXPORT_BUILDER_IMAGE}" \ + bash "${GITHUB_WORKSPACE}/export-utils/scripts/publish-export-staging.sh" + if [[ -f "${GITHUB_WORKSPACE}/.publish-export-staging.outputs" ]]; then + cat "${GITHUB_WORKSPACE}/.publish-export-staging.outputs" >> "${GITHUB_OUTPUT}" + fi + + - name: Log container image names + if: ${{ success() && steps.publish-staging.outputs.published-exports != '' }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_PUBLISHED_EXPORTS: ${{ steps.publish-staging.outputs.published-exports }} + INPUT_OVERLAY_ROOT: ${{ inputs.overlay-root }} + with: + script: | + const publishedExports = core.getMultilineInput('published_exports'); + const overlayRoot = core.getInput('overlay_root'); + core.summary + .addHeading(`Published container images for workspace '${overlayRoot}' :`, 4) + .addList(publishedExports) + .write(); + + - name: Check publish errors + if: ${{ failure() || steps.publish-staging.outputs.failed-exports != '' }} + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_FAILED_EXPORTS: ${{ steps.publish-staging.outputs.failed-exports }} + INPUT_OVERLAY_ROOT: ${{ inputs.overlay-root }} + with: + script: | + const overlayRoot = core.getInput('overlay_root'); + const failedExports = core.getMultilineInput('failed_exports'); + core.setFailed(`OCI publish for workspace '${overlayRoot}' failed for:\n${failedExports.join('\n')}`); + + export-metadata-gate: + name: Metadata validation gate + + needs: + - export-compile + - export-publish + if: >- + ${{ always() + && !inputs.skip-metadata-validation + && needs.export-compile.result == 'success' + && (needs.export-publish.result == 'success' + || needs.export-publish.result == 'skipped') }} + + runs-on: ubuntu-latest + + steps: + - name: Fail workflow on metadata validation errors + if: needs.export-compile.outputs.metadata-validation-passed == 'false' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + env: + INPUT_METADATA_VALIDATION_ERRORS: ${{ needs.export-compile.outputs.metadata-validation-errors }} + INPUT_METADATA_VALIDATION_ERROR_COUNT: ${{ needs.export-compile.outputs.metadata-validation-error-count }} + INPUT_OVERLAY_ROOT: ${{ inputs.overlay-root }} with: - overlay-root: ${{ github.workspace }}/overlay-repo/${{inputs.overlay-root}} - plugins-root: ${{ github.workspace }}/source-repo/${{inputs.plugins-root}} - target-backstage-version: ${{ inputs.target-backstage-version }} - image-repository-prefix: ${{ steps.set-image-tag-name.outputs.IMAGE_REPOSITORY_PREFIX }} - computed-image-tag-prefix: ${{ inputs.computed-image-tag-prefix }} + script: | + const overlayRoot = core.getInput('overlay_root'); + const errorCount = core.getInput('metadata_validation_error_count'); + const errorsJson = core.getInput('metadata_validation_errors'); + let detail = `${errorCount} error(s)`; + try { + const errors = JSON.parse(errorsJson || '[]'); + if (errors.length > 0) { + detail = errors.map(e => `${e.file}: ${e.message}`).join('\n'); + } + } catch { + detail = errorsJson || detail; + } + core.setFailed( + `Catalog metadata validation failed for workspace '${overlayRoot}' after OCI publish:\n${detail}`, + ); diff --git a/.github/workflows/export-workspaces-as-dynamic.yaml b/.github/workflows/export-workspaces-as-dynamic.yaml index 8191814..4cd4008 100644 --- a/.github/workflows/export-workspaces-as-dynamic.yaml +++ b/.github/workflows/export-workspaces-as-dynamic.yaml @@ -19,13 +19,13 @@ on: default: '' janus-cli-version: - description: Version of the janus-idp/cli package. + description: Version of @red-hat-developer-hub/cli (legacy input name; defaults to versions.json cli). type: string required: false default: '' cli-package: - description: Alternative CLI package to use for plugin export instead of @janus-idp/cli. + description: npm package for plugin export (@red-hat-developer-hub/cli). type: string required: false default: "" @@ -74,6 +74,44 @@ on: type: string required: false + force-export: + description: > + Export all plugins even when unchanged since last-publish-commit. + Recommended for fork smoke tests (with publish-container true). + type: boolean + required: false + default: false + + skip-metadata-validation: + description: > + Skip catalog metadata validation. When false, validation runs in compile + and a gate job fails the workflow after OCI publish. + type: boolean + required: false + default: false + + export-builder-ghcr-image: + description: > + ghcr.io image base for export-builder (without `:ubi9-node` tag). + Default is the upstream Red Hat image; fork smoke tests should override. + type: string + required: false + default: ghcr.io/redhat-developer/rhdh-plugin-export-utils/export-builder + + export-utils-repository: + description: > + Repository containing export-utils scripts (publish job checkout). + Defaults to upstream; fork smoke tests should pass owner/name of the fork repo. + type: string + required: false + default: redhat-developer/rhdh-plugin-export-utils + + export-utils-ref: + description: Git ref of export-utils for publish script checkout and same-repo workflow calls + type: string + required: false + default: main + outputs: published-exports: value: '${{ jobs.export.outputs.published-exports }}' @@ -92,7 +130,7 @@ on: metadata-validation-error-count: description: Number of metadata validation errors found value: '${{ jobs.export.outputs.metadata-validation-error-count }}' - + jobs: prepare: runs-on: ubuntu-latest @@ -103,6 +141,8 @@ jobs: janus-cli-version: ${{ steps.set-env-vars.outputs.JANUS_CLI_VERSION }} cli-package: ${{ steps.set-env-vars.outputs.CLI_PACKAGE }} backstage-version: ${{ steps.set-env-vars.outputs.BACKSTAGE_VERSION }} + export-builder-image: ${{ steps.set-env-vars.outputs.EXPORT_BUILDER_IMAGE }} + cli-caller: ${{ steps.set-env-vars.outputs.CLI_CALLER }} workspaces: ${{ steps.gather-workspaces.outputs.workspaces }} overlay-repo-ref: ${{ steps.set-overlay-repo-ref.outputs.OVERLAY_REPO_REF }} overlay-repo: ${{ steps.set-overlay-repo.outputs.OVERLAY_REPO }} @@ -123,24 +163,27 @@ jobs: id: set-overlay-repo-ref env: INPUT_OVERLAY_BRANCH: ${{ inputs.overlay-branch }} + HEAD_REF: ${{ github.head_ref }} + REF_NAME: ${{ github.ref_name }} run: | if [[ "${INPUT_OVERLAY_BRANCH}" != "" ]] then echo "OVERLAY_REPO_REF=${INPUT_OVERLAY_BRANCH}" >> $GITHUB_OUTPUT else - echo "OVERLAY_REPO_REF=${{ github.head_ref || github.ref_name }}" >> $GITHUB_OUTPUT + echo "OVERLAY_REPO_REF=${HEAD_REF:-${REF_NAME}}" >> $GITHUB_OUTPUT fi - name: Set overlay_repo id: set-overlay-repo env: INPUT_OVERLAY_REPO: ${{ inputs.overlay-repo }} + GITHUB_REPOSITORY: ${{ github.repository }} run: | if [[ "${INPUT_OVERLAY_REPO}" != "" ]] then echo "OVERLAY_REPO=${INPUT_OVERLAY_REPO}" >> $GITHUB_OUTPUT else - echo "OVERLAY_REPO=${{ github.repository }}" >> $GITHUB_OUTPUT + echo "OVERLAY_REPO=${GITHUB_REPOSITORY}" >> $GITHUB_OUTPUT fi - name: Checkout overlay repository @@ -156,21 +199,52 @@ jobs: INPUT_NODE_VERSION: ${{ inputs.node-version }} INPUT_JANUS_CLI_VERSION: ${{ inputs.janus-cli-version }} INPUT_CLI_PACKAGE: ${{ inputs.cli-package }} + EXPORT_BUILDER_GHCR_IMAGE: ${{ inputs.export-builder-ghcr-image }} run: | versions=$(cat versions.json) NODE_VERSION=$(echo ${versions} | jq -r "if (\"${INPUT_NODE_VERSION}\" == \"\") then (.node // \"20.x\") else \"${INPUT_NODE_VERSION}\" end") echo "NODE_VERSION=${NODE_VERSION}" >> $GITHUB_OUTPUT - JANUS_CLI_VERSION=$(echo ${versions} | jq -r "if (\"${INPUT_JANUS_CLI_VERSION}\" == \"\") then (.cli // \"^3.0.0\") else \"${INPUT_JANUS_CLI_VERSION}\" end") + JANUS_CLI_VERSION=$(echo ${versions} | jq -r "if (\"${INPUT_JANUS_CLI_VERSION}\" == \"\") then (.cli // \"^1.8.5\") else \"${INPUT_JANUS_CLI_VERSION}\" end") echo "JANUS_CLI_VERSION=$JANUS_CLI_VERSION" >> $GITHUB_OUTPUT - CLI_PACKAGE=$(echo ${versions} | jq -r "if (\"${INPUT_CLI_PACKAGE}\" == \"\") then (.\"cliPackage\" // \"@janus-idp/cli\") else \"${INPUT_CLI_PACKAGE}\" end") + CLI_PACKAGE=$(echo ${versions} | jq -r "if (\"${INPUT_CLI_PACKAGE}\" == \"\") then (.\"cliPackage\" // \"@red-hat-developer-hub/cli\") else \"${INPUT_CLI_PACKAGE}\" end") echo "CLI_PACKAGE=$CLI_PACKAGE" >> $GITHUB_OUTPUT BACKSTAGE_VERSION=$(echo ${versions} | jq -r ".backstage") echo "BACKSTAGE_VERSION=$BACKSTAGE_VERSION" >> $GITHUB_OUTPUT + node_major=$(echo "${NODE_VERSION}" | cut -d. -f1) + EXPORT_BUILDER_IMAGE="${EXPORT_BUILDER_GHCR_IMAGE}:ubi9-node${node_major}" + echo "EXPORT_BUILDER_IMAGE=${EXPORT_BUILDER_IMAGE}" >> $GITHUB_OUTPUT + + CLI_CALLER="/opt/rhdh-cli/${JANUS_CLI_VERSION}/bin/rhdh-cli" + echo "CLI_CALLER=${CLI_CALLER}" >> $GITHUB_OUTPUT + + - name: Log export toolchain + env: + NODE_VERSION: ${{ steps.set-env-vars.outputs.NODE_VERSION }} + CLI_PACKAGE: ${{ steps.set-env-vars.outputs.CLI_PACKAGE }} + JANUS_CLI_VERSION: ${{ steps.set-env-vars.outputs.JANUS_CLI_VERSION }} + EXPORT_BUILDER_IMAGE: ${{ steps.set-env-vars.outputs.EXPORT_BUILDER_IMAGE }} + CLI_CALLER: ${{ steps.set-env-vars.outputs.CLI_CALLER }} + BUILDER_GHCR_BASE: ${{ inputs.export-builder-ghcr-image }} + run: | + cat >> "$GITHUB_STEP_SUMMARY" <\`). + Rebuild via [\`publish-export-builder.yaml\`](https://github.com/redhat-developer/rhdh-plugin-export-utils/blob/main/.github/workflows/publish-export-builder.yaml) when \`versions.json\` \`node\` or \`cli\` changes. + EOF + - name: Install semver run: npm install semver -g @@ -182,8 +256,10 @@ jobs: OVERLAY_REPO: ${{ steps.set-overlay-repo.outputs.OVERLAY_REPO }} OVERLAY_REPO_REF: ${{ steps.set-overlay-repo-ref.outputs.OVERLAY_REPO_REF }} TARGET_BACKSTAGE_VERSION: ${{ steps.set-env-vars.outputs.BACKSTAGE_VERSION }} + GITHUB_REPOSITORY: ${{ github.repository }} + HEAD_REF: ${{ github.head_ref }} run: | - if [[ "${OVERLAY_REPO}" != "${{ github.repository }}" ]]; then + if [[ "${OVERLAY_REPO}" != "${GITHUB_REPOSITORY}" ]]; then echo "Exporting overlay workspaces from branch \`${OVERLAY_REPO_REF}\` of repository \`${OVERLAY_REPO}\`" else echo "Exporting overlay workspaces from branch \`${OVERLAY_REPO_REF}\`" @@ -194,9 +270,9 @@ jobs: if [[ "${INPUT_WORKSPACE_PATH}" != "" ]] then workspacePath="${INPUT_WORKSPACE_PATH}" - elif [[ "${{ github.head_ref }}" == "workspaces/"* ]] + elif [[ -n "${HEAD_REF}" && "${HEAD_REF}" == workspaces/* ]] then - workspacePath="$(echo '${{ github.head_ref }}' | sed -e 's:workspaces/[^_]*__\(.*\)$:workspaces/\1:')" + workspacePath="$(echo "${HEAD_REF}" | sed -e 's:workspaces/[^_]*__\(.*\)$:workspaces/\1:')" fi json=$( @@ -250,7 +326,7 @@ jobs: export: name: Export ${{ matrix.workspace.overlay-root }} needs: prepare - uses: redhat-developer/rhdh-plugin-export-utils/.github/workflows/export-dynamic.yaml@main + uses: ./.github/workflows/export-dynamic.yaml strategy: fail-fast: false matrix: @@ -274,7 +350,13 @@ jobs: computed-image-tag-prefix: ${{ matrix.workspace.computed-image-tag-prefix }} target-backstage-version: ${{ needs.prepare.outputs.backstage-version }} last-publish-commit: ${{ inputs.last-publish-commit }} - image-registry-user: ${{ inputs.image-registry-user }} + force-export: ${{ inputs.force-export }} + skip-metadata-validation: ${{ inputs.skip-metadata-validation }} + image-registry-user: ${{ inputs.image-registry-user != '' && inputs.image-registry-user || github.actor }} + export-builder-image: ${{ needs.prepare.outputs.export-builder-image }} + cli-caller: ${{ needs.prepare.outputs.cli-caller }} + export-utils-repository: ${{ inputs.export-utils-repository }} + export-utils-ref: ${{ inputs.export-utils-ref }} secrets: image-registry-password: ${{ secrets.image-registry-password }} diff --git a/.github/workflows/publish-export-builder.yaml b/.github/workflows/publish-export-builder.yaml new file mode 100644 index 0000000..4d1de0c --- /dev/null +++ b/.github/workflows/publish-export-builder.yaml @@ -0,0 +1,146 @@ +name: Publish export builder image + +on: + push: + branches: + - main + paths: + - build/** + - scripts/generate-export-builder-config.sh + - .github/workflows/publish-export-builder.yaml + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + inputs: + overlay-branches: + description: Comma-separated overlay branches to read versions.json from + required: false + default: main,release-1.10,release-1.9 + +# ghcr.io/${{ github.repository }}/export-builder — same image family as EXPORT_BUILDER_GHCR_IMAGE +# in export-workspaces-as-dynamic.yaml (hardcoded there because reusable workflows use caller context). +env: + EXPORT_BUILDER_IMAGE_NAME: export-builder + +jobs: + generate-config: + name: Generate builder config + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + node-majors: ${{ steps.generate.outputs.node-majors }} + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Generate export-builder manifests + id: generate + env: + GH_TOKEN: ${{ github.token }} + OVERLAY_BRANCHES: ${{ inputs.overlay-branches || 'main,release-1.10,release-1.9' }} + run: | + bash scripts/generate-export-builder-config.sh + node_majors=$(jq -c '.nodeMajors' build/generated/builder-matrix.json) + echo "node-majors=${node_majors}" >> "$GITHUB_OUTPUT" + + publish: + name: Build export-builder ubi9-node${{ matrix.node-major }} + needs: generate-config + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + node-major: ${{ fromJSON(needs.generate-config.outputs.node-majors) }} + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore generated builder config + env: + GH_TOKEN: ${{ github.token }} + OVERLAY_BRANCHES: ${{ inputs.overlay-branches || 'main,release-1.10,release-1.9' }} + run: | + bash scripts/generate-export-builder-config.sh + + - name: Set Node.js base image + id: node-base + env: + NODE_MAJOR: ${{ matrix.node-major }} + run: | + case "${NODE_MAJOR}" in + 22) + echo "image=registry.access.redhat.com/ubi9/nodejs-22:latest" >> "$GITHUB_OUTPUT" + ;; + 24) + echo "image=registry.access.redhat.com/ubi9/nodejs-24:1781010361@sha256:1938804c6eb623798504f7940bac7f09ca18766f62ce8b80353514a839e58426" >> "$GITHUB_OUTPUT" + ;; + *) + echo "Unsupported node major: ${NODE_MAJOR}" >&2 + exit 1 + ;; + esac + + - name: Log in to ghcr.io + uses: redhat-actions/podman-login@4934294ad0449894bcd1e9f191899d7292469603 # v1.7 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build export builder image + id: build + uses: redhat-actions/buildah-build@7a95fa7ee0f02d552a32753e7414641a04307056 # v2.13 + with: + image: ${{ env.EXPORT_BUILDER_IMAGE_NAME }} + tags: | + ghcr.io/${{ github.repository }}/${{ env.EXPORT_BUILDER_IMAGE_NAME }}:ubi9-node${{ matrix.node-major }} + ghcr.io/${{ github.repository }}/${{ env.EXPORT_BUILDER_IMAGE_NAME }}:${{ github.sha }}-node${{ matrix.node-major }} + containerfiles: build/containerfiles/export-builder.Containerfile + build-args: | + NODE_MAJOR=${{ matrix.node-major }} + NODEJS_BASE_IMAGE=${{ steps.node-base.outputs.image }} + + - name: Push export builder image + uses: redhat-actions/push-to-registry@5ed88d269cf581ea9ef6dd6806d01562096bee9c # v2.8 + with: + image: ${{ steps.build.outputs.image }} + tags: ${{ steps.build.outputs.tags }} + registry: ghcr.io/${{ github.repository }} + + publish-ubi9-alias: + name: Tag ubi9 alias to ubi9-node24 + needs: + - generate-config + - publish + if: contains(fromJSON(needs.generate-config.outputs.node-majors), 24) + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Install buildah + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq buildah + + - name: Log in to ghcr.io + uses: redhat-actions/podman-login@4934294ad0449894bcd1e9f191899d7292469603 # v1.7 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Retag ubi9-node24 as ubi9 + env: + GHCR_REPO: ghcr.io/${{ github.repository }}/${{ env.EXPORT_BUILDER_IMAGE_NAME }} + run: | + src="${GHCR_REPO}:ubi9-node24" + dst="${GHCR_REPO}:ubi9" + buildah pull "${src}" + buildah tag "${src}" "${dst}" + buildah push "${dst}" diff --git a/.github/workflows/test-export-smoke.yaml b/.github/workflows/test-export-smoke.yaml new file mode 100644 index 0000000..df19e2c --- /dev/null +++ b/.github/workflows/test-export-smoke.yaml @@ -0,0 +1,73 @@ +name: Smoke test export pipeline + +# Manual end-to-end test of the split export pipeline on a fork branch. +# See README.md "Testing on a fork (GitHub Actions)". +on: + workflow_dispatch: + inputs: + workspace-path: + description: Overlay workspace to export (e.g. workspaces/tech-radar) + type: string + default: workspaces/tech-radar + overlay-repo: + description: Overlay repository (owner/name) + type: string + default: redhat-developer/rhdh-plugin-export-overlays + overlay-branch: + description: Overlay git ref (branch, tag, or SHA) + type: string + default: main + publish-container: + description: Build and push OCI images after compile (requires builder image on ghcr) + type: boolean + default: true + force-export: + description: > + Export all plugins even if unchanged since last publish (recommended for + smoke tests). + type: boolean + default: true + skip-metadata-validation: + description: > + Skip catalog metadata validation. Default false to match production/overlays + behavior (validate after compile, fail workflow after OCI publish). + type: boolean + default: false + export-builder-ghcr-image: + description: > + ghcr.io base for export-builder without :ubi9-node tag. + Leave empty to use this fork (ghcr.io//rhdh-plugin-export-utils/export-builder). + type: string + default: "" + image-tag-prefix: + description: Prefix for OCI image tags + type: string + default: smoke__ + +jobs: + export: + name: Smoke export ${{ inputs.workspace-path }} + permissions: + contents: write + packages: write + attestations: write + id-token: write + uses: ./.github/workflows/export-workspaces-as-dynamic.yaml + with: + workspace-path: ${{ inputs.workspace-path }} + overlay-repo: ${{ inputs.overlay-repo }} + overlay-branch: ${{ inputs.overlay-branch }} + publish-container: ${{ inputs.publish-container }} + force-export: ${{ inputs.force-export }} + skip-metadata-validation: ${{ inputs.skip-metadata-validation }} + upload-project-on-error: true + image-repository-prefix: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} + image-tag-prefix: ${{ inputs.image-tag-prefix }} + export-builder-ghcr-image: >- + ${{ inputs.export-builder-ghcr-image != '' + && inputs.export-builder-ghcr-image + || format('ghcr.io/{0}/rhdh-plugin-export-utils/export-builder', github.repository_owner) }} + export-utils-repository: ${{ github.repository }} + export-utils-ref: ${{ github.ref_name }} + secrets: + image-registry-password: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index c2658d7..12d0157 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,7 @@ node_modules/ + +# actionlint binary cached by scripts/lint-workflows.sh +.cache/ + +# Local / CI output from scripts/generate-export-builder-config.sh +build/generated/ diff --git a/README.md b/README.md index 7a49131..eb7ac1a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,289 @@ # rhdh-plugin-export-utils + Utilities for exporting backstage plugins as dynamic plugins for installation in Red Hat Developer Hub +## Security model and split export pipeline + +Plugin export compiles **untrusted upstream source** (arbitrary git refs and +`yarn.lock` dependencies). OCI publish only packages pre-built `dist-dynamic` +artifacts (`FROM scratch` + `COPY`) and is much lower risk. + +The `export-dynamic.yaml` reusable workflow splits work across three jobs: + +| Job | Runner | Privileged | Registry secrets | What runs | +|-----|--------|------------|------------------|-----------| +| `export-compile` | `export-builder:ubi9-node{N}` | **no** | **no** | checkout, `override-sources`, monorepo `yarn install` (scripts **off**), `tsc`, `rhdh-cli plugin export` (per-plugin `dist-dynamic` install, scripts **on**), `.tgz` archives, catalog metadata validation (`continue-on-error`) | +| `export-publish` | `ubuntu-latest` + privileged `docker run` of `export-builder` | **no** | yes (`GITHUB_TOKEN` + `github.actor` for ghcr) | download staging artifact, baked `rhdh-cli plugin package`, `buildah push` | +| `export-metadata-gate` | `ubuntu-latest` | **no** | **no** | fails the workflow when compile-time metadata validation reported errors (OCI images already pushed — same semantics as `main`) | + +`export-workspaces-as-dynamic` **prepare** reads branch `versions.json` and passes +`export-builder-image` and `cli-caller` into **both** compile and publish jobs. Bumping `cli` in +`versions.json` requires rebuilding the export-builder images +(`publish-export-builder` workflow). + +### Two-phase `yarn install` and native dependencies + +Compile runs **two separate installs** with different script policies: + +| Phase | When | Purpose | Install scripts | +|-------|------|---------|-----------------| +| **Monorepo** | Before `yarn tsc` | Resolve workspace deps for TypeScript | **Disabled** (`NPM_CONFIG_IGNORE_SCRIPTS`, `YARN_ENABLE_SCRIPTS=false`) | +| **Per-plugin export** | Inside `rhdh-cli plugin export` (`dist-dynamic/`) | Production deps bundled into the dynamic plugin | **Enabled** (default; `export-dynamic.sh` unsets phase-1 env before calling the CLI) | + +Phase 1 skips install scripts on purpose: the job checks out **untrusted upstream** +source and a large `yarn.lock`. Running every package `postinstall` there would +execute arbitrary code across the whole monorepo. Phase 1 only needs enough of +the tree for `yarn tsc` — not a full native build of every optional transitive +(e.g. `cpu-features` from `ssh2`). + +Phase 2 is narrower: `rhdh-cli` derives a `package.json` in `dist-dynamic/`, +copies `yarn.lock`, and runs `yarn install` for **one plugin’s** production +dependencies. Allowed native modules compile here on the **UBI 9 / Node** ABI +(the export-builder image includes gcc, openssl, etc.). + +**Per-plugin native policy** is already configured in overlay `plugins-list.yaml` +via `rhdh-cli` export flags (not a separate workspace toggle): + +- `--allow-native-package ` — include and build a required native module + (e.g. `isolated-vm` on `plugins/scaffolder-backend`) +- `--suppress-native-package ` — replace an optional native dep with a stub + (e.g. `cpu-features` on `plugins/techdocs-backend`) + +Frontend-only workspaces (e.g. `tech-radar`) typically need no `--allow-native-package`. +Backend workspaces with natives are validated by exporting `workspaces/backstage`. + +## Export builder image (UBI 9) + +Plugin exports that include native Node modules (for example `isolated-vm` in +`@backstage/plugin-scaffolder-backend`) must be built on the same RHEL/UBI ABI as +the RHDH runtime. The **compile** job runs inside a UBI 9 Node.js builder image. + +- **Containerfile:** `build/containerfiles/export-builder.Containerfile` +- **Published images:** `ghcr.io/redhat-developer/rhdh-plugin-export-utils/export-builder:ubi9-node24` (and `:ubi9-node22` when needed) +- **Legacy alias:** `:ubi9` → `:ubi9-node24` (deprecated; remove after one release cycle) +- **Manifest:** `/etc/rhdh-export-builder/manifest.json` — baked `@red-hat-developer-hub/cli` versions per Node major +- **Generator:** `scripts/generate-export-builder-config.sh` (reads overlay branch `versions.json`) +- **Publish workflow:** `.github/workflows/publish-export-builder.yaml` + +Each Node-major image pre-installs CLI versions from active overlay release +branches (`main`, `release-1.10`, …). No runtime `npx` in CI compile jobs. + +CLI versions are installed from the **npm registry** (not `rhdh-cli` source + +`yarn.lock`). Installs use `--legacy-peer-deps` and `NPM_CONFIG_LOGLEVEL=error` +to avoid noisy peer-dependency warnings from `@backstage/cli` transitive deps. + +When RHDH migrates to UBI 10, add a `:ubi10-node{N}` image tag family. The split +job structure stays the same. + +### Local end-to-end test + +`scripts/local-test-export.sh` mirrors CI: builds the export-builder image for the +Node major in `versions.json`, runs **compile** inside the builder container +(without `--privileged`), then **publish** on the host with buildah. + +Default workdir: `~/tmp/rhdh-plugin-export-test` (override with `-d` or `WORKDIR`). + +```bash +# Default: tech-radar workspace (small smoke test) +./scripts/local-test-export.sh + +# Compile only — archives and staging, no OCI push +./scripts/local-test-export.sh --compile-only + +# Publish OCI from a prior staging directory +./scripts/local-test-export.sh --publish-only ~/tmp/rhdh-plugin-export-test/export-staging + +# Use the plugins-list.yaml from your overlays PR branch +./scripts/local-test-export.sh -f workspaces/backstage/plugins-list.yaml + +# Point at a non-default overlays checkout (e.g. your PR branch) +./scripts/local-test-export.sh -o ../rhdh-plugin-export-overlays \ + -f workspaces/backstage/plugins-list.yaml + +# Use a local rhdh-cli binary (publish step; compile uses baked CLI in image) +./scripts/local-test-export.sh --rhdh-cli /path/to/rhdh-cli + +# Archives only — skip registry and OCI push (same as --no-push / --compile-only) +./scripts/local-test-export.sh --no-registry +``` + +`-f` accepts any `plugins-list.yaml` — including one with export CLI arguments +(`--allow-native-package`, `--embed-package`, etc.) exactly as committed in +`rhdh-plugin-export-overlays`. The file is copied into the workspace overlay +before export, matching CI. + +#### How this compares to CI (`export-dynamic.yaml`) + +Per **workspace**, both CI and this script: + +1. Clone upstream **once**, apply overlay **once**, `override-sources` **once** +2. `yarn install` + `yarn tsc` **once** on the monorepo (compile job / builder container) +3. `export-dynamic.sh` loops `plugins-list.yaml` — per-plugin `rhdh-cli export` (including `yarn install` in each `dist-dynamic/`), write `.tgz` archives +4. **Publish job** (CI) or host `publish-export-staging.sh` (local) packages `dist-dynamic` with buildah + +`--keep-workdir` reuses clone and `node_modules` when iterating. + +#### Disk space (backstage workspace) + +Plan for **~25 GiB free** on the workdir and container storage filesystem. + +```bash +./scripts/local-test-export.sh --no-push -f workspaces/backstage/plugins-list.yaml +./scripts/local-test-export.sh --prune-podman -f workspaces/backstage/plugins-list.yaml +WORKDIR=/mnt/big/rhdh-plugin-export-backstage ./scripts/local-test-export-reset.sh +``` + +Reset local state (workdir, `build/generated/`, registry, test images, **aggressive `podman system prune -af`**): + +```bash +./scripts/local-test-export-reset.sh +./scripts/local-test-export-reset.sh --purge-builder # also remove the builder image +./scripts/local-test-export-reset.sh --no-prune # keep other unused podman images +``` + +Requires `podman`, `buildah` (for OCI publish), `git`, and `jq`. Expects +`rhdh-plugin-export-overlays` as a sibling directory (or set `OVERLAYS_DIR` / `-o`). + +Pull an image from the local registry: + +```bash +buildah pull --tls-verify=false localhost:5001/rhdh-plugin-export-test/:local__ +``` + +The builder entrypoint (`export-builder-entrypoint`) fixes GHA workspace ownership +then drops to UID 1001. The compile container does **not** need `--privileged`. + +OCI publish runs the **export-builder image** via a privileged `docker run` on the +host runner (GHA service containers cannot run buildah — no user namespaces for +rootless mode). Local `local-test-export.sh` uses host buildah for convenience. +The local registry uses **host networking** on port `5001`. + +For ghcr.io on GitHub Actions, publish authenticates with **`secrets.GITHUB_TOKEN`** +as the password and **`github.actor`** as the username (password via stdin only; +never passed on the command line). + +### Lint workflow files locally + +Before pushing workflow changes, run [actionlint](https://github.com/rhysd/actionlint) via: + +```bash +./scripts/lint-workflows.sh +``` + +Lint specific files or enable shellcheck integration: + +```bash +./scripts/lint-workflows.sh .github/workflows/export-dynamic.yaml +./scripts/lint-workflows.sh --shellcheck +``` + +The script uses `actionlint` from `PATH` when installed; otherwise it downloads a +pinned binary to `.cache/` (gitignored). This catches invalid expressions and many +`uses:` / context mistakes early. It does not replace a fork smoke test for nested +workflow permissions or registry access. + +### Testing on a fork (GitHub Actions) + +Local `scripts/local-test-export.sh` does not exercise GHA container jobs, artifact +handoff, or ghcr permissions. Use your **export-utils fork** to run the same +workflows as production without merging to upstream `main`. + +#### 1. Publish export-builder images on your fork + +Push your branch, then run **Publish export builder image** +(`.github/workflows/publish-export-builder.yaml`) via **Actions → Run workflow**. + +Images are pushed to: + +`ghcr.io//rhdh-plugin-export-utils/export-builder:ubi9-node24` + +Confirm the package appears under your fork’s **Packages** tab before exporting. + +#### 2. Smoke test the export pipeline (export-utils fork only) + +On the same branch, run **Smoke test export pipeline** +(`.github/workflows/test-export-smoke.yaml`): + +| Input | Default (smoke workflow) | Notes | +|-------|--------------------------|-------| +| `workspace-path` | `workspaces/tech-radar` | Use a small workspace first | +| `overlay-repo` | `redhat-developer/rhdh-plugin-export-overlays` | | +| `overlay-branch` | `main` | | +| `publish-container` | **`true`** | Push OCI images to your fork’s ghcr | +| `force-export` | **`true`** | Export even if unchanged since last publish | +| `skip-metadata-validation` | **`false`** | Same as production/overlays; set `true` to test compile+publish only | +| `export-builder-ghcr-image` | leave empty (uses your fork’s ghcr) | | + +Set **`force-export: true`** (smoke workflow default) so plugins are exported and +OCI images are pushed even when the overlay workspace is unchanged since a prior +release (`last-publish-commit` optimization). + +Metadata validation runs in the compile job (against post-`override-sources` trees) +and fails the workflow in a gate job **after** OCI publish — matching `main` +semantics. Use **`skip-metadata-validation: true`** only when you want to exercise +compile and publish without metadata checks (e.g. incomplete metadata on a large +workspace). + +The smoke workflow calls the same reusable workflows as production, using: + +- Your fork’s export-builder image (via `export-builder-ghcr-image`) +- Same-repo workflow and composite action paths (this commit, not upstream `@main`) +- Your fork’s ghcr for OCI plugin images when `publish-container` is true (default) + +CLI equivalent: + +```bash +# Default smoke run (tech-radar, publish OCI, metadata validation enabled): +gh workflow run test-export-smoke.yaml --ref ubi9-experiment + +# Compile only (no ghcr push): +gh workflow run test-export-smoke.yaml \ + --ref ubi9-experiment \ + -f publish-container=false + +# Skip metadata validation (e.g. workspaces/backstage with incomplete metadata): +gh workflow run test-export-smoke.yaml \ + --ref ubi9-experiment \ + -f workspace-path=workspaces/backstage \ + -f skip-metadata-validation=true +``` + +#### 3. Optional — test from your overlays fork + +Upstream overlays keeps calling `redhat-developer/rhdh-plugin-export-utils@main` +until your PR merges. To mimic `/publish` on a fork, create a branch on your +**overlays fork** that points at your export-utils branch and passes fork overrides: + +```yaml +jobs: + export: + uses: /rhdh-plugin-export-utils/.github/workflows/export-workspaces-as-dynamic.yaml@ + with: + overlay-branch: main + workspace-path: workspaces/tech-radar + publish-container: true + export-builder-ghcr-image: ghcr.io//rhdh-plugin-export-utils/export-builder + export-utils-repository: /rhdh-plugin-export-utils + export-utils-ref: + image-repository-prefix: ghcr.io// + secrets: + image-registry-password: ${{ secrets.GITHUB_TOKEN }} +``` + +Then run the overlays **Export Workspace as Dynamic Plugins Packages** workflow +via `workflow_dispatch`. + +#### Operational notes + +- **Upstream is unaffected** until export-utils merges and overlays updates its + `uses:` pin (if needed). Fork runs push plugin images only to **your** ghcr. +- **Reusable workflow inputs** `export-builder-ghcr-image`, `export-utils-repository`, + and `export-utils-ref` default to upstream production values so existing overlays + callers behave unchanged. +- After merge to upstream, run **Publish export builder image** once on + `redhat-developer/rhdh-plugin-export-utils` before overlays `/publish` uses the + new compile job. + ## Actions ### export-dynamic @@ -8,6 +291,7 @@ Utilities for exporting backstage plugins as dynamic plugins for installation in Exports plugins as dynamic plugin archives. This should be run **after** the `override-sources` action in order to support per-plugin source overlays, or if patch modifications are needed. **Usage:** + ```yaml - name: Export Dynamic Plugins uses: ./export-dynamic @@ -18,22 +302,23 @@ Exports plugins as dynamic plugin archives. This should be run **after** the `ov ``` **Key Features:** -- Exports plugins as dynamic plugin packages + - Handles both frontend and backend plugins -- Optional container image packaging +- Optional `staging-root` packs outputs for `publish-export-staging.sh` (OCI publish job) ### override-sources Applies patches and source overlays to modify plugin sources before export. This should be run **before** the `export-dynamic` action. **Features:** -- Applies patches from `/patches/` directory using `git apply` + - Copies source overlay files from `/plugins/{plugin-name}/overlay/` directories - Robust error handling and cleanup - Ordered patch application (by filename) - Configurable overlay subfolder name (defaults to "overlay") **Usage:** + ```yaml - name: Override Sources uses: ./override-sources @@ -43,12 +328,12 @@ Applies patches and source overlays to modify plugin sources before export. This ``` **Inputs:** -- `overlay-root`: Absolute path to the overlay root directory (expects `patches/` and `plugins/` subdirectories) + - `workspace-root`: Directory to apply changes to (defaults to ".") - `source-overlay-folder-name`: Name of subfolder within each plugin directory containing overlay files (defaults to "overlay") **Outputs:** -- `patches-applied`: Number of patches applied + - `source-overlay-applied`: Whether source overlay files were copied ### validate-metadata @@ -56,7 +341,7 @@ Applies patches and source overlays to modify plugin sources before export. This Validates catalog metadata files against plugin `package.json` files to ensure consistency. This should be run **after** the `export-dynamic` action. **Features:** -- Validates `packageName` corresponds to a plugin from `plugins-list.yaml` + - Validates `version` matches the plugin's `package.json` version - Validates OCI reference format in `dynamicArtifact` (tag and repository prefix) - Validates `backstage.supportedVersions` matches major.minor of `dist-dynamic/package.json` @@ -64,6 +349,7 @@ Validates catalog metadata files against plugin `package.json` files to ensure c - Provides JSON output for downstream workflow consumption **Usage:** + ```yaml - name: Validate Catalog Metadata uses: ./validate-metadata @@ -75,13 +361,13 @@ Validates catalog metadata files against plugin `package.json` files to ensure c ``` **Inputs:** -- `overlay-root`: Absolute path to the overlay workspace folder containing `metadata/` and `plugins-list.yaml` + - `plugins-root`: Absolute path to the source plugins folder containing plugin directories with `package.json` files - `target-backstage-version`: Target Backstage version for validating OCI tag format - `image-repository-prefix`: Repository prefix for validating OCI reference format (optional) **Outputs:** -- `validation-passed`: Whether the metadata validation passed (`true`/`false`) + - `validation-errors`: JSON array of validation errors (see [validate-metadata/README.md](validate-metadata/README.md) for format details) - `validation-error-count`: Number of validation errors found @@ -89,28 +375,46 @@ Validates catalog metadata files against plugin `package.json` files to ensure c ```yaml jobs: - export-plugins: + export-compile: runs-on: ubuntu-latest + container: + image: ghcr.io/redhat-developer/rhdh-plugin-export-utils/export-builder:ubi9-node24 + env: + NPM_CONFIG_IGNORE_SCRIPTS: "true" + YARN_ENABLE_SCRIPTS: "false" + INPUTS_CLI_CALLER: /opt/rhdh-cli/1.10.7/bin/rhdh-cli steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Override Sources (apply patches and overlays) - uses: ./override-sources + - uses: actions/checkout@v6 + - uses: ./override-sources with: overlay-root: ${{ github.workspace }}/overlay-repo workspace-root: . - - - name: Export Dynamic Plugins - uses: ./export-dynamic + - run: yarn install --immutable && yarn tsc + - uses: ./export-dynamic with: plugins-root: plugins plugins-file: ${{ github.workspace }}/plugins-list.yaml destination: ${{ github.workspace }}/archives + staging-root: ${{ github.workspace }}/export-staging + image-tag-prefix: bs_1.48.3__ + cli-caller: /opt/rhdh-cli/1.10.7/bin/rhdh-cli - - name: Validate Catalog Metadata - uses: ./validate-metadata + export-publish: + needs: export-compile + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v7 with: - overlay-root: ${{ github.workspace }}/overlay-repo - plugins-root: ${{ github.workspace }}/plugins -``` \ No newline at end of file + name: export-staging + path: export-staging + - run: sudo apt-get install -y buildah + - run: npm install -g @red-hat-developer-hub/cli@1.10.7 --ignore-scripts --omit=dev --legacy-peer-deps + env: + NPM_CONFIG_LOGLEVEL: error + - run: bash scripts/publish-export-staging.sh + env: + STAGING_ROOT: ${{ github.workspace }}/export-staging + INPUTS_IMAGE_REPOSITORY_PREFIX: ghcr.io/my-org/my-repo + INPUTS_CLI_CALLER: $(command -v rhdh-cli) + INPUTS_CONTAINER_BUILD_TOOL: buildah +``` diff --git a/build/containerfiles/entrypoint.sh b/build/containerfiles/entrypoint.sh new file mode 100755 index 0000000..7a430a9 --- /dev/null +++ b/build/containerfiles/entrypoint.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Drop privileges after fixing GHA workspace ownership when the job starts as root. +set -euo pipefail + +RUN_UID="${EXPORT_BUILDER_UID:-1001}" +RUN_GID="${EXPORT_BUILDER_GID:-1001}" + +fix_workspace_owner() { + local dir="$1" + if [[ -z "${dir}" || ! -d "${dir}" ]]; then + return 0 + fi + if [[ "$(id -u)" -eq 0 ]]; then + chown -R "${RUN_UID}:${RUN_GID}" "${dir}" 2>/dev/null || true + fi +} + +if [[ "$(id -u)" -eq 0 ]]; then + fix_workspace_owner "${GITHUB_WORKSPACE:-}" + fix_workspace_owner "${EXPORT_WORKSPACE_DIRS:-}" + if command -v setpriv >/dev/null 2>&1; then + exec setpriv --reuid="${RUN_UID}" --regid="${RUN_GID}" --init-groups "$@" + fi + if command -v gosu >/dev/null 2>&1; then + exec gosu "${RUN_UID}:${RUN_GID}" "$@" + fi + exec su -s /bin/bash -c 'exec "$@"' "default" -- "$@" +fi + +exec "$@" diff --git a/build/containerfiles/export-builder.Containerfile b/build/containerfiles/export-builder.Containerfile new file mode 100644 index 0000000..4cf146b --- /dev/null +++ b/build/containerfiles/export-builder.Containerfile @@ -0,0 +1,43 @@ +# Builder image for exporting Backstage dynamic plugins on the same RHEL/UBI ABI as RHDH. +# One image per Node major (ubi9-node22, ubi9-node24) with pre-installed rhdh-cli versions. +# +# Build args (set by publish-export-builder workflow): +# NODE_MAJOR — 22 or 24 +# NODEJS_BASE_IMAGE — pinned ubi9/nodejs-N image reference +# +ARG NODE_MAJOR=24 +ARG NODEJS_BASE_IMAGE=registry.access.redhat.com/ubi9/nodejs-24:1781010361@sha256:1938804c6eb623798504f7940bac7f09ca18766f62ce8b80353514a839e58426 + +FROM ${NODEJS_BASE_IMAGE} +ARG NODE_MAJOR=24 +USER 0 + +# Native build deps; buildah/skopeo for OCI publish (compile job does not nest containers). +RUN dnf config-manager --set-enabled ubi-9-codeready-builder-rpms \ + && dnf install -y -q --allowerasing \ + gcc gcc-c++ make git patch jq \ + libjpeg-turbo-devel \ + python3 libstdc++-devel zlib-devel openssl-devel \ + buildah skopeo \ + util-linux \ + && dnf clean all \ + && npm install -g corepack \ + && corepack enable + +COPY build/generated/ /tmp/export-builder-generated/ +RUN cp "/tmp/export-builder-generated/cli-install-node${NODE_MAJOR}.sh" /tmp/cli-install.sh \ + && mkdir -p /etc/rhdh-export-builder \ + && cp "/tmp/export-builder-generated/export-builder-node${NODE_MAJOR}.json" /etc/rhdh-export-builder/manifest.json \ + && rm -rf /tmp/export-builder-generated +RUN chmod +x /tmp/cli-install.sh && /tmp/cli-install.sh && rm -f /tmp/cli-install.sh \ + && chown -R 1001:1001 /opt/rhdh-cli /opt/app-root/src/.npm 2>/dev/null || true + +COPY build/containerfiles/entrypoint.sh /usr/local/bin/export-builder-entrypoint +RUN chmod +x /usr/local/bin/export-builder-entrypoint + +LABEL summary="RHDH dynamic plugin export builder (UBI 9 Node ${NODE_MAJOR})" \ + description="UBI 9 Node.js ${NODE_MAJOR} environment for exporting Backstage plugins" \ + maintainer="RHDH Team " + +USER 1001 +ENTRYPOINT ["/usr/local/bin/export-builder-entrypoint"] diff --git a/build/export-builder-manifest.schema.json b/build/export-builder-manifest.schema.json new file mode 100644 index 0000000..123eb96 --- /dev/null +++ b/build/export-builder-manifest.schema.json @@ -0,0 +1,45 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/redhat-developer/rhdh-plugin-export-utils/export-builder-manifest.schema.json", + "title": "RHDH export builder manifest", + "type": "object", + "required": ["node", "cliPackage", "cliVersions"], + "properties": { + "node": { + "type": "object", + "required": ["major", "ubiTag"], + "properties": { + "major": { "type": "integer", "minimum": 18 }, + "full": { "type": "string" }, + "ubiTag": { "type": "string" } + } + }, + "cliPackage": { "type": "string" }, + "cliVersions": { + "type": "array", + "items": { + "type": "object", + "required": ["version", "path"], + "properties": { + "version": { "type": "string" }, + "path": { "type": "string" }, + "overlayBranches": { + "type": "array", + "items": { "type": "string" } + } + } + } + }, + "builtFrom": { + "type": "object", + "properties": { + "overlayRepo": { "type": "string" }, + "overlayBranches": { + "type": "array", + "items": { "type": "string" } + }, + "generatedAt": { "type": "string", "format": "date-time" } + } + } + } +} diff --git a/export-dynamic/action.yaml b/export-dynamic/action.yaml index 47cf3de..1772b27 100644 --- a/export-dynamic/action.yaml +++ b/export-dynamic/action.yaml @@ -1,5 +1,8 @@ name: Export to dynamic plugin archives -description: Export plugins to dynamic plugin package archives +description: > + Export plugins to dynamic plugin package archives (dist-dynamic + .tgz). + OCI images are built in a separate publish job via create-export-staging.sh + and scripts/publish-export-staging.sh. inputs: plugins-root: description: Monorepo root relative folder, in the repository that contains the backstage plugins to be exported as dynamic. @@ -17,14 +20,14 @@ inputs: default: "" janus-cli-version: - description: Version of the janus-idp/cli package. + description: Version of @red-hat-developer-hub/cli (legacy input name; use versions.json cli in CI). required: false default: ^1.8.5 cli-package: - description: Alternative CLI package to use for plugin export instead of @janus-idp/cli. + description: npm package for plugin export (@red-hat-developer-hub/cli). required: false - default: "@janus-idp/cli" + default: "@red-hat-developer-hub/cli" app-config-file-name: description: @@ -49,13 +52,8 @@ inputs: required: false default: patch - image-repository-prefix: - description: Base image name to publish dynamic plugins as container image - default: "" - required: false - image-tag-prefix: - description: Optional prefix to prepend to the plugin version in the image tag + description: Optional prefix for OCI image tags (recorded in export-staging manifest for the publish job) default: "" required: false @@ -64,16 +62,28 @@ inputs: default: "" required: false + force-export: + description: Export all plugins even when unchanged since last-publish-commit + default: "false" + required: false + + cli-caller: + description: Path or command for pre-installed rhdh-cli (from export-builder manifest) + default: "" + required: false + + staging-root: + description: If set, create export-staging directory after export (for the export-publish job) + default: "" + required: false + outputs: failed-exports: description: "Failed exports" - value: ${{ steps.run-export-dynamic.outputs.FAILED_EXPORTS }} - published-exports: - description: "published container images" - value: ${{ steps.run-export-dynamic.outputs.PUBLISHED_EXPORTS }} + value: ${{ steps.run-export-dynamic.outputs.failed-exports }} workspace-skipped-unchanged-since: description: "workspace has been skipped, because unchanged since provided commit" - value: ${{ steps.run-export-dynamic.outputs.WORKSPACE_SKIPPED_UNCHANGED_SINCE }} + value: ${{ steps.run-export-dynamic.outputs.workspace-skipped-unchanged-since }} runs: using: "composite" @@ -85,7 +95,7 @@ runs: with: script: | const pluginsRoot = core.getInput('plugins_root'); - if (pluginsRoot.startsWith('/') || pluginsRoot.includes('..')) { + if (pluginsRoot.startsWith('/') || pluginsRoot.includes('..')) { core.setFailed(`Invalid plugins root: ${pluginsRoot}`); } @@ -94,19 +104,33 @@ runs: shell: bash working-directory: ${{ inputs.plugins-root }} env: - NPM_CONFIG_ignore-scripts: "true" YARN_ENABLE_IMMUTABLE_INSTALLS: "false" INPUTS_DESTINATION: "${{ inputs.destination }}" - INPUTS_JANUS_CLI_VERSION: "${{ inputs.janus-cli-version }}" + INPUTS_CLI_VERSION: "${{ inputs.janus-cli-version }}" INPUTS_CLI_PACKAGE: "${{ inputs.cli-package }}" INPUTS_PLUGINS_FILE: "${{ inputs.plugins-file }}" INPUTS_APP_CONFIG_FILE_NAME: "${{ inputs.app-config-file-name }}" INPUTS_SCALPRUM_CONFIG_FILE_NAME: "${{ inputs.scalprum-config-file-name }}" INPUTS_SOURCE_OVERLAY_FOLDER_NAME: "${{ inputs.source-overlay-folder-name }}" INPUTS_SOURCE_PATCH_FILE_NAME: "${{ inputs.source-patch-file-name }}" - INPUTS_IMAGE_REPOSITORY_PREFIX: "${{ inputs.image-repository-prefix }}" - INPUTS_IMAGE_TAG_PREFIX: "${{ inputs.image-tag-prefix }}" - INPUTS_PUSH_CONTAINER_IMAGE: "true" INPUTS_LAST_PUBLISH_COMMIT: "${{ inputs.last-publish-commit }}" - # for now always pass this step so we can get all the followup logging steps to still run even if the export fails - run: ${{ github.action_path }}/export-dynamic.sh || true + INPUTS_FORCE_EXPORT: "${{ inputs.force-export }}" + INPUTS_CLI_CALLER: "${{ inputs.cli-caller }}" + run: | + ${{ github.action_path }}/export-dynamic.sh || true + if [[ -f "${GITHUB_WORKSPACE}/.export-dynamic-outputs" ]]; then + cat "${GITHUB_WORKSPACE}/.export-dynamic-outputs" >> "${GITHUB_OUTPUT}" + fi + + - name: Create export staging + if: ${{ inputs.staging-root != '' }} + shell: bash + working-directory: ${{ inputs.plugins-root }} + env: + STAGING_ROOT: ${{ inputs.staging-root }} + INPUTS_PLUGINS_FILE: ${{ inputs.plugins-file }} + INPUTS_PLUGINS_ROOT: ${{ github.workspace }}/${{ inputs.plugins-root }} + INPUTS_DESTINATION: ${{ inputs.destination }} + INPUTS_IMAGE_TAG_PREFIX: ${{ inputs.image-tag-prefix }} + INPUTS_APP_CONFIG_FILE_NAME: ${{ inputs.app-config-file-name }} + run: ${{ github.action_path }}/create-export-staging.sh diff --git a/export-dynamic/create-export-staging.sh b/export-dynamic/create-export-staging.sh new file mode 100755 index 0000000..59d564f --- /dev/null +++ b/export-dynamic/create-export-staging.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# +# Pack export outputs into a staging directory for the publish job. +# +set -euo pipefail + +STAGING_ROOT="${STAGING_ROOT:?STAGING_ROOT is required}" +INPUTS_PLUGINS_FILE="${INPUTS_PLUGINS_FILE:?INPUTS_PLUGINS_FILE is required}" +INPUTS_PLUGINS_ROOT="${INPUTS_PLUGINS_ROOT:?INPUTS_PLUGINS_ROOT is required}" +INPUTS_APP_CONFIG_FILE_NAME="${INPUTS_APP_CONFIG_FILE_NAME:-app-config.dynamic.yaml}" +WORKSPACE_OVERLAY_FOLDER="${WORKSPACE_OVERLAY_FOLDER:-$(dirname "${INPUTS_PLUGINS_FILE}")}" + +rm -rf "${STAGING_ROOT}" +mkdir -p "${STAGING_ROOT}/plugins" "${STAGING_ROOT}/overlay" "${STAGING_ROOT}/archives" + +if [[ -d "${WORKSPACE_OVERLAY_FOLDER}" ]]; then + cp -a "${WORKSPACE_OVERLAY_FOLDER}/." "${STAGING_ROOT}/overlay/" +fi + +plugins_json='[]' + +while IFS= read -r plugin || [[ -n "${plugin}" ]]; do + if [[ -z "${plugin// /}" ]]; then + continue + fi + if [[ "$(echo "$plugin" | sed 's/^#.*//')" == "" ]]; then + continue + fi + pluginPath=$(echo "$plugin" | sed 's/^\(^[^:]*\): *\(.*\)$/\1/') + if [[ ! -d "${INPUTS_PLUGINS_ROOT}/${pluginPath}" ]]; then + continue + fi + if [[ ! -d "${INPUTS_PLUGINS_ROOT}/${pluginPath}/dist-dynamic" ]]; then + echo "Warning: skipping ${pluginPath} (no dist-dynamic)" >&2 + continue + fi + + dest="${STAGING_ROOT}/plugins/${pluginPath}" + mkdir -p "$(dirname "${dest}")" + cp -a "${INPUTS_PLUGINS_ROOT}/${pluginPath}/." "${dest}/" + + name="$(jq -r '.name' "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + version="$(jq -r '.version' "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + image_name="$(jq -r '.name | sub("^@"; "") | sub("[/@]"; "-")' \ + "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + + plugins_json="$(echo "${plugins_json}" | jq \ + --arg path "${pluginPath}" \ + --arg name "${name}" \ + --arg version "${version}" \ + --arg imageName "${image_name}" \ + '. + [{path: $path, name: $name, version: $version, imageName: $imageName}]')" +done < "${INPUTS_PLUGINS_FILE}" + +if [[ -n "${INPUTS_DESTINATION:-}" && -d "${INPUTS_DESTINATION}" ]]; then + cp -a "${INPUTS_DESTINATION}/." "${STAGING_ROOT}/archives/" 2>/dev/null || true +fi + +jq -n \ + --argjson plugins "${plugins_json}" \ + --arg imageTagPrefix "${INPUTS_IMAGE_TAG_PREFIX:-}" \ + --arg appConfigFileName "${INPUTS_APP_CONFIG_FILE_NAME}" \ + '{ + plugins: $plugins, + imageTagPrefix: $imageTagPrefix, + appConfigFileName: $appConfigFileName + }' > "${STAGING_ROOT}/manifest.json" + +echo "Created export staging at ${STAGING_ROOT} ($(echo "${plugins_json}" | jq length) plugins)" diff --git a/export-dynamic/export-dynamic.sh b/export-dynamic/export-dynamic.sh index d29c800..257cb77 100755 --- a/export-dynamic/export-dynamic.sh +++ b/export-dynamic/export-dynamic.sh @@ -1,56 +1,38 @@ #!/usr/bin/env bash +# +# Export plugins to dist-dynamic archives (.tgz). OCI image build/push is handled +# separately by scripts/publish-export-staging.sh (export-publish CI job). +# + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" errors=() -images=() IFS=$'\n' workspaceOverlayFolder="$(dirname "${INPUTS_PLUGINS_FILE}")" skipWorkspace=false -# INPUTS_CLI_VERSION must be set; or fall back to old default INPUTS_JANUS_CLI_VERSION - -INPUTS_CLI_PACKAGE=${INPUTS_CLI_PACKAGE:-"@red-hat-developer-hub/cli"} -# set command names based on CLI package +INPUTS_CLI_PACKAGE=${INPUTS_CLI_PACKAGE:-"@red-hat-developer-hub/cli"} EXPORT_COMMAND=("plugin" "export") -INPUTS_CONTAINER_BUILD_TOOL=${INPUTS_CONTAINER_BUILD_TOOL:-"podman"} -PACKAGE_COMMAND=("plugin" "package" "--container-tool" "${INPUTS_CONTAINER_BUILD_TOOL}") - -########################################################## -# start TODO remove this once fully migrated to rhdh-cli -# fall back to old Janus defaults -if [[ "${INPUTS_JANUS_CLI_VERSION}" ]] -then - INPUTS_CLI_VERSION="${INPUTS_JANUS_CLI_VERSION}" -fi -# fall back to old janus-idp/cli commands -if [[ "${INPUTS_CLI_PACKAGE}" == "@janus-idp/cli" ]] -then - EXPORT_COMMAND=("package" "export-dynamic-plugin") - PACKAGE_COMMAND=("package" "package-dynamic-plugins") +if [[ -z "${INPUTS_CLI_CALLER:-}" ]]; then + INPUTS_CLI_CALLER="$("${SCRIPT_DIR}/resolve-export-cli.sh")" fi -# end TODO remove this once fully migrated to rhdh-cli -########################################################## -# by default, run online with npx --yes installing the cli -# use a local binary (for airgapped/hermetic use cases) with: -# export INPUTS_CLI_CALLER=/path/to/node_modules/.bin/rhdh-cli -INPUTS_CLI_CALLER=${INPUTS_CLI_CALLER:-"npx --yes ${INPUTS_CLI_PACKAGE}@${INPUTS_CLI_VERSION}"} - -# Check local installation first, then fall back to npx --yes (requires network) run_cli() { + # Phase 2 install: rhdh-cli runs yarn install in dist-dynamic/ where + # --allow-native-package modules must compile (UBI/Node ABI). Unset phase-1 + # env from the monorepo compile job (NPM_CONFIG_IGNORE_SCRIPTS / YARN_ENABLE_SCRIPTS). + unset NPM_CONFIG_IGNORE_SCRIPTS YARN_ENABLE_SCRIPTS + local cli_args=("$@") local cli_bin=() - # split by spaces into an array so we can execute IFS=" " read -r -a cli_bin <<< "$INPUTS_CLI_CALLER" IFS=" " read -r -a cli_args_split <<< "${cli_args[@]}" - # use @ not * to ignore newlines and show array values as a single line # shellcheck disable=SC2145 echo " > ${cli_bin[@]} ${cli_args_split[@]}" - # suppress logging unless an error occurs; then dump full log for debugging purposes - # we WANT cli_args to split by spaces here # shellcheck disable=SC2068 if ! "${cli_bin[@]}" ${cli_args_split[@]} >/tmp/export-dynamic-cli.log 2>&1; then echo "Error running CLI: " @@ -58,7 +40,6 @@ run_cli() { cat /tmp/export-dynamic-cli.log echo "##########################################################" - # Search for both types of log files for backwards compatibility local yarn_install_logs=("yarn-install.log" "rhdh-cli.yarn-install.log") for log_file in "${yarn_install_logs[@]}"; do if [[ -f "$log_file" ]]; then @@ -73,20 +54,19 @@ run_cli() { } set -e -if [[ "${INPUTS_LAST_PUBLISH_COMMIT}" != "" ]] -then - pushd "${workspaceOverlayFolder}" > /dev/null +if [[ "${INPUTS_FORCE_EXPORT:-}" == "true" ]]; then + echo "Force export enabled — skipping unchanged-since-last-publish check" +elif [[ "${INPUTS_LAST_PUBLISH_COMMIT}" != "" ]]; then + pushd "${workspaceOverlayFolder}" > /dev/null workspaceLastCommit="$(git log -1 --format=%H .)" echo "Checking if workspace last commit (${workspaceLastCommit}) is an ancestor of the last published commit (${INPUTS_LAST_PUBLISH_COMMIT})" - if git merge-base --is-ancestor "${workspaceLastCommit}" "${INPUTS_LAST_PUBLISH_COMMIT}" - then + if git merge-base --is-ancestor "${workspaceLastCommit}" "${INPUTS_LAST_PUBLISH_COMMIT}"; then skipWorkspace=true fi - popd > /dev/null + popd > /dev/null fi -if [[ "${skipWorkspace}" == "true" ]] -then +if [[ "${skipWorkspace}" == "true" ]]; then echo "Skipping workspace since it didn't change since last published commit (${INPUTS_LAST_PUBLISH_COMMIT})" else overlay_backstage_json="${workspaceOverlayFolder}/backstage.json" @@ -98,16 +78,11 @@ else fi fi - # We use '|| [[ -n "$plugin" ]]' to catch the last line even if it lacks a newline. - while IFS= read -r plugin || [[ -n "$plugin" ]] - do - # echo "Processing plugin: $plugin" - # Skip empty lines + while IFS= read -r plugin || [[ -n "$plugin" ]]; do if [[ -z "${plugin// /}" ]]; then echo "Skip empty line" continue fi - # Skip commented lines # shellcheck disable=SC2001 if [[ "$(echo "$plugin" | sed 's/^#.*//')" == "" ]]; then echo "Skip commented line" @@ -117,31 +92,27 @@ else pluginPath=$(echo "$plugin" | sed 's/^\(^[^:]*\): *\(.*\)$/\1/') # shellcheck disable=SC2001 args=$(echo "$plugin" | sed 's/^\(^[^:]*\): *\(.*\)$/\2/') - - # check if folder exists; if not, skip this package + if [ ! -d "$pluginPath" ]; then echo "Skip missing package folder $pluginPath" continue fi pushd "$pluginPath" > /dev/null - - if [[ "$(grep -e '"role" *: *"frontend-plugin' package.json)" != "" ]] - then + + if [[ "$(grep -e '"role" *: *"frontend-plugin' package.json)" != "" ]]; then pluginType=frontend optionalScalprumConfigFile="${workspaceOverlayFolder}/${pluginPath}/${INPUTS_SCALPRUM_CONFIG_FILE_NAME}" - if [ -f "${optionalScalprumConfigFile}" ] - then + if [[ -f "${optionalScalprumConfigFile}" ]]; then args="$args --scalprum-config ${optionalScalprumConfigFile}" fi else pluginType=backend fi - + echo "========== Exporting $pluginType plugin $pluginPath ==========" - + optionalSourceOverlay="${workspaceOverlayFolder}/${pluginPath}/${INPUTS_SOURCE_OVERLAY_FOLDER_NAME}" - if [ -d "${optionalSourceOverlay}" ] - then + if [[ -d "${optionalSourceOverlay}" ]]; then echo " copying source overlay" cp -Rfv "${optionalSourceOverlay}"/* . fi @@ -163,36 +134,7 @@ else fi echo - # package the dynamic plugin in a container image - if [[ "${INPUTS_IMAGE_REPOSITORY_PREFIX}" != "" ]] - then - PLUGIN_NAME=$(jq -r '.name | sub("^@"; "") | sub("[/@]"; "-")' package.json) - PLUGIN_VERSION="${INPUTS_IMAGE_TAG_PREFIX}$(jq -r '.version' package.json)" - PLUGIN_CONTAINER_TAG="${INPUTS_IMAGE_REPOSITORY_PREFIX}/${PLUGIN_NAME}:${PLUGIN_VERSION}" - - echo "========== Packaging Container ${PLUGIN_CONTAINER_TAG} ==========" - if run_cli "${PACKAGE_COMMAND[@]}" --tag "${PLUGIN_CONTAINER_TAG}"; then - if [[ "${INPUTS_PUSH_CONTAINER_IMAGE}" == "true" ]] - then - echo "========== Publishing Container ${PLUGIN_CONTAINER_TAG} ==========" - if ${INPUTS_CONTAINER_BUILD_TOOL} push "$PLUGIN_CONTAINER_TAG"; then - images+=("${PLUGIN_CONTAINER_TAG}") - else - echo " Error pushing container image" - errors+=("${pluginPath}") - fi - else - images+=("${PLUGIN_CONTAINER_TAG}") - fi - else - echo " Error building container image" - errors+=("${pluginPath}") - fi - fi - echo - - if [[ "${INPUTS_DESTINATION}" != "" ]] - then + if [[ "${INPUTS_DESTINATION}" != "" ]]; then echo "========== Moving $pluginType plugin $pluginPath archive into ${INPUTS_DESTINATION} ==========" packDestination=${INPUTS_DESTINATION} @@ -206,13 +148,12 @@ else continue fi set -e - + filename=$(echo "$json" | jq -r '.[0].filename') integrity=$(echo "$json" | jq -r '.[0].integrity') echo "$integrity" > "$packDestination/${filename}.integrity" optionalConfigFile="${workspaceOverlayFolder}/${pluginPath}/${INPUTS_APP_CONFIG_FILE_NAME}" - if [ -f "${optionalConfigFile}" ] - then + if [[ -f "${optionalConfigFile}" ]]; then echo " copying default app-config" cp -v "${optionalConfigFile}" "$packDestination/${filename}.${INPUTS_APP_CONFIG_FILE_NAME}" fi @@ -227,35 +168,28 @@ fi FAILED_EXPORTS_OUTPUT=${FAILED_EXPORTS_OUTPUT:-"failed-exports-output"} touch "$FAILED_EXPORTS_OUTPUT" -for error in "${errors[@]}" -do +for error in "${errors[@]}"; do echo "$error" >> "$FAILED_EXPORTS_OUTPUT" done -PUBLISHED_EXPORTS_OUTPUT=${PUBLISHED_EXPORTS_OUTPUT:-"published-exports-output"} -touch "$PUBLISHED_EXPORTS_OUTPUT" -for image in "${images[@]}" -do - echo "$image" >> "$PUBLISHED_EXPORTS_OUTPUT" -done - -# write to a temp file if the GITHUB_OUTPUT pipe isn't set -if [[ ! "$GITHUB_OUTPUT" ]]; then GITHUB_OUTPUT=/tmp/github_output.txt; fi - -echo "FAILED_EXPORTS< "$EXPORT_STEP_OUTPUTS" + exit $((${#errors[@]})) diff --git a/export-dynamic/resolve-export-cli.sh b/export-dynamic/resolve-export-cli.sh new file mode 100755 index 0000000..581f7f7 --- /dev/null +++ b/export-dynamic/resolve-export-cli.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# Resolve INPUTS_CLI_CALLER from a baked export-builder manifest and CLI version. +# +set -euo pipefail + +if [[ -n "${INPUTS_CLI_CALLER:-}" ]]; then + if [[ -x "${INPUTS_CLI_CALLER}" ]] || command -v "${INPUTS_CLI_CALLER%% *}" >/dev/null 2>&1; then + echo "${INPUTS_CLI_CALLER}" + exit 0 + fi + echo "Error: INPUTS_CLI_CALLER is set but not executable: ${INPUTS_CLI_CALLER}" >&2 + exit 1 +fi + +cli_version="${INPUTS_CLI_VERSION:-}" +if [[ -z "${cli_version}" ]]; then + echo "Error: INPUTS_CLI_VERSION is required." >&2 + exit 1 +fi + +manifest="${EXPORT_BUILDER_MANIFEST:-/etc/rhdh-export-builder/manifest.json}" +default_path="/opt/rhdh-cli/${cli_version}/bin/rhdh-cli" + +if [[ -f "${manifest}" ]]; then + path="$(jq -r --arg v "${cli_version}" \ + '.cliVersions[] | select(.version == $v) | .path' "${manifest}" | head -n1)" + if [[ -n "${path}" && "${path}" != "null" && -x "${path}" ]]; then + echo "${path}" + exit 0 + fi + supported="$(jq -r '[.cliVersions[].version] | join(", ")' "${manifest}")" + echo "Error: CLI ${cli_version} not found in ${manifest}." >&2 + echo "Supported versions: ${supported}" >&2 + echo "Rebuild the export-builder image (publish-export-builder workflow)." >&2 + exit 1 +fi + +if [[ -x "${default_path}" ]]; then + echo "${default_path}" + exit 0 +fi + +echo "Error: no manifest at ${manifest} and ${default_path} is missing." >&2 +exit 1 diff --git a/scripts/create-export-staging.sh b/scripts/create-export-staging.sh new file mode 100755 index 0000000..c4d7536 --- /dev/null +++ b/scripts/create-export-staging.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# Pack export outputs into a staging directory for the publish job. +# +# Env: +# STAGING_ROOT — output directory (required) +# INPUTS_PLUGINS_FILE — plugins-list.yaml path (required) +# INPUTS_PLUGINS_ROOT — absolute path to monorepo plugins root (required) +# INPUTS_DESTINATION — dynamic-plugin-archives folder (optional) +# INPUTS_IMAGE_TAG_PREFIX — OCI tag prefix (optional) +# INPUTS_APP_CONFIG_FILE_NAME — default app-config.dynamic.yaml +# WORKSPACE_OVERLAY_FOLDER — overlay workspace folder (dirname of plugins file) +# +set -euo pipefail + +STAGING_ROOT="${STAGING_ROOT:?STAGING_ROOT is required}" +INPUTS_PLUGINS_FILE="${INPUTS_PLUGINS_FILE:?INPUTS_PLUGINS_FILE is required}" +INPUTS_PLUGINS_ROOT="${INPUTS_PLUGINS_ROOT:?INPUTS_PLUGINS_ROOT is required}" +INPUTS_APP_CONFIG_FILE_NAME="${INPUTS_APP_CONFIG_FILE_NAME:-app-config.dynamic.yaml}" +WORKSPACE_OVERLAY_FOLDER="${WORKSPACE_OVERLAY_FOLDER:-$(dirname "${INPUTS_PLUGINS_FILE}")}" + +rm -rf "${STAGING_ROOT}" +mkdir -p "${STAGING_ROOT}/plugins" "${STAGING_ROOT}/overlay" "${STAGING_ROOT}/archives" + +if [[ -d "${WORKSPACE_OVERLAY_FOLDER}" ]]; then + cp -a "${WORKSPACE_OVERLAY_FOLDER}/." "${STAGING_ROOT}/overlay/" +fi + +plugins_json='[]' + +while IFS= read -r plugin || [[ -n "${plugin}" ]]; do + if [[ -z "${plugin// /}" ]]; then + continue + fi + if [[ "$(echo "$plugin" | sed 's/^#.*//')" == "" ]]; then + continue + fi + pluginPath=$(echo "$plugin" | sed 's/^\(^[^:]*\): *\(.*\)$/\1/') + if [[ ! -d "${INPUTS_PLUGINS_ROOT}/${pluginPath}" ]]; then + continue + fi + if [[ ! -d "${INPUTS_PLUGINS_ROOT}/${pluginPath}/dist-dynamic" ]]; then + echo "Warning: skipping ${pluginPath} (no dist-dynamic)" >&2 + continue + fi + + dest="${STAGING_ROOT}/plugins/${pluginPath}" + mkdir -p "$(dirname "${dest}")" + cp -a "${INPUTS_PLUGINS_ROOT}/${pluginPath}/." "${dest}/" + + name="$(jq -r '.name' "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + version="$(jq -r '.version' "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + image_name="$(jq -r '.name | sub("^@"; "") | sub("[/@]"; "-")' \ + "${INPUTS_PLUGINS_ROOT}/${pluginPath}/package.json")" + + plugins_json="$(echo "${plugins_json}" | jq \ + --arg path "${pluginPath}" \ + --arg name "${name}" \ + --arg version "${version}" \ + --arg imageName "${image_name}" \ + '. + [{path: $path, name: $name, version: $version, imageName: $imageName}]')" +done < "${INPUTS_PLUGINS_FILE}" + +if [[ -n "${INPUTS_DESTINATION:-}" && -d "${INPUTS_DESTINATION}" ]]; then + cp -a "${INPUTS_DESTINATION}/." "${STAGING_ROOT}/archives/" 2>/dev/null || true +fi + +jq -n \ + --argjson plugins "${plugins_json}" \ + --arg imageTagPrefix "${INPUTS_IMAGE_TAG_PREFIX:-}" \ + --arg appConfigFileName "${INPUTS_APP_CONFIG_FILE_NAME}" \ + '{ + plugins: $plugins, + imageTagPrefix: $imageTagPrefix, + appConfigFileName: $appConfigFileName + }' > "${STAGING_ROOT}/manifest.json" + +echo "Created export staging at ${STAGING_ROOT} ($(echo "${plugins_json}" | jq length) plugins)" diff --git a/scripts/generate-export-builder-config.sh b/scripts/generate-export-builder-config.sh new file mode 100755 index 0000000..ae7d990 --- /dev/null +++ b/scripts/generate-export-builder-config.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +# +# Generate per-Node-major export-builder manifests and CLI install scripts from +# versions.json on active overlay release branches. +# +# Usage: +# ./scripts/generate-export-builder-config.sh +# OVERLAY_REPO=redhat-developer/rhdh-plugin-export-overlays \ +# OVERLAY_BRANCHES=main,release-1.10,release-1.9 \ +# ./scripts/generate-export-builder-config.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-${REPO_ROOT}/build/generated}" +OVERLAY_REPO="${OVERLAY_REPO:-redhat-developer/rhdh-plugin-export-overlays}" +OVERLAY_BRANCHES="${OVERLAY_BRANCHES:-main,release-1.10,release-1.9}" +OVERLAY_LOCAL_DIR="${OVERLAY_LOCAL_DIR:-}" + +mkdir -p "${OUTPUT_DIR}" + +fetch_versions_json() { + local branch="$1" + if [[ -n "${OVERLAY_LOCAL_DIR}" ]]; then + if git -C "${OVERLAY_LOCAL_DIR}" rev-parse --verify "${branch}" >/dev/null 2>&1; then + git -C "${OVERLAY_LOCAL_DIR}" show "${branch}:versions.json" + return 0 + fi + if [[ -f "${OVERLAY_LOCAL_DIR}/versions.json" ]]; then + cat "${OVERLAY_LOCAL_DIR}/versions.json" + return 0 + fi + fi + if command -v gh >/dev/null 2>&1; then + gh api "repos/${OVERLAY_REPO}/contents/versions.json?ref=${branch}" --jq '.content' \ + | base64 -d + return 0 + fi + echo "Error: cannot fetch versions.json for branch ${branch}. Set OVERLAY_LOCAL_DIR or install gh." >&2 + return 1 +} + +declare -A NODE_MAJORS +declare -A CLI_BY_NODE +declare -A BRANCHES_BY_CLI_NODE + +IFS=',' read -r -a branches <<< "${OVERLAY_BRANCHES}" +for branch in "${branches[@]}"; do + branch="${branch// /}" + [[ -z "${branch}" ]] && continue + echo "Reading versions.json from ${OVERLAY_REPO}@${branch}..." + versions_json="$(fetch_versions_json "${branch}")" + node_full="$(echo "${versions_json}" | jq -r '.node // empty')" + cli_ver="$(echo "${versions_json}" | jq -r '.cli // empty')" + cli_pkg="$(echo "${versions_json}" | jq -r '."cliPackage" // "@red-hat-developer-hub/cli"')" + if [[ -z "${node_full}" || -z "${cli_ver}" ]]; then + echo "Warning: skipping branch ${branch} (missing node or cli in versions.json)" >&2 + continue + fi + node_major="${node_full%%.*}" + NODE_MAJORS["${node_major}"]="${node_full}" + CLI_PACKAGE="${cli_pkg}" + key="${node_major}|${cli_ver}" + CLI_BY_NODE["${key}"]=1 + existing="${BRANCHES_BY_CLI_NODE["${key}"]:-}" + if [[ -n "${existing}" ]]; then + BRANCHES_BY_CLI_NODE["${key}"]="${existing},${branch}" + else + BRANCHES_BY_CLI_NODE["${key}"]="${branch}" + fi +done + +if [[ ${#NODE_MAJORS[@]} -eq 0 ]]; then + echo "Error: no Node majors discovered from overlay branches." >&2 + exit 1 +fi + +generated_at="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +mapfile -t sorted_majors < <(printf '%s\n' "${!NODE_MAJORS[@]}" | sort -n) + +for node_major in "${sorted_majors[@]}"; do + node_full="${NODE_MAJORS[${node_major}]}" + ubi_tag="ubi9-node${node_major}" + manifest_path="${OUTPUT_DIR}/export-builder-node${node_major}.json" + install_path="${OUTPUT_DIR}/cli-install-node${node_major}.sh" + + cli_versions_json='[]' + for key in "${!CLI_BY_NODE[@]}"; do + if [[ "${key%%|*}" != "${node_major}" ]]; then + continue + fi + cli_ver="${key#*|}" + cli_path="/opt/rhdh-cli/${cli_ver}/bin/rhdh-cli" + branches_csv="${BRANCHES_BY_CLI_NODE[${key}]}" + branches_json="$(echo "${branches_csv}" | jq -R 'split(",")')" + cli_versions_json="$(echo "${cli_versions_json}" | jq \ + --arg version "${cli_ver}" \ + --arg path "${cli_path}" \ + --argjson overlayBranches "${branches_json}" \ + '. + [{version: $version, path: $path, overlayBranches: $overlayBranches}]')" + done + + jq -n \ + --argjson major "${node_major}" \ + --arg full "${node_full}" \ + --arg ubiTag "${ubi_tag}" \ + --arg cliPackage "${CLI_PACKAGE}" \ + --argjson cliVersions "${cli_versions_json}" \ + --arg overlayRepo "${OVERLAY_REPO}" \ + --argjson overlayBranches "$(printf '%s\n' "${branches[@]}" | jq -R . | jq -s .)" \ + --arg generatedAt "${generated_at}" \ + '{ + node: {major: $major, full: $full, ubiTag: $ubiTag}, + cliPackage: $cliPackage, + cliVersions: $cliVersions, + builtFrom: { + overlayRepo: $overlayRepo, + overlayBranches: $overlayBranches, + generatedAt: $generatedAt + } + }' > "${manifest_path}" + + { + echo '#!/usr/bin/env bash' + echo 'set -euo pipefail' + echo 'export NPM_CONFIG_LOGLEVEL="${NPM_CONFIG_LOGLEVEL:-error}"' + echo "CLI_PACKAGE='${CLI_PACKAGE}'" + echo "${cli_versions_json}" | jq -r '.[].version' | while read -r ver; do + [[ -z "${ver}" ]] && continue + cat < "${install_path}" + chmod +x "${install_path}" + + echo "Wrote ${manifest_path}" + echo "Wrote ${install_path}" +done + +# Summary for CI matrix +jq -n \ + --argjson majors "$(printf '%s\n' "${sorted_majors[@]}" | jq -R 'tonumber' | jq -s .)" \ + '{nodeMajors: $majors}' > "${OUTPUT_DIR}/builder-matrix.json" +echo "Wrote ${OUTPUT_DIR}/builder-matrix.json" diff --git a/scripts/lint-workflows.sh b/scripts/lint-workflows.sh new file mode 100755 index 0000000..b319699 --- /dev/null +++ b/scripts/lint-workflows.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# +# Lint GitHub Actions workflow YAML with actionlint before push. +# +# Usage: +# ./scripts/lint-workflows.sh +# ./scripts/lint-workflows.sh .github/workflows/export-dynamic.yaml +# ./scripts/lint-workflows.sh --shellcheck +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +ACTIONLINT_VERSION="${ACTIONLINT_VERSION:-1.7.12}" +CACHE_DIR="${REPO_ROOT}/.cache" +CACHED_ACTIONLINT="${CACHE_DIR}/actionlint" + +ENABLE_SHELLCHECK=false +WORKFLOW_PATHS=() + +usage() { + cat <<'EOF' +Lint GitHub Actions workflow files with actionlint. + +By default runs shellcheck integration off (-shellcheck=) so output focuses on +workflow syntax/expression errors. Pass --shellcheck to enable shell lint too. + +Uses actionlint from PATH when available; otherwise downloads a pinned binary +to .cache/actionlint (gitignored). + +Examples: + ./scripts/lint-workflows.sh + ./scripts/lint-workflows.sh .github/workflows/test-export-smoke.yaml + ./scripts/lint-workflows.sh --shellcheck +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --shellcheck) + ENABLE_SHELLCHECK=true + shift + ;; + --) + shift + WORKFLOW_PATHS+=("$@") + break + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + *) + WORKFLOW_PATHS+=("$1") + shift + ;; + esac +done + +if [[ ${#WORKFLOW_PATHS[@]} -eq 0 ]]; then + WORKFLOW_PATHS=("${REPO_ROOT}"/.github/workflows/*.yaml) +fi + +resolve_actionlint() { + if command -v actionlint >/dev/null 2>&1; then + command -v actionlint + return 0 + fi + + if [[ -x "${CACHED_ACTIONLINT}" ]]; then + echo "${CACHED_ACTIONLINT}" + return 0 + fi + + mkdir -p "${CACHE_DIR}" + local os arch ext archive url tmp + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + arch="$(uname -m)" + case "${arch}" in + x86_64) arch=amd64 ;; + aarch64|arm64) arch=arm64 ;; + *) + echo "Error: unsupported architecture for actionlint download: ${arch}" >&2 + echo "Install actionlint manually: https://github.com/rhysd/actionlint" >&2 + exit 1 + ;; + esac + case "${os}" in + linux) ext=tar.gz ;; + darwin) ext=tar.gz ;; + *) + echo "Error: unsupported OS for actionlint download: ${os}" >&2 + echo "Install actionlint manually: https://github.com/rhysd/actionlint" >&2 + exit 1 + ;; + esac + + archive="actionlint_${ACTIONLINT_VERSION}_${os}_${arch}.${ext}" + url="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${archive}" + tmp="$(mktemp "${CACHE_DIR}/actionlint.XXXXXX.${ext}")" + echo "Downloading actionlint v${ACTIONLINT_VERSION} to ${CACHED_ACTIONLINT}..." >&2 + curl -fsSL "${url}" -o "${tmp}" + tar -xzf "${tmp}" -C "${CACHE_DIR}" + rm -f "${tmp}" + chmod +x "${CACHED_ACTIONLINT}" + echo "${CACHED_ACTIONLINT}" +} + +ACTIONLINT_BIN="$(resolve_actionlint)" +args=("${ACTIONLINT_BIN}" -color) +if [[ "${ENABLE_SHELLCHECK}" == "false" ]]; then + args+=(-shellcheck=) +fi +args+=("${WORKFLOW_PATHS[@]}") + +echo "Running: ${args[*]}" +"${args[@]}" diff --git a/scripts/local-test-export-common.sh b/scripts/local-test-export-common.sh new file mode 100644 index 0000000..fa4624d --- /dev/null +++ b/scripts/local-test-export-common.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# +# Shared helpers for local-test-export.sh and local-test-export-reset.sh +# + +# Under $HOME/tmp (not system /tmp): user-owned, fewer SELinux surprises with podman bind mounts. +DEFAULT_EXPORT_WORKDIR="${HOME}/tmp/rhdh-plugin-export-test" +# Host-writable publish logs (compile workdir is container subuid-owned under rootless podman). +DEFAULT_PUBLISH_OUTPUT_DIR="${HOME}/tmp/rhdh-plugin-export-publish-output" + +# Repo-local output from generate-export-builder-config.sh (local-test-export.sh). +_EXPORT_UTILS_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUILD_GENERATED_DIR="${BUILD_GENERATED_DIR:-${_EXPORT_UTILS_REPO_ROOT}/build/generated}" + +remove_build_generated() { + local dir="${1:-${BUILD_GENERATED_DIR}}" + if [[ -d "${dir}" ]]; then + rm -rf "${dir}" + fi + return 0 +} + +# Rootless podman creates bind-mount files as subuids (e.g. 525287). chown from a container +# does not remap them on the host; podman unshare operates in the user namespace that owns them. +remove_export_workdir() { + local dir="$1" + if [[ ! -e "${dir}" ]]; then + return 0 + fi + if rm -rf "${dir}" 2>/dev/null; then + return 0 + fi + echo " Workdir not removable as $(id -un); removing via podman unshare..." + if podman unshare rm -rf "${dir}" 2>/dev/null; then + return 0 + fi + echo "Error: could not remove ${dir}." >&2 + return 1 +} diff --git a/scripts/local-test-export-reset.sh b/scripts/local-test-export-reset.sh new file mode 100755 index 0000000..10b84ee --- /dev/null +++ b/scripts/local-test-export-reset.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# +# Reset local state created by scripts/local-test-export.sh so the next run +# starts from a clean slate. +# +# Usage: +# ./scripts/local-test-export-reset.sh +# ./scripts/local-test-export-reset.sh --purge-builder +# ./scripts/local-test-export-reset.sh --keep-registry +# ./scripts/local-test-export-reset.sh --no-prune +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/local-test-export-common.sh +source "${SCRIPT_DIR}/local-test-export-common.sh" + +WORKDIR="${WORKDIR:-${DEFAULT_EXPORT_WORKDIR}}" +REGISTRY_NAME="${REGISTRY_NAME:-rhdh-export-local-registry}" +REGISTRY_VOLUME="${REGISTRY_VOLUME:-${REGISTRY_NAME}-data}" +REGISTRY_PORT="${REGISTRY_PORT:-5001}" +BUILDER_IMAGE="${BUILDER_IMAGE:-localhost/rhdh-plugin-export-builder:ubi9-node24}" +IMAGE_REPOSITORY_PREFIX="localhost:${REGISTRY_PORT}/rhdh-plugin-export-test" + +KEEP_REGISTRY=false +PURGE_BUILDER=false +PRUNE_PODMAN=true + +usage() { + cat <<'EOF' +Reset local plugin export test state. + +By default removes: + - build/generated/ in this repo (export-builder config from local-test-export.sh) + - The test workdir (clones, node_modules, archives, export-staging) + - Publish result files in ~/tmp/rhdh-plugin-export-publish-output/ + - The local registry container (fresh catalog on next test run) + - Locally cached test plugin images (localhost:5001/rhdh-plugin-export-test/*) + - All unused podman images and build cache (podman system prune -af) + +Options: + --keep-registry Leave the registry container running (only clear workdir/images) + --purge-builder Also remove export-builder images (ubi9-node* tags) + --no-prune Skip aggressive podman prune (keeps unrelated unused images) + -d, --workdir DIR Workdir to remove (default: ~/tmp/rhdh-plugin-export-test) + -h, --help Show this help + +Environment (same as local-test-export.sh): + WORKDIR, REGISTRY_NAME, REGISTRY_PORT, BUILDER_IMAGE + +Warning: default prune removes ALL unused podman images on this machine, not only +export-test artifacts. Use --no-prune if you rely on other local podman images. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --keep-registry) KEEP_REGISTRY=true; shift ;; + --purge-builder) PURGE_BUILDER=true; shift ;; + --no-prune) PRUNE_PODMAN=false; shift ;; + -d|--workdir) WORKDIR="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +# The export container entrypoint may leave root-owned files on the bind mount. + +echo "=== Resetting local plugin export test state ===" + +if [[ -d "${BUILD_GENERATED_DIR}" ]]; then + echo "Removing repo-generated builder config: ${BUILD_GENERATED_DIR}" + remove_build_generated "${BUILD_GENERATED_DIR}" +else + echo "Repo-generated builder config not present: ${BUILD_GENERATED_DIR}" +fi + +if [[ -d "${WORKDIR}" ]]; then + echo "Removing workdir: ${WORKDIR}" + remove_export_workdir "${WORKDIR}" +else + echo "Workdir not present: ${WORKDIR}" +fi + +if [[ -d "${DEFAULT_PUBLISH_OUTPUT_DIR}" ]]; then + echo "Removing publish output dir: ${DEFAULT_PUBLISH_OUTPUT_DIR}" + rm -rf "${DEFAULT_PUBLISH_OUTPUT_DIR}" +fi + +if [[ "${KEEP_REGISTRY}" == "true" ]]; then + echo "Keeping registry container: ${REGISTRY_NAME}" +else + if podman container exists "${REGISTRY_NAME}" 2>/dev/null; then + echo "Stopping and removing registry container: ${REGISTRY_NAME}" + podman rm -f "${REGISTRY_NAME}" >/dev/null + else + echo "Registry container not present: ${REGISTRY_NAME}" + fi + if podman volume exists "${REGISTRY_VOLUME}" 2>/dev/null; then + echo "Removing registry data volume: ${REGISTRY_VOLUME}" + podman volume rm "${REGISTRY_VOLUME}" >/dev/null + fi +fi + +echo "Removing local test plugin images: ${IMAGE_REPOSITORY_PREFIX}/*" +mapfile -t test_images < <(podman images --format '{{.Repository}}:{{.Tag}}' \ + | grep -E "^${IMAGE_REPOSITORY_PREFIX}/" || true) +if [[ ${#test_images[@]} -gt 0 ]]; then + podman rmi -f "${test_images[@]}" +else + echo " (none found)" +fi + +if [[ "${PURGE_BUILDER}" == "true" ]]; then + mapfile -t builder_images < <(podman images --format '{{.Repository}}:{{.Tag}}' \ + | grep -E '^localhost/rhdh-plugin-export-builder:ubi9(-node[0-9]+)?$' || true) + if [[ ${#builder_images[@]} -gt 0 ]]; then + echo "Removing export-builder images:" + podman rmi -f "${builder_images[@]}" + elif podman image exists "${BUILDER_IMAGE}" 2>/dev/null; then + echo "Removing export-builder image: ${BUILDER_IMAGE}" + podman rmi -f "${BUILDER_IMAGE}" + else + echo "Builder image not present: ${BUILDER_IMAGE}" + fi +else + echo "Keeping export-builder image(s) (use --purge-builder to remove)" +fi + +if [[ "${PRUNE_PODMAN}" == "true" ]]; then + echo "" + echo "=== Aggressive podman prune (all unused images and build cache) ===" + podman system prune -af + echo "Podman prune complete." +else + echo "" + echo "Skipping podman prune (--no-prune)." +fi + +echo "" +echo "Done. Run ./scripts/local-test-export.sh for a fresh test." diff --git a/scripts/local-test-export.sh b/scripts/local-test-export.sh new file mode 100755 index 0000000..db95fe0 --- /dev/null +++ b/scripts/local-test-export.sh @@ -0,0 +1,370 @@ +#!/usr/bin/env bash +# +# Local end-to-end test for the UBI 9 plugin export pipeline. +# +# Builds the export-builder image, optionally starts a local OCI registry, +# checks out upstream sources, applies overlay patches, and runs export-dynamic.sh +# inside the builder container (matching CI). +# +# Usage: +# ./scripts/local-test-export.sh +# ./scripts/local-test-export.sh -w workspaces/tech-radar +# ./scripts/local-test-export.sh -f workspaces/backstage/plugins-list.yaml +# ./scripts/local-test-export.sh -w workspaces/backstage -f workspaces/backstage/plugins-list.yaml -d ~/tmp/rhdh-plugin-export-backstage +# ./scripts/local-test-export.sh --no-push --no-registry +# +# Reset all local test state before a fresh run (includes aggressive podman prune): +# ./scripts/local-test-export-reset.sh +# +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/local-test-export-common.sh +source "${SCRIPT_DIR}/local-test-export-common.sh" +UTILS_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)" +OVERLAYS_DIR="${OVERLAYS_DIR:-$(cd "${UTILS_DIR}/../rhdh-plugin-export-overlays" 2>/dev/null && pwd || true)}" +WORKDIR="${WORKDIR:-${DEFAULT_EXPORT_WORKDIR}}" +WORKSPACE="workspaces/tech-radar" +PLUGINS_FILE="" +BUILDER_IMAGE="localhost/rhdh-plugin-export-builder:ubi9-node24" +REGISTRY_NAME="rhdh-export-local-registry" +REGISTRY_VOLUME="${REGISTRY_NAME}-data" +REGISTRY_PORT="5001" +START_REGISTRY=true +PUSH_OCI=true +IMAGE_TAG_PREFIX="local__" +RHDH_CLI_CALLER="" +KEEP_WORKDIR=false +PRUNE_PODMAN=false +WORKSPACE_EXPLICIT=false + +usage() { + sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//' + echo "" + echo "Options:" + echo " -w, --workspace PATH Overlay workspace (default: workspaces/tech-radar)" + echo " -f, --plugins-file PATH plugins-list.yaml (default: /plugins-list.yaml)" + echo " Relative paths resolve under the overlays repo (-o)." + echo " If the path contains workspaces//, -w is inferred." + echo " -o, --overlays DIR Path to rhdh-plugin-export-overlays clone" + echo " -d, --workdir DIR Working directory (default: ~/tmp/rhdh-plugin-export-test)" + echo " --keep-workdir Reuse existing workdir (skip rm -rf; faster re-exports)" + echo " --prune-podman Run podman system prune -af before export (free disk)" + echo " --no-registry Skip starting the local registry container" + echo " --no-push Skip OCI image build/push (compile-only; saves disk)" + echo " --compile-only Same as --no-push" + echo " --publish-only DIR Publish OCI from existing staging directory" + echo " --registry-port PORT Local registry port (default: 5001)" + echo " --builder-image IMAGE Export builder image tag (default: localhost/...:ubi9-node24)" + echo " --rhdh-cli PATH Use a local rhdh-cli binary for the publish step" + echo " -h, --help Show this help" +} + +PUBLISH_ONLY="" +while [[ $# -gt 0 ]]; do + case "$1" in + -w|--workspace) WORKSPACE="$2"; WORKSPACE_EXPLICIT=true; shift 2 ;; + -f|--plugins-file|--plugins-list) PLUGINS_FILE="$2"; shift 2 ;; + -o|--overlays) OVERLAYS_DIR="$2"; shift 2 ;; + -d|--workdir) WORKDIR="$2"; shift 2 ;; + --keep-workdir) KEEP_WORKDIR=true; shift ;; + --prune-podman) PRUNE_PODMAN=true; shift ;; + --no-registry) START_REGISTRY=false; PUSH_OCI=false; shift ;; + --no-push|--compile-only) PUSH_OCI=false; shift ;; + --publish-only) PUBLISH_ONLY="$2"; PUSH_OCI=true; shift 2 ;; + --registry-port) REGISTRY_PORT="$2"; shift 2 ;; + --builder-image) BUILDER_IMAGE="$2"; shift 2 ;; + --rhdh-cli) RHDH_CLI_CALLER="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +if [[ -z "${OVERLAYS_DIR}" || ! -d "${OVERLAYS_DIR}" ]]; then + echo "Error: rhdh-plugin-export-overlays not found. Set OVERLAYS_DIR or use -o." >&2 + exit 1 +fi + +# Resolve -f to an absolute path: cwd first, then overlays repo root. +resolve_plugins_file() { + local path="$1" + if [[ -z "${path}" ]]; then + return 1 + fi + if [[ -f "${path}" ]]; then + realpath "${path}" + return 0 + fi + if [[ -f "${OVERLAYS_DIR}/${path}" ]]; then + realpath "${OVERLAYS_DIR}/${path}" + return 0 + fi + return 1 +} + +infer_workspace_from_plugins_path() { + local path="$1" + if [[ "${path}" =~ workspaces/([^/]+)/ ]]; then + echo "workspaces/${BASH_REMATCH[1]}" + fi +} + +if [[ -n "${PLUGINS_FILE}" ]]; then + plugins_arg="${PLUGINS_FILE}" + if ! PLUGINS_FILE="$(resolve_plugins_file "${plugins_arg}")"; then + echo "Error: plugins-list not found: ${plugins_arg}" >&2 + echo " (tried cwd and \${OVERLAYS_DIR}/${plugins_arg})" >&2 + exit 1 + fi + if [[ "${WORKSPACE_EXPLICIT}" == "false" ]]; then + inferred="$(infer_workspace_from_plugins_path "${PLUGINS_FILE}")" + if [[ -n "${inferred}" ]]; then + WORKSPACE="${inferred}" + fi + fi +fi + +OVERLAY_ROOT="${OVERLAYS_DIR}/${WORKSPACE}" +SOURCE_JSON="${OVERLAY_ROOT}/source.json" +PLUGINS_FILE="${PLUGINS_FILE:-${OVERLAY_ROOT}/plugins-list.yaml}" +VERSIONS_JSON="${OVERLAYS_DIR}/versions.json" + +for required in "${SOURCE_JSON}" "${PLUGINS_FILE}" "${VERSIONS_JSON}"; do + if [[ ! -f "${required}" ]]; then + echo "Error: required file not found: ${required}" >&2 + exit 1 + fi +done + +CLI_PACKAGE="$(jq -r '.cliPackage' "${VERSIONS_JSON}")" +CLI_VERSION="$(jq -r '.cli' "${VERSIONS_JSON}")" +NODE_VERSION="$(jq -r '.node' "${VERSIONS_JSON}")" +NODE_MAJOR="${NODE_VERSION%%.*}" +if [[ "${BUILDER_IMAGE}" == "localhost/rhdh-plugin-export-builder:ubi9-node24" ]] || \ + [[ "${BUILDER_IMAGE}" == "localhost/rhdh-plugin-export-builder:ubi9" ]]; then + BUILDER_IMAGE="localhost/rhdh-plugin-export-builder:ubi9-node${NODE_MAJOR}" +fi +if [[ -z "${RHDH_CLI_CALLER}" ]]; then + RHDH_CLI_CALLER="/opt/rhdh-cli/${CLI_VERSION}/bin/rhdh-cli" +fi +SOURCE_REPO="$(jq -r '.repo' "${SOURCE_JSON}")" +SOURCE_REF="$(jq -r '."repo-ref"' "${SOURCE_JSON}")" +SOURCE_FLAT="$(jq -r '."repo-flat"' "${SOURCE_JSON}")" +if [[ "${SOURCE_FLAT}" == "true" ]]; then + PLUGINS_ROOT="." +else + PLUGINS_ROOT="${WORKSPACE}" +fi + +REGISTRY="localhost:${REGISTRY_PORT}" +IMAGE_REPOSITORY_PREFIX="${REGISTRY}/rhdh-plugin-export-test" + +warn_low_disk_space() { + local target dir + target="$(df -Pk "${1}" 2>/dev/null | awk 'NR==2 {print $4}')" || return 0 + dir="$(dirname "${1}")" + if [[ -z "${target}" ]]; then + target="$(df -Pk "${dir}" 2>/dev/null | awk 'NR==2 {print $4}')" || return 0 + fi + # df -Pk reports 1K blocks; 25G ≈ 26214400 blocks + if [[ "${target}" -lt 26214400 ]]; then + echo "WARNING: less than ~25 GiB free on ${1} (df reports ${target} 1K blocks)." >&2 + echo " The backstage workspace needs a full monorepo clone, yarn install, tsc," >&2 + echo " and OCI builds. Use -d on a larger filesystem, --prune-podman, export a" >&2 + echo " smaller plugins-list via -f, --no-push for archives-only, or run" >&2 + echo " ./scripts/local-test-export-reset.sh to reclaim disk (aggressive prune)." >&2 + fi +} + +if [[ "${PRUNE_PODMAN}" == "true" ]]; then + echo "=== Pruning unused podman data (all unused images) ===" + podman system prune -af +fi + +warn_low_disk_space "${WORKDIR}" + +echo "=== RHDH plugin export local test ===" +echo "Utils: ${UTILS_DIR}" +echo "Overlays: ${OVERLAY_ROOT}" +echo "Plugins: ${PLUGINS_FILE}" +echo "Source: ${SOURCE_REPO} @ ${SOURCE_REF}" +echo "Plugins: ${PLUGINS_ROOT} (repo-flat=${SOURCE_FLAT})" +echo "Builder: ${BUILDER_IMAGE}" +echo "Workdir: ${WORKDIR}" +echo "Registry: ${REGISTRY} (push=${PUSH_OCI})" +echo "" + +echo "=== Building export-builder image ===" +OVERLAY_LOCAL_DIR="${OVERLAYS_DIR}" bash "${UTILS_DIR}/scripts/generate-export-builder-config.sh" +podman build \ + --build-arg "NODE_MAJOR=${NODE_MAJOR}" \ + -f "${UTILS_DIR}/build/containerfiles/export-builder.Containerfile" \ + -t "${BUILDER_IMAGE}" \ + "${UTILS_DIR}" + +ensure_local_registry() { + if [[ "${START_REGISTRY}" != "true" ]]; then + return 0 + fi + echo "=== Ensuring local OCI registry on port ${REGISTRY_PORT} ===" + local recreate_registry=false + if podman container exists "${REGISTRY_NAME}" 2>/dev/null; then + local network_mode + network_mode="$(podman inspect -f '{{.HostConfig.NetworkMode}}' "${REGISTRY_NAME}")" + if [[ "${network_mode}" != "host" ]]; then + echo "Recreating ${REGISTRY_NAME}: switching from port mapping to host networking" + podman rm -f "${REGISTRY_NAME}" >/dev/null + recreate_registry=true + elif [[ "$(podman inspect -f '{{.State.Running}}' "${REGISTRY_NAME}")" != "true" ]]; then + podman start "${REGISTRY_NAME}" + fi + else + recreate_registry=true + fi + if [[ "${recreate_registry}" == "true" ]]; then + podman run -d \ + --name "${REGISTRY_NAME}" \ + --network host \ + -e "REGISTRY_HTTP_ADDR=0.0.0.0:${REGISTRY_PORT}" \ + -v "${REGISTRY_VOLUME}:/var/lib/registry" \ + --restart unless-stopped \ + docker.io/library/registry:2 + fi +} + +run_local_publish() { + local staging_root="$1" + echo "=== Publishing OCI images from staging (host buildah) ===" + ensure_local_registry + if ! command -v buildah >/dev/null 2>&1; then + echo "Error: buildah is required for local publish. Install buildah or use podman." >&2 + exit 1 + fi + if [[ -z "${RHDH_CLI_CALLER}" ]] || [[ ! -x "${RHDH_CLI_CALLER}" ]]; then + if command -v rhdh-cli >/dev/null 2>&1; then + RHDH_CLI_CALLER="$(command -v rhdh-cli)" + else + NPM_CONFIG_LOGLEVEL="${NPM_CONFIG_LOGLEVEL:-error}" \ + npm install -g "${CLI_PACKAGE}@${CLI_VERSION}" \ + --ignore-scripts --omit=dev --legacy-peer-deps + RHDH_CLI_CALLER="$(command -v rhdh-cli)" + fi + fi + buildah login "${REGISTRY}" -u test -p test --tls-verify=false 2>/dev/null || true + mkdir -p "${DEFAULT_PUBLISH_OUTPUT_DIR}" + STAGING_ROOT="${staging_root}" \ + INPUTS_IMAGE_REPOSITORY_PREFIX="${IMAGE_REPOSITORY_PREFIX}" \ + INPUTS_IMAGE_TAG_PREFIX="${IMAGE_TAG_PREFIX}" \ + INPUTS_CLI_CALLER="${RHDH_CLI_CALLER}" \ + INPUTS_CONTAINER_BUILD_TOOL=buildah \ + INPUTS_PUSH_CONTAINER_IMAGE=true \ + PUBLISHED_EXPORTS_OUTPUT="${DEFAULT_PUBLISH_OUTPUT_DIR}/published-exports-output" \ + FAILED_EXPORTS_OUTPUT="${DEFAULT_PUBLISH_OUTPUT_DIR}/failed-exports-output" \ + bash "${UTILS_DIR}/scripts/publish-export-staging.sh" +} + +if [[ -n "${PUBLISH_ONLY}" ]]; then + run_local_publish "${PUBLISH_ONLY}" + exit 0 +fi + +ensure_local_registry + +echo "=== Preparing source checkout ===" +mkdir -p "$(dirname "${WORKDIR}")" +if [[ "${KEEP_WORKDIR}" == "true" ]]; then + echo "Keeping existing workdir: ${WORKDIR}" +else + remove_export_workdir "${WORKDIR}" +fi +mkdir -p "${WORKDIR}/archives" "${WORKDIR}/overlay-repo/${WORKSPACE}" "${WORKDIR}/source-repo" + +cp -a "${OVERLAY_ROOT}/." "${WORKDIR}/overlay-repo/${WORKSPACE}/" +cp "${PLUGINS_FILE}" "${WORKDIR}/overlay-repo/${WORKSPACE}/plugins-list.yaml" + +if [[ ! -d "${WORKDIR}/source-repo/.git" ]]; then + git clone "${SOURCE_REPO}" "${WORKDIR}/source-repo" + git -C "${WORKDIR}/source-repo" fetch --depth 1 origin "${SOURCE_REF}" 2>/dev/null || \ + git -C "${WORKDIR}/source-repo" fetch origin "${SOURCE_REF}" 2>/dev/null || true + git -C "${WORKDIR}/source-repo" checkout -q "${SOURCE_REF}" +elif [[ "${KEEP_WORKDIR}" == "true" ]]; then + echo "Reusing existing source checkout in ${WORKDIR}/source-repo" +else + git -C "${WORKDIR}/source-repo" fetch --depth 1 origin "${SOURCE_REF}" 2>/dev/null || \ + git -C "${WORKDIR}/source-repo" fetch origin "${SOURCE_REF}" 2>/dev/null || true + git -C "${WORKDIR}/source-repo" checkout -q "${SOURCE_REF}" +fi + +echo "=== Applying overlay patches and source overlays ===" +bash "${UTILS_DIR}/override-sources/override-sources.sh" \ + "${WORKDIR}/overlay-repo/${WORKSPACE}" \ + "${WORKDIR}/source-repo/${PLUGINS_ROOT}" + +SOURCE_WORKDIR="${WORKDIR}/source-repo/${PLUGINS_ROOT}" +if [[ "${PLUGINS_ROOT}" == "." ]]; then + SOURCE_WORKDIR="${WORKDIR}/source-repo" +fi + +CLI_ENV=() +if [[ -n "${RHDH_CLI_CALLER}" ]]; then + CLI_ENV+=(-e "INPUTS_CLI_CALLER=${RHDH_CLI_CALLER}") +fi + +echo "=== Running compile export inside UBI builder (no privileged) ===" +STAGING_DIR="${WORKDIR}/export-staging" +EXPORT_PODMAN_ARGS=( + run --rm --network host --user 0 + -v "${WORKDIR}:/work:z" + -v "${UTILS_DIR}:/utils:ro,z" + -e "GITHUB_WORKSPACE=/work" + -w "/work/source-repo/${PLUGINS_ROOT}" +) +EXPORT_PODMAN_ARGS+=("${CLI_ENV[@]}") +EXPORT_PODMAN_ARGS+=(-e "NPM_CONFIG_IGNORE_SCRIPTS=true" -e "YARN_ENABLE_SCRIPTS=false") +EXPORT_PODMAN_ARGS+=(-e "NPM_CONFIG_cache=/work/.npm-cache") +EXPORT_PODMAN_ARGS+=(-e "NODE_OPTIONS=--max-old-space-size=8192") + +podman "${EXPORT_PODMAN_ARGS[@]}" "${BUILDER_IMAGE}" \ + bash -lc " + set -euo pipefail + corepack enable + yarn --version + yarn install --immutable + yarn tsc + export INPUTS_PLUGINS_FILE='/work/overlay-repo/${WORKSPACE}/plugins-list.yaml' + export INPUTS_DESTINATION='/work/archives' + export INPUTS_CLI_PACKAGE='${CLI_PACKAGE}' + export INPUTS_CLI_VERSION='${CLI_VERSION}' + export INPUTS_IMAGE_TAG_PREFIX='${IMAGE_TAG_PREFIX}' + bash /utils/export-dynamic/export-dynamic.sh + STAGING_ROOT='/work/export-staging' \ + INPUTS_PLUGINS_ROOT='/work/source-repo/${PLUGINS_ROOT}' \ + INPUTS_PLUGINS_FILE='/work/overlay-repo/${WORKSPACE}/plugins-list.yaml' \ + INPUTS_DESTINATION='/work/archives' \ + INPUTS_IMAGE_TAG_PREFIX='${IMAGE_TAG_PREFIX}' \ + bash /utils/export-dynamic/create-export-staging.sh + " + +echo "" +echo "=== Compile export complete ===" +echo "Archives: ${WORKDIR}/archives" +ls -la "${WORKDIR}/archives" 2>/dev/null || true + +if [[ "${PUSH_OCI}" == "true" ]]; then + run_local_publish "${STAGING_DIR}" + if [[ -f "${DEFAULT_PUBLISH_OUTPUT_DIR}/published-exports-output" ]]; then + echo "" + echo "Published OCI images:" + cat "${DEFAULT_PUBLISH_OUTPUT_DIR}/published-exports-output" + fi + echo "" + echo "Registry catalog (${REGISTRY}):" + curl -s "http://${REGISTRY}/v2/_catalog" | jq . 2>/dev/null || \ + echo "(install jq or inspect with: curl -s http://${REGISTRY}/v2/_catalog)" + echo "" + echo "Pull example:" + echo " buildah pull --tls-verify=false ${IMAGE_REPOSITORY_PREFIX}/:${IMAGE_TAG_PREFIX}" +fi + +echo "" +echo "Tip: ./scripts/local-test-export-reset.sh reclaims disk (workdir, test images, podman prune -af)." diff --git a/scripts/publish-export-staging.sh b/scripts/publish-export-staging.sh new file mode 100755 index 0000000..78add8a --- /dev/null +++ b/scripts/publish-export-staging.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# +# Build and push OCI images from an export-staging artifact (compile job output). +# +# Env: +# STAGING_ROOT — unpacked staging directory (required) +# INPUTS_IMAGE_REPOSITORY_PREFIX — ghcr.io/org/repo (required when pushing) +# INPUTS_IMAGE_TAG_PREFIX — tag prefix (optional) +# INPUTS_CLI_CALLER — rhdh-cli command (required) +# INPUTS_CONTAINER_BUILD_TOOL — buildah (default) +# INPUTS_PUSH_CONTAINER_IMAGE — true|false (default true) +# IMAGE_REGISTRY_USER — registry username (default: GITHUB_ACTOR for ghcr.io) +# IMAGE_REGISTRY_PASSWORD — registry password (use GITHUB_TOKEN for ghcr.io; never logged) +# GITHUB_WORKSPACE — workspace root for CI step outputs (optional; see PUBLISH_STEP_OUTPUTS) +# PUBLISH_STEP_OUTPUTS — path for GHA-style step outputs (default: alongside FAILED_EXPORTS_OUTPUT) +# +set -euo pipefail + +STAGING_ROOT="${STAGING_ROOT:?STAGING_ROOT is required}" +INPUTS_IMAGE_REPOSITORY_PREFIX="${INPUTS_IMAGE_REPOSITORY_PREFIX:?INPUTS_IMAGE_REPOSITORY_PREFIX is required}" +INPUTS_CONTAINER_BUILD_TOOL="${INPUTS_CONTAINER_BUILD_TOOL:-buildah}" +INPUTS_PUSH_CONTAINER_IMAGE="${INPUTS_PUSH_CONTAINER_IMAGE:-true}" +INPUTS_CLI_CALLER="${INPUTS_CLI_CALLER:-}" + +if [[ -z "${INPUTS_CLI_CALLER}" ]]; then + # shellcheck source=scripts/resolve-export-cli.sh + INPUTS_CLI_CALLER="$(bash "$(dirname "$0")/resolve-export-cli.sh")" +fi + +manifest="${STAGING_ROOT}/manifest.json" +if [[ ! -f "${manifest}" ]]; then + echo "Error: missing ${manifest}" >&2 + exit 1 +fi + +IFS=" " read -r -a cli_bin <<< "${INPUTS_CLI_CALLER}" + +run_cli() { + local cli_args=("$@") + local cli_args_split=() + IFS=" " read -r -a cli_args_split <<< "${cli_args[*]}" + echo " > ${cli_bin[*]} ${cli_args_split[*]}" + # shellcheck disable=SC2068 + "${cli_bin[@]}" ${cli_args_split[@]} +} + +if [[ "${INPUTS_PUSH_CONTAINER_IMAGE}" == "true" ]]; then + registry="$(echo "${INPUTS_IMAGE_REPOSITORY_PREFIX}" | cut -d/ -f1)" + registry_user="${IMAGE_REGISTRY_USER:-${GITHUB_ACTOR:-}}" + if [[ -n "${IMAGE_REGISTRY_PASSWORD:-}" ]]; then + if [[ -z "${registry_user}" ]]; then + echo "Error: IMAGE_REGISTRY_PASSWORD is set but no registry username (set IMAGE_REGISTRY_USER or GITHUB_ACTOR)" >&2 + exit 1 + fi + echo "Logging in to ${registry} as ${registry_user}" + echo "${IMAGE_REGISTRY_PASSWORD}" | ${INPUTS_CONTAINER_BUILD_TOOL} login "${registry}" \ + -u "${registry_user}" --password-stdin + fi +fi + +errors=() +images=() +image_tag_prefix="$(jq -r '.imageTagPrefix // ""' "${manifest}")" + +mapfile -t plugin_paths < <(jq -r '.plugins[].path' "${manifest}") + +for pluginPath in "${plugin_paths[@]}"; do + plugin_dir="${STAGING_ROOT}/plugins/${pluginPath}" + if [[ ! -d "${plugin_dir}/dist-dynamic" ]]; then + echo "Error: missing dist-dynamic for ${pluginPath}" >&2 + errors+=("${pluginPath}") + continue + fi + + image_name="$(jq -r --arg p "${pluginPath}" \ + '.plugins[] | select(.path == $p) | .imageName' "${manifest}")" + version="$(jq -r --arg p "${pluginPath}" \ + '.plugins[] | select(.path == $p) | .version' "${manifest}")" + tag="${INPUTS_IMAGE_REPOSITORY_PREFIX}/${image_name}:${image_tag_prefix}${version}" + + echo "========== Packaging Container ${tag} ==========" + pushd "${plugin_dir}" > /dev/null + if run_cli plugin package --container-tool "${INPUTS_CONTAINER_BUILD_TOOL}" --tag "${tag}"; then + if [[ "${INPUTS_PUSH_CONTAINER_IMAGE}" == "true" ]]; then + echo "========== Publishing Container ${tag} ==========" + if ${INPUTS_CONTAINER_BUILD_TOOL} push "${tag}"; then + images+=("${tag}") + else + errors+=("${pluginPath}") + fi + else + images+=("${tag}") + fi + else + errors+=("${pluginPath}") + fi + popd > /dev/null +done + +FAILED_EXPORTS_OUTPUT="${FAILED_EXPORTS_OUTPUT:-failed-exports-output}" +PUBLISHED_EXPORTS_OUTPUT="${PUBLISHED_EXPORTS_OUTPUT:-published-exports-output}" +: > "${FAILED_EXPORTS_OUTPUT}" +: > "${PUBLISHED_EXPORTS_OUTPUT}" + +for e in "${errors[@]}"; do echo "${e}" >> "${FAILED_EXPORTS_OUTPUT}"; done +for i in "${images[@]}"; do echo "${i}" >> "${PUBLISHED_EXPORTS_OUTPUT}"; done + +publish_output_dir="$(dirname "${FAILED_EXPORTS_OUTPUT}")" +mkdir -p "${publish_output_dir}" +PUBLISH_STEP_OUTPUTS="${PUBLISH_STEP_OUTPUTS:-${publish_output_dir}/.publish-export-staging.outputs}" +{ + if [[ -s "${FAILED_EXPORTS_OUTPUT}" ]]; then + echo "failed-exports<&2 + exit "${#errors[@]}" +fi diff --git a/scripts/resolve-export-cli.sh b/scripts/resolve-export-cli.sh new file mode 100755 index 0000000..c25e37f --- /dev/null +++ b/scripts/resolve-export-cli.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Resolve INPUTS_CLI_CALLER from a baked export-builder manifest and CLI version. +# +# Env: +# INPUTS_CLI_VERSION — required unless INPUTS_CLI_CALLER set +# EXPORT_BUILDER_MANIFEST — default /etc/rhdh-export-builder/manifest.json +# +set -euo pipefail + +if [[ -n "${INPUTS_CLI_CALLER:-}" ]]; then + if [[ -x "${INPUTS_CLI_CALLER}" ]] || command -v "${INPUTS_CLI_CALLER%% *}" >/dev/null 2>&1; then + echo "${INPUTS_CLI_CALLER}" + exit 0 + fi + echo "Error: INPUTS_CLI_CALLER is set but not executable: ${INPUTS_CLI_CALLER}" >&2 + exit 1 +fi + +cli_version="${INPUTS_CLI_VERSION:-}" +if [[ -z "${cli_version}" ]]; then + echo "Error: INPUTS_CLI_VERSION is required." >&2 + exit 1 +fi + +manifest="${EXPORT_BUILDER_MANIFEST:-/etc/rhdh-export-builder/manifest.json}" +default_path="/opt/rhdh-cli/${cli_version}/bin/rhdh-cli" + +if [[ -f "${manifest}" ]]; then + path="$(jq -r --arg v "${cli_version}" \ + '.cliVersions[] | select(.version == $v) | .path' "${manifest}" | head -n1)" + if [[ -n "${path}" && "${path}" != "null" && -x "${path}" ]]; then + echo "${path}" + exit 0 + fi + supported="$(jq -r '[.cliVersions[].version] | join(", ")' "${manifest}")" + echo "Error: CLI ${cli_version} not found in ${manifest}." >&2 + echo "Supported versions: ${supported}" >&2 + echo "Rebuild the export-builder image (publish-export-builder workflow)." >&2 + exit 1 +fi + +if [[ -x "${default_path}" ]]; then + echo "${default_path}" + exit 0 +fi + +echo "Error: no manifest at ${manifest} and ${default_path} is missing." >&2 +exit 1