From f6c53febb1b250dc17021723e64aa67621e48ee5 Mon Sep 17 00:00:00 2001 From: Patrick Knight Date: Tue, 26 May 2026 10:09:36 -0400 Subject: [PATCH] bulk-import-git-repos: align universal-pr with RHDH orchestrator e2e Assisted-by: Cursor AI Signed-off-by: Patrick Knight --- .github/workflows/bulk-import-git-repos.yml | 2 +- Makefile | 11 +++- make.md | 1 + scripts/gen_manifests.sh | 52 ++++++++++++++++++- workflows/bulk-import-git-repos/README.md | 50 ++++++++++++++---- .../01-configmap_universal-pr-props.yaml | 15 +----- ...ap_01-universal-pr-resources-schemas.yaml} | 1 - .../02-secret_universal-pr-secrets.yaml | 14 ----- ...gmap_02-universal-pr-resources-specs.yaml} | 1 - ...r.yaml => 04-sonataflow_universal-pr.yaml} | 12 +---- .../src/main/resources/application.properties | 14 +---- .../src/main/resources/secret.properties | 1 - 12 files changed, 108 insertions(+), 66 deletions(-) rename workflows/bulk-import-git-repos/manifests/{03-configmap_01-universal-pr-resources-schemas.yaml => 02-configmap_01-universal-pr-resources-schemas.yaml} (98%) delete mode 100755 workflows/bulk-import-git-repos/manifests/02-secret_universal-pr-secrets.yaml rename workflows/bulk-import-git-repos/manifests/{04-configmap_02-universal-pr-resources-specs.yaml => 03-configmap_02-universal-pr-resources-specs.yaml} (99%) rename workflows/bulk-import-git-repos/manifests/{05-sonataflow_universal-pr.yaml => 04-sonataflow_universal-pr.yaml} (97%) delete mode 100644 workflows/bulk-import-git-repos/src/main/resources/secret.properties diff --git a/.github/workflows/bulk-import-git-repos.yml b/.github/workflows/bulk-import-git-repos.yml index 635dceee..c833552d 100644 --- a/.github/workflows/bulk-import-git-repos.yml +++ b/.github/workflows/bulk-import-git-repos.yml @@ -5,7 +5,7 @@ on: push: branches: [ "main" ] paths: - - 'workflows/experimentals/bulk-import-git-repos/**' + - 'workflows/bulk-import-git-repos/**' - 'pipeline/**' - 'scripts/**' - .github/workflows/bulk-import-git-repos.yml diff --git a/Makefile b/Makefile index 2532b941..20f49bfc 100644 --- a/Makefile +++ b/Makefile @@ -169,13 +169,22 @@ save-oci: build-image # Optional: WORKFLOW_SUBDIR — when set, first arg is workflows/$(WORKFLOW_SUBDIR); when unset, workflows/$(WORKFLOW_ID) # make WORKFLOW_ID=bulk-import-git-repos WORKFLOW_SUBDIR=bulk-import-git-repos/src/main/resources gen-manifests GEN_MANIFESTS_WORKFLOW_FOLDER = workflows/$(if $(WORKFLOW_SUBDIR),$(WORKFLOW_SUBDIR),$(WORKFLOW_ID)) +ifeq ($(WORKFLOW_ID),bulk-import-git-repos) +WORKFLOW_SUBDIR ?= bulk-import-git-repos/src/main/resources +RHDH_PERSISTENCE ?= true +endif gen-manifests: prepare-workdir @# Ensure WORKDIR exists and is accessible (important for macOS Podman) @test -d $(WORKDIR) || mkdir -p $(WORKDIR) @# Use absolute path for volume mount (required for Podman on macOS) @# On macOS, use realpath or fallback to WORKDIR if realpath fails @$(CONTAINER_ENGINE) run --rm -v "$(shell realpath $(WORKDIR) 2>/dev/null || echo $(WORKDIR)):/workdir:Z" -w /workdir \ - $(LINUX_IMAGE) /bin/bash -c "ENABLE_PERSISTENCE=$(ENABLE_PERSISTENCE) WORKFLOW_IMAGE_TAG=$(IMAGE_TAG) ${SCRIPTS_DIR}/gen_manifests.sh $(GEN_MANIFESTS_WORKFLOW_FOLDER) $(WORKFLOW_ID)" + $(LINUX_IMAGE) /bin/bash -c "ENABLE_PERSISTENCE=$(ENABLE_PERSISTENCE) RHDH_PERSISTENCE=$(RHDH_PERSISTENCE) WORKFLOW_IMAGE_TAG=$(IMAGE_TAG) ${SCRIPTS_DIR}/gen_manifests.sh $(GEN_MANIFESTS_WORKFLOW_FOLDER) $(WORKFLOW_ID)" +ifeq ($(WORKFLOW_ID),bulk-import-git-repos) + @mkdir -p $(WORKDIR)/workflows/$(WORKFLOW_ID)/manifests + @cp -f $(WORKDIR)/$(GEN_MANIFESTS_WORKFLOW_FOLDER)/manifests/* $(WORKDIR)/workflows/$(WORKFLOW_ID)/manifests/ + @echo "Manifests copied to $(WORKDIR)/workflows/$(WORKFLOW_ID)/manifests" +endif @echo "Manifests are available in workdir $(WORKDIR)/$(GEN_MANIFESTS_WORKFLOW_FOLDER)/manifests" remove-trailing-whitespaces: diff --git a/make.md b/make.md index 0b0a2325..bd04d62e 100644 --- a/make.md +++ b/make.md @@ -45,6 +45,7 @@ Variables can be used to configure the behavior of the [Makefile](./Makefile): | IMAGE_PREFIX | Automatically added image prefix | `serverless-workflow` | | IMAGE_TAG | Automatically added image tag | 8 chars commit hash of the latest commit | | ENABLE_PERSISTENCE | Enables the addition of persistence to the generated manifests. Useful for local testing | `false` | +| RHDH_PERSISTENCE | Uses backstage Postgres refs for RHDH `installOrchestrator`; removes GHTOKEN secret from generated manifests. Set automatically for `bulk-import-git-repos` | `false` | Override the default values with: ```bash diff --git a/scripts/gen_manifests.sh b/scripts/gen_manifests.sh index e761487e..76616c86 100755 --- a/scripts/gen_manifests.sh +++ b/scripts/gen_manifests.sh @@ -10,11 +10,20 @@ WORKFLOW_IMAGE_NAMESPACE="${WORKFLOW_IMAGE_NAMESPACE:-orchestrator}" WORKFLOW_IMAGE_REPO="${WORKFLOW_IMAGE_REPO:-serverless-workflow-${WORKFLOW_ID}}" WORKFLOW_IMAGE_TAG="${WORKFLOW_IMAGE_TAG:-latest}" +# Red Hat Developer Hub (RHDH) orchestrator e2e persistence — backstage Postgres from installOrchestrator() +RHDH_PG_SECRET_NAME="${RHDH_PG_SECRET_NAME:-backstage-psql-secret}" +RHDH_PG_USER_KEY="${RHDH_PG_USER_KEY:-POSTGRES_USER}" +RHDH_PG_PASSWORD_KEY="${RHDH_PG_PASSWORD_KEY:-POSTGRES_PASSWORD}" +RHDH_PG_SERVICE_NAME="${RHDH_PG_SERVICE_NAME:-backstage-psql}" +RHDH_PG_DATABASE="${RHDH_PG_DATABASE:-backstage_plugin_orchestrator}" +RHDH_PG_NAMESPACE="${RHDH_PG_NAMESPACE:-orchestrator}" + # helper binaries should be either on the developer machine or in the helper # image quay.io/orchestrator/ubi9-pipeline from setup/Dockerfile, which we use # to exeute this script. See the Makefile gen-manifests target. command -v kn-workflow command -v kubectl +command -v yq cd "${WORKFLOW_FOLDER}" @@ -50,9 +59,50 @@ fi # gen-manifests are now sorted by name. We need to take *-sonataflow-$workflow_id.yaml to resolve that. SONATAFLOW_CR=$(printf '%s' manifests/*-sonataflow_"${workflow_id}".yaml) +cleanup_generated_manifests() { + for manifest in manifests/*.yaml; do + [ -f "${manifest}" ] || continue + yq --inplace 'del(.metadata.creationTimestamp) | del(.status)' "${manifest}" + done +} + +apply_rhdh_manifest_cleanup() { + echo "Applying RHDH manifest cleanup (token propagation only, no GHTOKEN secret)..." + yq --inplace ' + .spec.podTemplate.container.env |= ( + (. // []) | map(select(.name != "GHTOKEN")) + ) + | (if .spec.podTemplate.container.env == [] then del(.spec.podTemplate.container.env) else . end) + ' "${SONATAFLOW_CR}" + rm -f manifests/*secret*.yaml + cleanup_generated_manifests +} + # The following properties are set in the Sonataflow CR, for each workflow to enable persistence. # TODO: It should be replaced with a single definition in the SonataflowPlatform CR -if [ "${ENABLE_PERSISTENCE}" = true ]; then +if [ "${RHDH_PERSISTENCE}" = true ]; then + yq --inplace ".spec |= ( + . + { + \"persistence\": { + \"postgresql\": { + \"secretRef\": { + \"name\": \"${RHDH_PG_SECRET_NAME}\", + \"userKey\": \"${RHDH_PG_USER_KEY}\", + \"passwordKey\": \"${RHDH_PG_PASSWORD_KEY}\" + }, + \"serviceRef\": { + \"name\": \"${RHDH_PG_SERVICE_NAME}\", + \"port\": 5432, + \"databaseName\": \"${RHDH_PG_DATABASE}\", + \"databaseSchema\": \"${WORKFLOW_ID}\", + \"namespace\": \"${RHDH_PG_NAMESPACE}\" + } + } + } + } + )" "${SONATAFLOW_CR}" + apply_rhdh_manifest_cleanup +elif [ "${ENABLE_PERSISTENCE}" = true ]; then yq --inplace ".spec |= ( . + { \"persistence\": { diff --git a/workflows/bulk-import-git-repos/README.md b/workflows/bulk-import-git-repos/README.md index 85aec38c..626cb987 100644 --- a/workflows/bulk-import-git-repos/README.md +++ b/workflows/bulk-import-git-repos/README.md @@ -2,13 +2,18 @@ This workflow creates pull requests (GitHub) or merge requests (GitLab) based on the `approvalTool` parameter. +Workflow id: **`universal-pr`** (do not rename without bulk-import plugin changes). + ## Overview The workflow supports both GitHub and GitLab repositories and can create PRs/MRs with multiple files. +GitHub authentication uses **token propagation** (`X-Authorization-Github`) from Red Hat Developer Hub — no `GHTOKEN` secret in manifests. + ## Input Schema The workflow expects the following input parameters: + - `approvalTool`: Either "GIT" for GitHub or "GITLAB" for GitLab - `owner`: The owner/namespace of the repository - `repo`: The repository name @@ -24,26 +29,51 @@ The workflow expects the following input parameters: ## Output -- For GitHub: Returns PR URL in `PR_URL` output +- For GitHub: Returns PR URL in **`PR_URL`** output (link format) - For GitLab: Returns MR URL in `MR_URL` output -# Development +## Deploy on Red Hat Developer Hub (RHDH) -Java artifacts build(prerequisites: pre-installed java and maven): +After [e2e-test-utils `installOrchestrator`](https://github.com/redhat-developer/rhdh-plugin-export-overlays) in namespace **`orchestrator`**: +```bash +oc apply -n orchestrator -f workflows/bulk-import-git-repos/manifests/ +oc rollout status deployment/universal-pr -n orchestrator --timeout=600s ``` -mvn clean install -``` -Generate manifests, from the root of the repository: +Manifests use **backstage Postgres** (`backstage-psql-secret`, `backstage-psql`, database `backstage_plugin_orchestrator`) created by `installOrchestrator`. Workflow image: `quay.io/orchestrator/serverless-workflow-bulk-import-git-repos`. + +Verify data-index lists the workflow: +```bash +oc exec -n orchestrator deploy/sonataflow-platform-data-index-service -- \ + curl -sf -X POST -H 'Content-Type: application/json' \ + -d '{"query":"{ ProcessDefinitions { id } }"}' \ + http://localhost:8080/graphql | grep universal-pr ``` -make WORKFLOW_ID=bulk-import-git-repos WORKFLOW_SUBDIR=bulk-import-git-repos/src/main/resources gen-manifests -cp -rf /tmp/serverless-workflows/workflows/bulk-import-git-repos/src/main/resources/manifests ./workflows/bulk-import-git-repos + +**Context:** [RHIDP-9350](https://issues.redhat.com/browse/RHIDP-9350), [serverless-workflows PR #774](https://github.com/rhdhorchestrator/serverless-workflows/pull/774). + +## Development + +Java artifacts build (prerequisites: pre-installed java and maven): + +```bash +mvn clean install ``` -Build image: +Generate manifests (RHDH persistence profile is enabled automatically for this workflow): +```bash +make WORKFLOW_ID=bulk-import-git-repos gen-manifests +cp -rf /tmp/serverless-workflows/workflows/bulk-import-git-repos/src/main/resources/manifests/* \ + ./workflows/bulk-import-git-repos/manifests/ ``` -make WORKFLOW_ID=bulk-import-git-repos build-image + +Build and push image: + +```bash +make WORKFLOW_ID=bulk-import-git-repos build-image push-image ``` + +For non-RHDH clusters, use `RHDH_PERSISTENCE=false` and the default `sonataflow-psql-*` persistence from `ENABLE_PERSISTENCE=true`. diff --git a/workflows/bulk-import-git-repos/manifests/01-configmap_universal-pr-props.yaml b/workflows/bulk-import-git-repos/manifests/01-configmap_universal-pr-props.yaml index 5928ec1c..a8724e8e 100755 --- a/workflows/bulk-import-git-repos/manifests/01-configmap_universal-pr-props.yaml +++ b/workflows/bulk-import-git-repos/manifests/01-configmap_universal-pr-props.yaml @@ -5,14 +5,10 @@ data: # of information logged on HTTP requests based on the severity of the events. # Possible values: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL. # and see https://quarkus.io/guides/logging for documentation - quarkus.log.category."org.apache.http".level=DEBUG - quarkus.log.level=DEBUG + quarkus.log.category."org.apache.http".level=INFO + quarkus.log.level=INFO - # Datasource quarkus.datasource.db-kind=postgresql - quarkus.datasource.jdbc.url=jdbc:postgresql://postgres.showcase.svc.cluster.local:5432/sonata - quarkus.datasource.username=sonata - quarkus.datasource.password=sonata # Persistence kogito.persistence.type=jdbc @@ -24,12 +20,6 @@ data: quarkus.kogito.devservices.enabled=false quarkus.openapi-generator.github_yaml.auth.BearerToken.token-propagation=true quarkus.openapi-generator.github_yaml.auth.BearerToken.header-name=X-Authorization-Github - # quarkus.rest-client.github_yaml.headers.Accept=application/vnd.github+json - # quarkus.openapi-generator.github_yaml.auth.BearerToken.header-name=X-Authorization-github - # quarkus.openapi-generator.github_yaml.auth.BearerToken.bearer-token=${ghToken} - - # quarkus.openapi-generator.githubtwo_yaml.auth.BearerToken.token-propagation=true - # quarkus.openapi-generator.githubtwo_yaml.auth.BearerToken.header-name=X-Authorization-github-two quarkus.openapi-generator.gitlab_yaml.auth.BearerToken.token-propagation=true quarkus.openapi-generator.gitlab_yaml.auth.BearerToken.header-name=X-Authorization-gitlab @@ -37,7 +27,6 @@ data: kie.flyway.enabled=true kind: ConfigMap metadata: - creationTimestamp: null labels: app: universal-pr app.kubernetes.io/component: serverless-workflow diff --git a/workflows/bulk-import-git-repos/manifests/03-configmap_01-universal-pr-resources-schemas.yaml b/workflows/bulk-import-git-repos/manifests/02-configmap_01-universal-pr-resources-schemas.yaml similarity index 98% rename from workflows/bulk-import-git-repos/manifests/03-configmap_01-universal-pr-resources-schemas.yaml rename to workflows/bulk-import-git-repos/manifests/02-configmap_01-universal-pr-resources-schemas.yaml index e1bc69b9..9d4ae023 100755 --- a/workflows/bulk-import-git-repos/manifests/03-configmap_01-universal-pr-resources-schemas.yaml +++ b/workflows/bulk-import-git-repos/manifests/02-configmap_01-universal-pr-resources-schemas.yaml @@ -44,5 +44,4 @@ data: } kind: ConfigMap metadata: - creationTimestamp: null name: 01-universal-pr-resources-schemas diff --git a/workflows/bulk-import-git-repos/manifests/02-secret_universal-pr-secrets.yaml b/workflows/bulk-import-git-repos/manifests/02-secret_universal-pr-secrets.yaml deleted file mode 100755 index ba7b346b..00000000 --- a/workflows/bulk-import-git-repos/manifests/02-secret_universal-pr-secrets.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - creationTimestamp: null - labels: - app: universal-pr - app.kubernetes.io/component: serverless-workflow - app.kubernetes.io/managed-by: sonataflow-operator - app.kubernetes.io/name: universal-pr - sonataflow.org/workflow-app: universal-pr - sonataflow.org/workflow-namespace: "" - name: universal-pr-secrets -stringData: - ghToken: ghp_ABC123 diff --git a/workflows/bulk-import-git-repos/manifests/04-configmap_02-universal-pr-resources-specs.yaml b/workflows/bulk-import-git-repos/manifests/03-configmap_02-universal-pr-resources-specs.yaml similarity index 99% rename from workflows/bulk-import-git-repos/manifests/04-configmap_02-universal-pr-resources-specs.yaml rename to workflows/bulk-import-git-repos/manifests/03-configmap_02-universal-pr-resources-specs.yaml index 21413170..c088d56a 100755 --- a/workflows/bulk-import-git-repos/manifests/04-configmap_02-universal-pr-resources-specs.yaml +++ b/workflows/bulk-import-git-repos/manifests/03-configmap_02-universal-pr-resources-specs.yaml @@ -490,5 +490,4 @@ data: description: Bearer Token authentication kind: ConfigMap metadata: - creationTimestamp: null name: 02-universal-pr-resources-specs diff --git a/workflows/bulk-import-git-repos/manifests/05-sonataflow_universal-pr.yaml b/workflows/bulk-import-git-repos/manifests/04-sonataflow_universal-pr.yaml similarity index 97% rename from workflows/bulk-import-git-repos/manifests/05-sonataflow_universal-pr.yaml rename to workflows/bulk-import-git-repos/manifests/04-sonataflow_universal-pr.yaml index 634105dd..837cd675 100755 --- a/workflows/bulk-import-git-repos/manifests/05-sonataflow_universal-pr.yaml +++ b/workflows/bulk-import-git-repos/manifests/04-sonataflow_universal-pr.yaml @@ -6,7 +6,6 @@ metadata: sonataflow.org/expressionLang: jq sonataflow.org/profile: gitops sonataflow.org/version: "1.0" - creationTimestamp: null labels: app: universal-pr app.kubernetes.io/component: serverless-workflow @@ -341,13 +340,7 @@ spec: type: operation podTemplate: container: - env: - - name: GHTOKEN - valueFrom: - secretKeyRef: - key: ghToken - name: universal-pr-secrets - image: quay.io/aandriienko/serverless-workflow-bulk-import-git-repos:latest + image: quay.io/orchestrator/serverless-workflow-bulk-import-git-repos:latest resources: {} resources: configMaps: @@ -369,6 +362,3 @@ spec: databaseName: backstage_plugin_orchestrator databaseSchema: bulk-import-git-repos namespace: orchestrator -status: - address: {} - lastTimeRecoverAttempt: null diff --git a/workflows/bulk-import-git-repos/src/main/resources/application.properties b/workflows/bulk-import-git-repos/src/main/resources/application.properties index 3e147352..9f8a1b7a 100644 --- a/workflows/bulk-import-git-repos/src/main/resources/application.properties +++ b/workflows/bulk-import-git-repos/src/main/resources/application.properties @@ -2,14 +2,10 @@ # of information logged on HTTP requests based on the severity of the events. # Possible values: OFF, FATAL, ERROR, WARN, INFO, DEBUG, ALL. # and see https://quarkus.io/guides/logging for documentation -quarkus.log.category."org.apache.http".level=DEBUG -quarkus.log.level=DEBUG +quarkus.log.category."org.apache.http".level=INFO +quarkus.log.level=INFO -# Datasource quarkus.datasource.db-kind=postgresql -quarkus.datasource.jdbc.url=jdbc:postgresql://postgres.showcase.svc.cluster.local:5432/sonata -quarkus.datasource.username=sonata -quarkus.datasource.password=sonata # Persistence kogito.persistence.type=jdbc @@ -21,12 +17,6 @@ quarkus.oidc.tenant-enabled=false quarkus.kogito.devservices.enabled=false quarkus.openapi-generator.github_yaml.auth.BearerToken.token-propagation=true quarkus.openapi-generator.github_yaml.auth.BearerToken.header-name=X-Authorization-Github -# quarkus.rest-client.github_yaml.headers.Accept=application/vnd.github+json -# quarkus.openapi-generator.github_yaml.auth.BearerToken.header-name=X-Authorization-github -# quarkus.openapi-generator.github_yaml.auth.BearerToken.bearer-token=${ghToken} - -# quarkus.openapi-generator.githubtwo_yaml.auth.BearerToken.token-propagation=true -# quarkus.openapi-generator.githubtwo_yaml.auth.BearerToken.header-name=X-Authorization-github-two quarkus.openapi-generator.gitlab_yaml.auth.BearerToken.token-propagation=true quarkus.openapi-generator.gitlab_yaml.auth.BearerToken.header-name=X-Authorization-gitlab diff --git a/workflows/bulk-import-git-repos/src/main/resources/secret.properties b/workflows/bulk-import-git-repos/src/main/resources/secret.properties deleted file mode 100644 index 4a977c3f..00000000 --- a/workflows/bulk-import-git-repos/src/main/resources/secret.properties +++ /dev/null @@ -1 +0,0 @@ -ghToken=ghp_ABC123