diff --git a/.changelog/pr-141.txt b/.changelog/pr-141.txt new file mode 100644 index 0000000..0ffe5be --- /dev/null +++ b/.changelog/pr-141.txt @@ -0,0 +1,3 @@ +```release-note:internal +Added a provisioned-environment E2E test tier (`terraform-test-data/`, `tools/tf-regression-provision`, `make e2e`) that provisions a throwaway PingOne environment via Terraform, applies hand-authored fixture resources, and runs the existing base-vs-PR export/compare pipeline against it. Additive to the existing `tests/regression/` tier; not yet wired into CI. +``` diff --git a/.gitignore b/.gitignore index f0dad61..91e7c82 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.so *.dylib davinci-convert +/tf-regression-provision # Test binary, built with `go test -c` *.test @@ -44,8 +45,19 @@ dist/ !**/testdata/**/*.json !tests/regression/matrix.json !**/testdata/**/*.tf +!terraform-test-data/**/*.tf localsecrets +# Terraform working directories (terraform-test-data/ is applied by +# tools/tf-regression-provision; these are always local/ephemeral) +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.* +terraform.tfvars.json +crash.log +crash.*.log + pingcli-terraformer # Local regression test reports diff --git a/Makefile b/Makefile index ea7a9ea..95902bc 100644 --- a/Makefile +++ b/Makefile @@ -83,8 +83,12 @@ regression-local: build if [ $$EXIT_CODE -ne 0 ]; then exit $$EXIT_CODE; fi; \ echo "==> Regression test passed." +e2e: build + @echo "==> Running provisioned-environment E2E test..." + @./tests/regression-provisioned/run-local.sh + devcheck: build vet fmt lint test testacc devchecknotest: build vet fmt lint test -.PHONY: build install test testacc testcoverage vet depscheck lint golangcilint fmt clean regression-local devcheck devchecknotest +.PHONY: build install test testacc testcoverage vet depscheck lint golangcilint fmt clean regression-local e2e devcheck devchecknotest diff --git a/terraform-test-data/root/main.tf b/terraform-test-data/root/main.tf new file mode 100644 index 0000000..d00dfcd --- /dev/null +++ b/terraform-test-data/root/main.tf @@ -0,0 +1,30 @@ +# Composition root for terraform-test-data fixtures. Applied by +# tools/tf-regression-provision using org-admin credentials, which both +# create the throwaway environment (pingone_environment below) and +# authenticate every fixture resource inside it - one provider config, one +# dependency graph, one state. Fixture modules get added here in dependency +# order as terraform-test-data grows (see contributing docs for the rollout +# sequencing). + +provider "pingone" { + environment_id = var.org_admin_environment_id + client_id = var.org_admin_client_id + client_secret = var.org_admin_client_secret + region_code = var.region_code +} + +resource "pingone_environment" "this" { + name = var.environment_name + description = "Throwaway environment for the pingcli-plugin-terraformer provisioned-environment E2E test. Safe to delete." + type = "SANDBOX" + license_id = var.license_id + + services = [ + { type = "SSO" }, + ] +} + +module "sso_population" { + source = "../sso/population" + environment_id = pingone_environment.this.id +} diff --git a/terraform-test-data/root/outputs.tf b/terraform-test-data/root/outputs.tf new file mode 100644 index 0000000..403dd5b --- /dev/null +++ b/terraform-test-data/root/outputs.tf @@ -0,0 +1,4 @@ +output "environment_id" { + description = "ID of the throwaway environment created by this apply." + value = pingone_environment.this.id +} diff --git a/terraform-test-data/root/variables.tf b/terraform-test-data/root/variables.tf new file mode 100644 index 0000000..a1e1316 --- /dev/null +++ b/terraform-test-data/root/variables.tf @@ -0,0 +1,30 @@ +variable "org_admin_environment_id" { + description = "Home environment of the org-admin worker application used to authenticate the provider." + type = string +} + +variable "org_admin_client_id" { + description = "Client ID of the org-admin worker application." + type = string +} + +variable "org_admin_client_secret" { + description = "Client secret of the org-admin worker application." + type = string + sensitive = true +} + +variable "region_code" { + description = "PingOne region code (NA, EU, AP, CA, AU, SG)." + type = string +} + +variable "license_id" { + description = "License ID to assign to the throwaway environment created by this run." + type = string +} + +variable "environment_name" { + description = "Name for the throwaway environment created by this run." + type = string +} diff --git a/terraform-test-data/root/versions.tf b/terraform-test-data/root/versions.tf new file mode 100644 index 0000000..cd9de88 --- /dev/null +++ b/terraform-test-data/root/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + pingone = { + source = "pingidentity/pingone" + version = "1.16.0-beta.2" + } + } +} diff --git a/terraform-test-data/sso/population/main.tf b/terraform-test-data/sso/population/main.tf new file mode 100644 index 0000000..ec61731 --- /dev/null +++ b/terraform-test-data/sso/population/main.tf @@ -0,0 +1,10 @@ +variable "environment_id" { + description = "ID of the environment to create this population in." + type = string +} + +resource "pingone_population" "e2e" { + environment_id = var.environment_id + name = "pingcli-terraformer-e2e-population" + description = "Minimal population fixture for the provisioned-environment E2E test." +} diff --git a/terraform-test-data/sso/population/versions.tf b/terraform-test-data/sso/population/versions.tf new file mode 100644 index 0000000..2d33b13 --- /dev/null +++ b/terraform-test-data/sso/population/versions.tf @@ -0,0 +1,7 @@ +terraform { + required_providers { + pingone = { + source = "pingidentity/pingone" + } + } +} diff --git a/tests/regression-provisioned/run-local.sh b/tests/regression-provisioned/run-local.sh new file mode 100755 index 0000000..ab679a9 --- /dev/null +++ b/tests/regression-provisioned/run-local.sh @@ -0,0 +1,359 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ────────────────────────────────────────────────────────────────────────────── +# Provisioned-environment E2E test runner. +# +# Creates a throwaway PingOne environment via tools/tf-regression-provision. +# The environment itself and every fixture inside it are defined as +# Terraform in terraform-test-data/root, all authenticated with a single +# org-admin credential. Once applied, runs the same base-vs-PR export +# comparison as tests/regression/run-local.sh against that fresh environment. +# The environment is always torn down on exit, success or failure. +# ────────────────────────────────────────────────────────────────────────────── + +# --------------------------------------------------------------------------- +# Color helpers (gracefully degrade when not a TTY) +# --------------------------------------------------------------------------- +if [ -t 1 ]; then + RED=$(tput setaf 1 2>/dev/null || printf '') + GREEN=$(tput setaf 2 2>/dev/null || printf '') + YELLOW=$(tput setaf 3 2>/dev/null || printf '') + BOLD=$(tput bold 2>/dev/null || printf '') + RESET=$(tput sgr0 2>/dev/null || printf '') +else + RED='' GREEN='' YELLOW='' BOLD='' RESET='' +fi + +info() { printf '%s[INFO]%s %s\n' "${BOLD}" "${RESET}" "$*"; } +success() { printf '%s[PASS]%s %s\n' "${GREEN}" "${RESET}" "$*"; } +warn() { printf '%s[WARN]%s %s\n' "${YELLOW}" "${RESET}" "$*"; } +fail() { printf '%s[FAIL]%s %s\n' "${RED}" "${RESET}" "$*" >&2; } +die() { fail "$*"; exit 1; } + +# --------------------------------------------------------------------------- +# Prerequisites +# --------------------------------------------------------------------------- +check_prerequisites() { + local missing=0 + + for var in \ + PINGCLI_PINGONE_ORGADMIN_CLIENT_ID \ + PINGCLI_PINGONE_ORGADMIN_CLIENT_SECRET \ + PINGCLI_PINGONE_ORGADMIN_ENVIRONMENT_ID \ + PINGCLI_PINGONE_ORGADMIN_LICENSE_ID; do + if [ -z "${!var:-}" ]; then + fail "Required environment variable not set: ${var}" + missing=1 + fi + done + + for tool in jq terraform; do + if ! command -v "${tool}" &>/dev/null; then + fail "Required tool not found: ${tool}" + missing=1 + fi + done + + [ "$missing" -eq 0 ] || exit 1 +} + +# --------------------------------------------------------------------------- +# Optional env vars with defaults +# --------------------------------------------------------------------------- +apply_defaults() { + : "${PINGCLI_PINGONE_ORGADMIN_REGION_CODE:=NA}" + : "${REGRESSION_BASE:=main}" + : "${E2E_KEEP_ENVIRONMENT:=0}" + + export PINGCLI_PINGONE_ORGADMIN_REGION_CODE + export REGRESSION_BASE + export E2E_KEEP_ENVIRONMENT +} + +# --------------------------------------------------------------------------- +# Globals (set after apply_defaults) +# --------------------------------------------------------------------------- +REPO_ROOT="" +TMPDIR_LOCAL="" +WORKTREE_DIR="" +TF_DIR="" +PROVISIONED_ENV_ID="" + +setup_dirs() { + REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + if [ ! -f "${REPO_ROOT}/Makefile" ]; then + die "Could not locate repo root (Makefile not found at ${REPO_ROOT})" + fi + + TMPDIR_LOCAL="$(mktemp -d "${TMPDIR:-/tmp}/pingcli-e2e.XXXXXX")" + WORKTREE_DIR="${TMPDIR_LOCAL}/worktree-base" + TF_DIR="${REPO_ROOT}/terraform-test-data/root" + + info "Repo root : ${REPO_ROOT}" + info "Temp dir : ${TMPDIR_LOCAL}" + info "Base branch: ${REGRESSION_BASE}" +} + +# --------------------------------------------------------------------------- +# Cleanup trap - always tears down the provisioned environment, even on +# failure, unless the developer explicitly asked to keep it for debugging. +# +# Trigger condition is the presence of terraform.tfvars.json in TF_DIR, NOT +# PROVISIONED_ENV_ID: `terraform apply` can create the environment and then +# fail on a later resource, in which case create() never reaches the point +# of setting PROVISIONED_ENV_ID, but a real (now-orphaned) environment and +# its tfvars/state already exist in TF_DIR and must still be torn down. +# --------------------------------------------------------------------------- +cleanup() { + if [ -f "${TF_DIR}/terraform.tfvars.json" ]; then + if [ "${E2E_KEEP_ENVIRONMENT}" = "1" ]; then + warn "E2E_KEEP_ENVIRONMENT=1 set - leaving provisioned environment in place." + warn "Remember to destroy it manually when done (terraform destroy in ${TF_DIR})." + else + info "Tearing down provisioned environment${PROVISIONED_ENV_ID:+ ${PROVISIONED_ENV_ID}}..." + "${TMPDIR_LOCAL}/tf-regression-provision" \ + --action destroy \ + --terraform-dir "${TF_DIR}" \ + || warn "Teardown reported an error - verify manually in the PingOne admin console." + fi + fi + + if [ -n "${WORKTREE_DIR}" ] && [ -d "${WORKTREE_DIR}" ]; then + git -C "${REPO_ROOT}" worktree remove --force "${WORKTREE_DIR}" 2>/dev/null || true + fi + if [ -n "${TMPDIR_LOCAL}" ] && [ -d "${TMPDIR_LOCAL}" ]; then + rm -rf "${TMPDIR_LOCAL}" + fi +} + +# --------------------------------------------------------------------------- +# Provision a throwaway environment and apply terraform-test-data/root. +# Populates PINGCLI_PINGONE_* env vars for the export steps below from the +# tool's JSON stdout - never echoed, only parsed with jq. +# --------------------------------------------------------------------------- +provision() { + info "Building tf-regression-provision..." + (cd "${REPO_ROOT}" && go build -o "${TMPDIR_LOCAL}/tf-regression-provision" ./tools/tf-regression-provision/) + + info "Provisioning throwaway PingOne environment (this can take a minute)..." + local result="${TMPDIR_LOCAL}/provision-result.json" + "${TMPDIR_LOCAL}/tf-regression-provision" \ + --action create \ + --terraform-dir "${TF_DIR}" \ + --name-prefix "pingcli-terraformer-e2e" \ + >"${result}" + + PROVISIONED_ENV_ID=$(jq -r '.target_environment_id' "${result}") + # Auth environment (where the org-admin credential's OAuth token is + # acquired) is distinct from the export target (the throwaway environment + # just created) - mirrors internal/platform/pingone.NewFromCredentials's + # workerEnvID vs exportEnvID split. + PINGCLI_PINGONE_ENVIRONMENT_ID=$(jq -r '.auth_environment_id' "${result}") + PINGCLI_PINGONE_EXPORT_ENVIRONMENT_ID="${PROVISIONED_ENV_ID}" + PINGCLI_PINGONE_CLIENT_CREDENTIALS_CLIENT_ID=$(jq -r '.client_id' "${result}") + PINGCLI_PINGONE_CLIENT_CREDENTIALS_CLIENT_SECRET=$(jq -r '.client_secret' "${result}") + PINGCLI_PINGONE_REGION_CODE=$(jq -r '.region_code' "${result}") + export PINGCLI_PINGONE_ENVIRONMENT_ID PINGCLI_PINGONE_EXPORT_ENVIRONMENT_ID + export PINGCLI_PINGONE_CLIENT_CREDENTIALS_CLIENT_ID PINGCLI_PINGONE_CLIENT_CREDENTIALS_CLIENT_SECRET + export PINGCLI_PINGONE_REGION_CODE + rm -f "${result}" + + success "Provisioned environment ${PROVISIONED_ENV_ID}." +} + +# --------------------------------------------------------------------------- +# Build export/compare binaries (base branch vs current branch) +# --------------------------------------------------------------------------- +build_binaries() { + info "Creating git worktree for base branch '${REGRESSION_BASE}'..." + git -C "${REPO_ROOT}" worktree add --detach "${WORKTREE_DIR}" "origin/${REGRESSION_BASE}" \ + 2>/dev/null || \ + git -C "${REPO_ROOT}" worktree add --detach "${WORKTREE_DIR}" "${REGRESSION_BASE}" + + info "Building base binary from '${REGRESSION_BASE}'..." + (cd "${WORKTREE_DIR}" && go build -o "${TMPDIR_LOCAL}/binary-base" .) + + info "Building PR binary from current branch..." + (cd "${REPO_ROOT}" && go build -o "${TMPDIR_LOCAL}/binary-pr" .) + + info "Building regression-compare tool..." + (cd "${REPO_ROOT}" && go build -o "${TMPDIR_LOCAL}/regression-compare" ./tools/regression-compare/) + + success "All binaries built." +} + +# --------------------------------------------------------------------------- +# Build export CLI args from a matrix entry +# --------------------------------------------------------------------------- +build_args() { + local format="$1" + local skip_deps="$2" + local include_imports="$3" + local include_values="$4" + local outdir="$5" + + local args="export --output-format ${format} --out ${outdir} --module-name e2e-test --module-dir e2e-module" + + [ "${skip_deps}" = "true" ] && args="${args} --skip-dependencies" + [ "${include_imports}" = "true" ] && args="${args} --include-imports" + [ "${include_values}" = "true" ] && args="${args} --include-values" + + printf '%s' "${args}" +} + +# --------------------------------------------------------------------------- +# Run a single matrix entry (base binary vs PR binary against the +# provisioned environment). Returns 0 if no breaking changes, 1 otherwise. +# --------------------------------------------------------------------------- +run_entry() { + local name="$1" + local format="$2" + local skip_deps="$3" + local include_imports="$4" + local include_values="$5" + + local outdir_base="${TMPDIR_LOCAL}/output-base-${name}" + local outdir_pr="${TMPDIR_LOCAL}/output-pr-${name}" + local report="${TMPDIR_LOCAL}/report-${name}.json" + + mkdir -p "${outdir_base}" "${outdir_pr}" + + info "Running matrix entry: ${name}" + + local base_args pr_args + base_args=$(build_args "${format}" "${skip_deps}" "${include_imports}" "${include_values}" "${outdir_base}") + "${TMPDIR_LOCAL}/binary-base" ${base_args} + + pr_args=$(build_args "${format}" "${skip_deps}" "${include_imports}" "${include_values}" "${outdir_pr}") + "${TMPDIR_LOCAL}/binary-pr" ${pr_args} + + "${TMPDIR_LOCAL}/regression-compare" \ + --base-dir "${outdir_base}" \ + --pr-dir "${outdir_pr}" \ + --report-file "${report}" || true # compare exits non-zero on breaking; handled below + + if [ -f "${report}" ] && jq -e '.has_breaking == true' "${report}" &>/dev/null; then + return 1 + fi + return 0 +} + +# --------------------------------------------------------------------------- +# Print summary table +# --------------------------------------------------------------------------- +print_summary() { + local -a names=("${!1}") + local -a statuses=("${!2}") + local -a breaking=("${!3}") + local -a acceptable=("${!4}") + + local col_name=20 col_status=10 col_break=10 col_acc=10 + + printf '\n%s══════════════════════════════════════════════%s\n' "${BOLD}" "${RESET}" + printf '%s E2E Test Summary%s\n' "${BOLD}" "${RESET}" + printf '%s══════════════════════════════════════════════%s\n' "${BOLD}" "${RESET}" + printf ' %-*s %-*s %-*s %-*s\n' \ + "${col_name}" "Matrix Entry" \ + "${col_status}" "Status" \ + "${col_break}" "Breaking" \ + "${col_acc}" "Acceptable" + printf ' %s %s %s %s\n' \ + "$(printf '─%.0s' $(seq 1 ${col_name}))" \ + "$(printf '─%.0s' $(seq 1 ${col_status}))" \ + "$(printf '─%.0s' $(seq 1 ${col_break}))" \ + "$(printf '─%.0s' $(seq 1 ${col_acc}))" + + for i in "${!names[@]}"; do + local name="${names[$i]}" + local status="${statuses[$i]}" + local brk="${breaking[$i]}" + local acc="${acceptable[$i]}" + + if [ "${status}" = "PASS" ]; then + printf ' %-*s %s%-*s%s %-*s %-*s\n' \ + "${col_name}" "${name}" \ + "${GREEN}" "${col_status}" "✅ PASS" "${RESET}" \ + "${col_break}" "${brk}" \ + "${col_acc}" "${acc}" + else + printf ' %-*s %s%-*s%s %-*s %-*s\n' \ + "${col_name}" "${name}" \ + "${RED}" "${col_status}" "❌ FAIL" "${RESET}" \ + "${col_break}" "${brk}" \ + "${col_acc}" "${acc}" + fi + done + + printf '%s══════════════════════════════════════════════%s\n\n' "${BOLD}" "${RESET}" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +main() { + check_prerequisites + apply_defaults + setup_dirs + + trap cleanup EXIT + + provision + build_binaries + + local matrix_file="${REPO_ROOT}/tests/regression/matrix.json" + [ -f "${matrix_file}" ] || die "Matrix file not found: ${matrix_file}" + + local entry_count + entry_count=$(jq 'length' "${matrix_file}") + + local -a names=() + local -a statuses=() + local -a breaking_counts=() + local -a acceptable_counts=() + local overall_exit=0 + + for (( i=0; i]") + os.Exit(2) + } +} + +func runCreate(cfg orgAdminConfig, tfDir, namePrefix string) error { + envName := fmt.Sprintf("%s-%d", namePrefix, time.Now().Unix()) + if err := writeTFVars(tfDir, cfg, envName); err != nil { + return fmt.Errorf("write %s: %w", tfvarsFileName, err) + } + + if err := runTerraform(tfDir, "init", "-input=false"); err != nil { + return fmt.Errorf("terraform init: %w", err) + } + if err := runTerraform(tfDir, "apply", "-auto-approve", "-input=false"); err != nil { + return fmt.Errorf("terraform apply: %w", err) + } + + targetEnvID, err := terraformOutput(tfDir, "environment_id") + if err != nil { + return fmt.Errorf("read environment_id output: %w", err) + } + + result := createResult{ + AuthEnvironmentID: cfg.environmentID, + TargetEnvironmentID: targetEnvID, + ClientID: cfg.clientID, + ClientSecret: cfg.clientSecret, + RegionCode: cfg.regionCode, + } + enc := json.NewEncoder(os.Stdout) + return enc.Encode(result) +} + +func runDestroy(tfDir string) error { + if err := runTerraform(tfDir, "init", "-input=false"); err != nil { + return fmt.Errorf("terraform init: %w", err) + } + if err := runTerraform(tfDir, "destroy", "-auto-approve", "-input=false"); err != nil { + return fmt.Errorf("terraform destroy: %w", err) + } + if err := os.Remove(filepath.Join(tfDir, tfvarsFileName)); err != nil && !os.IsNotExist(err) { + log.Printf("warning: failed to remove %s: %v", tfvarsFileName, err) + } + return nil +} + +// writeTFVars persists the org-admin credentials and the desired +// environment name as a terraform.tfvars.json file in tfDir, which +// Terraform loads automatically on every subsequent init/apply/destroy in +// that directory - including a later `destroy` invocation, which runs as a +// separate process with no access to create's in-memory values. +func writeTFVars(tfDir string, cfg orgAdminConfig, envName string) error { + vars := map[string]string{ + "org_admin_environment_id": cfg.environmentID, + "org_admin_client_id": cfg.clientID, + "org_admin_client_secret": cfg.clientSecret, + "region_code": cfg.regionCode, + "license_id": cfg.licenseID, + "environment_name": envName, + } + data, err := json.MarshalIndent(vars, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(tfDir, tfvarsFileName), data, 0600) +} + +func terraformOutput(tfDir, name string) (string, error) { + cmd := exec.Command("terraform", "output", "-raw", name) + cmd.Dir = tfDir + out, err := cmd.Output() + if err != nil { + return "", err + } + return string(out), nil +} + +// runTerraform sends the child's stdout to our stderr, not our stdout: +// `create` prints exactly one JSON line to stdout as its machine-readable +// result, and Terraform's own progress output must never share that stream. +func runTerraform(dir string, args ...string) error { + cmd := exec.Command("terraform", args...) + cmd.Dir = dir + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + return cmd.Run() +}