diff --git a/.github/workflows/jsonschema.yaml b/.github/workflows/jsonschema.yaml index b9c1424..0bfc121 100644 --- a/.github/workflows/jsonschema.yaml +++ b/.github/workflows/jsonschema.yaml @@ -32,4 +32,4 @@ jobs: - name: Verify secrets json schema run: | set -e - for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$i"; check-jsonschema --fill-defaults --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$i.yaml"; done diff --git a/.github/workflows/superlinter.yml b/.github/workflows/superlinter.yml index 9492da0..509672f 100644 --- a/.github/workflows/superlinter.yml +++ b/.github/workflows/superlinter.yml @@ -27,6 +27,8 @@ jobs: VALIDATE_ALL_CODEBASE: true DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Skip installed collection copies under .ansible/ (duplicate paths break PYTHON_MYPY and other tools). + FILTER_REGEX_EXCLUDE: '(^|/)\.ansible/' # These are the validation we disable atm VALIDATE_ANSIBLE: false VALIDATE_BIOME_FORMAT: false diff --git a/Makefile b/Makefile index f62ebfb..85d263e 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,9 @@ help: ## This help message .PHONY: super-linter super-linter: ## Runs super linter locally - rm -rf .mypy_cache + rm -rf .mypy_cache .ansible podman run -e RUN_LOCAL=true -e USE_FIND_ALGORITHM=true \ + -e FILTER_REGEX_EXCLUDE='(^|/)\\.ansible/' \ -e VALIDATE_ANSIBLE=false \ -e VALIDATE_BASH=false \ -e VALIDATE_BIOME_FORMAT=false \ @@ -48,4 +49,4 @@ test: ansible-sanitytest ansible-unittest .PHONY: check-jsonschema check-jsonschema: ## Runs check-jsonschema against all unit test files except known broken ones set -e; \ - for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done + for i in values-secret-v2-base values-secret-v2-generic-onlygenerate values-secret-v2-block-yamlstring values-secret-v2-bootstrap-mixed; do echo "$$i"; check-jsonschema --schemafile ./roles/vault_utils/values-secrets.v2.schema.json "tests/unit/v2/$$i.yaml"; done diff --git a/README.md b/README.md index 68af1ea..f68b110 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,92 @@ The main purpose of this collections are to: loading local secrets files into VP secrets stores. 2. Help manage imperative and other utility functions of the cluster + +## Secrets loading + +Secrets are loaded from a **single primary** values-secret file (plus optional `values-secret.yaml.template` under the +pattern tree as a last-resort discovery path). There are **no** separate `*-bootstrap.yaml` files or `VALUES_SECRET_BOOTSTRAP` +paths; early cluster bootstrap uses **per-entry** `bootstrap` fields on v2 secrets in that same primary file. + +### Primary values-secret + +- **Backing store** comes from `values-global.yaml`: `.global.secretStore.backend` (default `vault`). That drives parsing + and whether secrets go to Vault or Kubernetes. +- **Discovery order** when `VALUES_SECRET` is unset (first existing file wins): + `~/.config/hybrid-cloud-patterns/values-secret-.yaml`, + `~/.config/validated-patterns/values-secret-.yaml`, + `~/values-secret-.yaml`, + `~/values-secret.yaml`, + then `/values-secret.yaml.template`. +- When `VALUES_SECRET` is set to an existing path, that file is used for the primary load. + +Files may be plain YAML or `ansible-vault` encrypted. + +### Per-secret `bootstrap` in v2 primary files + +On schema **2.0** primary values-secret files, each secret may set `bootstrap`: + +- **`bootstrap: true`** (or string equivalents such as `yes`, `both`) — the secret is included in the **early** + Kubernetes inject pass (`none` backend) and is **also** parsed in the **primary** pass into the configured backend + (Vault or Kubernetes as in `values-global.yaml`). It must not use `onMissingValue: generate` on any field (the early + pass cannot generate in Vault). +- **`bootstrap: only`** (or `early`) — the secret is **only** in the early inject pass; the primary pass **omits** it. +- **Unset / false** — normal primary-only secret. + +Invalid `bootstrap` scalars fail parsing with a clear error. + +Early inject runs **before** the primary backend load: during `playbooks/install.yml`, immediately after the +pattern-install manifests are applied (`operator_deploy.yml`), then again inside `load_secrets` unless that early pass +already completed (duplicate inject is skipped). + +### Playbooks and flows + +- **`playbooks/load_secrets.yml`** + Respects `.global.secretLoader.disabled` in `values-global.yaml`. When enabled: `cluster_pre_check`, primary file + discovery, early Kubernetes inject for bootstrap-tagged v2 entries (when present), then parse and load the rest into + the configured backend. + +- **`playbooks/load_bootstrap_secrets.yml`** + **Early bootstrap inject only**: `determine_pattern_dir`, `determine_pattern_name`, `pattern_settings`, then only the + Kubernetes inject for bootstrap-tagged secrets in the primary file (with retries). **Fails** if no primary file exists + or there are no bootstrap-tagged v2 entries. Does **not** read `secretLoader.disabled` or load into Vault / primary + backend. For the full early-then-primary flow, use `load_secrets.yml` (or `install.yml`). + +- **`playbooks/display_secrets_info.yml`** + Loads and displays parsed secrets (using the backend from `values-global`). For v2 files with any bootstrap-tagged + entries, output is split into **`early_bootstrap_inject`** (none backend, early K8s view; includes `bootstrap: true` + and `bootstrap: only`) and **`primary_backend`** (configured backend; includes normal secrets and **`bootstrap: true`** + again so dual-mode entries appear in both groups). Otherwise a single parse is shown as before. + +Typical usage passes the pattern checkout as `pattern_dir` (for example `-e pattern_dir=/path/to/pattern`). If you omit +it, the same resolution as `pattern_settings` applies: `PATTERN_DIR`, then `PWD`, then the `pwd` command. + +`playbooks/install.yml` imports `load_secrets.yml` after the pattern install playbook. When secret loading is enabled, +early bootstrap inject from the primary file runs at the end of `operator_deploy.yml` (right after apply), then +`load_secrets.yml` continues without repeating that inject when it already succeeded. + +### Early bootstrap inject retries + +Outer retries (parse plus Kubernetes apply) are controlled on the role defaults / extra-vars: + +- `vp_secrets_bootstrap_retry_max` (default `20`) +- `vp_secrets_bootstrap_retry_delay` (seconds between attempts, default `30`) + +These apply to the early inject path inside `load_secrets` and to `load_bootstrap_secrets.yml`. + +Per-secret namespace readiness (before each `kubernetes.core.k8s` apply) uses role defaults on `k8s_secret_utils`: + +- `k8s_secret_namespace_check_retries` (default `5`) and `k8s_secret_namespace_check_delay` (seconds between attempts, default `45`). + +If the namespace still does not exist after those attempts, the inject fails and the **outer** retry re-runs parse plus +all secret injections from the start. + +### Roles (implementation notes) + +- `roles/load_secrets/tasks/main.yml` implements the **combined** flow (early inject from primary file, then primary + backend load). +- `roles/load_secrets/tasks/bootstrap_only.yml` is used when you invoke the `load_secrets` role with + `tasks_from: bootstrap_only.yml` (as `playbooks/load_bootstrap_secrets.yml` does). +- `roles/find_vp_secrets` resolves the primary file (`tasks/main.yml`). +- v2 parsing and phase filters (`bootstrap_only`, `exclude_bootstrap`, `all`) are implemented in + `plugins/module_utils/parse_secrets_v2.py` (single `bootstrap` normalizer: off / dual / early-only). diff --git a/playbooks/determine_pattern_dir.yml b/playbooks/determine_pattern_dir.yml index 17b2cd7..98c3adb 100644 --- a/playbooks/determine_pattern_dir.yml +++ b/playbooks/determine_pattern_dir.yml @@ -1,17 +1,24 @@ --- +# Resolves pattern_dir the same way as the pattern_settings role (extra-vars, PATTERN_DIR, PWD, pwd), +# then fails if still unset. Used by display_secrets_info, load_values_global, load_bootstrap_secrets, etc. - name: Determine pattern dir hosts: localhost connection: local gather_facts: false become: false - vars: - pattern_dir: '' tasks: - - name: Fail if directory is not set + - name: Resolve pattern_dir from extra-vars, PATTERN_DIR, PWD, or pwd + ansible.builtin.include_role: + name: pattern_settings + tasks_from: resolve_overrides.yml + + - name: Fail if pattern directory is not set after resolution ansible.builtin.fail: - msg: "pattern_dir variable must be set" - when: pattern_dir | length == 0 + msg: >- + pattern_dir is not set. Pass -e pattern_dir=/path/to/pattern, export PATTERN_DIR to that path, + or run the playbook from the pattern directory so PWD is correct. + when: pattern_dir | default('') | string | trim | length == 0 - name: Set pattern_dir fact for future plays ansible.builtin.set_fact: - pattern_dir: '{{ pattern_dir }}' + pattern_dir: "{{ pattern_dir | string | trim }}" diff --git a/playbooks/display_secrets_info.yml b/playbooks/display_secrets_info.yml index 8ce8619..93db004 100644 --- a/playbooks/display_secrets_info.yml +++ b/playbooks/display_secrets_info.yml @@ -29,12 +29,70 @@ ansible.builtin.set_fact: secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" - - name: Parse secrets data + - name: Detect inline bootstrap secrets in primary v2 file + ansible.builtin.set_fact: + _vp_has_inline_bootstrap_secrets: >- + {{ + (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + and ( + (secrets_yaml.secrets | default([]) + | selectattr('bootstrap', 'defined') + | selectattr('bootstrap') + | list + | length) > 0 + ) + }} + + - name: Parse secrets data (v2 with bootstrap — two display groups) + when: _vp_has_inline_bootstrap_secrets | bool + block: + - name: Parse early-bootstrap inject portion for display (none backend) + no_log: '{{ hide_sensitive_output }}' + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_data }}" + secrets_backing_store: none + secrets_parse_filter: bootstrap_only + register: _display_bootstrap_parse + + - name: Parse primary-backend portion for display (configured backend) + no_log: '{{ hide_sensitive_output }}' + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_data }}" + secrets_backing_store: "{{ secrets_backing_store }}" + secrets_parse_filter: exclude_bootstrap + register: _display_primary_parse + + - name: Build two-group secrets display (dual bootstrap entries appear in both) + ansible.builtin.set_fact: + secrets_results: + early_bootstrap_inject: + parsed_secrets: "{{ _display_bootstrap_parse.parsed_secrets }}" + kubernetes_secret_objects: "{{ _display_bootstrap_parse.kubernetes_secret_objects }}" + vault_policies: "{{ _display_bootstrap_parse.vault_policies | default({}) }}" + unique_vault_prefixes: "{{ _display_bootstrap_parse.unique_vault_prefixes | default([]) }}" + backing_store: none + primary_backend: + parsed_secrets: "{{ _display_primary_parse.parsed_secrets }}" + kubernetes_secret_objects: "{{ _display_primary_parse.kubernetes_secret_objects }}" + vault_policies: "{{ _display_primary_parse.vault_policies | default({}) }}" + secret_store_namespace: "{{ _display_primary_parse.secret_store_namespace }}" + unique_vault_prefixes: "{{ _display_primary_parse.unique_vault_prefixes | default([]) }}" + secrets_backing_store: "{{ secrets_backing_store }}" + + # Do not register: secrets_results here — a skipped task still overwrites the register + # and would wipe the two-group set_fact when bootstrap secrets are present. + - name: Parse secrets data (single phase) + when: not (_vp_has_inline_bootstrap_secrets | bool) no_log: '{{ hide_sensitive_output }}' parse_secrets_info: values_secrets_plaintext: "{{ values_secrets_data }}" secrets_backing_store: "{{ secrets_backing_store }}" - register: secrets_results + register: _display_single_phase_parse + + - name: Set secrets_results from single-phase parse + when: not (_vp_has_inline_bootstrap_secrets | bool) + ansible.builtin.set_fact: + secrets_results: "{{ _display_single_phase_parse }}" - name: Display secrets data ansible.builtin.debug: diff --git a/playbooks/load_bootstrap_secrets.yml b/playbooks/load_bootstrap_secrets.yml new file mode 100644 index 0000000..02d9737 --- /dev/null +++ b/playbooks/load_bootstrap_secrets.yml @@ -0,0 +1,25 @@ +--- +# Inject only bootstrap-tagged secrets from the primary values-secret file (none backend, with retries). +# Does not load secrets into Vault or the primary Kubernetes backend. Does not honor +# secretLoader.disabled from values-global. Fails if no primary file exists or there are no +# bootstrap-tagged v2 entries. +# Optional extra-vars: vp_secrets_bootstrap_retry_max, vp_secrets_bootstrap_retry_delay (seconds). +- name: Determine pattern directory + ansible.builtin.import_playbook: ./determine_pattern_dir.yml + +- name: Determine pattern name + ansible.builtin.import_playbook: ./determine_pattern_name.yml + +- name: Load bootstrap secrets + hosts: localhost + connection: local + gather_facts: false + become: false + roles: + - pattern_settings + + tasks: + - name: Run bootstrap-only secrets load + ansible.builtin.include_role: + name: load_secrets + tasks_from: bootstrap_only.yml diff --git a/playbooks/operator_deploy.yml b/playbooks/operator_deploy.yml index 9758b4d..7a3442d 100644 --- a/playbooks/operator_deploy.yml +++ b/playbooks/operator_deploy.yml @@ -51,3 +51,29 @@ msg: | Failed to install pattern after 10 retries. Error: {{ _apply.error | default(_apply.msg) | default('Unknown error') }} + + # Bootstrap-tagged secrets in the primary values-secret file run immediately after apply + # (before full load_secrets and Argo health wait). + - name: Evaluate secret loader setting for bootstrap timing + ansible.builtin.set_fact: + secret_loader_disabled: "{{ values_global.global.secretLoader.disabled | default(false) | bool }}" + + - name: Run cluster pre-check and early bootstrap from primary file after pattern apply + when: not secret_loader_disabled + block: + - name: Run cluster pre-check before bootstrap + ansible.builtin.include_role: + name: cluster_pre_check + + - name: Remember cluster pre-check completed (avoid duplicate in load_secrets) + ansible.builtin.set_fact: + vp_cluster_pre_check_done: true + + - name: Early inject of bootstrap-tagged secrets from primary values-secret + ansible.builtin.include_role: + name: load_secrets + tasks_from: early_bootstrap_from_primary.yml + + - name: Remember early bootstrap inject for duplicate skip in load_secrets + ansible.builtin.set_fact: + vp_early_primary_bootstrap_done: "{{ vp_early_primary_file_loaded | default(false) | bool }}" diff --git a/playbooks/process_secrets.yml b/playbooks/process_secrets.yml index 6329dda..b612d5a 100644 --- a/playbooks/process_secrets.yml +++ b/playbooks/process_secrets.yml @@ -18,10 +18,10 @@ - find_vp_secrets # find_vp_secrets will return a plaintext data structure called values_secrets_data - # This will allow us to determine schema version and which backend to use - - name: Determine how to load secrets - ansible.builtin.set_fact: - secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" + - name: Early inject of bootstrap-tagged secrets from primary file (v2) + ansible.builtin.include_role: + name: load_secrets + tasks_from: inject_early_bootstrap_primary_entries.yml - name: Parse secrets data no_log: '{{ hide_sensitive_output | default(true) }}' diff --git a/plugins/module_utils/parse_secrets_v2.py b/plugins/module_utils/parse_secrets_v2.py index 960de5b..1d86f94 100644 --- a/plugins/module_utils/parse_secrets_v2.py +++ b/plugins/module_utils/parse_secrets_v2.py @@ -36,13 +36,28 @@ class ParseSecretsV2(SecretsV2Base): - def __init__(self, module, syaml, secrets_backing_store): + def __init__( + self, + module, + syaml, + secrets_backing_store, + secrets_parse_filter="exclude_bootstrap", + ): super().__init__(module, syaml) self.secrets_backing_store = str(secrets_backing_store) + if secrets_parse_filter not in ("all", "bootstrap_only", "exclude_bootstrap"): + self.module.fail_json( + msg=( + "secrets_parse_filter must be one of 'all', " + f"'bootstrap_only', 'exclude_bootstrap' (got {secrets_parse_filter!r})" + ) + ) + self.secrets_parse_filter = secrets_parse_filter self.secret_store_namespace = None self.parsed_secrets = {} self.kubernetes_secret_objects = [] self.vault_policies = {} + self._effective_backing_for_current_secret = self.secrets_backing_store def _get_backingstore(self): """ @@ -81,13 +96,90 @@ def _get_vault_policies(self, enable_default_vp_policies=True): return policies - def _get_secrets(self): + def _all_secrets_raw(self): secrets = self.syaml.get("secrets", []) # We check for "None" here because the yaml file is currently # filtered thru' from_yaml in module # We also check for None here to cover when there is no jinja filter is used (unit tests) return [] if secrets == "None" or secrets is None else secrets + def _bootstrap_mode(self, s): + """ + Normalize per-secret bootstrap behavior (single place for the "three-way switch"). + + Returns: + "off" — not part of the early bootstrap inject phase; primary parse includes the secret. + "dual" — C(bootstrap: true) (or equivalent string): early K8s inject (none) and primary load + using the configured backend. + "early_only" — C(bootstrap: only): early K8s inject only; omitted from C(exclude_bootstrap) primary parse. + """ + if "bootstrap" not in s: + return "off" + val = s.get("bootstrap") + if val is None or val is False: + return "off" + if val is True: + return "dual" + if isinstance(val, str): + key = val.strip().lower() + if key in ("", "false", "no", "0", "off"): + return "off" + if key in ("only", "early"): + return "early_only" + if key in ("true", "yes", "1", "both", "dual"): + return "dual" + self.module.fail_json( + msg=( + f"Secret {s.get('name', '?')}: invalid `bootstrap` value {val!r}. " + "Use boolean true (early inject plus configured backend), false/absent for normal " + "primary-only secrets, or the string 'only' (early inject only, excluded from primary load)." + ) + ) + + def _secret_in_early_bootstrap_phase(self, s): + """Secrets that participate in the optional early (none-backend) inject phase.""" + return self._bootstrap_mode(s) != "off" + + def _secret_exclude_from_primary_parse(self, s): + """Secrets dropped from the default primary parse (C(exclude_bootstrap) filter).""" + return self._bootstrap_mode(s) == "early_only" + + def _effective_backing_for_secret(self, s): + """ + Backing store used while parsing this secret's fields. Early-only and dual secrets use the + none backend only during C(bootstrap_only) parsing; dual secrets use the pattern backend + during primary / C(all) parsing so vault generate paths still work. + """ + bmode = self._bootstrap_mode(s) + if bmode == "early_only": + return "none" + if bmode == "dual" and self.secrets_parse_filter == "bootstrap_only": + return "none" + return self._get_backingstore() + + def _filter_secrets_for_phase(self, secrets): + mode = self.secrets_parse_filter + if mode == "all": + return list(secrets) + if mode == "bootstrap_only": + return [x for x in secrets if self._secret_in_early_bootstrap_phase(x)] + return [x for x in secrets if not self._secret_exclude_from_primary_parse(x)] + + def _get_secrets(self): + return self._filter_secrets_for_phase(self._all_secrets_raw()) + + def _secrets_subject_to_phase_validation(self): + """ + Secrets that receive backing-store, namespace, and field validation for this parse. + + For bootstrap_only (early K8s inject, caller backing store none), only validate entries that + participate in that phase. Other secrets in the same file are parsed in a separate primary + pass with the real backend and must not inherit none-backend namespace rules here. + """ + if self.secrets_parse_filter == "bootstrap_only": + return self._get_secrets() + return list(self._all_secrets_raw()) + def _get_field_annotations(self, f): return f.get("annotations", {}) @@ -161,13 +253,17 @@ def parse(self): total_secrets = 0 # Counter for all the secrets uploaded if len(secrets) == 0: - self.module.warn("No secrets were parsed") + if self.secrets_parse_filter != "bootstrap_only": + self.module.warn("No secrets were parsed") return total_secrets for s in secrets: total_secrets += 1 counter = 0 # This counter is to use kv put on first secret and kv patch on latter sname = s.get("name") + self._effective_backing_for_current_secret = ( + self._effective_backing_for_secret(s) + ) fields = s.get("fields", []) vault_prefixes = self._get_vault_prefixes(s) secret_type = s.get("type", "Opaque") @@ -214,22 +310,25 @@ def parse(self): return total_secrets def _validate_secrets(self): - backing_store = self._get_backingstore() - secrets = self._get_secrets() + secrets = self._all_secrets_raw() if len(secrets) == 0: self.module.warn("No secrets found") return (True, "") names = [] for s in secrets: - # These fields are mandatory for i in ["name"]: try: unused = s[i] except KeyError: - return (False, f"Secret {s['name']} is missing {i}") + return (False, f"Secret {s.get('name', '?')} is missing {i}") names.append(s["name"]) + dupes = find_dupes(names) + if len(dupes) > 0: + return (False, f"You cannot have duplicate secret names: {dupes}") + + for s in self._secrets_subject_to_phase_validation(): vault_prefixes = s.get("vaultPrefixes", ["hub"]) # This checks for the case when vaultPrefixes: is specified but empty if vault_prefixes is None or len(vault_prefixes) == 0: @@ -239,10 +338,11 @@ def _validate_secrets(self): if not isinstance(namespaces, list): return (False, f"Secret {s['name']} targetNamespaces must be a list") - if backing_store == "none" and namespaces == []: + effective_backing = self._effective_backing_for_secret(s) + if effective_backing == "none" and namespaces == []: return ( False, - f"Secret {s['name']} targetNamespaces cannot be empty for secrets backend {backing_store}", + f"Secret {s['name']} targetNamespaces cannot be empty for secrets backend {effective_backing}", ) # noqa: E501 labels = s.get("labels", {}) @@ -259,6 +359,16 @@ def _validate_secrets(self): field_names = [] for i in fields: + if ( + self._secret_in_early_bootstrap_phase(s) + and self._get_field_on_missing_value(i) == "generate" + ): + return ( + False, + f"Secret {s['name']} field {i['name']}: secrets that use the early bootstrap " + "inject phase (bootstrap: true or bootstrap: only) cannot use onMissingValue " + "'generate' (that phase uses the 'none' backend)", + ) (ret, msg) = self._validate_field(i) if not ret: return (False, msg) @@ -267,9 +377,6 @@ def _validate_secrets(self): if len(field_dupes) > 0: return (False, f"You cannot have duplicate field names: {field_dupes}") - dupes = find_dupes(names) - if len(dupes) > 0: - return (False, f"You cannot have duplicate secret names: {dupes}") return (True, "") def sanitize_values(self): @@ -314,7 +421,7 @@ def _inject_field(self, secret_name, f): if kind in ["value", ""]: if on_missing_value == "generate": self.parsed_secrets[secret_name]["generate"].append(f["name"]) - if self._get_backingstore() != "vault": + if self._effective_backing_for_current_secret != "vault": self.module.fail_json( "You cannot have onMissingValue set to 'generate' unless using vault backingstore " f"for secret {secret_name} field {f['name']}" diff --git a/plugins/modules/parse_secrets_info.py b/plugins/modules/parse_secrets_info.py index 0097ac6..e129c6a 100644 --- a/plugins/modules/parse_secrets_info.py +++ b/plugins/modules/parse_secrets_info.py @@ -85,6 +85,21 @@ required: false default: vault type: str + secrets_parse_filter: + description: + - Controls which v2 secrets entries are parsed. For v2 secrets, C(bootstrap) is normalized to a mode + C(off) (unset/false), C(dual) (C(bootstrap) true or equivalent string), or C(early_only) (C(bootstrap) only)). + C(bootstrap_only) returns every secret that participates in the early inject phase (C(dual) and C(early_only)). + C(exclude_bootstrap) (the default primary parse) omits only C(early_only) secrets; C(dual) secrets are + parsed with the configured backend. Use C(all) when parsing a dedicated bootstrap file that lists secrets + without per-entry C(bootstrap) flags, or to merge phases for display. + required: false + default: exclude_bootstrap + type: str + choices: + - all + - bootstrap_only + - exclude_bootstrap """ RETURN = """ @@ -107,6 +122,13 @@ values_secrets_plaintext: '{{ }}' secrets_backing_store: 'none' register: secrets_info + +- name: Parse only v2 secrets marked bootstrap (none backend inject phase) + parse_secrets_info: + values_secrets_plaintext: '{{ }}' + secrets_backing_store: 'none' + secrets_parse_filter: 'bootstrap_only' + register: bootstrap_secrets_info """ import traceback @@ -136,13 +158,16 @@ def run(module): args = module.params values_secrets_plaintext = args.get("values_secrets_plaintext", "") secrets_backing_store = args.get("secrets_backing_store", "vault") + secrets_parse_filter = args.get("secrets_parse_filter", "exclude_bootstrap") syaml = yaml.safe_load(values_secrets_plaintext) if syaml is None: syaml = {} - parsed_secret_obj = ParseSecretsV2(module, syaml, secrets_backing_store) + parsed_secret_obj = ParseSecretsV2( + module, syaml, secrets_backing_store, secrets_parse_filter + ) parsed_secret_obj.parse() results["failed"] = False diff --git a/roles/cluster_pre_check/tasks/main.yml b/roles/cluster_pre_check/tasks/main.yml index 1dc5f44..036dd7e 100644 --- a/roles/cluster_pre_check/tasks/main.yml +++ b/roles/cluster_pre_check/tasks/main.yml @@ -1,6 +1,10 @@ --- - name: Check if the kubernetes python module is usable from ansible - ansible.builtin.command: "{{ ansible_python_interpreter }} -c 'import kubernetes'" + ansible.builtin.command: + argv: + - "{{ ansible_python_interpreter | default(ansible_playbook_python | default('python3')) }}" + - "-c" + - "import kubernetes" changed_when: false - name: Check if KUBECONFIG is correctly set diff --git a/roles/find_vp_secrets/tasks/main.yml b/roles/find_vp_secrets/tasks/main.yml index 9c99590..6bb15f4 100644 --- a/roles/find_vp_secrets/tasks/main.yml +++ b/roles/find_vp_secrets/tasks/main.yml @@ -5,9 +5,9 @@ ansible.builtin.set_fact: secret_template: "{{ pattern_dir }}/values-secret.yaml.template" -- name: Is a VALUES_SECRET env variable set? +- name: Resolve VALUES_SECRET for primary secrets search ansible.builtin.set_fact: - custom_env_values_secret: "{{ lookup('ansible.builtin.env', 'VALUES_SECRET') }}" + custom_env_values_secret: "{{ lookup('ansible.builtin.env', 'VALUES_SECRET') | default('', true) }}" - name: Check if VALUES_SECRET file exists ansible.builtin.stat: @@ -37,52 +37,5 @@ - "{{ pattern_dir }}/values-secret.yaml.template" when: custom_env_values_secret | default('') | length == 0 -- name: Is found values secret file encrypted - no_log: "{{ hide_sensitive_output | default(true) }}" - ansible.builtin.shell: | - set -o pipefail - head -1 "{{ found_file }}" | grep -q \$ANSIBLE_VAULT - changed_when: false - register: encrypted - failed_when: (encrypted.rc not in [0, 1]) - -# When HOME is set we replace it with '~' in this debug message -# because when run from inside the container the HOME is /pattern-home -# which is confusing for users -- name: Is found values secret file encrypted - ansible.builtin.debug: - msg: "Using {{ (lookup('env', 'HOME') | length > 0) | ternary(found_file | regex_replace('^' + lookup('env', 'HOME'), '~'), found_file) }} to parse secrets" - -- name: Set encryption bool fact - no_log: "{{ hide_sensitive_output | default(true) }}" - ansible.builtin.set_fact: - is_encrypted: "{{ encrypted.rc == 0 | bool }}" - -- name: Get password for "{{ found_file }}" - ansible.builtin.pause: - prompt: "Input the password for {{ found_file }}" - echo: false - when: is_encrypted - register: vault_pass - -- name: Get decrypted content if {{ found_file }} was encrypted - no_log: "{{ hide_sensitive_output | default(true) }}" - ansible.builtin.shell: - ansible-vault view --vault-password-file <(cat <<<"{{ vault_pass.user_input }}") "{{ found_file }}" - register: values_secret_plaintext - when: is_encrypted - changed_when: false - -- name: Normalize secrets format (un-encrypted) - no_log: '{{ hide_sensitive_output | default(true) }}' - ansible.builtin.set_fact: - values_secrets_data: "{{ lookup('file', found_file) | from_yaml }}" - when: not is_encrypted - changed_when: false - -- name: Normalize secrets format (encrypted) - no_log: '{{ hide_sensitive_output | default(true) }}' - ansible.builtin.set_fact: - values_secrets_data: "{{ values_secret_plaintext.stdout | from_yaml }}" - when: is_encrypted - changed_when: false +- name: Read secrets from discovered file + ansible.builtin.include_tasks: read_secret_from_path.yml diff --git a/roles/find_vp_secrets/tasks/read_secret_from_path.yml b/roles/find_vp_secrets/tasks/read_secret_from_path.yml new file mode 100644 index 0000000..57d982b --- /dev/null +++ b/roles/find_vp_secrets/tasks/read_secret_from_path.yml @@ -0,0 +1,52 @@ +--- +# Reads YAML from found_file (vault-encrypted or plain) into values_secrets_data. +# Expects: found_file, hide_sensitive_output (optional) +- name: Is found values secret file encrypted + no_log: "{{ hide_sensitive_output | default(true) }}" + ansible.builtin.shell: | + set -o pipefail + head -1 "{{ found_file }}" | grep -q \$ANSIBLE_VAULT + changed_when: false + register: encrypted + failed_when: (encrypted.rc not in [0, 1]) + +# When HOME is set we replace it with '~' in this debug message +# because when run from inside the container the HOME is /pattern-home +# which is confusing for users +- name: Is found values secret file encrypted + ansible.builtin.debug: + msg: "Using {{ (lookup('env', 'HOME') | length > 0) | ternary(found_file | regex_replace('^' + lookup('env', 'HOME'), '~'), found_file) }} to parse secrets" + +- name: Set encryption bool fact + no_log: "{{ hide_sensitive_output | default(true) }}" + ansible.builtin.set_fact: + is_encrypted: "{{ encrypted.rc == 0 | bool }}" + +- name: Get password for "{{ found_file }}" + ansible.builtin.pause: + prompt: "Input the password for {{ found_file }}" + echo: false + when: is_encrypted + register: vault_pass + +- name: Get decrypted content if {{ found_file }} was encrypted + no_log: "{{ hide_sensitive_output | default(true) }}" + ansible.builtin.shell: + ansible-vault view --vault-password-file <(cat <<<"{{ vault_pass.user_input }}") "{{ found_file }}" + register: values_secret_plaintext + when: is_encrypted + changed_when: false + +- name: Normalize secrets format (un-encrypted) + no_log: "{{ hide_sensitive_output | default(true) }}" + ansible.builtin.set_fact: + values_secrets_data: "{{ lookup('file', found_file) | from_yaml }}" + when: not is_encrypted + changed_when: false + +- name: Normalize secrets format (encrypted) + no_log: "{{ hide_sensitive_output | default(true) }}" + ansible.builtin.set_fact: + values_secrets_data: "{{ values_secret_plaintext.stdout | from_yaml }}" + when: is_encrypted + changed_when: false diff --git a/roles/k8s_secret_utils/defaults/main.yml b/roles/k8s_secret_utils/defaults/main.yml index 7ebda20..b674cca 100644 --- a/roles/k8s_secret_utils/defaults/main.yml +++ b/roles/k8s_secret_utils/defaults/main.yml @@ -1,2 +1,5 @@ --- secrets_ns: 'validated-patterns-secrets' +# Namespace wait before injecting each Secret (then outer bootstrap retry can re-run the full inject). +k8s_secret_namespace_check_retries: 5 +k8s_secret_namespace_check_delay: 45 diff --git a/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml b/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml index 410e1a0..f25e2ad 100644 --- a/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml +++ b/roles/k8s_secret_utils/tasks/inject_k8s_secret.yml @@ -1,15 +1,38 @@ --- -- name: Check for secrets namespace +- name: >- + Check for secrets namespace {{ item.metadata.namespace | default('unknown') }} + ({{ k8s_secret_namespace_check_retries | default(5) }} attempts) no_log: '{{ hide_sensitive_output | default(true) }}' kubernetes.core.k8s_info: kind: Namespace name: "{{ item['metadata']['namespace'] }}" register: secrets_ns_rc until: secrets_ns_rc.resources | length > 0 - retries: 20 - delay: 45 + retries: "{{ k8s_secret_namespace_check_retries | default(5) | int }}" + delay: "{{ k8s_secret_namespace_check_delay | default(45) | int }}" -- name: Inject k8s secret - no_log: '{{ hide_sensitive_output | default(True) }}' +- name: >- + Report namespace ready for {{ item.kind | default('Secret') }} {{ item.metadata.name | default('unknown') }} + (namespace {{ item.metadata.namespace | default('unknown') }}) + ansible.builtin.debug: + msg: >- + Namespace '{{ item.metadata.namespace | default('unknown') }}' is ready for + {{ item.kind | default('Secret') }} '{{ item.metadata.name | default('unknown') }}'. + +- name: >- + Inject Kubernetes {{ item.kind | default('Secret') }} {{ item.metadata.name | default('unknown') }} in namespace + {{ item.metadata.namespace | default('unknown') }} + no_log: '{{ hide_sensitive_output | default(true) }}' kubernetes.core.k8s: definition: '{{ item }}' + wait: true + register: k8s_secret_apply_result + changed_when: k8s_secret_apply_result is changed + +- name: >- + Report {{ item.kind | default('Secret') }} apply result for {{ item.metadata.namespace | default('unknown') }}/ + {{ item.metadata.name | default('unknown') }} + ansible.builtin.debug: + msg: >- + Applied {{ item.kind | default('Secret') }} namespace='{{ item.metadata.namespace | default('unknown') }}' + name='{{ item.metadata.name | default('unknown') }}' (changed={{ k8s_secret_apply_result.changed | default(false) }}). diff --git a/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml b/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml index fab658f..126a1c6 100644 --- a/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml +++ b/roles/k8s_secret_utils/tasks/inject_k8s_secrets.yml @@ -1,5 +1,7 @@ --- - name: Inject secrets - no_log: '{{ hide_sensitive_output | default(True) }}' + no_log: '{{ hide_sensitive_output | default(true) }}' ansible.builtin.include_tasks: inject_k8s_secret.yml loop: '{{ kubernetes_secret_objects }}' + loop_control: + label: "{{ item.metadata.namespace | default('?') }}/{{ item.kind | default('Secret') }}/{{ item.metadata.name | default('?') }}" diff --git a/roles/k8s_secret_utils/tasks/parse_secrets.yml b/roles/k8s_secret_utils/tasks/parse_secrets.yml index 2fa4cb2..a6b4de0 100644 --- a/roles/k8s_secret_utils/tasks/parse_secrets.yml +++ b/roles/k8s_secret_utils/tasks/parse_secrets.yml @@ -1,4 +1,39 @@ --- +- name: Cache values-secrets as YAML for v2 inline bootstrap detection + ansible.builtin.set_fact: + secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" + +- name: Detect inline bootstrap secrets in primary v2 file + ansible.builtin.set_fact: + _vp_has_inline_bootstrap_secrets: >- + {{ + (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + and ( + (secrets_yaml.secrets | default([]) + | selectattr('bootstrap', 'defined') + | selectattr('bootstrap') + | list + | length) > 0 + ) + }} + +- name: Parse and inject inline bootstrap secrets from primary file (v2) + when: _vp_has_inline_bootstrap_secrets | bool + block: + - name: Parse primary file bootstrap-only secrets for none backend + no_log: '{{ hide_sensitive_output | default(true) }}' + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_data }}" + secrets_backing_store: none + secrets_parse_filter: bootstrap_only + register: inline_bootstrap_secrets_results + + - name: Inject inline bootstrap secrets into the cluster + ansible.builtin.include_tasks: inject_k8s_secrets.yml + vars: + kubernetes_secret_objects: "{{ inline_bootstrap_secrets_results.kubernetes_secret_objects }}" + when: inline_bootstrap_secrets_results.parsed_secrets | length > 0 + - name: Parse secrets data no_log: '{{ hide_sensitive_output | default(true) }}' parse_secrets_info: diff --git a/roles/load_secrets/defaults/main.yml b/roles/load_secrets/defaults/main.yml index 0999947..bf5dbcb 100644 --- a/roles/load_secrets/defaults/main.yml +++ b/roles/load_secrets/defaults/main.yml @@ -1,3 +1,5 @@ -secrets_role: "vault_utils" -tasks_from: "push_parsed_secrets" -hide_sensitive_output: true +--- +vp_secrets_bootstrap_retry_max: 20 +vp_secrets_bootstrap_retry_delay: 30 +secrets_role: vault_utils +tasks_from: push_parsed_secrets diff --git a/roles/load_secrets/tasks/bootstrap_inject_retry.yml b/roles/load_secrets/tasks/bootstrap_inject_retry.yml new file mode 100644 index 0000000..f5446cb --- /dev/null +++ b/roles/load_secrets/tasks/bootstrap_inject_retry.yml @@ -0,0 +1,51 @@ +--- +- name: Ensure bootstrap inject attempt counter + ansible.builtin.set_fact: + _bootstrap_inject_attempt: "{{ _bootstrap_inject_attempt | default(1) | int }}" + +- name: Determine bootstrap secrets YAML shape + ansible.builtin.set_fact: + secrets_bootstrap_yaml: "{{ values_secrets_bootstrap_data if values_secrets_bootstrap_data is not string else values_secrets_bootstrap_data | from_yaml }}" + +- name: Fail when bootstrap schema is too old for none backend loading + ansible.builtin.fail: + msg: Bootstrap secrets require values-secret schema version 2.0 or higher when using the none backend. + when: (secrets_bootstrap_yaml.version | default('2.0')) is version('2.0', '<') + +- name: Parse and inject bootstrap secrets (attempt {{ _bootstrap_inject_attempt }}/{{ vp_secrets_bootstrap_retry_max | default(20) }}) + block: + - name: Parse bootstrap secrets data + no_log: "{{ hide_sensitive_output | default(true) }}" + parse_secrets_info: + values_secrets_plaintext: "{{ values_secrets_bootstrap_data }}" + secrets_backing_store: none + secrets_parse_filter: bootstrap_only + register: bootstrap_secrets_results + + - name: Inject bootstrap secrets into the cluster + ansible.builtin.include_role: + name: k8s_secret_utils + tasks_from: inject_k8s_secrets + vars: + kubernetes_secret_objects: "{{ bootstrap_secrets_results.kubernetes_secret_objects }}" + vault_policies: "{{ bootstrap_secrets_results.vault_policies }}" + parsed_secrets: "{{ bootstrap_secrets_results.parsed_secrets }}" + unique_vault_prefixes: "{{ bootstrap_secrets_results.unique_vault_prefixes | default([]) }}" + + rescue: + - name: Fail when bootstrap secrets inject retries are exhausted + ansible.builtin.fail: + msg: | + Bootstrap secrets loading failed after {{ vp_secrets_bootstrap_retry_max | default(20) }} attempt(s). + when: (_bootstrap_inject_attempt | int) >= (vp_secrets_bootstrap_retry_max | default(20) | int) + + - name: Wait before retrying bootstrap secrets inject + ansible.builtin.pause: + seconds: "{{ vp_secrets_bootstrap_retry_delay | default(30) | int }}" + + - name: Bump bootstrap secrets inject attempt counter + ansible.builtin.set_fact: + _bootstrap_inject_attempt: "{{ (_bootstrap_inject_attempt | int) + 1 }}" + + - name: Retry bootstrap secrets inject + ansible.builtin.include_tasks: bootstrap_inject_retry.yml diff --git a/roles/load_secrets/tasks/bootstrap_only.yml b/roles/load_secrets/tasks/bootstrap_only.yml new file mode 100644 index 0000000..d2426a9 --- /dev/null +++ b/roles/load_secrets/tasks/bootstrap_only.yml @@ -0,0 +1,29 @@ +--- +# Early Kubernetes inject only for v2 primary values-secret entries tagged with bootstrap. +# Fails if no primary file is found or the file has no bootstrap-tagged secrets. +- name: Run cluster pre-check before bootstrap-only secrets load + ansible.builtin.include_role: + name: cluster_pre_check + +- name: Find and read primary values-secret file + ansible.builtin.include_role: + name: find_vp_secrets + +- name: Require primary values-secret content + ansible.builtin.assert: + that: + - values_secrets_data is defined + fail_msg: >- + No primary values-secret file was found or read. Set VALUES_SECRET to an existing file or install + a file at one of the standard paths (see collection README). + +- name: Inject bootstrap-tagged secrets from primary file (with retries) + ansible.builtin.include_tasks: inject_early_bootstrap_primary_entries.yml + +- name: Require at least one bootstrap-tagged secret in primary file + ansible.builtin.assert: + that: + - _vp_has_inline_bootstrap_secrets | default(false) | bool + fail_msg: >- + No v2 secrets with bootstrap: true / bootstrap: only (or equivalent) were found in the primary + values-secret file. Nothing to inject for bootstrap-only mode. diff --git a/roles/load_secrets/tasks/early_bootstrap_from_primary.yml b/roles/load_secrets/tasks/early_bootstrap_from_primary.yml new file mode 100644 index 0000000..b593a06 --- /dev/null +++ b/roles/load_secrets/tasks/early_bootstrap_from_primary.yml @@ -0,0 +1,35 @@ +--- +# Discover primary values-secret and inject bootstrap-tagged entries (none backend, with retries). +# Used from operator_deploy after pattern manifests apply. Rescue allows install to continue when +# no primary file exists yet; load_secrets.yml will surface the same discovery rules later. +- name: Early bootstrap from primary values-secret file + block: + - name: Find and read primary values-secret file + ansible.builtin.include_role: + name: find_vp_secrets + + - name: Assert primary values-secret content is available + ansible.builtin.assert: + that: + - values_secrets_data is defined + fail_msg: >- + Primary values-secret discovery succeeded but no content was loaded. + Check file permissions and format. + + - name: Inject bootstrap-tagged entries from primary file + ansible.builtin.include_tasks: inject_early_bootstrap_primary_entries.yml + + - name: Record that primary values-secret was loaded for early bootstrap timing + ansible.builtin.set_fact: + vp_early_primary_file_loaded: true + + rescue: + - name: Note skipping early bootstrap from primary file + ansible.builtin.debug: + msg: >- + Skipping early bootstrap from primary values-secret (file not found or not readable yet). + Full secrets loading will run in load_secrets.yml when secret loading is enabled. + + - name: Record that primary values-secret was not loaded for early bootstrap + ansible.builtin.set_fact: + vp_early_primary_file_loaded: false diff --git a/roles/load_secrets/tasks/inject_early_bootstrap_primary_entries.yml b/roles/load_secrets/tasks/inject_early_bootstrap_primary_entries.yml new file mode 100644 index 0000000..9f4ba1c --- /dev/null +++ b/roles/load_secrets/tasks/inject_early_bootstrap_primary_entries.yml @@ -0,0 +1,49 @@ +--- +# Requires values_secrets_data from find_vp_secrets. When vp_early_primary_bootstrap_done is true, +# duplicate inject is skipped (early pass already ran after pattern apply in operator_deploy). +- name: Cache values-secrets as YAML for v2 inline bootstrap detection + ansible.builtin.set_fact: + secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" + +- name: Detect inline bootstrap secrets in primary v2 file + ansible.builtin.set_fact: + _vp_has_inline_bootstrap_secrets: >- + {{ + (secrets_yaml.version | default('2.0')) is version('2.0', '>=') + and ( + (secrets_yaml.secrets | default([]) + | selectattr('bootstrap', 'defined') + | selectattr('bootstrap') + | list + | length) > 0 + ) + }} + +- name: Parse and inject bootstrap-phase secrets with retries + when: + - _vp_has_inline_bootstrap_secrets | bool + - not (vp_early_primary_bootstrap_done | default(false) | bool) + block: + - name: Stage primary plaintext for bootstrap inject helper + ansible.builtin.set_fact: + values_secrets_bootstrap_data: "{{ values_secrets_data }}" + + - name: Initialize bootstrap secrets inject attempt counter + ansible.builtin.set_fact: + _bootstrap_inject_attempt: 1 + + - name: Inject bootstrap-tagged secrets into the cluster with retries on failure + ansible.builtin.include_tasks: bootstrap_inject_retry.yml + +- name: Clear bootstrap inject staging fact + ansible.builtin.set_fact: + values_secrets_bootstrap_data: '' + +- name: Note duplicate early bootstrap inject skipped + when: + - _vp_has_inline_bootstrap_secrets | bool + - vp_early_primary_bootstrap_done | default(false) | bool + ansible.builtin.debug: + msg: >- + Early bootstrap from the primary values-secret file already ran after pattern apply; + skipping duplicate inject. diff --git a/roles/load_secrets/tasks/main.yml b/roles/load_secrets/tasks/main.yml index 7d79b09..8a31897 100644 --- a/roles/load_secrets/tasks/main.yml +++ b/roles/load_secrets/tasks/main.yml @@ -3,12 +3,14 @@ ansible.builtin.set_fact: secrets_backing_store: "{{ values_global.global.secretStore.backend | default('vault') }}" -- name: Run secret-loading pre-requisites +- name: Run cluster pre-check once before bootstrap-tagged and primary secrets loading ansible.builtin.include_role: - name: "{{ item }}" - loop: - - cluster_pre_check - - find_vp_secrets + name: cluster_pre_check + when: not (vp_cluster_pre_check_done | default(false) | bool) + +- name: Find primary values-secret file and read it + ansible.builtin.include_role: + name: find_vp_secrets - name: Fail if values_secrets_data is missing ansible.builtin.shell: | @@ -19,6 +21,9 @@ exit 1 when: values_secrets_data is not defined +- name: Early Kubernetes inject for bootstrap-tagged secrets in primary file + ansible.builtin.include_tasks: inject_early_bootstrap_primary_entries.yml + - name: Determine how to load secrets ansible.builtin.set_fact: secrets_yaml: "{{ values_secrets_data if values_secrets_data is not string else values_secrets_data | from_yaml }}" @@ -46,4 +51,9 @@ kubernetes_secret_objects: "{{ secrets_results.kubernetes_secret_objects }}" vault_policies: "{{ secrets_results.vault_policies }}" parsed_secrets: "{{ secrets_results.parsed_secrets }}" - unique_vault_prefixes: "{{ secrets_results.unique_vault_prefixes }}" + unique_vault_prefixes: "{{ secrets_results.unique_vault_prefixes | default([]) }}" + +- name: Clear early-bootstrap timing facts for later plays + ansible.builtin.set_fact: + vp_early_primary_bootstrap_done: false + vp_early_primary_file_loaded: false diff --git a/roles/vault_utils/README.md b/roles/vault_utils/README.md index 50dbec1..791f37d 100644 --- a/roles/vault_utils/README.md +++ b/roles/vault_utils/README.md @@ -69,6 +69,16 @@ By default, the first file that will looked up is The paths can be overridden by setting the environment variable `VALUES_SECRET` to the path of the secret file. +Optional **early bootstrap** behavior (Kubernetes inject for `bootstrap`-tagged v2 secrets in the **primary** +values-secret file only), the early-then-primary loading order, `load_bootstrap_secrets.yml`, and +`display_secrets_info.yml` are documented under **Secrets loading** in the collection `README.md` at the repository root. + +For **v2.0** primary files, each `secrets[]` entry may set `bootstrap`: use boolean `true` (or strings like `yes`, +`both`) for **early Kubernetes inject plus** load into the configured primary backend (Vault or Kubernetes); use the +string `only` or `early` for **early inject only** (primary parse skips that entry). See **Per-secret `bootstrap` in v2 +primary files** in the root `README.md`. The machine-readable rules live in `values-secrets.v2.schema.json` in this role +(`bootstrap` is a boolean or a non-empty string). + The values secret YAML files can be encrypted with `ansible-vault`. If the role detects they are encrypted, the password to decrypt them will be prompted when needed. diff --git a/roles/vault_utils/values-secrets.v2.schema.json b/roles/vault_utils/values-secrets.v2.schema.json index b5582c3..bdd4c52 100644 --- a/roles/vault_utils/values-secrets.v2.schema.json +++ b/roles/vault_utils/values-secrets.v2.schema.json @@ -205,6 +205,14 @@ "minItems": 1, "uniqueItems": true } + }, + "bootstrap": { + "description": "When boolean true (or string equivalents such as 'yes', 'both'), this secret is included in the early bootstrap Kubernetes inject pass and also in the primary load using the configured backend. When the string 'only' or 'early', it is only in the early inject pass and omitted from the primary parse. When false or omitted, this is a normal primary-only secret.", + "anyOf": [ + {"type": "boolean"}, + {"type": "string", "minLength": 1} + ], + "default": false } } }, diff --git a/tests/unit/test_parse_secrets.py b/tests/unit/test_parse_secrets.py index 2d10455..3ee9e09 100644 --- a/tests/unit/test_parse_secrets.py +++ b/tests/unit/test_parse_secrets.py @@ -41,6 +41,9 @@ def set_module_args(args): """prepare arguments so that they will be picked up during module creation""" args = json.dumps({"ANSIBLE_MODULE_ARGS": args}) basic._ANSIBLE_ARGS = to_bytes(args) + # Ansible 2.19+ module_utils.basic._load_params requires a serialization profile when args are injected. + if hasattr(basic, "_ANSIBLE_PROFILE"): + basic._ANSIBLE_PROFILE = "legacy" class BytesEncoder(json.JSONEncoder): @@ -972,6 +975,133 @@ def test_ensure_success_null_secrets(self, getpass): and (len(ret["kubernetes_secret_objects"]) == 0) ) + def test_invalid_secrets_parse_filter(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-mixed.yaml") + ) + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_parse_filter": "bogus", + } + ) + parse_secrets_info.main() + ret = ansible_err.exception.args[0] + self.assertEqual(ret["failed"], True) + self.assertIn("secrets_parse_filter must be one of", ret["msg"]) + + def test_bootstrap_only_parse_returns_bootstrap_secrets(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-phase-only.yaml") + ) + with self.assertRaises(AnsibleExitJson) as result: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "none", + "secrets_parse_filter": "bootstrap_only", + } + ) + parse_secrets_info.main() + ret = result.exception.args[0] + self.assertFalse(ret["failed"]) + self.assertEqual( + set(ret["parsed_secrets"].keys()), {"boot-token", "early-only-token"} + ) + self.assertEqual( + ret["parsed_secrets"]["boot-token"]["fields"]["token"], + "inline-bootstrap-value", + ) + self.assertEqual( + ret["parsed_secrets"]["early-only-token"]["fields"]["key"], + "early-only-inline", + ) + self.assertEqual(len(ret["kubernetes_secret_objects"]), 2) + ns_set = {o["metadata"]["namespace"] for o in ret["kubernetes_secret_objects"]} + self.assertEqual(ns_set, {"openshift-gitops", "openshift-config"}) + + def test_exclude_bootstrap_parse_omits_bootstrap_secrets(self, getpass): + getpass.return_value = os.path.expanduser("~/empty") + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-mixed.yaml") + ) + with self.assertRaises(AnsibleExitJson) as result: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + "secrets_parse_filter": "exclude_bootstrap", + } + ) + parse_secrets_info.main() + ret = result.exception.args[0] + self.assertFalse(ret["failed"]) + self.assertEqual( + set(ret["parsed_secrets"].keys()), {"boot-token", "main-vault-secret"} + ) + self.assertIn("secret", ret["parsed_secrets"]["main-vault-secret"]["generate"]) + + def test_parse_all_includes_bootstrap_and_primary(self, getpass): + getpass.return_value = os.path.expanduser("~/empty") + testfile_output = self.get_file_as_stdout( + os.path.join(self.testdir_v2, "values-secret-v2-bootstrap-mixed.yaml") + ) + with self.assertRaises(AnsibleExitJson) as result: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + "secrets_parse_filter": "all", + } + ) + parse_secrets_info.main() + ret = result.exception.args[0] + self.assertFalse(ret["failed"]) + self.assertEqual( + set(ret["parsed_secrets"].keys()), + {"boot-token", "early-only-token", "main-vault-secret"}, + ) + + def test_bootstrap_secret_may_not_use_generate(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-generate-invalid.yaml" + ) + ) + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + } + ) + parse_secrets_info.main() + ret = ansible_err.exception.args[0] + self.assertTrue(ret["failed"]) + msg = ret.get("msg") or " ".join(str(a) for a in ret.get("args", ())) + self.assertIn("bootstrap", msg) + self.assertIn("generate", msg) + + def test_invalid_bootstrap_scalar_fails(self, getpass): + testfile_output = self.get_file_as_stdout( + os.path.join( + self.testdir_v2, "values-secret-v2-bootstrap-invalid-scalar.yaml" + ) + ) + with self.assertRaises(AnsibleFailJson) as ansible_err: + set_module_args( + { + "values_secrets_plaintext": testfile_output, + "secrets_backing_store": "vault", + } + ) + parse_secrets_info.main() + ret = ansible_err.exception.args[0] + self.assertTrue(ret["failed"]) + msg = ret.get("msg") or " ".join(str(a) for a in ret.get("args", ())) + self.assertIn("invalid `bootstrap` value", msg) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_vault_load_parsed_secrets.py b/tests/unit/test_vault_load_parsed_secrets.py index 3ddd042..48972e5 100644 --- a/tests/unit/test_vault_load_parsed_secrets.py +++ b/tests/unit/test_vault_load_parsed_secrets.py @@ -34,6 +34,8 @@ def set_module_args(args): """prepare arguments so that they will be picked up during module creation""" args = json.dumps({"ANSIBLE_MODULE_ARGS": args}) basic._ANSIBLE_ARGS = to_bytes(args) + if hasattr(basic, "_ANSIBLE_PROFILE"): + basic._ANSIBLE_PROFILE = "legacy" class AnsibleExitJson(Exception): diff --git a/tests/unit/test_vault_load_secrets.py b/tests/unit/test_vault_load_secrets.py index 43bea39..eb6ea33 100644 --- a/tests/unit/test_vault_load_secrets.py +++ b/tests/unit/test_vault_load_secrets.py @@ -33,6 +33,8 @@ def set_module_args(args): """prepare arguments so that they will be picked up during module creation""" args = json.dumps({"ANSIBLE_MODULE_ARGS": args}) basic._ANSIBLE_ARGS = to_bytes(args) + if hasattr(basic, "_ANSIBLE_PROFILE"): + basic._ANSIBLE_PROFILE = "legacy" class AnsibleExitJson(Exception): diff --git a/tests/unit/test_vault_load_secrets_v2.py b/tests/unit/test_vault_load_secrets_v2.py index 0cc3f40..caa167a 100644 --- a/tests/unit/test_vault_load_secrets_v2.py +++ b/tests/unit/test_vault_load_secrets_v2.py @@ -33,6 +33,8 @@ def set_module_args(args): """prepare arguments so that they will be picked up during module creation""" args = json.dumps({"ANSIBLE_MODULE_ARGS": args}) basic._ANSIBLE_ARGS = to_bytes(args) + if hasattr(basic, "_ANSIBLE_PROFILE"): + basic._ANSIBLE_PROFILE = "legacy" class AnsibleExitJson(Exception): diff --git a/tests/unit/v2/values-secret-v2-bootstrap-generate-invalid.yaml b/tests/unit/v2/values-secret-v2-bootstrap-generate-invalid.yaml new file mode 100644 index 0000000..ee7428c --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-generate-invalid.yaml @@ -0,0 +1,14 @@ +version: "2.0" + +vaultPolicies: + basicPolicy: "length=10\nrule \"charset\" { charset = \"abcdefghijklmnopqrstuvwxyz\" min-chars = 1 }\n" + +secrets: + - name: bad-bootstrap + bootstrap: only + targetNamespaces: + - openshift-gitops + fields: + - name: secret + onMissingValue: generate + vaultPolicy: basicPolicy diff --git a/tests/unit/v2/values-secret-v2-bootstrap-invalid-scalar.yaml b/tests/unit/v2/values-secret-v2-bootstrap-invalid-scalar.yaml new file mode 100644 index 0000000..2b74666 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-invalid-scalar.yaml @@ -0,0 +1,13 @@ +version: "2.0" + +backingStore: vault + +secrets: + - name: bad-bootstrap-value + bootstrap: nonsense + targetNamespaces: + - openshift-gitops + fields: + - name: token + value: x + onMissingValue: error diff --git a/tests/unit/v2/values-secret-v2-bootstrap-mixed.yaml b/tests/unit/v2/values-secret-v2-bootstrap-mixed.yaml new file mode 100644 index 0000000..819fd39 --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-mixed.yaml @@ -0,0 +1,31 @@ +version: "2.0" + +vaultPolicies: + basicPolicy: "length=10\nrule \"charset\" { charset = \"abcdefghijklmnopqrstuvwxyz\" min-chars = 1 }\n" + +secrets: + - name: boot-token + bootstrap: true + targetNamespaces: + - openshift-gitops + fields: + - name: token + value: inline-bootstrap-value + onMissingValue: error + + - name: early-only-token + bootstrap: only + targetNamespaces: + - openshift-config + fields: + - name: key + value: early-only-inline + onMissingValue: error + + - name: main-vault-secret + vaultPrefixes: + - hub + fields: + - name: secret + onMissingValue: generate + vaultPolicy: basicPolicy diff --git a/tests/unit/v2/values-secret-v2-bootstrap-phase-only.yaml b/tests/unit/v2/values-secret-v2-bootstrap-phase-only.yaml new file mode 100644 index 0000000..574a44e --- /dev/null +++ b/tests/unit/v2/values-secret-v2-bootstrap-phase-only.yaml @@ -0,0 +1,23 @@ +version: "2.0" + +vaultPolicies: + basicPolicy: "length=10\nrule \"charset\" { charset = \"abcdefghijklmnopqrstuvwxyz\" min-chars = 1 }\n" + +secrets: + - name: boot-token + bootstrap: true + targetNamespaces: + - openshift-gitops + fields: + - name: token + value: inline-bootstrap-value + onMissingValue: error + + - name: early-only-token + bootstrap: only + targetNamespaces: + - openshift-config + fields: + - name: key + value: early-only-inline + onMissingValue: error