From 4048f039f1dec01181ade0f72545ab499dfd7949 Mon Sep 17 00:00:00 2001 From: as51340 Date: Thu, 23 Jul 2026 11:48:35 +0200 Subject: [PATCH 01/34] feat: kubebuilder scaffold for the MemgraphCluster operator Replace the discarded prior attempt (preserved on archive/pre-operator-mvp) with a clean kubebuilder v4 scaffold: - MemgraphCluster API skeleton: group memgraph.com, version v1alpha1, short name mgc; spec fields land in subsequent slices - Hello-world reconciler wired through the manager - Generated CRD manifests (verified to install on a local kind cluster; kubectl get mgc resolves) - CI skeleton: lint, unit, and envtest suites run on every pull request - specs/operator-mvp/ (PRD + issue slices) carried into the new history Implements specs/operator-mvp/issues/01-repo-reset-scaffold.md --- .custom-gcl.yml | 11 + .devcontainer/devcontainer.json | 35 ++ .devcontainer/post-install.sh | 153 +++++++++ .dockerignore | 12 +- .github/CODEOWNERS | 1 - .github/config_files/config_lint.yaml | 11 - .github/workflows/kubelint.yaml | 65 ---- .github/workflows/lint.yml | 31 ++ .github/workflows/pre-commit.yaml | 28 -- .github/workflows/test.yml | 48 +++ .gitignore | 20 +- .gitmodules | 4 - .golangci.yml | 73 ++-- .pre-commit-config.yaml | 18 - AGENTS.md | 320 +++++++++++++++++ Dockerfile | 10 +- Makefile | 299 +++++++--------- PROJECT | 15 +- README.md | 141 +++++++- api/v1/memgraphha_types.go | 135 -------- api/v1/zz_generated.deepcopy.go | 298 ---------------- api/{v1 => v1alpha1}/groupversion_info.go | 29 +- api/v1alpha1/memgraphcluster_types.go | 92 +++++ api/v1alpha1/zz_generated.deepcopy.go | 122 +++++++ cmd/main.go | 129 +++++-- .../bases/memgraph.com_memgraphclusters.yaml | 123 +++++++ .../crd/bases/memgraph.com_memgraphhas.yaml | 232 ------------- config/crd/kustomization.yaml | 13 +- config/crd/kustomizeconfig.yaml | 7 - .../default/cert_metrics_manager_patch.yaml | 30 ++ config/default/kustomization.yaml | 230 ++++++++++++- config/default/manager_metrics_patch.yaml | 4 + config/default/metrics_service.yaml | 18 + config/manager/kustomization.yaml | 10 - config/manager/manager.yaml | 88 ++++- config/manager/namespace.yaml | 4 - config/manifests/kustomization.yaml | 7 - .../network-policy/allow-metrics-traffic.yaml | 27 ++ config/network-policy/kustomization.yaml | 2 + config/prometheus/kustomization.yaml | 11 + config/prometheus/monitor.yaml | 27 ++ config/prometheus/monitor_tls_patch.yaml | 19 ++ config/rbac/kustomization.yaml | 24 ++ config/rbac/leader_election_role.yaml | 40 +++ config/rbac/leader_election_role_binding.yaml | 15 + config/rbac/memgraphcluster_admin_role.yaml | 27 ++ config/rbac/memgraphcluster_editor_role.yaml | 33 ++ config/rbac/memgraphcluster_viewer_role.yaml | 29 ++ config/rbac/metrics_auth_role.yaml | 17 + config/rbac/metrics_auth_role_binding.yaml | 12 + config/rbac/metrics_reader_role.yaml | 9 + config/rbac/role.yaml | 47 +-- config/rbac/role_binding.yaml | 11 +- config/rbac/service_account.yaml | 7 +- config/samples/kustomization.yaml | 4 +- config/samples/memgraph_v1_ha.yaml | 114 ------- config/samples/v1alpha1_memgraphcluster.yaml | 8 + config/scorecard/bases/config.yaml | 7 - config/scorecard/kustomization.yaml | 16 - config/scorecard/patches/basic.config.yaml | 10 - config/scorecard/patches/olm.config.yaml | 50 --- docs/installation.md | 65 ---- go.mod | 124 ++++--- go.sum | 323 +++++++++++------- hack/boilerplate.go.txt | 4 +- helm-charts | 1 - .../controller/memgraphcluster_controller.go | 65 ++++ .../memgraphcluster_controller_test.go | 87 +++++ internal/controller/memgraphha_constants.go | 23 -- internal/controller/memgraphha_controller.go | 152 --------- internal/controller/memgraphha_coord.go | 236 ------------- .../controller/memgraphha_coord_services.go | 156 --------- .../controller/memgraphha_data_instance.go | 229 ------------- .../controller/memgraphha_data_services.go | 155 --------- internal/controller/memgraphha_reconciler.go | 28 -- internal/controller/memgraphha_setup_job.go | 115 ------- internal/controller/suite_test.go | 118 +++++++ specs/operator-mvp/PRD.md | 129 +++++++ .../issues/01-repo-reset-scaffold.md | 25 ++ .../02-provisioning-walking-skeleton.md | 26 ++ .../issues/03-bootstrap-registration.md | 26 ++ .../issues/04-kind-e2e-harness.md | 25 ++ .../issues/05-continuous-re-registration.md | 27 ++ .../issues/06-status-and-conditions.md | 25 ++ .../issues/07-cel-immutability-validation.md | 23 ++ .../issues/08-storage-configuration.md | 24 ++ .../issues/09-pod-tuning-knobs.md | 26 ++ .../issues/10-operator-install-chart.md | 24 ++ .../issues/11-release-cross-publish.md | 24 ++ .../operator-mvp/issues/12-quickstart-docs.md | 24 ++ test/e2e/e2e_suite_test.go | 93 ++++- test/e2e/e2e_test.go | 322 ++++++++++++++--- test/utils/utils.go | 160 +++++++-- 93 files changed, 3532 insertions(+), 2784 deletions(-) create mode 100644 .custom-gcl.yml create mode 100644 .devcontainer/devcontainer.json create mode 100644 .devcontainer/post-install.sh delete mode 100644 .github/CODEOWNERS delete mode 100644 .github/config_files/config_lint.yaml delete mode 100644 .github/workflows/kubelint.yaml create mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/test.yml delete mode 100644 .gitmodules delete mode 100644 .pre-commit-config.yaml create mode 100644 AGENTS.md delete mode 100644 api/v1/memgraphha_types.go delete mode 100644 api/v1/zz_generated.deepcopy.go rename api/{v1 => v1alpha1}/groupversion_info.go (50%) create mode 100644 api/v1alpha1/memgraphcluster_types.go create mode 100644 api/v1alpha1/zz_generated.deepcopy.go create mode 100644 config/crd/bases/memgraph.com_memgraphclusters.yaml delete mode 100644 config/crd/bases/memgraph.com_memgraphhas.yaml create mode 100644 config/default/cert_metrics_manager_patch.yaml create mode 100644 config/default/manager_metrics_patch.yaml create mode 100644 config/default/metrics_service.yaml delete mode 100644 config/manager/namespace.yaml delete mode 100644 config/manifests/kustomization.yaml create mode 100644 config/network-policy/allow-metrics-traffic.yaml create mode 100644 config/network-policy/kustomization.yaml create mode 100644 config/prometheus/kustomization.yaml create mode 100644 config/prometheus/monitor.yaml create mode 100644 config/prometheus/monitor_tls_patch.yaml create mode 100644 config/rbac/leader_election_role.yaml create mode 100644 config/rbac/leader_election_role_binding.yaml create mode 100644 config/rbac/memgraphcluster_admin_role.yaml create mode 100644 config/rbac/memgraphcluster_editor_role.yaml create mode 100644 config/rbac/memgraphcluster_viewer_role.yaml create mode 100644 config/rbac/metrics_auth_role.yaml create mode 100644 config/rbac/metrics_auth_role_binding.yaml create mode 100644 config/rbac/metrics_reader_role.yaml delete mode 100644 config/samples/memgraph_v1_ha.yaml create mode 100644 config/samples/v1alpha1_memgraphcluster.yaml delete mode 100644 config/scorecard/bases/config.yaml delete mode 100644 config/scorecard/kustomization.yaml delete mode 100644 config/scorecard/patches/basic.config.yaml delete mode 100644 config/scorecard/patches/olm.config.yaml delete mode 100644 docs/installation.md delete mode 160000 helm-charts create mode 100644 internal/controller/memgraphcluster_controller.go create mode 100644 internal/controller/memgraphcluster_controller_test.go delete mode 100644 internal/controller/memgraphha_constants.go delete mode 100644 internal/controller/memgraphha_controller.go delete mode 100644 internal/controller/memgraphha_coord.go delete mode 100644 internal/controller/memgraphha_coord_services.go delete mode 100644 internal/controller/memgraphha_data_instance.go delete mode 100644 internal/controller/memgraphha_data_services.go delete mode 100644 internal/controller/memgraphha_reconciler.go delete mode 100644 internal/controller/memgraphha_setup_job.go create mode 100644 internal/controller/suite_test.go create mode 100644 specs/operator-mvp/PRD.md create mode 100644 specs/operator-mvp/issues/01-repo-reset-scaffold.md create mode 100644 specs/operator-mvp/issues/02-provisioning-walking-skeleton.md create mode 100644 specs/operator-mvp/issues/03-bootstrap-registration.md create mode 100644 specs/operator-mvp/issues/04-kind-e2e-harness.md create mode 100644 specs/operator-mvp/issues/05-continuous-re-registration.md create mode 100644 specs/operator-mvp/issues/06-status-and-conditions.md create mode 100644 specs/operator-mvp/issues/07-cel-immutability-validation.md create mode 100644 specs/operator-mvp/issues/08-storage-configuration.md create mode 100644 specs/operator-mvp/issues/09-pod-tuning-knobs.md create mode 100644 specs/operator-mvp/issues/10-operator-install-chart.md create mode 100644 specs/operator-mvp/issues/11-release-cross-publish.md create mode 100644 specs/operator-mvp/issues/12-quickstart-docs.md diff --git a/.custom-gcl.yml b/.custom-gcl.yml new file mode 100644 index 0000000..d9ae33b --- /dev/null +++ b/.custom-gcl.yml @@ -0,0 +1,11 @@ +# This file configures golangci-lint with module plugins. +# When you run 'make lint', it will automatically build a custom golangci-lint binary +# with all the plugins listed below. +# +# See: https://golangci-lint.run/plugins/module-plugins/ +version: v2.12.2 +plugins: + # logcheck validates structured logging calls and parameters (e.g., balanced key-value pairs) + - module: "sigs.k8s.io/logtools" + import: "sigs.k8s.io/logtools/logcheck/gclplugin" + version: latest diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..a96838b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,35 @@ +{ + "name": "Kubebuilder DevContainer", + "image": "golang:1.26", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:2": { + "moby": false, + "dockerDefaultAddressPool": "base=172.30.0.0/16,size=24" + }, + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/common-utils:2": { + "upgradePackages": true + } + }, + + "runArgs": ["--privileged", "--init"], + + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "ms-kubernetes-tools.vscode-kubernetes-tools", + "ms-azuretools.vscode-docker" + ] + } + }, + + "remoteEnv": { + "GO111MODULE": "on" + }, + + "onCreateCommand": "bash .devcontainer/post-install.sh" +} + diff --git a/.devcontainer/post-install.sh b/.devcontainer/post-install.sh new file mode 100644 index 0000000..6d75a49 --- /dev/null +++ b/.devcontainer/post-install.sh @@ -0,0 +1,153 @@ +#!/bin/bash +set -euo pipefail + +echo "====================================" +echo "Kubebuilder DevContainer Setup" +echo "====================================" + +# Verify running as root (required for installing to /usr/local/bin and /etc) +if [ "$(id -u)" -ne 0 ]; then + echo "ERROR: This script must be run as root" + exit 1 +fi + +echo "" +echo "Detecting system architecture..." +# Detect architecture using uname +MACHINE=$(uname -m) +case "${MACHINE}" in + x86_64) + ARCH="amd64" + ;; + aarch64|arm64) + ARCH="arm64" + ;; + *) + echo "WARNING: Unsupported architecture ${MACHINE}, defaulting to amd64" + ARCH="amd64" + ;; +esac +echo "Architecture: ${ARCH}" + +echo "" +echo "------------------------------------" +echo "Setting up bash completion..." +echo "------------------------------------" + +BASH_COMPLETIONS_DIR="/usr/share/bash-completion/completions" + +# Enable bash-completion in root's .bashrc (devcontainer runs as root) +if ! grep -q "source /usr/share/bash-completion/bash_completion" ~/.bashrc 2>/dev/null; then + echo 'source /usr/share/bash-completion/bash_completion' >> ~/.bashrc + echo "Added bash-completion to .bashrc" +fi + +echo "" +echo "------------------------------------" +echo "Installing development tools..." +echo "------------------------------------" + +# Install kind +if ! command -v kind &> /dev/null; then + echo "Installing kind..." + curl -Lo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/latest/kind-linux-${ARCH}" + chmod +x /usr/local/bin/kind + echo "kind installed successfully" +fi + +# Generate kind bash completion +if command -v kind &> /dev/null; then + if kind completion bash > "${BASH_COMPLETIONS_DIR}/kind" 2>/dev/null; then + echo "kind completion installed" + else + echo "WARNING: Failed to generate kind completion" + fi +fi + +# Install kubebuilder +if ! command -v kubebuilder &> /dev/null; then + echo "Installing kubebuilder..." + curl -Lo /usr/local/bin/kubebuilder "https://go.kubebuilder.io/dl/latest/linux/${ARCH}" + chmod +x /usr/local/bin/kubebuilder + echo "kubebuilder installed successfully" +fi + +# Generate kubebuilder bash completion +if command -v kubebuilder &> /dev/null; then + if kubebuilder completion bash > "${BASH_COMPLETIONS_DIR}/kubebuilder" 2>/dev/null; then + echo "kubebuilder completion installed" + else + echo "WARNING: Failed to generate kubebuilder completion" + fi +fi + +# Install kubectl +if ! command -v kubectl &> /dev/null; then + echo "Installing kubectl..." + KUBECTL_VERSION=$(curl -Ls https://dl.k8s.io/release/stable.txt) + curl -Lo /usr/local/bin/kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${ARCH}/kubectl" + chmod +x /usr/local/bin/kubectl + echo "kubectl installed successfully" +fi + +# Generate kubectl bash completion +if command -v kubectl &> /dev/null; then + if kubectl completion bash > "${BASH_COMPLETIONS_DIR}/kubectl" 2>/dev/null; then + echo "kubectl completion installed" + else + echo "WARNING: Failed to generate kubectl completion" + fi +fi + +# Generate Docker bash completion +if command -v docker &> /dev/null; then + if docker completion bash > "${BASH_COMPLETIONS_DIR}/docker" 2>/dev/null; then + echo "docker completion installed" + else + echo "WARNING: Failed to generate docker completion" + fi +fi + +echo "" +echo "------------------------------------" +echo "Configuring Docker environment..." +echo "------------------------------------" + +# Wait for Docker to be ready +echo "Waiting for Docker to be ready..." +for i in {1..30}; do + if docker info >/dev/null 2>&1; then + echo "Docker is ready" + break + fi + if [ "$i" -eq 30 ]; then + echo "WARNING: Docker not ready after 30s" + fi + sleep 1 +done + +# Create kind network (ignore if already exists) +if ! docker network inspect kind >/dev/null 2>&1; then + if docker network create kind >/dev/null 2>&1; then + echo "Created kind network" + else + echo "WARNING: Failed to create kind network (may already exist)" + fi +fi + +echo "" +echo "------------------------------------" +echo "Verifying installations..." +echo "------------------------------------" +kind version +kubebuilder version +kubectl version --client +docker --version +go version + +echo "" +echo "====================================" +echo "DevContainer ready!" +echo "====================================" +echo "All development tools installed successfully." +echo "You can now start building Kubernetes operators." diff --git a/.dockerignore b/.dockerignore index a3aab7a..9af8280 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,11 @@ # More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file -# Ignore build and test binaries. -bin/ +# Ignore everything by default and re-include only needed files +** + +# Re-include Go source files (but not *_test.go) +!**/*.go +**/*_test.go + +# Re-include Go module files +!go.mod +!go.sum diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index adcfaf6..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1 +0,0 @@ -as51340 diff --git a/.github/config_files/config_lint.yaml b/.github/config_files/config_lint.yaml deleted file mode 100644 index ac873af..0000000 --- a/.github/config_files/config_lint.yaml +++ /dev/null @@ -1,11 +0,0 @@ -checks: - addAllBuiltIn: true - exclude: - - "non-existent-service-account" # because the service account is created in another file - - "minimum-three-replicas" # because the deployment contains only 1 replica of the operator - - "no-liveness-probe" # not necessary - - "no-readiness-probe" # no necessary - - "use-namespace" - - "dnsconfig-options" - - "no-node-affinity" - - "non-isolated-pod" diff --git a/.github/workflows/kubelint.yaml b/.github/workflows/kubelint.yaml deleted file mode 100644 index cc12160..0000000 --- a/.github/workflows/kubelint.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: Kubelinter-check - -on: - push: - branches: - - main - paths-ignore: - - docs/** - pull_request: - branches: - - main - workflow_dispatch: {} - -env: - KUBELINTER_VERSION: "143183121" - -jobs: - Kubelinter-check: - name: Run Kube-linter check - runs-on: ubuntu-latest - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - - name: Scan directory ./config/crd/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/crd - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION - - - name: Scan directory ./config/default/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/default - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION - - - name: Scan directory ./config/manager/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/manager - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION - - - name: Scan directory ./config/manifests/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/manifests - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION - - - name: Scan directory ./config/samples/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/samples - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION - - - name: Scan directory ./config/scorecard/ with kube-linter - uses: stackrox/kube-linter-action@v1.0.3 - with: - directory: config/scorecard - config: ${GITHUB_WORKSPACE}/.github/config_files/config_lint.yaml - version: $KUBELINTER_VERSION diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..06b19df --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,31 @@ +name: Lint + +on: + push: + branches: + - main + pull_request: + +permissions: {} + +jobs: + lint: + permissions: + contents: read + name: Run on Ubuntu + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Check linter configuration + run: make lint-config + - name: Run linter + run: make lint diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml deleted file mode 100644 index 3e45bf4..0000000 --- a/.github/workflows/pre-commit.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: Pre-commit - -on: pull_request - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 2 # fetches all history so pre-commit can run properly - - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.9' # Use Python 3.9 - - - name: Install pre-commit - run: pip install pre-commit - - - name: Run pre-commit - run: pre-commit run --all-files --show-diff-on-failure - - - name: Cache the pre-commit environment - uses: actions/cache@v3 - with: - path: ~/.cache/pre-commit - key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..04f5442 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,48 @@ +name: Tests + +on: + push: + branches: + - main + pull_request: + +permissions: {} + +jobs: + unit: + permissions: + contents: read + name: Unit tests + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Run unit tests + run: make test-unit + + envtest: + permissions: + contents: read + name: Envtest + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Run tests against envtest + run: make test diff --git a/.gitignore b/.gitignore index 6a96658..9f0f3a1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,30 @@ - # Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib -bin +bin/* +Dockerfile.cross + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Go workspace file +go.work + +# Kubernetes Generated files - skip generated files, except for vendored files +!vendor/**/zz_generated.* # editor and IDE paraphernalia .idea +.vscode *.swp *.swo *~ -./manager +# Kubeconfig might contain secrets +*.kubeconfig diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 0163692..0000000 --- a/.gitmodules +++ /dev/null @@ -1,4 +0,0 @@ -[submodule "helm-charts"] - path = helm-charts - url = https://github.com/memgraph/helm-charts.git - branch = main diff --git a/.golangci.yml b/.golangci.yml index aed8644..b139f79 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,40 +1,69 @@ +version: "2" run: - deadline: 5m allow-parallel-runners: true - -issues: - # don't skip warning about doc comments - # don't exclude the default set of lint - exclude-use-default: false - # restore some of the defaults - # (fill in the rest as needed) - exclude-rules: - - path: "api/*" - linters: - - lll - - path: "internal/*" - linters: - - dupl - - lll linters: - disable-all: true + default: none enable: + - copyloopvar + - depguard - dupl - errcheck - - exportloopref + - ginkgolinter - goconst - gocyclo - - gofmt - - goimports - - gosimple - govet - ineffassign - lll + - modernize - misspell - nakedret - prealloc + - revive - staticcheck - - typecheck - unconvert - unparam - unused + - logcheck + settings: + custom: + logcheck: + type: "module" + description: Checks Go logging calls for Kubernetes logging conventions. + depguard: + rules: + forbid-sort-pkg: + deny: + - pkg: sort + desc: Should be replaced with slices package + revive: + rules: + - name: comment-spacings + - name: import-shadowing + modernize: + disable: + - omitzero + - newexpr + exclusions: + generated: lax + rules: + - linters: + - lll + path: api/* + - linters: + - dupl + - lll + path: internal/* + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 2ca863e..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - exclude: ^charts/memgraph/templates/ - - id: check-json - - id: mixed-line-ending - - id: check-merge-conflict - - id: detect-private-key - -- repo: https://github.com/tekwizely/pre-commit-golang - rev: v1.0.0-rc.1 - hooks: - - id: go-mod-tidy - - id: go-fmt diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b1080a0 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,320 @@ +# kubernetes-operator - AI Agent Guide + +## Project Structure + +**Single-group layout (default):** +``` +cmd/main.go Manager entry (registers controllers/webhooks) +api//*_types.go CRD schemas (+kubebuilder markers) +api//zz_generated.* Auto-generated (DO NOT EDIT) +internal/controller/* Reconciliation logic +internal/webhook/* Validation/defaulting (if present) +config/crd/bases/* Generated CRDs (DO NOT EDIT) +config/rbac/role.yaml Generated RBAC (DO NOT EDIT) +config/samples/* Example CRs (edit these) +Makefile Build/test/deploy commands +PROJECT Kubebuilder metadata Auto-generated (DO NOT EDIT) +``` + +**Multi-group layout** (for projects with multiple API groups): +``` +api///*_types.go CRD schemas by group +internal/controller//* Controllers by group +internal/webhook///* Webhooks by group and version (if present) +``` + +Multi-group layout organizes APIs by group name (e.g., `batch`, `apps`). Check the `PROJECT` file for `multigroup: true`. + +**To convert to multi-group layout:** +1. Run: `kubebuilder edit --multigroup=true` +2. Move APIs: `mkdir -p api/ && mv api/ api//` +3. Move controllers: `mkdir -p internal/controller/ && mv internal/controller/*.go internal/controller//` +4. Move webhooks (if present): `mkdir -p internal/webhook/ && mv internal/webhook/ internal/webhook//` +5. Update import paths in all files +6. Fix `path` in `PROJECT` file for each resource +7. Update test suite CRD paths (add one more `..` to relative paths) + +## Critical Rules + +### Never Edit These (Auto-Generated) +- `config/crd/bases/*.yaml` - from `make manifests` +- `config/rbac/role.yaml` - from `make manifests` +- `config/webhook/manifests.yaml` - from `make manifests` +- `**/zz_generated.*.go` - from `make generate` +- `PROJECT` - from `kubebuilder [OPTIONS]` + +### Never Remove Scaffold Markers +Do NOT delete `// +kubebuilder:scaffold:*` comments. CLI injects code at these markers. + +### Keep Project Structure +Do not move files around. The CLI expects files in specific locations. + +### Always Use CLI Commands +Always use `kubebuilder create api` and `kubebuilder create webhook` to scaffold. Do NOT create files manually. + +### E2E Tests Require an Isolated Kind Cluster +The e2e tests are designed to validate the solution in an isolated environment (similar to GitHub Actions CI). +Ensure you run them against a dedicated [Kind](https://kind.sigs.k8s.io/) cluster (not your “real” dev/prod cluster). + +## After Making Changes + +**After editing `*_types.go` or markers:** +``` +make manifests # Regenerate CRDs/RBAC from markers +make generate # Regenerate DeepCopy methods +``` + +**After editing `*.go` files:** +``` +make lint-fix # Auto-fix code style +make test # Run unit tests +``` + +## CLI Commands Cheat Sheet + +### Create API (your own types) +```bash +kubebuilder create api --group --version --kind +``` + +### Deploy Image Plugin (scaffold to deploy/manage ANY container image) + +Generate a controller that deploys and manages a container image (nginx, redis, memcached, your app, etc.): + +```bash +# Example: deploying memcached +kubebuilder create api --group example.com --version v1alpha1 --kind Memcached \ + --image=memcached:alpine \ + --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Scaffolds good-practice code: reconciliation logic, status conditions, finalizers, RBAC. Use as a reference implementation. + + +### Create Webhooks +```bash +# Validation + defaulting +kubebuilder create webhook --group --version --kind \ + --defaulting --programmatic-validation + +# Conversion webhook (for multi-version APIs) +kubebuilder create webhook --group --version v1 --kind \ + --conversion --spoke v2 +``` + +### Controller for Core Kubernetes Types +```bash +# Watch Pods +kubebuilder create api --group core --version v1 --kind Pod \ + --controller=true --resource=false + +# Watch Deployments +kubebuilder create api --group apps --version v1 --kind Deployment \ + --controller=true --resource=false +``` + +### Controller for External Types (e.g., from other operators) + +Watch resources from external APIs (cert-manager, Argo CD, Istio, etc.): + +```bash +# Example: watching cert-manager Certificate resources +kubebuilder create api \ + --group cert-manager --version v1 --kind Certificate \ + --controller=true --resource=false \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +**Note:** Use `--external-api-module=@` only if you need a specific version. Otherwise, omit `@` to use what's in go.mod. + +### Webhook for External Types + +```bash +# Example: validating external resources +kubebuilder create webhook \ + --group cert-manager --version v1 --kind Issuer \ + --defaulting \ + --external-api-path=github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1 \ + --external-api-domain=io \ + --external-api-module=github.com/cert-manager/cert-manager +``` + +## Testing & Development + +```bash +make test # Run unit tests (uses envtest: real K8s API + etcd) +make run # Run locally (uses current kubeconfig context) +``` + +Tests use **Ginkgo + Gomega** (BDD style). Check `suite_test.go` for setup. + +## Deployment Workflow + +```bash +# 1. Regenerate manifests +make manifests generate + +# 2. Build & deploy +export IMG=/:tag +make docker-build docker-push IMG=$IMG # Or: kind load docker-image $IMG --name +make deploy IMG=$IMG + +# 3. Test +kubectl apply -k config/samples/ + +# 4. Debug +kubectl logs -n -system deployment/-controller-manager -c manager -f +``` + +### API Design + +**Key markers for** `api//*_types.go`: + +```go +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Namespaced +// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=".status.conditions[?(@.type=='Ready')].status" + +// On fields: +// +kubebuilder:validation:Required +// +kubebuilder:validation:Minimum=1 +// +kubebuilder:validation:MaxLength=100 +// +kubebuilder:validation:Pattern="^[a-z]+$" +// +kubebuilder:default="value" +``` + +- **Use** `metav1.Condition` for status (not custom string fields) +- **Use predefined types**: `metav1.Time` instead of `string` for dates +- **Follow K8s API conventions**: Standard field names (`spec`, `status`, `metadata`) + +### Controller Design + +**RBAC markers in** `internal/controller/*_controller.go`: + +```go +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=mygroup.example.com,resources=mykinds/finalizers,verbs=update +// +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch +// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +``` + +**Implementation rules:** +- **Idempotent reconciliation**: Safe to run multiple times +- **Re-fetch before updates**: `r.Get(ctx, req.NamespacedName, obj)` before `r.Update` to avoid conflicts +- **Structured logging**: `log := log.FromContext(ctx); log.Info("msg", "key", val)` +- **Owner references**: Enable automatic garbage collection (`SetControllerReference`) +- **Watch secondary resources**: Use `.Owns()` or `.Watches()`, not just `RequeueAfter` +- **Finalizers**: Clean up external resources (buckets, VMs, DNS entries) + +### Logging + +**Follow Kubernetes logging message style guidelines:** + +- Start from a capital letter +- Do not end the message with a period +- Active voice: subject present (`"Deployment could not create Pod"`) or omitted (`"Could not create Pod"`) +- Past tense: `"Could not delete Pod"` not `"Cannot delete Pod"` +- Specify object type: `"Deleted Pod"` not `"Deleted"` +- Balanced key-value pairs + +```go +log.Info("Starting reconciliation") +log.Info("Created Deployment", "name", deploy.Name) +log.Error(err, "Failed to create Pod", "name", name) +``` + +**Reference:** https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines + +### Webhooks +- **Create all types together**: `--defaulting --programmatic-validation --conversion` +- **When`--force`is used**: Backup custom logic first, then restore after scaffolding +- **For multi-version APIs**: Use hub-and-spoke pattern (`--conversion --spoke v2`) + - Hub version: Usually oldest stable version (v1) + - Spoke versions: Newer versions that convert to/from hub (v2, v3) + - Example: `--group crew --version v1 --kind Captain --conversion --spoke v2` (v1 is hub, v2 is spoke) + +### Learning from Examples + +The **deploy-image plugin** scaffolds a complete controller following good practices. Use it as a reference implementation: + +```bash +kubebuilder create api --group example --version v1alpha1 --kind MyApp \ + --image= --plugins=deploy-image.go.kubebuilder.io/v1-alpha +``` + +Generated code includes: status conditions (`metav1.Condition`), finalizers, owner references, events, idempotent reconciliation. + +## Distribution Options + +### Option 1: YAML Bundle (Kustomize) + +```bash +# Generate dist/install.yaml from Kustomize manifests +make build-installer IMG=/:tag +``` + +**Key points:** +- The `dist/install.yaml` is generated from Kustomize manifests (CRDs, RBAC, Deployment) +- Commit this file to your repository for easy distribution +- Users only need `kubectl` to install (no additional tools required) + +**Example:** Users install with a single command: +```bash +kubectl apply -f https://raw.githubusercontent.com////dist/install.yaml +``` + +### Option 2: Helm Chart + +```bash +kubebuilder edit --plugins=helm/v2-alpha # Generates dist/chart/ (default) +kubebuilder edit --plugins=helm/v2-alpha --output-dir=charts # Generates charts/chart/ +``` + +**For development:** +```bash +make helm-deploy IMG=/: # Deploy manager via Helm +make helm-deploy IMG=$IMG HELM_EXTRA_ARGS="--set ..." # Deploy with custom values +make helm-status # Show release status +make helm-uninstall # Remove release +make helm-history # View release history +make helm-rollback # Rollback to previous version +``` + +**For end users/production:** +```bash +helm install my-release .//chart/ --namespace --create-namespace +``` + +**Important:** If you add webhooks or modify manifests after initial chart generation: +1. Backup any customizations in `/chart/values.yaml` and `/chart/manager/manager.yaml` +2. Re-run: `kubebuilder edit --plugins=helm/v2-alpha --force` (use same `--output-dir` if customized) +3. Manually restore your custom values from the backup + +### Publish Container Image + +```bash +export IMG=/: +make docker-build docker-push IMG=$IMG +``` + +## References + +### Essential Reading +- **Kubebuilder Book**: https://book.kubebuilder.io (comprehensive guide) +- **controller-runtime FAQ**: https://github.com/kubernetes-sigs/controller-runtime/blob/main/FAQ.md (common patterns and questions) +- **Good Practices**: https://book.kubebuilder.io/reference/good-practices.html (why reconciliation is idempotent, status conditions, etc.) +- **Logging Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md#message-style-guidelines (message style, verbosity levels) + +### API Design & Implementation +- **API Conventions**: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md +- **Operator Pattern**: https://kubernetes.io/docs/concepts/extend-kubernetes/operator/ +- **Markers Reference**: https://book.kubebuilder.io/reference/markers.html + +### Tools & Libraries +- **controller-runtime**: https://github.com/kubernetes-sigs/controller-runtime +- **controller-tools**: https://github.com/kubernetes-sigs/controller-tools +- **Kubebuilder Repo**: https://github.com/kubernetes-sigs/kubebuilder diff --git a/Dockerfile b/Dockerfile index a48973e..5b59f51 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.22 AS builder +FROM golang:1.26 AS builder ARG TARGETOS ARG TARGETARCH @@ -11,13 +11,11 @@ COPY go.sum go.sum # and so that source changes don't invalidate our downloaded layer RUN go mod download -# Copy the go source -COPY cmd/main.go cmd/main.go -COPY api/ api/ -COPY internal/controller/ internal/controller/ +# Copy the Go source (relies on .dockerignore to filter) +COPY . . # Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command +# the GOARCH has no default value to allow the binary to be built according to the host where the command # was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO # the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, # by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. diff --git a/Makefile b/Makefile index a4234e4..d61b31c 100644 --- a/Makefile +++ b/Makefile @@ -1,59 +1,7 @@ -# VERSION defines the project version for the bundle. -# Update this value when you upgrade the version of your project. -# To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 1.0.0 - -# CHANNELS define the bundle channels used in the bundle. -# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") -# To re-generate a bundle for other specific channels without changing the standard setup, you can: -# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) -# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=$(CHANNELS) -endif - -# DEFAULT_CHANNEL defines the default channel used in the bundle. -# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") -# To re-generate a bundle for any other default channel without changing the default setup, you can: -# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) -# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. -# This variable is used to construct full image tags for bundle and catalog images. -# -# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# com/kubernetes-operator-bundle:$VERSION and com/kubernetes-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= memgraph/kubernetes-operator - -# BUNDLE_IMG defines the image:tag used for the bundle. -# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) - -# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command -BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - -# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests -# You can enable this value if you would like to use SHA Based Digests -# To enable set flag to true -USE_IMAGE_DIGESTS ?= false -ifeq ($(USE_IMAGE_DIGESTS), true) - BUNDLE_GEN_FLAGS += --use-image-digests -endif - -# Set the Operator SDK version to use. By default, what is installed on the system is used. -# This is useful for CI or a project to utilize a specific version of the operator-sdk toolkit. -OPERATOR_SDK_VERSION ?= v1.35.0 - # Image URL to use all building/pushing image targets -IMG ?= $(IMAGE_TAG_BASE):$(VERSION) -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.28.3 +IMG ?= controller:latest +# YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header. +YEAR ?= $(shell date +%Y) # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -96,12 +44,12 @@ help: ## Display this help. ##@ Development .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration and CustomResourceDefinition objects. - $(CONTROLLER_GEN) crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + "$(CONTROLLER_GEN)" rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + "$(CONTROLLER_GEN)" object:headerFile="hack/boilerplate.go.txt",year=$(YEAR) paths="./..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -111,30 +59,56 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... +.PHONY: test-unit +test-unit: manifests generate fmt vet ## Run unit tests (pure packages, no envtest binaries required). + go test $$(go list ./... | grep -v /e2e | grep -v /internal/controller) -coverprofile cover-unit.out + .PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v - -GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint -GOLANGCI_LINT_VERSION ?= v1.54.2 -golangci-lint: - @[ -f $(GOLANGCI_LINT) ] || { \ - set -e ;\ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell dirname $(GOLANGCI_LINT)) $(GOLANGCI_LINT_VERSION) ;\ +test: manifests generate fmt vet setup-envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + +# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# kubectl kuberc is disabled by default for test isolation; enable with: +# - KUBECTL_KUBERC=true +# CertManager is installed by default; skip with: +# - CERT_MANAGER_INSTALL_SKIP=true +KIND_CLUSTER ?= kubernetes-operator-test-e2e + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) .PHONY: lint -lint: golangci-lint ## Run golangci-lint linter & yamllint - $(GOLANGCI_LINT) run +lint: golangci-lint ## Run golangci-lint linter + "$(GOLANGCI_LINT)" run .PHONY: lint-fix lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix + "$(GOLANGCI_LINT)" run --fix + +.PHONY: lint-config +lint-config: golangci-lint ## Verify golangci-lint linter configuration + "$(GOLANGCI_LINT)" config verify ##@ Build @@ -168,12 +142,18 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le docker-buildx: ## Build and push docker image for the manager for cross-platform support # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name project-v3-builder - $(CONTAINER_TOOL) buildx use project-v3-builder + - $(CONTAINER_TOOL) buildx create --name kubernetes-operator-builder + $(CONTAINER_TOOL) buildx use kubernetes-operator-builder - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm project-v3-builder + - $(CONTAINER_TOOL) buildx rm kubernetes-operator-builder rm Dockerfile.cross +.PHONY: build-installer +build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. + mkdir -p dist + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default > dist/install.yaml + ##@ Deployment ifndef ignore-not-found @@ -182,127 +162,102 @@ endif .PHONY: install install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" apply -f -; else echo "No CRDs to install; skipping."; fi .PHONY: uninstall uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - + @out="$$( "$(KUSTOMIZE)" build config/crd 2>/dev/null || true )"; \ + if [ -n "$$out" ]; then echo "$$out" | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f -; else echo "No CRDs to delete; skipping."; fi .PHONY: deploy deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - + cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" apply -f - .PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - +undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + "$(KUSTOMIZE)" build config/default | "$(KUBECTL)" delete --ignore-not-found=$(ignore-not-found) -f - -##@ Build Dependencies +##@ Dependencies ## Location to install dependencies to LOCALBIN ?= $(shell pwd)/bin $(LOCALBIN): - mkdir -p $(LOCALBIN) + mkdir -p "$(LOCALBIN)" ## Tool Binaries KUBECTL ?= kubectl +KIND ?= kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest +GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions -KUSTOMIZE_VERSION ?= v5.2.1 -CONTROLLER_TOOLS_VERSION ?= v0.15.0 +KUSTOMIZE_VERSION ?= v5.8.1 +CONTROLLER_TOOLS_VERSION ?= v0.21.0 + +#ENVTEST_VERSION is the controller-runtime version to use for setup-envtest, derived from go.mod +ENVTEST_VERSION ?= $(shell v='$(call gomodver,sigs.k8s.io/controller-runtime)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_VERSION manually (controller-runtime replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v") +#ENVTEST_K8S_VERSION is the version of Kubernetes to use for setting up ENVTEST binaries (i.e. 1.31) +ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ + [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ + printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') + +GOLANGCI_LINT_VERSION ?= v2.12.2 .PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) - @if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \ - echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \ - rm -rf $(LOCALBIN)/kustomize; \ - fi - test -s $(LOCALBIN)/kustomize || GOBIN=$(LOCALBIN) GO111MODULE=on go install sigs.k8s.io/kustomize/kustomize/v5@$(KUSTOMIZE_VERSION) + $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) .PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. $(CONTROLLER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) + +.PHONY: setup-envtest +setup-envtest: envtest ## Download the binaries required for ENVTEST in the local bin directory. + @echo "Setting up envtest binaries for Kubernetes version $(ENVTEST_K8S_VERSION)..." + @"$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path || { \ + echo "Error: Failed to set up envtest binaries for version $(ENVTEST_K8S_VERSION)."; \ + exit 1; \ + } .PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest - -.PHONY: operator-sdk -OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk -operator-sdk: ## Download operator-sdk locally if necessary. -ifeq (,$(wildcard $(OPERATOR_SDK))) -ifeq (, $(shell which operator-sdk 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPERATOR_SDK)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$${OS}_$${ARCH} ;\ - chmod +x $(OPERATOR_SDK) ;\ - } -else -OPERATOR_SDK = $(shell which operator-sdk) -endif -endif - -.PHONY: bundle -bundle: manifests kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. - $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) - $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) - $(OPERATOR_SDK) bundle validate ./bundle - -.PHONY: bundle-build -bundle-build: ## Build the bundle image. - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . - -.PHONY: bundle-push -bundle-push: ## Push the bundle image. - $(MAKE) docker-push IMG=$(BUNDLE_IMG) - -.PHONY: opm -OPM = $(LOCALBIN)/opm -opm: ## Download opm locally if necessary. -ifeq (,$(wildcard $(OPM))) -ifeq (,$(shell which opm 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPM)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ - chmod +x $(OPM) ;\ - } -else -OPM = $(shell which opm) -endif -endif - -# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). -# These images MUST exist in a registry and be pull-able. -BUNDLE_IMGS ?= $(BUNDLE_IMG) - -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) - -# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. -ifneq ($(origin CATALOG_BASE_IMG), undefined) -FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) -endif - -# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. -# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: -# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator -.PHONY: catalog-build -catalog-build: opm ## Build a catalog image. - $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) - -# Push the catalog image. -.PHONY: catalog-push -catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMG=$(CATALOG_IMG) + $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) + +.PHONY: golangci-lint +golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +$(GOLANGCI_LINT): $(LOCALBIN) + $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) + @test -f .custom-gcl.yml && { \ + echo "Building custom golangci-lint with plugins..." && \ + $(GOLANGCI_LINT) custom --destination $(LOCALBIN) --name golangci-lint-custom && \ + mv -f $(LOCALBIN)/golangci-lint-custom $(GOLANGCI_LINT); \ + } || true + +# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist +# $1 - target path with name of binary +# $2 - package url which can be installed +# $3 - specific version of package +define go-install-tool +@[ -f "$(1)-$(3)" ] && [ "$$(readlink -- "$(1)" 2>/dev/null)" = "$(1)-$(3)" ] || { \ +set -e; \ +package=$(2)@$(3) ;\ +echo "Downloading $${package}" ;\ +rm -f "$(1)" ;\ +GOBIN="$(LOCALBIN)" go install $${package} ;\ +mv "$(LOCALBIN)/$$(basename "$(1)")" "$(1)-$(3)" ;\ +} ;\ +ln -sf "$$(realpath "$(1)-$(3)")" "$(1)" +endef + +define gomodver +$(shell go list -m -f '{{if .Replace}}{{.Replace.Version}}{{else}}{{.Version}}{{end}}' $(1) 2>/dev/null) +endef diff --git a/PROJECT b/PROJECT index be3ed36..5f1df6f 100644 --- a/PROJECT +++ b/PROJECT @@ -2,12 +2,10 @@ # This file is used to track the info used to scaffold your project # and allow the plugins properly work. # More info: https://book.kubebuilder.io/reference/project-config.html -domain: com +cliVersion: 4.15.0 +domain: memgraph.com layout: - go.kubebuilder.io/v4 -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} projectName: kubernetes-operator repo: github.com/memgraph/kubernetes-operator resources: @@ -15,9 +13,8 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: com - group: memgraph - kind: MemgraphHA - path: github.com/memgraph/kubernetes-operator/api/v1 - version: v1 + domain: memgraph.com + kind: MemgraphCluster + path: github.com/memgraph/kubernetes-operator/api/v1alpha1 + version: v1alpha1 version: "3" diff --git a/README.md b/README.md index 4e63d94..660d29a 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,140 @@ # Memgraph Kubernetes Operator -## Introduction +A Kubernetes operator for running [Memgraph](https://memgraph.com) high-availability clusters. It exposes a `MemgraphCluster` custom resource (API group `memgraph.com/v1alpha1`, short name `mgc`): declare the cluster topology in a single resource and the operator provisions the workloads, bootstraps HA registration, and continuously reconciles registration state. -Memgraph Kubernetes Operator is WIP. You can currently install the operator and manage the deployment of Memgraph's High Availability cluster -through it. +> **Status: early development.** This repository was reset for a fresh operator effort; the previous attempt is preserved on the `archive/pre-operator-mvp` branch. The product requirements and issue slices driving the current work live in [`specs/operator-mvp/`](specs/operator-mvp/PRD.md). -## Table of Contents +## Description -- [Prerequisites](#prerequisites) -- [Documentation](#documentation) -- [License](#license) +The operator replaces the `memgraph-high-availability` Helm chart's fire-and-forget registration Job with a controller that continuously drives the cluster toward its declared topology: one StatefulSet per role (coordinators, data instances), automatic bootstrap and MAIN promotion, and automatic re-registration of instances that lose their registration state. See the [PRD](specs/operator-mvp/PRD.md) for the full design. -## Prerequisites +## Getting Started -We use Go version 1.22.5 (not needed at the moment). Check out here how to [install Go](https://go.dev/doc/install). -The current Helm version used is v3.14.4. +### Prerequisites +- go version v1.24.6+ +- docker version 17.03+. +- kubectl version v1.11.3+. +- Access to a Kubernetes v1.11.3+ cluster. -## Documentation +### To Deploy on the cluster +**Build and push your image to the location specified by `IMG`:** -Check our [Documentation](/docs) to start using our Kubernetes operator. +```sh +make docker-build docker-push IMG=/kubernetes-operator:tag +``` -1. [Install the Memgraph Kubernetes Operator](docs/installation.md) +**NOTE:** This image ought to be published in the personal registry you specified. +And it is required to have access to pull the image from the working environment. +Make sure you have the proper permission to the registry if the above commands don’t work. + +**Install the CRDs into the cluster:** + +```sh +make install +``` + +**Deploy the Manager to the cluster with the image specified by `IMG`:** + +```sh +make deploy IMG=/kubernetes-operator:tag +``` + +> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin +privileges or be logged in as admin. + +**Create instances of your solution** +You can apply the samples (examples) from the config/sample: + +```sh +kubectl apply -k config/samples/ +``` + +>**NOTE**: Ensure that the samples has default values to test it out. + +### To Uninstall +**Delete the instances (CRs) from the cluster:** + +```sh +kubectl delete -k config/samples/ +``` + +**Delete the APIs(CRDs) from the cluster:** + +```sh +make uninstall +``` + +**UnDeploy the controller from the cluster:** + +```sh +make undeploy +``` + +## Project Distribution + +Following the options to release and provide this solution to the users. + +### By providing a bundle with all YAML files + +1. Build the installer for the image built and published in the registry: + +```sh +make build-installer IMG=/kubernetes-operator:tag +``` + +**NOTE:** The makefile target mentioned above generates an 'install.yaml' +file in the dist directory. This file contains all the resources built +with Kustomize, which are necessary to install this project without its +dependencies. + +2. Using the installer + +Users can just run 'kubectl apply -f ' to install +the project, i.e.: + +```sh +kubectl apply -f https://raw.githubusercontent.com//kubernetes-operator//dist/install.yaml +``` + +### By providing a Helm Chart + +1. Build the chart using the optional helm plugin + +```sh +kubebuilder edit --plugins=helm/v2-alpha +``` + +2. See that a chart was generated under 'dist/chart', and users +can obtain this solution from there. + +**NOTE:** If you change the project, you need to update the Helm Chart +using the same command above to sync the latest changes. Furthermore, +if you create webhooks, you need to use the above command with +the '--force' flag and manually ensure that any custom configuration +previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml' +is manually re-applied afterwards. + +## Contributing + +Development is sliced into PR-gated issues under [`specs/operator-mvp/issues/`](specs/operator-mvp/issues). Every pull request runs lint, unit, and envtest suites. + +**NOTE:** Run `make help` for more information on all potential `make` targets + +More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) ## License -Please check the [LICENSE](LICENSE) file +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/api/v1/memgraphha_types.go b/api/v1/memgraphha_types.go deleted file mode 100644 index ade40e6..0000000 --- a/api/v1/memgraphha_types.go +++ /dev/null @@ -1,135 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -In some way this file simulates types.go from k8s.io/api/apps/v1 to define new resources -we are using. -*/ - -package v1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// MemgraphHASpec defines the desired state of MemgraphHA -type MemgraphHASpec struct { - Coordinators []Coordinator `json:"coordinators"` - Data []DataItem `json:"data"` - Memgraph MemgraphConfig `json:"memgraph"` -} - -type Coordinator struct { - ID string `json:"id"` - BoltPort int `json:"boltPort"` - ManagementPort int `json:"managementPort"` - CoordinatorPort int `json:"coordinatorPort"` - Args []string `json:"args"` -} - -type DataItem struct { - ID string `json:"id"` - BoltPort int `json:"boltPort"` - ManagementPort int `json:"managementPort"` - ReplicationPort int `json:"replicationPort"` - Args []string `json:"args"` -} - -type MemgraphConfig struct { - Data MemgraphDataConfig `json:"data"` - Coordinators MemgraphCoordinatorsConfig `json:"coordinators"` - Env map[string]string `json:"env"` - Image ImageConfig `json:"image"` - Probes MemgraphProbesConfig `json:"probes"` -} - -type MemgraphDataConfig struct { - VolumeClaim VolumeClaimConfig `json:"volumeClaim"` -} - -type MemgraphCoordinatorsConfig struct { - VolumeClaim VolumeClaimConfig `json:"volumeClaim"` -} - -type VolumeClaimConfig struct { - StoragePVCClassName string `json:"storagePVCClassName"` - StoragePVC bool `json:"storagePVC"` - StoragePVCSize string `json:"storagePVCSize"` - LogPVCClassName string `json:"logPVCClassName"` - LogPVC bool `json:"logPVC"` - LogPVCSize string `json:"logPVCSize"` -} - -type ImageConfig struct { - PullPolicy string `json:"pullPolicy"` - Repository string `json:"repository"` - Tag string `json:"tag"` -} - -type MemgraphProbesConfig struct { - Liveness ProbeConfig `json:"liveness"` - Readiness ProbeConfig `json:"readiness"` - Startup ProbeConfig `json:"startup"` -} - -// ProbeConfig configures individual probes -type ProbeConfig struct { - InitialDelaySeconds int `json:"initialDelaySeconds"` - PeriodSeconds int `json:"periodSeconds"` - FailureThreshold int `json:"failureThreshold,omitempty"` -} - -// MemgraphHAStatus defines the observed state of MemgraphHA -type MemgraphHAStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file -} - -//+kubebuilder:object:root=true -//+kubebuilder:subresource:status - -// MemgraphHA is the Schema for the memgraphhas API -/* -Every Kind needs to have two structures: metav1.TypeMeta and metav1.ObjectMeta. -TypeMeta structure contains information about the GVK of the Kind. -ObjectMeta contains metadata for the Kind. -*/ -type MemgraphHA struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec MemgraphHASpec `json:"spec,omitempty"` - Status MemgraphHAStatus `json:"status,omitempty"` -} - -//+kubebuilder:object:root=true - -// MemgraphHAList contains a list of MemgraphHA -type MemgraphHAList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []MemgraphHA `json:"items"` -} - -func init() { - // A Scheme is an abstraction used to register the API objects - // as Group-Version-Kinds, convert between API Objects of various - // versions and serialize/deserialize API Objects - SchemeBuilder.Register(&MemgraphHA{}, &MemgraphHAList{}) -} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go deleted file mode 100644 index 160c81e..0000000 --- a/api/v1/zz_generated.deepcopy.go +++ /dev/null @@ -1,298 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -This file is generated by the deepcopy-gen generator. It contains the generated definition -of the DeepCopyObject method for each type defined in the package. This method is necessary -for the structures to implement the runtime.Object interface, which is defined in the API -Machinery Library and the API Machinery expects that all Kind structures will implement -this runtime.Object interface. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Coordinator) DeepCopyInto(out *Coordinator) { - *out = *in - if in.Args != nil { - in, out := &in.Args, &out.Args - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Coordinator. -func (in *Coordinator) DeepCopy() *Coordinator { - if in == nil { - return nil - } - out := new(Coordinator) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DataItem) DeepCopyInto(out *DataItem) { - *out = *in - if in.Args != nil { - in, out := &in.Args, &out.Args - *out = make([]string, len(*in)) - copy(*out, *in) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataItem. -func (in *DataItem) DeepCopy() *DataItem { - if in == nil { - return nil - } - out := new(DataItem) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImageConfig) DeepCopyInto(out *ImageConfig) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageConfig. -func (in *ImageConfig) DeepCopy() *ImageConfig { - if in == nil { - return nil - } - out := new(ImageConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphConfig) DeepCopyInto(out *MemgraphConfig) { - *out = *in - out.Data = in.Data - out.Coordinators = in.Coordinators - if in.Env != nil { - in, out := &in.Env, &out.Env - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - out.Image = in.Image - out.Probes = in.Probes -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphConfig. -func (in *MemgraphConfig) DeepCopy() *MemgraphConfig { - if in == nil { - return nil - } - out := new(MemgraphConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphCoordinatorsConfig) DeepCopyInto(out *MemgraphCoordinatorsConfig) { - *out = *in - out.VolumeClaim = in.VolumeClaim -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphCoordinatorsConfig. -func (in *MemgraphCoordinatorsConfig) DeepCopy() *MemgraphCoordinatorsConfig { - if in == nil { - return nil - } - out := new(MemgraphCoordinatorsConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphDataConfig) DeepCopyInto(out *MemgraphDataConfig) { - *out = *in - out.VolumeClaim = in.VolumeClaim -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphDataConfig. -func (in *MemgraphDataConfig) DeepCopy() *MemgraphDataConfig { - if in == nil { - return nil - } - out := new(MemgraphDataConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphHA) DeepCopyInto(out *MemgraphHA) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphHA. -func (in *MemgraphHA) DeepCopy() *MemgraphHA { - if in == nil { - return nil - } - out := new(MemgraphHA) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MemgraphHA) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphHAList) DeepCopyInto(out *MemgraphHAList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]MemgraphHA, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphHAList. -func (in *MemgraphHAList) DeepCopy() *MemgraphHAList { - if in == nil { - return nil - } - out := new(MemgraphHAList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *MemgraphHAList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphHASpec) DeepCopyInto(out *MemgraphHASpec) { - *out = *in - if in.Coordinators != nil { - in, out := &in.Coordinators, &out.Coordinators - *out = make([]Coordinator, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - if in.Data != nil { - in, out := &in.Data, &out.Data - *out = make([]DataItem, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - in.Memgraph.DeepCopyInto(&out.Memgraph) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphHASpec. -func (in *MemgraphHASpec) DeepCopy() *MemgraphHASpec { - if in == nil { - return nil - } - out := new(MemgraphHASpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphHAStatus) DeepCopyInto(out *MemgraphHAStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphHAStatus. -func (in *MemgraphHAStatus) DeepCopy() *MemgraphHAStatus { - if in == nil { - return nil - } - out := new(MemgraphHAStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MemgraphProbesConfig) DeepCopyInto(out *MemgraphProbesConfig) { - *out = *in - out.Liveness = in.Liveness - out.Readiness = in.Readiness - out.Startup = in.Startup -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphProbesConfig. -func (in *MemgraphProbesConfig) DeepCopy() *MemgraphProbesConfig { - if in == nil { - return nil - } - out := new(MemgraphProbesConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ProbeConfig) DeepCopyInto(out *ProbeConfig) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeConfig. -func (in *ProbeConfig) DeepCopy() *ProbeConfig { - if in == nil { - return nil - } - out := new(ProbeConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VolumeClaimConfig) DeepCopyInto(out *VolumeClaimConfig) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimConfig. -func (in *VolumeClaimConfig) DeepCopy() *VolumeClaimConfig { - if in == nil { - return nil - } - out := new(VolumeClaimConfig) - in.DeepCopyInto(out) - return out -} diff --git a/api/v1/groupversion_info.go b/api/v1alpha1/groupversion_info.go similarity index 50% rename from api/v1/groupversion_info.go rename to api/v1alpha1/groupversion_info.go index 7ac142d..f41a982 100644 --- a/api/v1/groupversion_info.go +++ b/api/v1alpha1/groupversion_info.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Memgraph Ltd. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,26 +14,31 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package v1 contains API Schema definitions for the memgraph v1 API group +// Package v1alpha1 contains API Schema definitions for the v1alpha1 API group. // +kubebuilder:object:generate=true // +groupName=memgraph.com -package v1 +package v1alpha1 import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "memgraph.com", Version: "v1"} + // SchemeGroupVersion is group version used to register these objects. + // This name is used by applyconfiguration generators (e.g. controller-gen). + SchemeGroupVersion = schema.GroupVersion{Group: "memgraph.com", Version: "v1alpha1"} - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + // GroupVersion is an alias for SchemeGroupVersion, for backward compatibility. + GroupVersion = SchemeGroupVersion - /* AddToScheme adds the types in this group-version to the given scheme. - Scheme is an abstraction used in the API Machinery to create a mapping between Go - structures and Group-Version-Kinds. - */ + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = runtime.NewSchemeBuilder(func(scheme *runtime.Scheme) error { + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil + }) + + // AddToScheme adds the types in this group-version to the given scheme. AddToScheme = SchemeBuilder.AddToScheme ) diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go new file mode 100644 index 0000000..2deacb7 --- /dev/null +++ b/api/v1alpha1/memgraphcluster_types.go @@ -0,0 +1,92 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// MemgraphClusterSpec defines the desired state of MemgraphCluster. +// +// Topology, image, storage, and pod-tuning fields land in subsequent +// slices of the operator MVP (see specs/operator-mvp/PRD.md). +type MemgraphClusterSpec struct { +} + +// MemgraphClusterStatus defines the observed state of MemgraphCluster. +type MemgraphClusterStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file + + // For Kubernetes API conventions, see: + // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties + + // conditions represent the current state of the MemgraphCluster resource. + // Each condition has a unique type and reflects the status of a specific aspect of the resource. + // + // Standard condition types include: + // - "Available": the resource is fully functional + // - "Progressing": the resource is being created or updated + // - "Degraded": the resource failed to reach or maintain its desired state + // + // The status of each condition is one of True, False, or Unknown. + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:shortName=mgc + +// MemgraphCluster is the Schema for the memgraphclusters API +type MemgraphCluster struct { + metav1.TypeMeta `json:",inline"` + + // metadata is a standard object metadata + // +optional + metav1.ObjectMeta `json:"metadata,omitzero"` + + // spec defines the desired state of MemgraphCluster + // +required + Spec MemgraphClusterSpec `json:"spec"` + + // status defines the observed state of MemgraphCluster + // +optional + Status MemgraphClusterStatus `json:"status,omitzero"` +} + +// +kubebuilder:object:root=true + +// MemgraphClusterList contains a list of MemgraphCluster +type MemgraphClusterList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitzero"` + Items []MemgraphCluster `json:"items"` +} + +func init() { + SchemeBuilder.Register(func(s *runtime.Scheme) error { + s.AddKnownTypes(SchemeGroupVersion, &MemgraphCluster{}, &MemgraphClusterList{}) + return nil + }) +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..6aea644 --- /dev/null +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,122 @@ +//go:build !ignore_autogenerated + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemgraphCluster) DeepCopyInto(out *MemgraphCluster) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphCluster. +func (in *MemgraphCluster) DeepCopy() *MemgraphCluster { + if in == nil { + return nil + } + out := new(MemgraphCluster) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MemgraphCluster) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemgraphClusterList) DeepCopyInto(out *MemgraphClusterList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]MemgraphCluster, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterList. +func (in *MemgraphClusterList) DeepCopy() *MemgraphClusterList { + if in == nil { + return nil + } + out := new(MemgraphClusterList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *MemgraphClusterList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterSpec. +func (in *MemgraphClusterSpec) DeepCopy() *MemgraphClusterSpec { + if in == nil { + return nil + } + out := new(MemgraphClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MemgraphClusterStatus) DeepCopyInto(out *MemgraphClusterStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterStatus. +func (in *MemgraphClusterStatus) DeepCopy() *MemgraphClusterStatus { + if in == nil { + return nil + } + out := new(MemgraphClusterStatus) + in.DeepCopyInto(out) + return out +} diff --git a/cmd/main.go b/cmd/main.go index d9afff3..035ca0c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Memgraph Ltd. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -31,12 +31,13 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" "sigs.k8s.io/controller-runtime/pkg/webhook" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" "github.com/memgraph/kubernetes-operator/internal/controller" - //+kubebuilder:scaffold:imports + // +kubebuilder:scaffold:imports ) var ( @@ -47,19 +48,35 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(memgraphv1.AddToScheme(scheme)) - //+kubebuilder:scaffold:scheme + utilruntime.Must(memgraphcomv1alpha1.AddToScheme(scheme)) + // +kubebuilder:scaffold:scheme } +// nolint:gocyclo func main() { var metricsAddr string + var metricsCertPath, metricsCertName, metricsCertKey string + var webhookCertPath, webhookCertName, webhookCertKey string + var enableLeaderElection bool var probeAddr string var secureMetrics bool var enableHTTP2 bool - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + var tlsOpts []func(*tls.Config) + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ + "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") - flag.BoolVar(&secureMetrics, "metrics-secure", false, - "If set the metrics endpoint is served securely") + flag.BoolVar(&enableLeaderElection, "leader-elect", false, + "Enable leader election for controller manager. "+ + "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") + flag.StringVar(&webhookCertPath, "webhook-cert-path", "", "The directory that contains the webhook certificate.") + flag.StringVar(&webhookCertName, "webhook-cert-name", "tls.crt", "The name of the webhook certificate file.") + flag.StringVar(&webhookCertKey, "webhook-cert-key", "tls.key", "The name of the webhook key file.") + flag.StringVar(&metricsCertPath, "metrics-cert-path", "", + "The directory that contains the metrics server certificate.") + flag.StringVar(&metricsCertName, "metrics-cert-name", "tls.crt", "The name of the metrics server certificate file.") + flag.StringVar(&metricsCertKey, "metrics-cert-key", "tls.key", "The name of the metrics server key file.") flag.BoolVar(&enableHTTP2, "enable-http2", false, "If set, HTTP/2 will be enabled for the metrics and webhook servers") opts := zap.Options{ @@ -72,60 +89,116 @@ func main() { // if the enable-http2 flag is false (the default), http/2 should be disabled // due to its vulnerabilities. More specifically, disabling http/2 will - // prevent from being vulnerable to the HTTP/2 Stream Cancelation and + // prevent from being vulnerable to the HTTP/2 Stream Cancellation and // Rapid Reset CVEs. For more information see: // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 // - https://github.com/advisories/GHSA-4374-p667-p6c8 disableHTTP2 := func(c *tls.Config) { - setupLog.Info("disabling http/2") + setupLog.Info("Disabling HTTP/2") c.NextProtos = []string{"http/1.1"} } - tlsOpts := []func(*tls.Config){} if !enableHTTP2 { tlsOpts = append(tlsOpts, disableHTTP2) } - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, - }) + // Initial webhook TLS options + webhookTLSOpts := tlsOpts + webhookServerOptions := webhook.Options{ + TLSOpts: webhookTLSOpts, + } + + if len(webhookCertPath) > 0 { + setupLog.Info("Initializing webhook certificate watcher using provided certificates", + "webhook-cert-path", webhookCertPath, "webhook-cert-name", webhookCertName, "webhook-cert-key", webhookCertKey) + + webhookServerOptions.CertDir = webhookCertPath + webhookServerOptions.CertName = webhookCertName + webhookServerOptions.KeyName = webhookCertKey + } + + webhookServer := webhook.NewServer(webhookServerOptions) + + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. + // More info: + // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/server + // - https://book.kubebuilder.io/reference/metrics.html + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + SecureServing: secureMetrics, + TLSOpts: tlsOpts, + } + + if secureMetrics { + // FilterProvider is used to protect the metrics endpoint with authn/authz. + // These configurations ensure that only authorized users and service accounts + // can access the metrics endpoint. The RBAC are configured in 'config/rbac/kustomization.yaml'. More info: + // https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.24.1/pkg/metrics/filters#WithAuthenticationAndAuthorization + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + } + + // If the certificate is not specified, controller-runtime will automatically + // generate self-signed certificates for the metrics server. While convenient for development and testing, + // this setup is not recommended for production. + // + // TODO(user): If you enable certManager, uncomment the following lines: + // - [METRICS-WITH-CERTS] at config/default/kustomization.yaml to generate and use certificates + // managed by cert-manager for the metrics server. + // - [PROMETHEUS-WITH-CERTS] at config/prometheus/kustomization.yaml for TLS certification. + if len(metricsCertPath) > 0 { + setupLog.Info("Initializing metrics certificate watcher using provided certificates", + "metrics-cert-path", metricsCertPath, "metrics-cert-name", metricsCertName, "metrics-cert-key", metricsCertKey) + + metricsServerOptions.CertDir = metricsCertPath + metricsServerOptions.CertName = metricsCertName + metricsServerOptions.KeyName = metricsCertKey + } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - SecureServing: secureMetrics, - TLSOpts: tlsOpts, - }, + Scheme: scheme, + Metrics: metricsServerOptions, WebhookServer: webhookServer, HealthProbeBindAddress: probeAddr, + LeaderElection: enableLeaderElection, + LeaderElectionID: "a5adec69.memgraph.com", + // LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily + // when the Manager ends. This requires the binary to immediately end when the + // Manager is stopped, otherwise, this setting is unsafe. Setting this significantly + // speeds up voluntary leader transitions as the new leader don't have to wait + // LeaseDuration time first. + // + // In the default scaffold provided, the program ends immediately after + // the manager stops, so would be fine to enable this option. However, + // if you are doing or is intended to do any operation such as perform cleanups + // after the manager stops then its usage might be unsafe. + // LeaderElectionReleaseOnCancel: true, }) if err != nil { - setupLog.Error(err, "unable to start manager") + setupLog.Error(err, "Failed to start manager") os.Exit(1) } - if err = (&controller.MemgraphHAReconciler{ + if err := (&controller.MemgraphClusterReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "MemgraphHA") + setupLog.Error(err, "Failed to create controller", "controller", "memgraphcluster") os.Exit(1) } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") + setupLog.Error(err, "Failed to set up health check") os.Exit(1) } if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") + setupLog.Error(err, "Failed to set up ready check") os.Exit(1) } - setupLog.Info("starting manager") + setupLog.Info("Starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { - setupLog.Error(err, "problem running manager") + setupLog.Error(err, "Failed to run manager") os.Exit(1) } } diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml new file mode 100644 index 0000000..eb56325 --- /dev/null +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -0,0 +1,123 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: memgraphclusters.memgraph.com +spec: + group: memgraph.com + names: + kind: MemgraphCluster + listKind: MemgraphClusterList + plural: memgraphclusters + shortNames: + - mgc + singular: memgraphcluster + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: MemgraphCluster is the Schema for the memgraphclusters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of MemgraphCluster + type: object + status: + description: status defines the observed state of MemgraphCluster + properties: + conditions: + description: |- + conditions represent the current state of the MemgraphCluster resource. + Each condition has a unique type and reflects the status of a specific aspect of the resource. + + Standard condition types include: + - "Available": the resource is fully functional + - "Progressing": the resource is being created or updated + - "Degraded": the resource failed to reach or maintain its desired state + + The status of each condition is one of True, False, or Unknown. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/crd/bases/memgraph.com_memgraphhas.yaml b/config/crd/bases/memgraph.com_memgraphhas.yaml deleted file mode 100644 index becc8f3..0000000 --- a/config/crd/bases/memgraph.com_memgraphhas.yaml +++ /dev/null @@ -1,232 +0,0 @@ ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.15.0 - name: memgraphhas.memgraph.com -spec: - group: memgraph.com - names: - kind: MemgraphHA - listKind: MemgraphHAList - plural: memgraphhas - singular: memgraphha - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: |- - MemgraphHA is the Schema for the memgraphhas API - - - Every Kind needs to have two structures: metav1.TypeMeta and metav1.ObjectMeta. - TypeMeta structure contains information about the GVK of the Kind. - ObjectMeta contains metadata for the Kind. - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: MemgraphHASpec defines the desired state of MemgraphHA - properties: - coordinators: - items: - properties: - args: - items: - type: string - type: array - boltPort: - type: integer - coordinatorPort: - type: integer - id: - type: string - managementPort: - type: integer - required: - - args - - boltPort - - coordinatorPort - - id - - managementPort - type: object - type: array - data: - items: - properties: - args: - items: - type: string - type: array - boltPort: - type: integer - id: - type: string - managementPort: - type: integer - replicationPort: - type: integer - required: - - args - - boltPort - - id - - managementPort - - replicationPort - type: object - type: array - memgraph: - properties: - coordinators: - properties: - volumeClaim: - properties: - logPVC: - type: boolean - logPVCClassName: - type: string - logPVCSize: - type: string - storagePVC: - type: boolean - storagePVCClassName: - type: string - storagePVCSize: - type: string - required: - - logPVC - - logPVCClassName - - logPVCSize - - storagePVC - - storagePVCClassName - - storagePVCSize - type: object - required: - - volumeClaim - type: object - data: - properties: - volumeClaim: - properties: - logPVC: - type: boolean - logPVCClassName: - type: string - logPVCSize: - type: string - storagePVC: - type: boolean - storagePVCClassName: - type: string - storagePVCSize: - type: string - required: - - logPVC - - logPVCClassName - - logPVCSize - - storagePVC - - storagePVCClassName - - storagePVCSize - type: object - required: - - volumeClaim - type: object - env: - additionalProperties: - type: string - type: object - image: - properties: - pullPolicy: - type: string - repository: - type: string - tag: - type: string - required: - - pullPolicy - - repository - - tag - type: object - probes: - properties: - liveness: - description: ProbeConfig configures individual probes - properties: - failureThreshold: - type: integer - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - required: - - initialDelaySeconds - - periodSeconds - type: object - readiness: - description: ProbeConfig configures individual probes - properties: - failureThreshold: - type: integer - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - required: - - initialDelaySeconds - - periodSeconds - type: object - startup: - description: ProbeConfig configures individual probes - properties: - failureThreshold: - type: integer - initialDelaySeconds: - type: integer - periodSeconds: - type: integer - required: - - initialDelaySeconds - - periodSeconds - type: object - required: - - liveness - - readiness - - startup - type: object - required: - - coordinators - - data - - env - - image - - probes - type: object - required: - - coordinators - - data - - memgraph - type: object - status: - description: MemgraphHAStatus defines the observed state of MemgraphHA - type: object - type: object - served: true - storage: true - subresources: - status: {} diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index f059d30..4507dd8 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -2,22 +2,15 @@ # since it depends on service name and namespace that are out of this kustomize package. # It should be run by config/default resources: -- bases/memgraph.com_memgraphhas.yaml -#+kubebuilder:scaffold:crdkustomizeresource +- bases/memgraph.com_memgraphclusters.yaml +# +kubebuilder:scaffold:crdkustomizeresource patches: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. # patches here are for enabling the conversion webhook for each CRD -#- path: patches/webhook_in_memgraphhas.yaml -#+kubebuilder:scaffold:crdkustomizewebhookpatch - -# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. -# patches here are for enabling the CA injection for each CRD -#- path: patches/cainjection_in_memgraphhas.yaml -#+kubebuilder:scaffold:crdkustomizecainjectionpatch +# +kubebuilder:scaffold:crdkustomizewebhookpatch # [WEBHOOK] To enable webhook, uncomment the following section # the following config is for teaching kustomize how to do kustomization for CRDs. - #configurations: #- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml index ec5c150..61361ff 100644 --- a/config/crd/kustomizeconfig.yaml +++ b/config/crd/kustomizeconfig.yaml @@ -8,12 +8,5 @@ nameReference: group: apiextensions.k8s.io path: spec/conversion/webhook/clientConfig/service/name -namespace: -- kind: CustomResourceDefinition - version: v1 - group: apiextensions.k8s.io - path: spec/conversion/webhook/clientConfig/service/namespace - create: false - varReference: - path: metadata/annotations diff --git a/config/default/cert_metrics_manager_patch.yaml b/config/default/cert_metrics_manager_patch.yaml new file mode 100644 index 0000000..d975015 --- /dev/null +++ b/config/default/cert_metrics_manager_patch.yaml @@ -0,0 +1,30 @@ +# This patch adds the args, volumes, and ports to allow the manager to use the metrics-server certs. + +# Add the volumeMount for the metrics-server certs +- op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + mountPath: /tmp/k8s-metrics-server/metrics-certs + name: metrics-certs + readOnly: true + +# Add the --metrics-cert-path argument for the metrics server +- op: add + path: /spec/template/spec/containers/0/args/- + value: --metrics-cert-path=/tmp/k8s-metrics-server/metrics-certs + +# Add the metrics-server certs volume configuration +- op: add + path: /spec/template/spec/volumes/- + value: + name: metrics-certs + secret: + secretName: metrics-server-cert + optional: false + items: + - key: ca.crt + path: ca.crt + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index da8437b..5415254 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -1,8 +1,234 @@ -namespace: memgraph-operator-system +# Adds namespace to all resources. +namespace: kubernetes-operator-system -namePrefix: "" +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kubernetes-operator- + +# Labels to add to all resources and selectors. +#labels: +#- includeSelectors: true +# pairs: +# someName: someValue resources: - ../crd - ../rbac - ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml +# [NETWORK POLICY] Protect the /metrics endpoint and Webhook Server with NetworkPolicy. +# Only Pod(s) running a namespace labeled with 'metrics: enabled' will be able to gather the metrics. +# Only CR(s) which requires webhooks and are applied on namespaces labeled with 'webhooks: enabled' will +# be able to communicate with the Webhook Server. +#- ../network-policy + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment + +# Uncomment the patches line if you enable Metrics and CertManager +# [METRICS-WITH-CERTS] To enable metrics protected with certManager, uncomment the following line. +# This patch will protect the metrics with certManager self-signed certs. +#- path: cert_metrics_manager_patch.yaml +# target: +# kind: Deployment + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- path: manager_webhook_patch.yaml +# target: +# kind: Deployment + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +# Uncomment the following replacements to add the cert-manager CA injection annotations +#replacements: +# - source: # Uncomment the following block to enable certificates for metrics +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.name +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - select: # Uncomment the following to set the Service name for TLS config in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 0 +# create: true + +# - source: +# kind: Service +# version: v1 +# name: controller-manager-metrics-service +# fieldPath: metadata.namespace +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: metrics-certs +# fieldPaths: +# - spec.dnsNames.0 +# - spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true +# - select: # Uncomment the following to set the Service namespace for TLS in Prometheus ServiceMonitor +# kind: ServiceMonitor +# group: monitoring.coreos.com +# version: v1 +# name: controller-manager-metrics-monitor +# fieldPaths: +# - spec.endpoints.0.tlsConfig.serverName +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have any webhook +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.name # Name of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 0 +# create: true +# - source: +# kind: Service +# version: v1 +# name: webhook-service +# fieldPath: .metadata.namespace # Namespace of the service +# targets: +# - select: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPaths: +# - .spec.dnsNames.0 +# - .spec.dnsNames.1 +# options: +# delimiter: '.' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ValidatingWebhook (--programmatic-validation) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # This name should match the one in certificate.yaml +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: ValidatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a DefaultingWebhook (--defaulting ) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 0 +# create: true +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: +# - select: +# kind: MutatingWebhookConfiguration +# fieldPaths: +# - .metadata.annotations.[cert-manager.io/inject-ca-from] +# options: +# delimiter: '/' +# index: 1 +# create: true + +# - source: # Uncomment the following block if you have a ConversionWebhook (--conversion) +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.namespace # Namespace of the certificate CR +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionns +# - source: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert +# fieldPath: .metadata.name +# targets: # Do not remove or uncomment the following scaffold marker; required to generate code for target CRD. +# +kubebuilder:scaffold:crdkustomizecainjectionname diff --git a/config/default/manager_metrics_patch.yaml b/config/default/manager_metrics_patch.yaml new file mode 100644 index 0000000..2aaef65 --- /dev/null +++ b/config/default/manager_metrics_patch.yaml @@ -0,0 +1,4 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 diff --git a/config/default/metrics_service.yaml b/config/default/metrics_service.yaml new file mode 100644 index 0000000..fa7ebe2 --- /dev/null +++ b/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index c4879ff..5c5f0b8 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,12 +1,2 @@ resources: - manager.yaml -- namespace.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: controller - newName: memgraph/kubernetes-operator - newTag: 1.0.0 -- name: memgraph-kubernetes-operator - newName: memgraph/kubernetes-operator - newTag: 1.0.0 diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 2455004..a727fcc 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -1,38 +1,94 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: system +--- apiVersion: apps/v1 kind: Deployment metadata: - namespace: memgraph-operator-system - annotations: - email: engineering@memgraph.io + name: controller-manager + namespace: system labels: - owner: Memgraph - name: memgraph-kubernetes-operator + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize spec: - replicas: 1 selector: matchLabels: - name: memgraph-kubernetes-operator - strategy: - rollingUpdate: - maxUnavailable: 1 - type: RollingUpdate + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + replicas: 1 template: metadata: + annotations: + kubectl.kubernetes.io/default-container: manager labels: - name: memgraph-kubernetes-operator + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator spec: + # TODO(user): Uncomment the following code to configure the nodeAffinity expression + # according to the platforms which are supported by your solution. + # It is considered best practice to support multiple architectures. You can + # build your manager image using the makefile target docker-buildx. + # affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: kubernetes.io/arch + # operator: In + # values: + # - amd64 + # - arm64 + # - ppc64le + # - s390x + # - key: kubernetes.io/os + # operator: In + # values: + # - linux securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted runAsNonRoot: true + seccompProfile: + type: RuntimeDefault containers: - - args: - image: memgraph/kubernetes-operator:1.0.0 + - command: + - /manager + args: + - --leader-elect + - --health-probe-bind-address=:8081 + image: controller:latest name: manager + ports: + - containerPort: 8081 + name: health + protocol: TCP securityContext: readOnlyRootFilesystem: true allowPrivilegeEscalation: false capabilities: drop: - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ resources: limits: cpu: 500m @@ -40,5 +96,7 @@ spec: requests: cpu: 10m memory: 64Mi - serviceAccountName: memgraph-kubernetes-operator + volumeMounts: [] + volumes: [] + serviceAccountName: controller-manager terminationGracePeriodSeconds: 10 diff --git a/config/manager/namespace.yaml b/config/manager/namespace.yaml deleted file mode 100644 index 64458ee..0000000 --- a/config/manager/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: memgraph-operator-system diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml deleted file mode 100644 index e8b968a..0000000 --- a/config/manifests/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# These resources constitute the fully configured set of manifests -# used to generate the 'manifests/' directory in a bundle. -resources: -- bases/kubernetes-operator.clusterserviceversion.yaml -- ../default -- ../samples -- ../scorecard diff --git a/config/network-policy/allow-metrics-traffic.yaml b/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000..cee0b4f --- /dev/null +++ b/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 0000000..ec0fb5e --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 0000000..fdc5481 --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,11 @@ +resources: +- monitor.yaml + +# [PROMETHEUS-WITH-CERTS] The following patch configures the ServiceMonitor in ../prometheus +# to securely reference certificates created and managed by cert-manager. +# Additionally, ensure that you uncomment the [METRICS WITH CERTMANAGER] patch under config/default/kustomization.yaml +# to mount the "metrics-server-cert" secret in the Manager Deployment. +#patches: +# - path: monitor_tls_patch.yaml +# target: +# kind: ServiceMonitor diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 0000000..7b75f9b --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,27 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + # TODO(user): The option insecureSkipVerify: true is not recommended for production since it disables + # certificate verification, exposing the system to potential man-in-the-middle attacks. + # For production environments, it is recommended to use cert-manager for automatic TLS certificate management. + # To apply this configuration, enable cert-manager and use the patch located at config/prometheus/servicemonitor_tls_patch.yaml, + # which securely references the certificate from the 'metrics-server-cert' secret. + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: kubernetes-operator diff --git a/config/prometheus/monitor_tls_patch.yaml b/config/prometheus/monitor_tls_patch.yaml new file mode 100644 index 0000000..5bf84ce --- /dev/null +++ b/config/prometheus/monitor_tls_patch.yaml @@ -0,0 +1,19 @@ +# Patch for Prometheus ServiceMonitor to enable secure TLS configuration +# using certificates managed by cert-manager +- op: replace + path: /spec/endpoints/0/tlsConfig + value: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + serverName: SERVICE_NAME.SERVICE_NAMESPACE.svc + insecureSkipVerify: false + ca: + secret: + name: metrics-server-cert + key: ca.crt + cert: + secret: + name: metrics-server-cert + key: tls.crt + keySecret: + name: metrics-server-cert + key: tls.key diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index 664fcac..9169566 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,4 +1,28 @@ resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. - service_account.yaml - role.yaml - role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml +# For each CRD, "Admin", "Editor" and "Viewer" roles are scaffolded by +# default, aiding admins in cluster management. Those roles are +# not used by the kubernetes-operator itself. You can comment the following lines +# if you do not want those helpers be installed with your Project. +- memgraphcluster_admin_role.yaml +- memgraphcluster_editor_role.yaml +- memgraphcluster_viewer_role.yaml + diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000..19c7f16 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000..56d6f7b --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/memgraphcluster_admin_role.yaml b/config/rbac/memgraphcluster_admin_role.yaml new file mode 100644 index 0000000..4931023 --- /dev/null +++ b/config/rbac/memgraphcluster_admin_role.yaml @@ -0,0 +1,27 @@ +# This rule is not used by the project kubernetes-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants full permissions ('*') over memgraph.com. +# This role is intended for users authorized to modify roles and bindings within the cluster, +# enabling them to delegate specific permissions to other users or groups as needed. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: memgraphcluster-admin-role +rules: +- apiGroups: + - memgraph.com + resources: + - memgraphclusters + verbs: + - '*' +- apiGroups: + - memgraph.com + resources: + - memgraphclusters/status + verbs: + - get diff --git a/config/rbac/memgraphcluster_editor_role.yaml b/config/rbac/memgraphcluster_editor_role.yaml new file mode 100644 index 0000000..e36e59e --- /dev/null +++ b/config/rbac/memgraphcluster_editor_role.yaml @@ -0,0 +1,33 @@ +# This rule is not used by the project kubernetes-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants permissions to create, update, and delete resources within the memgraph.com. +# This role is intended for users who need to manage these resources +# but should not control RBAC or manage permissions for others. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: memgraphcluster-editor-role +rules: +- apiGroups: + - memgraph.com + resources: + - memgraphclusters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - memgraph.com + resources: + - memgraphclusters/status + verbs: + - get diff --git a/config/rbac/memgraphcluster_viewer_role.yaml b/config/rbac/memgraphcluster_viewer_role.yaml new file mode 100644 index 0000000..40a9c6a --- /dev/null +++ b/config/rbac/memgraphcluster_viewer_role.yaml @@ -0,0 +1,29 @@ +# This rule is not used by the project kubernetes-operator itself. +# It is provided to allow the cluster admin to help manage permissions for users. +# +# Grants read-only access to memgraph.com resources. +# This role is intended for users who need visibility into these resources +# without permissions to modify them. It is ideal for monitoring purposes and limited-access viewing. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: memgraphcluster-viewer-role +rules: +- apiGroups: + - memgraph.com + resources: + - memgraphclusters + verbs: + - get + - list + - watch +- apiGroups: + - memgraph.com + resources: + - memgraphclusters/status + verbs: + - get diff --git a/config/rbac/metrics_auth_role.yaml b/config/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000..32d2e4e --- /dev/null +++ b/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/metrics_auth_role_binding.yaml b/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000..e775d67 --- /dev/null +++ b/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/metrics_reader_role.yaml b/config/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000..51a75db --- /dev/null +++ b/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e28a724..7496194 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -2,51 +2,12 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: memgraph-kubernetes-operator + name: manager-role rules: -- apiGroups: - - "" - resources: - - pods - - services - - configmaps - - secrets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - apps - resources: - - statefulsets - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - memgraph.com resources: - - memgraphhas + - memgraphclusters verbs: - create - delete @@ -58,13 +19,13 @@ rules: - apiGroups: - memgraph.com resources: - - memgraphhas/finalizers + - memgraphclusters/finalizers verbs: - update - apiGroups: - memgraph.com resources: - - memgraphhas/status + - memgraphclusters/status verbs: - get - patch diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 9fded4b..8619f0d 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -1,12 +1,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: memgraph-kubernetes-operator + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: memgraph-kubernetes-operator + name: manager-role subjects: - kind: ServiceAccount - name: memgraph-kubernetes-operator - namespace: memgraph-operator-system + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml index f81938c..0a20477 100644 --- a/config/rbac/service_account.yaml +++ b/config/rbac/service_account.yaml @@ -1,5 +1,8 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: memgraph-kubernetes-operator - namespace: memgraph-operator-system + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml index e0823f5..1ceab16 100644 --- a/config/samples/kustomization.yaml +++ b/config/samples/kustomization.yaml @@ -1,2 +1,4 @@ +## Append samples of your project ## resources: -- memgraph_v1_ha.yaml +- v1alpha1_memgraphcluster.yaml +# +kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/samples/memgraph_v1_ha.yaml b/config/samples/memgraph_v1_ha.yaml deleted file mode 100644 index b48bfcd..0000000 --- a/config/samples/memgraph_v1_ha.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: memgraph.com/v1 -kind: MemgraphHA -metadata: - name: memgraphha-sample -spec: - coordinators: - - id: "1" - boltPort: 7687 - managementPort: 10000 - coordinatorPort: 12000 - args: - - --experimental-enabled=high-availability - - --coordinator-id=1 - - --coordinator-port=12000 - - --management-port=10000 - - --bolt-port=7687 - - --also-log-to-stderr - - --log-level=TRACE - - --coordinator-hostname=memgraph-coordinator-1.default.svc.cluster.local - - --log-file=/var/log/memgraph/memgraph.log - - - id: "2" - boltPort: 7687 - managementPort: 10000 - coordinatorPort: 12000 - args: - - --experimental-enabled=high-availability - - --coordinator-id=2 - - --coordinator-port=12000 - - - --management-port=10000 - - --bolt-port=7687 - - --also-log-to-stderr - - --log-level=TRACE - - --coordinator-hostname=memgraph-coordinator-2.default.svc.cluster.local - - --log-file=/var/log/memgraph/memgraph.log - - - id: "3" - boltPort: 7687 - managementPort: 10000 - coordinatorPort: 12000 - args: - - --experimental-enabled=high-availability - - --coordinator-id=3 - - --coordinator-port=12000 - - --management-port=10000 - - --bolt-port=7687 - - --also-log-to-stderr - - --log-level=TRACE - - --coordinator-hostname=memgraph-coordinator-3.default.svc.cluster.local - - --log-file=/var/log/memgraph/memgraph.log - - - data: - - id: "0" - boltPort: 7687 - managementPort: 10000 - replicationPort: 20000 - args: - - --experimental-enabled=high-availability - - --management-port=10000 - - --bolt-port=7687 - - --also-log-to-stderr - - --log-level=TRACE - - --log-file=/var/log/memgraph/memgraph.log - - - id: "1" - boltPort: 7687 - managementPort: 10000 - replicationPort: 20000 - args: - - --experimental-enabled=high-availability - - --management-port=10000 - - --bolt-port=7687 - - --also-log-to-stderr - - --log-level=TRACE - - --log-file=/var/log/memgraph/memgraph.log - - memgraph: - data: - volumeClaim: - logPVCClassName: "" - logPVC: true - logPVCSize: 256Mi - storagePVCClassName: "" - storagePVC: true - storagePVCSize: 1Gi - coordinators: - volumeClaim: - logPVCClassName: "" - logPVC: true - logPVCSize: 256Mi - storagePVCClassName: "" - storagePVC: true - storagePVCSize: 1Gi - - env: # This can be removed I think - MEMGRAPH_ENTERPRISE_LICENSE: "${MEMGRAPH_ENTERPRISE_LICENSE}" - MEMGRAPH_ORGANIZATION_NAME: "${MEMGRAPH_ORGANIZATION_NAME}" - image: - pullPolicy: IfNotPresent - repository: memgraph/memgraph - tag: 2.18.1 # I think we should read this value in controller code. - probes: - liveness: - initialDelaySeconds: 30 - periodSeconds: 10 - readiness: - initialDelaySeconds: 5 - periodSeconds: 5 - startup: - initialDelaySeconds: 5 - failureThreshold: 30 - periodSeconds: 10 diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml new file mode 100644 index 0000000..a59e619 --- /dev/null +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -0,0 +1,8 @@ +apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + labels: + app.kubernetes.io/name: kubernetes-operator + app.kubernetes.io/managed-by: kustomize + name: memgraphcluster-sample +spec: {} diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml deleted file mode 100644 index c770478..0000000 --- a/config/scorecard/bases/config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: scorecard.operatorframework.io/v1alpha3 -kind: Configuration -metadata: - name: config -stages: -- parallel: true - tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml deleted file mode 100644 index 50cd2d0..0000000 --- a/config/scorecard/kustomization.yaml +++ /dev/null @@ -1,16 +0,0 @@ -resources: -- bases/config.yaml -patchesJson6902: -- path: patches/basic.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -- path: patches/olm.config.yaml - target: - group: scorecard.operatorframework.io - version: v1alpha3 - kind: Configuration - name: config -#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml deleted file mode 100644 index 893ebd2..0000000 --- a/config/scorecard/patches/basic.config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - basic-check-spec - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: basic - test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml deleted file mode 100644 index 6cf777b..0000000 --- a/config/scorecard/patches/olm.config.yaml +++ /dev/null @@ -1,50 +0,0 @@ -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-bundle-validation - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: olm - test: olm-bundle-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-validation - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: olm - test: olm-crds-have-validation-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-crds-have-resources - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: olm - test: olm-crds-have-resources-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-spec-descriptors - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: olm - test: olm-spec-descriptors-test -- op: add - path: /stages/0/tests/- - value: - entrypoint: - - scorecard-test - - olm-status-descriptors - image: quay.io/operator-framework/scorecard-test:v1.35.0 - labels: - suite: olm - test: olm-status-descriptors-test diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index 820c431..0000000 --- a/docs/installation.md +++ /dev/null @@ -1,65 +0,0 @@ -# Install Memgraph Kubernetes Operator - -All described installation options will run the Operator inside the cluster. - -Make sure to clone this repository with its submodule (helm-charts). - -```bash -git clone --recurse-submodules git@github.com:memgraph/kubernetes-operator.git -``` - -## Install K8 Resources - -```bash -make deploy -``` - -This command will use operator's image from Memgraph's DockerHub and create all necessary Kubernetes resources for running an operator. - -## Verify Installation - -Installation using any of the options described above will create a Kubernetes ServiceAccount, RoleBinding, Role, Deployment, and Pods all in the newly created namespace `memgraph-operator-system`. You can check your resources with: - -```bash -kubectl get serviceaccounts -n memgraph-operator-system -kubectl get clusterrolebindings -n memgraph-operator-system -kubectl get clusterroles -n memgraph-operator-system -kubectl get deployments -n memgraph-operator-system -kubectl get pods -n memgraph-operator-system -kubectl get services -n memgraph-operator-system -``` - -CustomResourceDefinition `memgraphhas.memgraph.com`, whose job is to monitor CustomResource `MemgraphHA`, will also get created and you can verify -this with: - -```bash -kubectl get crds -A -``` - -## Start Memgraph High Availability Cluster - -We already provide a sample cluster in `config/samples/memgraph_v1_ha.yaml`. You only need to set your license information by -creating a Kubernetes Secret containing licensing info. You can do this in a following way: - -```bash - kubectl create secret generic memgraph-secrets \ ---from-literal=MEMGRAPH_ENTERPRISE_LICENSE="" \ ---from-literal=MEMGRAPH_ORGANIZATION_NAME="" -``` - -Start Memgraph HA cluster with `kubectl apply -f config/samples/memgraph_v1_ha.yaml`. - -After approximately 60 seconds, you should be able to see instances in the output of `kubectl get pods -A`. - -You can now find the URL of any coordinator instances by running e.g `minikube service list` and connect to see the state of the cluster by running -`show instances;`: -![image](https://github.com/memgraph/kubernetes-operator/assets/53269502/c68d52e2-19f7-4e45-8ff0-fc2ee662c64b) - -## Clear Resources - -```bash -kubectl delete -f config/samples/memgraph_v1_ha.yaml # For deleting cluster -kubectl delete pvc --all # Or leave them if you want to use persistent storage -kubectl delete secret memgraph-secrets -make undeploy -``` diff --git a/go.mod b/go.mod index f541a53..3448f3d 100644 --- a/go.mod +++ b/go.mod @@ -1,72 +1,100 @@ module github.com/memgraph/kubernetes-operator -go 1.22.0 - -toolchain go1.22.5 +go 1.26.0 require ( - github.com/go-logr/logr v1.4.2 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 - k8s.io/api v0.30.3 - k8s.io/apimachinery v0.30.3 - k8s.io/client-go v0.30.3 - sigs.k8s.io/controller-runtime v0.18.4 + github.com/onsi/ginkgo/v2 v2.27.4 + github.com/onsi/gomega v1.39.0 + k8s.io/apimachinery v0.36.0 + k8s.io/client-go v0.36.0 + sigs.k8s.io/controller-runtime v0.24.1 ) require ( + cel.dev/expr v0.25.1 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.12.1 // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 // indirect + github.com/google/cel-go v0.26.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.23.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/sdk v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.23.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/text v0.33.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.41.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect + google.golang.org/grpc v1.79.3 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.30.3 // indirect - k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + k8s.io/api v0.36.0 // indirect + k8s.io/apiextensions-apiserver v0.36.0 // indirect + k8s.io/apiserver v0.36.0 // indirect + k8s.io/component-base v0.36.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.0 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 244baea..690c70d 100644 --- a/go.sum +++ b/go.sum @@ -1,175 +1,258 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= -github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= -github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= +github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 h1:ssNFCCVmib/GQSzx3uCWyfMgOamLGWuGqlMS77Y1m3Y= -github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y= +github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= +github.com/onsi/gomega v1.39.0/go.mod h1:ZCU1pkQcXDO5Sl9/VVEGlDyp+zm0m1cmeG5TOzLgdh4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0 h1:7iP2uCb7sGddAr30RRS6xjKy7AZ2JtTOPA3oolgVSw8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.65.0/go.mod h1:c7hN3ddxs/z6q9xwvfLPk+UHlWRQyaeR1LdgfL/66l0= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0 h1:QKdN8ly8zEMrByybbQgv8cWBcdAarwmIPZ6FThrWXJs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.40.0/go.mod h1:bTdK1nhqF76qiPoCCdyFIV+N/sRHYXYCTQc+3VCi3MI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0 h1:DvJDOPmSWQHWywQS6lKL+pb8s3gBLOZUtw4N+mavW1I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.40.0/go.mod h1:EtekO9DEJb4/jRyN4v4Qjc2yA7AtfCBuz2FynRUWTXs= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93 h1:fQsdNF2N+/YewlRZiricy4P1iimyPKZ/xwniHj8Q2a0= +golang.org/x/exp v0.0.0-20251219203646-944ab1f22d93/go.mod h1:EPRbTFwzwjXj9NpYyyrvenVh9Y+GFeEvMNh7Xuz7xgU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409 h1:merA0rdPeUV3YIIfHHcH4qBkiQAc1nfCKSI7lB4cV2M= +google.golang.org/genproto/googleapis/api v0.0.0-20260128011058-8636f8732409/go.mod h1:fl8J1IvUjCilwZzQowmw2b7HQB2eAuYBabMXzWurF+I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= -k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= -k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= -k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= -k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= -k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f h1:2sXuKesAYbRHxL3aE2PN6zX/gcJr22cjrsej+W784Tc= -k8s.io/kube-openapi v0.0.0-20240709000822-3c01b740850f/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +k8s.io/api v0.36.0 h1:SgqDhZzHdOtMk40xVSvCXkP9ME0H05hPM3p9AB1kL80= +k8s.io/api v0.36.0/go.mod h1:m1LVrGPNYax5NBHdO+QuAedXyuzTt4RryI/qnmNvs34= +k8s.io/apiextensions-apiserver v0.36.0 h1:Wt7E8J+VBCbj4FjiBfDTK/neXDDjyJVJc7xfuOHImZ0= +k8s.io/apiextensions-apiserver v0.36.0/go.mod h1:kGDjH0msuiIB3tgsYRV0kS9GqpMYMUsQ3GHv7TApyug= +k8s.io/apimachinery v0.36.0 h1:jZyPzhd5Z+3h9vJLt0z9XdzW9VzNzWAUw+P1xZ9PXtQ= +k8s.io/apimachinery v0.36.0/go.mod h1:FklypaRJt6n5wUIwWXIP6GJlIpUizTgfo1T/As+Tyxc= +k8s.io/apiserver v0.36.0 h1:Jg5OFAENUACByUCg15CmhZAYrr5ZyJ+jodyA1mHl3YE= +k8s.io/apiserver v0.36.0/go.mod h1:mHvwdHf+qKEm+1/hYm756SV+oREOKSPnsjagOpx6Vho= +k8s.io/client-go v0.36.0 h1:pOYi7C4RHChYjMiHpZSpSbIM6ZxVbRXBy7CuiIwqA3c= +k8s.io/client-go v0.36.0/go.mod h1:ZKKcpwF0aLYfkHFCjillCKaTK/yBkEDHTDXCFY6AS9Y= +k8s.io/component-base v0.36.0 h1:hFjEktssxiJhrK1zfybkH4kJOi8iZuF+mIDCqS5+jRo= +k8s.io/component-base v0.36.0/go.mod h1:JZvIfcNHk+uck+8LhJzhSBtydWXaZNQwX2OdL+Mnwsk= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.0 h1:agnTxU+NFulUrtYzXUGKO3ndEa8jKwht1Kwn9nu9x+4= +k8s.io/streaming v0.36.0/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 h1:hSfpvjjTQXQY2Fol2CS0QHMNs/WI1MOSGzCm1KhM5ec= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= +sigs.k8s.io/controller-runtime v0.24.1 h1:miPEwrmirImAvgME1L9qebGHrOnGJoVmVdtOU9fRfo4= +sigs.k8s.io/controller-runtime v0.24.1/go.mod h1:vFkfY5fGt5xAC/sKb8IBFKgWPNKG9OUG29dR8Y2wImw= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index 759b82a..af737e6 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /* -Copyright 2024 Memgraph Ltd. +Copyright YEAR. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -*/ +*/ \ No newline at end of file diff --git a/helm-charts b/helm-charts deleted file mode 160000 index a2f3d48..0000000 --- a/helm-charts +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a2f3d480cfa4efd3cd32935cd58391c914fb8296 diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go new file mode 100644 index 0000000..2a68548 --- /dev/null +++ b/internal/controller/memgraphcluster_controller.go @@ -0,0 +1,65 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +// MemgraphClusterReconciler reconciles a MemgraphCluster object +type MemgraphClusterReconciler struct { + client.Client + Scheme *runtime.Scheme +} + +// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// +// Placeholder implementation: fetches the MemgraphCluster and logs a +// greeting. Provisioning arrives with the walking-skeleton slice +// (specs/operator-mvp/issues/02). +func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + var cluster memgraphcomv1alpha1.MemgraphCluster + if err := r.Get(ctx, req.NamespacedName, &cluster); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + log.Info("hello from the MemgraphCluster reconciler", "memgraphcluster", req.NamespacedName) + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *MemgraphClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&memgraphcomv1alpha1.MemgraphCluster{}). + Named("memgraphcluster"). + Complete(r) +} diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go new file mode 100644 index 0000000..293ed64 --- /dev/null +++ b/internal/controller/memgraphcluster_controller_test.go @@ -0,0 +1,87 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +var _ = Describe("MemgraphCluster Controller", func() { + Context("When reconciling a resource", func() { + const ( + resourceName = "test-resource" + resourceNamespace = "default" + ) + + ctx := context.Background() + + typeNamespacedName := types.NamespacedName{ + Name: resourceName, + Namespace: resourceNamespace, + } + memgraphcluster := &memgraphcomv1alpha1.MemgraphCluster{} + + BeforeEach(func() { + By("creating the custom resource for the Kind MemgraphCluster") + err := k8sClient.Get(ctx, typeNamespacedName, memgraphcluster) + if err != nil && errors.IsNotFound(err) { + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: resourceNamespace, + }, + // TODO(user): Specify other spec details if needed. + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + } + }) + + AfterEach(func() { + // TODO(user): Cleanup logic after each test, like removing the resource instance. + resource := &memgraphcomv1alpha1.MemgraphCluster{} + err := k8sClient.Get(ctx, typeNamespacedName, resource) + Expect(err).NotTo(HaveOccurred()) + + By("Cleanup the specific resource instance MemgraphCluster") + Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + }) + It("should successfully reconcile the resource", func() { + By("Reconciling the created resource") + controllerReconciler := &MemgraphClusterReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), + } + + _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: typeNamespacedName, + }) + Expect(err).NotTo(HaveOccurred()) + // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. + // Example: If you expect a certain status condition after reconciliation, verify it here. + }) + }) +}) diff --git a/internal/controller/memgraphha_constants.go b/internal/controller/memgraphha_constants.go deleted file mode 100644 index 5c3160a..0000000 --- a/internal/controller/memgraphha_constants.go +++ /dev/null @@ -1,23 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -var boltPort int = 7687 -var coordinatorPort int = 12000 -var mgmtPort int = 10000 -var replicationPort int = 20000 -var image string = "memgraph/memgraph:2.18.1" diff --git a/internal/controller/memgraphha_controller.go b/internal/controller/memgraphha_controller.go deleted file mode 100644 index 041f2aa..0000000 --- a/internal/controller/memgraphha_controller.go +++ /dev/null @@ -1,152 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - - "k8s.io/apimachinery/pkg/api/errors" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/log" - - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" -) - -//+kubebuilder:rbac:groups=memgraph.com,resources=memgraphhas,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=memgraph.com,resources=memgraphhas/status,verbs=get;update;patch -//+kubebuilder:rbac:groups=memgraph.com,resources=memgraphhas/finalizers,verbs=update - -// For more details, check Reconcile and its Result here: -// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.16.3/pkg/reconcile -func (r *MemgraphHAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - logger := log.FromContext(ctx) - - memgraphha := &memgraphv1.MemgraphHA{} - err := r.Get(ctx, req.NamespacedName, memgraphha) - if err != nil { - if errors.IsNotFound(err) { - logger.Info("MemgraphHA resource not found. Ignoring since object must be deleted.") - return ctrl.Result{}, nil - } - logger.Error(err, "Failed to get MemgraphHA") - return ctrl.Result{}, err - } - - logger.Info("Started reconciliation MemgrahHA") - - for coordId := 1; coordId <= 3; coordId++ { - // ClusterIP - coordClusterIPStatus, coordClusterIPErr := r.reconcileCoordClusterIPService(ctx, memgraphha, &logger, coordId) - if coordClusterIPErr != nil { - logger.Info("Error returned when reconciling ClusterIP Returning empty Result with error.", "coordId", coordId) - return ctrl.Result{}, coordClusterIPErr - } - - if coordClusterIPStatus == true { - logger.Info("ClusterIP has been created. Returning Result with the request for requeing with error set to nil.", "coordId", coordId) - return ctrl.Result{Requeue: true}, nil - } - - // NodePort - coordNodePortStatus, coordNodePortErr := r.reconcileCoordNodePortService(ctx, memgraphha, &logger, coordId) - if coordNodePortErr != nil { - logger.Info("Error returned when reconciling NodePort. Returning empty Result with error.", "coordId", coordId) - return ctrl.Result{}, coordNodePortErr - } - - if coordNodePortStatus == true { - logger.Info("NodePort has been created. Returning Result with the request for requeing with error set to nil.", "coordId", coordId) - return ctrl.Result{Requeue: true}, nil - } - - // Coordinator - coordStatus, coordErr := r.reconcileCoordinator(ctx, memgraphha, &logger, coordId) - if coordErr != nil { - logger.Info("Error returned when reconciling coordinator. Returning empty Result with error.", "coordId", coordId) - return ctrl.Result{}, coordErr - } - - if coordStatus == true { - logger.Info("Coordinator has been created. Returning Result with the request for requeing with error set to nil.", "coordId", coordId) - return ctrl.Result{Requeue: true}, nil - } - } - - logger.Info("Reconciliation of coordinators finished without actions needed.") - - for dataInstanceId := 0; dataInstanceId <= 1; dataInstanceId++ { - // ClusterIP - dataInstanceClusterIPStatus, dataInstanceClusterIPErr := r.reconcileDataInstanceClusterIPService(ctx, memgraphha, &logger, dataInstanceId) - if dataInstanceClusterIPErr != nil { - logger.Info("Error returned when reconciling ClusterIP. Returning empty Result with error.", "dataInstanceId", dataInstanceId) - return ctrl.Result{}, dataInstanceClusterIPErr - } - - if dataInstanceClusterIPStatus == true { - logger.Info("ClusterIP has been created. Returning Result with the request for requeing with error set to nil.", "dataInstanceId", dataInstanceId) - return ctrl.Result{Requeue: true}, nil - } - - // NodePort - dataInstanceNodePortStatus, dataInstanceNodePortErr := r.reconcileDataInstanceNodePortService(ctx, memgraphha, &logger, dataInstanceId) - if dataInstanceNodePortErr != nil { - logger.Info("Error returned when reconciling NodePort. Returning empty Result with error.", "dataInstanceId", dataInstanceId) - return ctrl.Result{}, dataInstanceNodePortErr - } - - if dataInstanceNodePortStatus == true { - logger.Info("NodePort has been created. Returning Result with the request for requeing with error set to nil.", "dataInstanceId", dataInstanceId) - return ctrl.Result{Requeue: true}, nil - } - - // Data instance - dataInstancesStatus, dataInstancesErr := r.reconcileDataInstance(ctx, memgraphha, &logger, dataInstanceId) - if dataInstancesErr != nil { - logger.Info("Error returned when reconciling data instance. Returning empty Result with error.", "dataInstanceId", dataInstanceId) - return ctrl.Result{}, dataInstancesErr - } - - if dataInstancesStatus == true { - logger.Info("Data instance has been created. Returning Result with the request for requeing with error=nil.", "dataInstanceId", dataInstanceId) - return ctrl.Result{Requeue: true}, nil - } - } - - logger.Info("Reconciliation of data instances finished without actions needed.") - - setupJobStatus, setupJobErr := r.reconcileSetupJob(ctx, memgraphha, &logger) - if setupJobErr != nil { - logger.Info("Error returned when reconciling coordinator. Returning empty Result with error.") - return ctrl.Result{}, setupJobErr - } - - // Since it is currently the last step, we don't need to requeue - if setupJobStatus == true { - logger.Info("SetupJob has been created.") - } - - logger.Info("Reconciliation of MemgraphHA finished.") - // The resource doesn't need to be reconciled anymore - return ctrl.Result{}, nil -} - -// SetupWithManager sets up the controller with the Manager. -func (r *MemgraphHAReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&memgraphv1.MemgraphHA{}). - Complete(r) -} diff --git a/internal/controller/memgraphha_coord.go b/internal/controller/memgraphha_coord.go deleted file mode 100644 index cba91b8..0000000 --- a/internal/controller/memgraphha_coord.go +++ /dev/null @@ -1,236 +0,0 @@ -/* -copyright 2024 memgraph ltd. - -licensed under the apache license, version 2.0 (the "license"); -you may not use this file except in compliance with the license. -you may obtain a copy of the license at - - http://www.apache.org/licenses/license-2.0 - -unless required by applicable law or agreed to in writing, software -distributed under the license is distributed on an "as is" basis, -without warranties or conditions of any kind, either express or implied. -see the license for the specific language governing permissions and -limitations under the license. -*/ - -package controller - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" -) - -/* -Returns bool, error tuple. If error exists, the caller should return with error and status will always be set to true. -If there is no error, we must look at bool status which when true will say that the coordinator was createdand we need to requeue -or that nothing was done and we can continue with the next step of reconciliation. -*/ -func (r *MemgraphHAReconciler) reconcileCoordinator(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, coordId int) (bool, error) { - name := fmt.Sprintf("memgraph-coordinator-%d", coordId) - logger.Info("Started reconciling", "StatefulSet", name) - coordStatefulSet := &appsv1.StatefulSet{} - err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: memgraphha.Namespace}, coordStatefulSet) - - if err == nil { - logger.Info("StatefulSet already exists.", "StatefulSet", name) - return false, nil - } - - if errors.IsNotFound(err) { - coord := r.createStatefulSetForCoord(memgraphha, coordId) - logger.Info("Creating a new StatefulSet", "StatefulSet.Namespace", coord.Namespace, "StatefulSet.Name", coord.Name) - err := r.Create(ctx, coord) - if err != nil { - logger.Error(err, "Failed to create new StatefulSet", "StatefulSet.Namespace", coord.Namespace, "StatefulSet.Name", coord.Name) - return true, err - } - logger.Info("StatefulSet is created.", "StatefulSet", name) - return true, nil - } - - logger.Error(err, "Failed to fetch StatefulSet", "StatefulSet", name) - return true, err -} - -func (r *MemgraphHAReconciler) createStatefulSetForCoord(memgraphha *memgraphv1.MemgraphHA, coordId int) *appsv1.StatefulSet { - coordName := fmt.Sprintf("memgraph-coordinator-%d", coordId) - serviceName := coordName // service has the same name as the coordinator - labels := createCoordLabels(coordName) - replicas := int32(1) - containerName := "memgraph-coordinator" - args := []string{ - fmt.Sprintf("--coordinator-id=%d", coordId), - fmt.Sprintf("--coordinator-port=%d", coordinatorPort), - fmt.Sprintf("--management-port=%d", mgmtPort), - fmt.Sprintf("--bolt-port=%d", boltPort), - fmt.Sprintf("--coordinator-hostname=%s.default.svc.cluster.local", coordName), - "--experimental-enabled=high-availability", - "--also-log-to-stderr", - "--log-level=TRACE", - "--log-file=/var/log/memgraph/memgraph.log", - } - volumeLibName := fmt.Sprintf("%s-lib-storage", coordName) - volumeLibSize := "1Gi" - volumeLogName := fmt.Sprintf("%s-log-storage", coordName) - volumeLogSize := "256Mi" - initContainerName := "init" - initContainerCommand := []string{ - "/bin/sh", - "-c", - } - initContainerArgs := []string{"chown -R memgraph:memgraph /var/log; chown -R memgraph:memgraph /var/lib"} - initContainerPrivileged := true - initContainerReadOnlyRootFilesystem := false - initContainerRunAsNonRoot := false - initContainerRunAsUser := int64(0) - - coord := &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: coordName, - Namespace: memgraphha.Namespace, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: serviceName, - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: initContainerName, - Image: image, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeLibName, - MountPath: "/var/lib/memgraph", - }, - { - Name: volumeLogName, - MountPath: "/var/log/memgraph", - }, - }, - Command: initContainerCommand, - Args: initContainerArgs, - SecurityContext: &corev1.SecurityContext{ - Privileged: &initContainerPrivileged, - ReadOnlyRootFilesystem: &initContainerReadOnlyRootFilesystem, - RunAsNonRoot: &initContainerRunAsNonRoot, - RunAsUser: &initContainerRunAsUser, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"all"}, - Add: []corev1.Capability{"CHOWN"}, - }, - }, - }, - }, - - Containers: []corev1.Container{{ - Name: containerName, - Image: image, - ImagePullPolicy: corev1.PullAlways, // set to PullIfNotPresent when testing with local image - Ports: []corev1.ContainerPort{ - { - ContainerPort: int32(boltPort), - Name: "bolt", - }, - { - ContainerPort: int32(mgmtPort), - Name: "management", - }, - { - ContainerPort: int32(coordinatorPort), - Name: "coordinator", - }, - }, - Args: args, - Env: []corev1.EnvVar{ - { - Name: "MEMGRAPH_ENTERPRISE_LICENSE", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "memgraph-secrets", - }, - Key: "MEMGRAPH_ENTERPRISE_LICENSE", - }, - }, - }, - { - Name: "MEMGRAPH_ORGANIZATION_NAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "memgraph-secrets", - }, - Key: "MEMGRAPH_ORGANIZATION_NAME", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeLibName, - MountPath: "/var/lib/memgraph", - }, - { - Name: volumeLogName, - MountPath: "/var/log/memgraph", - }, - }, - }}, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: volumeLibName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(volumeLibSize), - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: volumeLogName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(volumeLogSize), - }, - }, - }, - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, coord, r.Scheme) - return coord -} - -func createCoordLabels(coordName string) map[string]string { - return map[string]string{"app": coordName} -} diff --git a/internal/controller/memgraphha_coord_services.go b/internal/controller/memgraphha_coord_services.go deleted file mode 100644 index 659a43f..0000000 --- a/internal/controller/memgraphha_coord_services.go +++ /dev/null @@ -1,156 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - corev1 "k8s.io/api/core/v1" - ctrl "sigs.k8s.io/controller-runtime" - - "github.com/go-logr/logr" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" -) - -func (r *MemgraphHAReconciler) reconcileCoordNodePortService(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, coordId int) (bool, error) { - serviceName := fmt.Sprintf("memgraph-coordinator-%d-external", coordId) - logger.Info("Started reconciling NodePort service", "NodePort", serviceName) - - coordNodePortService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: memgraphha.Namespace}, coordNodePortService) - - if err == nil { - logger.Info("NodePort already exists.", "NodePort", serviceName) - return false, nil - } - - if errors.IsNotFound(err) { - nodePort := r.createCoordNodePort(memgraphha, coordId) - logger.Info("Creating a new NodePort", "NodePort.Namespace", nodePort.Namespace, "NodePort.Name", nodePort.Name) - err := r.Create(ctx, nodePort) - if err != nil { - logger.Error(err, "Failed to create new NodePort", "NodePort.Namespace", nodePort.Namespace, "NodePort.Name", nodePort.Name) - return true, err - } - logger.Info("NodePort is created.", "NodePort", serviceName) - return true, nil - } - - logger.Error(err, "Failed to fetch NodePort", "NodePort", serviceName) - return true, err - -} - -func (r *MemgraphHAReconciler) createCoordNodePort(memgraphha *memgraphv1.MemgraphHA, coordId int) *corev1.Service { - serviceName := fmt.Sprintf("memgraph-coordinator-%d-external", coordId) - coordName := fmt.Sprintf("memgraph-coordinator-%d", coordId) - - coordNodePort := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: memgraphha.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeNodePort, - Selector: createCoordLabels(coordName), - Ports: []corev1.ServicePort{ - { - Name: "bolt", - Protocol: corev1.ProtocolTCP, - Port: int32(boltPort), - TargetPort: intstr.FromInt(boltPort), - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, coordNodePort, r.Scheme) - return coordNodePort -} - -func (r *MemgraphHAReconciler) reconcileCoordClusterIPService(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, coordId int) (bool, error) { - serviceName := fmt.Sprintf("memgraph-coordinator-%d", coordId) - logger.Info("Started reconciling ClusterIP service", "ClusterIP", serviceName) - - coordClusterIPService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: memgraphha.Namespace}, coordClusterIPService) - - if err == nil { - logger.Info("ClusterIP already exists.", "ClusterIP", serviceName) - return false, nil - } - - if errors.IsNotFound(err) { - clusterIP := r.createCoordClusterIP(memgraphha, coordId) - logger.Info("Creating a new ClusterIP", "ClusterIP.Namespace", clusterIP.Namespace, "ClusterIP.Name", clusterIP.Name) - err := r.Create(ctx, clusterIP) - if err != nil { - logger.Error(err, "Failed to create new ClusterIP", "ClusterIP.Namespace", clusterIP.Namespace, "ClusterIP.Name", clusterIP.Name) - return true, err - } - logger.Info("ClusterIP is created.", "ClusterIP", serviceName) - return true, nil - } - - logger.Error(err, "Failed to fetch ClusterIP", "ClusterIP", serviceName) - return true, err - -} - -func (r *MemgraphHAReconciler) createCoordClusterIP(memgraphha *memgraphv1.MemgraphHA, coordId int) *corev1.Service { - serviceName := fmt.Sprintf("memgraph-coordinator-%d", coordId) - coordName := serviceName - - coordClusterIP := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: memgraphha.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: createCoordLabels(coordName), - Ports: []corev1.ServicePort{ - { - Name: "bolt", - Protocol: corev1.ProtocolTCP, - Port: int32(boltPort), - TargetPort: intstr.FromInt(boltPort), - }, - { - Name: "coordinator", - Protocol: corev1.ProtocolTCP, - Port: int32(coordinatorPort), - TargetPort: intstr.FromInt(coordinatorPort), - }, - { - Name: "management", - Protocol: corev1.ProtocolTCP, - Port: int32(mgmtPort), - TargetPort: intstr.FromInt(mgmtPort), - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, coordClusterIP, r.Scheme) - return coordClusterIP -} diff --git a/internal/controller/memgraphha_data_instance.go b/internal/controller/memgraphha_data_instance.go deleted file mode 100644 index fd2a5ee..0000000 --- a/internal/controller/memgraphha_data_instance.go +++ /dev/null @@ -1,229 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" -) - -func (r *MemgraphHAReconciler) reconcileDataInstance(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, dataInstanceId int) (bool, error) { - name := fmt.Sprintf("memgraph-data-%d", dataInstanceId) - logger.Info("Started reconciling", "StatefulSet", name) - dataInstanceStatefulSet := &appsv1.StatefulSet{} - err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: memgraphha.Namespace}, dataInstanceStatefulSet) - - if err == nil { - logger.Info("StatefulSet already exists.", "StatefulSet", name) - return false, nil - } - - if errors.IsNotFound(err) { - dataInstance := r.createStatefulSetForDataInstance(memgraphha, dataInstanceId) - logger.Info("Creating a new StatefulSet", "StatefulSet.Namespace", dataInstance.Namespace, "StatefulSet.Name", dataInstance.Name) - err := r.Create(ctx, dataInstance) - if err != nil { - logger.Error(err, "Failed to create new StatefulSet", "StatefulSet.Namespace", dataInstance.Namespace, "StatefulSet.Name", dataInstance.Name) - return true, err - } - logger.Info("StatefulSet is created.", "StatefulSet", name) - return true, nil - } - - logger.Error(err, "Failed to fetch StatefulSet", "StatefulSet", name) - return true, err - -} - -func (r *MemgraphHAReconciler) createStatefulSetForDataInstance(memgraphha *memgraphv1.MemgraphHA, dataInstanceId int) *appsv1.StatefulSet { - dataInstanceName := fmt.Sprintf("memgraph-data-%d", dataInstanceId) - serviceName := dataInstanceName - labels := createDataInstanceLabels(dataInstanceName) - replicas := int32(1) - containerName := "memgraph-data" - args := []string{ - fmt.Sprintf("--management-port=%d", mgmtPort), - fmt.Sprintf("--bolt-port=%d", boltPort), - "--experimental-enabled=high-availability", - "--also-log-to-stderr", - "--log-level=TRACE", - "--log-file=/var/log/memgraph/memgraph.log", - } - volumeLibName := fmt.Sprintf("%s-lib-storage", dataInstanceName) - volumeLibSize := "1Gi" - volumeLogName := fmt.Sprintf("%s-log-storage", dataInstanceName) - volumeLogSize := "256Mi" - initContainerName := "init" - initContainerCommand := []string{ - "/bin/sh", - "-c", - } - initContainerArgs := []string{"chown -R memgraph:memgraph /var/log; chown -R memgraph:memgraph /var/lib"} - initContainerPrivileged := true - initContainerReadOnlyRootFilesystem := false - initContainerRunAsNonRoot := false - initContainerRunAsUser := int64(0) - - data := &appsv1.StatefulSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: dataInstanceName, - Namespace: memgraphha.Namespace, - }, - Spec: appsv1.StatefulSetSpec{ - ServiceName: serviceName, - Replicas: &replicas, - Selector: &metav1.LabelSelector{ - MatchLabels: labels, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - }, - Spec: corev1.PodSpec{ - InitContainers: []corev1.Container{ - { - Name: initContainerName, - Image: image, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeLibName, - MountPath: "/var/lib/memgraph", - }, - { - Name: volumeLogName, - MountPath: "/var/log/memgraph", - }, - }, - Command: initContainerCommand, - Args: initContainerArgs, - SecurityContext: &corev1.SecurityContext{ - Privileged: &initContainerPrivileged, - ReadOnlyRootFilesystem: &initContainerReadOnlyRootFilesystem, - RunAsNonRoot: &initContainerRunAsNonRoot, - RunAsUser: &initContainerRunAsUser, - Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{"all"}, - Add: []corev1.Capability{"CHOWN"}, - }, - }, - }, - }, - - Containers: []corev1.Container{{ - Name: containerName, - Image: image, - ImagePullPolicy: corev1.PullAlways, // set to PullIfNotPresent when testing with local image - Ports: []corev1.ContainerPort{ - { - ContainerPort: int32(boltPort), - Name: "bolt", - }, - { - ContainerPort: int32(mgmtPort), - Name: "management", - }, - { - ContainerPort: int32(replicationPort), - Name: "replication", - }, - }, - Args: args, - Env: []corev1.EnvVar{ - { - Name: "MEMGRAPH_ENTERPRISE_LICENSE", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "memgraph-secrets", - }, - Key: "MEMGRAPH_ENTERPRISE_LICENSE", - }, - }, - }, - { - Name: "MEMGRAPH_ORGANIZATION_NAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "memgraph-secrets", - }, - Key: "MEMGRAPH_ORGANIZATION_NAME", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: volumeLibName, - MountPath: "/var/lib/memgraph", - }, - { - Name: volumeLogName, - MountPath: "/var/log/memgraph", - }, - }, - }}, - }, - }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: volumeLibName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(volumeLibSize), - }, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: volumeLogName, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - Resources: corev1.VolumeResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(volumeLogSize), - }, - }, - }, - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, data, r.Scheme) - return data -} - -func createDataInstanceLabels(dataInstanceName string) map[string]string { - return map[string]string{"app": dataInstanceName} -} diff --git a/internal/controller/memgraphha_data_services.go b/internal/controller/memgraphha_data_services.go deleted file mode 100644 index 253228a..0000000 --- a/internal/controller/memgraphha_data_services.go +++ /dev/null @@ -1,155 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - ctrl "sigs.k8s.io/controller-runtime" -) - -func (r *MemgraphHAReconciler) reconcileDataInstanceNodePortService(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, dataInstanceId int) (bool, error) { - serviceName := fmt.Sprintf("memgraph-data-%d-external", dataInstanceId) - logger.Info("Started reconciling NodePort service", "NodePort", serviceName) - - dataInstanceNodePortService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: memgraphha.Namespace}, dataInstanceNodePortService) - - if err == nil { - logger.Info("NodePort already exists.", "NodePort", serviceName) - return false, nil - } - - if errors.IsNotFound(err) { - nodePort := r.createDataInstanceNodePort(memgraphha, dataInstanceId) - logger.Info("Creating a new NodePort", "NodePort.Namespace", nodePort.Namespace, "NodePort.Name", nodePort.Name) - err := r.Create(ctx, nodePort) - if err != nil { - logger.Error(err, "Failed to create new NodePort", "NodePort.Namespace", nodePort.Namespace, "NodePort.Name", nodePort.Name) - return true, err - } - logger.Info("NodePort is created.", "NodePort", serviceName) - return true, nil - } - - logger.Error(err, "Failed to fetch NodePort", "NodePort", serviceName) - return true, err - -} - -func (r *MemgraphHAReconciler) createDataInstanceNodePort(memgraphha *memgraphv1.MemgraphHA, dataInstanceId int) *corev1.Service { - serviceName := fmt.Sprintf("memgraph-data-%d-external", dataInstanceId) - dataInstanceName := fmt.Sprintf("memgraph-data-%d", dataInstanceId) - - dataInstanceNodePort := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: memgraphha.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeNodePort, - Selector: createDataInstanceLabels(dataInstanceName), - Ports: []corev1.ServicePort{ - { - Name: "bolt", - Protocol: corev1.ProtocolTCP, - Port: int32(boltPort), - TargetPort: intstr.FromInt(boltPort), - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, dataInstanceNodePort, r.Scheme) - return dataInstanceNodePort -} - -func (r *MemgraphHAReconciler) reconcileDataInstanceClusterIPService(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger, dataInstanceId int) (bool, error) { - serviceName := fmt.Sprintf("memgraph-data-%d", dataInstanceId) - logger.Info("Started reconciling ClusterIP service", "ClusterIP", serviceName) - - dataInstanceClusterIPService := &corev1.Service{} - err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: memgraphha.Namespace}, dataInstanceClusterIPService) - - if err == nil { - logger.Info("ClusterIP already exists.", "ClusterIP", serviceName) - return false, nil - } - - if errors.IsNotFound(err) { - clusterIP := r.createDataInstanceClusterIP(memgraphha, dataInstanceId) - logger.Info("Creating a new ClusterIP", "ClusterIP.Namespace", clusterIP.Namespace, "ClusterIP.Name", clusterIP.Name) - err := r.Create(ctx, clusterIP) - if err != nil { - logger.Error(err, "Failed to create new ClusterIP", "ClusterIP.Namespace", clusterIP.Namespace, "ClusterIP.Name", clusterIP.Name) - return true, err - } - logger.Info("ClusterIP is created.", "ClusterIP", serviceName) - return true, nil - } - - logger.Error(err, "Failed to fetch ClusterIP", "ClusterIP", serviceName) - return true, err - -} - -func (r *MemgraphHAReconciler) createDataInstanceClusterIP(memgraphha *memgraphv1.MemgraphHA, dataInstanceId int) *corev1.Service { - serviceName := fmt.Sprintf("memgraph-data-%d", dataInstanceId) - dataInstanceName := serviceName - - dataInstanceClusterIP := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Namespace: memgraphha.Namespace, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeClusterIP, - Selector: createDataInstanceLabels(dataInstanceName), - Ports: []corev1.ServicePort{ - { - Name: "bolt", - Protocol: corev1.ProtocolTCP, - Port: int32(boltPort), - TargetPort: intstr.FromInt(boltPort), - }, - { - Name: "replication", - Protocol: corev1.ProtocolTCP, - Port: int32(replicationPort), - TargetPort: intstr.FromInt(replicationPort), - }, - { - Name: "management", - Protocol: corev1.ProtocolTCP, - Port: int32(mgmtPort), - TargetPort: intstr.FromInt(mgmtPort), - }, - }, - }, - } - - ctrl.SetControllerReference(memgraphha, dataInstanceClusterIP, r.Scheme) - return dataInstanceClusterIP -} diff --git a/internal/controller/memgraphha_reconciler.go b/internal/controller/memgraphha_reconciler.go deleted file mode 100644 index 49c3067..0000000 --- a/internal/controller/memgraphha_reconciler.go +++ /dev/null @@ -1,28 +0,0 @@ -/* -Copyright 2024 Memgraph Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// MemgraphHAReconciler reconciles a MemgraphHA object -type MemgraphHAReconciler struct { - client.Client - Scheme *runtime.Scheme -} diff --git a/internal/controller/memgraphha_setup_job.go b/internal/controller/memgraphha_setup_job.go deleted file mode 100644 index 6607335..0000000 --- a/internal/controller/memgraphha_setup_job.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -copyright 2024 memgraph ltd. - -licensed under the apache license, version 2.0 (the "license"); -you may not use this file except in compliance with the license. -you may obtain a copy of the license at - - http://www.apache.org/licenses/license-2.0 - -unless required by applicable law or agreed to in writing, software -distributed under the license is distributed on an "as is" basis, -without warranties or conditions of any kind, either express or implied. -see the license for the specific language governing permissions and -limitations under the license. -*/ - -package controller - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - memgraphv1 "github.com/memgraph/kubernetes-operator/api/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - ctrl "sigs.k8s.io/controller-runtime" -) - -func (r *MemgraphHAReconciler) reconcileSetupJob(ctx context.Context, memgraphha *memgraphv1.MemgraphHA, logger *logr.Logger) (bool, error) { - name := fmt.Sprintf("memgraph-setup") - logger.Info("Started reconciling", "Job", name) - setupJob := &batchv1.Job{} - err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: memgraphha.Namespace}, setupJob) - - if err == nil { - logger.Info("SetupJob already exists.", "Job", name) - return false, nil - } - - if errors.IsNotFound(err) { - job := r.createSetupJob(memgraphha) - logger.Info("Creating a new SetupJob", "SetupJob.Namespace", job.Namespace, "SetupJob.Name", job.Name) - err := r.Create(ctx, job) - if err != nil { - logger.Error(err, "Failed to create new SetupJob", "SetupJob.Namespace", job.Namespace, "SetupJob.Name", job.Name) - return true, err - } - logger.Info("SetupJob is created.", "Job", name) - return true, nil - } - - logger.Error(err, "Failed to fetch SetupJob", "Job", name) - return true, err - -} - -func (r *MemgraphHAReconciler) createSetupJob(memgraphha *memgraphv1.MemgraphHA) *batchv1.Job { - containerName := "memgraph-setup" - runAsUser := int64(0) - backoffLimit := int32(4) - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: containerName, - Namespace: memgraphha.Namespace, - }, - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: containerName, - Image: image, - Command: []string{"/bin/bash", "-c"}, - Args: []string{` - echo "Installing netcat..." - apt-get update && apt-get install -y netcat-openbsd - echo "Waiting for pods to become available for Bolt connection. Time: $(date +'%H:%M:%S')" - until nc -z memgraph-coordinator-1.default.svc.cluster.local 7687; do sleep 1; done - until nc -z memgraph-coordinator-2.default.svc.cluster.local 7687; do sleep 1; done - until nc -z memgraph-coordinator-3.default.svc.cluster.local 7687; do sleep 1; done - until nc -z memgraph-data-0.default.svc.cluster.local 7687; do sleep 1; done - until nc -z memgraph-data-1.default.svc.cluster.local 7687; do sleep 1; done - echo "Pods are available for Bolt connection. Running registration queries! Time: $(date +'%H:%M:%S')" - echo 'ADD COORDINATOR 2 WITH CONFIG {"bolt_server": "memgraph-coordinator-2.default.svc.cluster.local:7687", "management_server": "memgraph-coordinator-2.default.svc.cluster.local:10000", "coordinator_server": "memgraph-coordinator-2.default.svc.cluster.local:12000"};' | mgconsole --host memgraph-coordinator-1.default.svc.cluster.local --port 7687 - echo "Coordinator 2 added. Time: $(date +'%H:%M:%S')" - echo 'ADD COORDINATOR 3 WITH CONFIG {"bolt_server": "memgraph-coordinator-3.default.svc.cluster.local:7687", "management_server": "memgraph-coordinator-3.default.svc.cluster.local:10000", "coordinator_server": "memgraph-coordinator-3.default.svc.cluster.local:12000"};' | mgconsole --host memgraph-coordinator-1.default.svc.cluster.local --port 7687 - echo "Coordinator 3 added. Time: $(date +'%H:%M:%S')" - echo 'REGISTER INSTANCE instance_1 WITH CONFIG {"bolt_server": "memgraph-data-0.default.svc.cluster.local:7687", "management_server": "memgraph-data-0.default.svc.cluster.local:10000", "replication_server": "memgraph-data-0.default.svc.cluster.local:20000"};' | mgconsole --host memgraph-coordinator-1.default.svc.cluster.local --port 7687 - echo "Instance 1 added. Time: $(date +'%H:%M:%S')" - echo 'REGISTER INSTANCE instance_2 WITH CONFIG {"bolt_server": "memgraph-data-1.default.svc.cluster.local:7687", "management_server": "memgraph-data-1.default.svc.cluster.local:10000", "replication_server": "memgraph-data-1.default.svc.cluster.local:20000"};' | mgconsole --host memgraph-coordinator-1.default.svc.cluster.local --port 7687 - echo "Instance 2 added. Time: $(date +'%H:%M:%S')" - echo 'SET INSTANCE instance_1 TO MAIN;' | mgconsole --host memgraph-coordinator-1.default.svc.cluster.local --port 7687 - echo "Instance 1 set to main. Time: $(date +'%H:%M:%S')" - echo "Setup finished!" - `}, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &runAsUser, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - }, - }, - BackoffLimit: &backoffLimit, - }, - } - - ctrl.SetControllerReference(memgraphha, job, r.Scheme) - return job -} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go new file mode 100644 index 0000000..2ed9dea --- /dev/null +++ b/internal/controller/suite_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = memgraphcomv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // Retrieve the first found binary directory to allow running tests from IDEs + if getFirstFoundEnvTestBinaryDir() != "" { + testEnv.BinaryAssetsDirectory = getFirstFoundEnvTestBinaryDir() + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + Eventually(func() error { + return testEnv.Stop() + }, time.Minute, time.Second).Should(Succeed()) +}) + +// getFirstFoundEnvTestBinaryDir locates the first binary in the specified path. +// ENVTEST-based tests depend on specific binaries, usually located in paths set by +// controller-runtime. When running tests directly (e.g., via an IDE) without using +// Makefile targets, the 'BinaryAssetsDirectory' must be explicitly configured. +// +// This function streamlines the process by finding the required binaries, similar to +// setting the 'KUBEBUILDER_ASSETS' environment variable. To ensure the binaries are +// properly set up, run 'make setup-envtest' beforehand. +func getFirstFoundEnvTestBinaryDir() string { + basePath := filepath.Join("..", "..", "bin", "k8s") + entries, err := os.ReadDir(basePath) + if err != nil { + logf.Log.Error(err, "Failed to read directory", "path", basePath) + return "" + } + for _, entry := range entries { + if entry.IsDir() { + return filepath.Join(basePath, entry.Name()) + } + } + return "" +} diff --git a/specs/operator-mvp/PRD.md b/specs/operator-mvp/PRD.md new file mode 100644 index 0000000..fb3d6e3 --- /dev/null +++ b/specs/operator-mvp/PRD.md @@ -0,0 +1,129 @@ +# PRD: Memgraph Kubernetes Operator — v1alpha1 MVP + +## Problem Statement + +Operating a Memgraph high-availability cluster on Kubernetes today means installing the `memgraph-high-availability` Helm chart, which has two structural problems from the user's perspective: + +1. **Cluster registration is fire-and-forget.** A post-install Job registers coordinators and data instances once, then exits. If a pod is later rescheduled and loses its registration state, nothing re-registers it — the user must notice the degraded cluster and intervene manually with mgconsole. +2. **The configuration surface fights the user.** Every coordinator and data instance is its own copy-pasted values block backing its own StatefulSet and Services. Growing the cluster means duplicating a ~20-line block and hand-assigning IDs; the topology is spread across many near-identical resources instead of being expressed as "3 coordinators, 2 data instances." + +Users of comparable databases (MongoDB, Elasticsearch, CockroachDB) expect a Kubernetes operator: declare the cluster as a single custom resource, and a controller continuously drives reality toward it. + +## Solution + +A Go operator (kubebuilder) exposing a `MemgraphCluster` custom resource in the `memgraph.com/v1alpha1` API group (short name `mgc`). The user writes one resource declaring topology as two numbers — coordinator count and data-instance count — plus standard pod-level knobs, and the operator: + +- Provisions **one StatefulSet per role** (coordinators, data instances) with headless Services, deriving per-pod identity (coordinator ID, advertised addresses) from pod ordinals — no per-instance configuration blocks. +- **Bootstraps the HA cluster**: adds coordinators, registers data instances, and promotes the initial MAIN. +- **Continuously reconciles registration**: every reconcile compares `SHOW INSTANCES` on the coordinator leader against the declared topology and re-issues only the missing registrations, so a wiped or rescheduled pod rejoins the cluster without human action. +- **Reports cluster state** on the CR status: which instance is MAIN, registration convergence, and readiness conditions. + +The operator is a redesign, not a port of the chart. It will reach feature parity with the HA chart over subsequent releases, after which the chart is frozen and deprecated with a documented migration path. The standalone `memgraph` and `memgraph-lab` charts are unaffected. + +This MVP is deliberately "provision, bootstrap, observe": both replica counts are immutable after creation, and day-2 operations are excluded. + +## User Stories + +1. As a database operator, I want to declare my entire HA cluster as a single `MemgraphCluster` resource, so that the cluster topology lives in one reviewable, GitOps-committable object. +2. As a database operator, I want to set the number of coordinators and data instances as single integer fields, so that I don't copy-paste per-instance configuration blocks. +3. As a database operator, I want the operator to register all coordinators and data instances automatically after the pods start, so that I never run mgconsole registration commands by hand. +4. As a database operator, I want the operator to promote an initial MAIN automatically during bootstrap, so that the cluster is writable without manual promotion. +5. As a database operator, I want a data instance that lost its registration (e.g. after rescheduling onto a fresh node) to be re-registered automatically, so that transient infrastructure events don't silently degrade my cluster. +6. As a database operator, I want the operator to leave failover decisions entirely to the Raft coordinators after bootstrap, so that two control systems never fight over which instance is MAIN. +7. As a database operator, I want to see which instance is currently MAIN in the CR status, so that I can inspect cluster health with `kubectl get mgc` instead of querying coordinators. +8. As a database operator, I want status conditions telling me whether the cluster is converged (all declared instances registered and healthy), so that my monitoring and GitOps tooling can gate on it. +9. As a database operator, I want to specify the Memgraph image repository, tag, and pull policy, so that I control exactly which Memgraph version runs. +10. As a database operator, I want to reference my existing Kubernetes Secret for the enterprise license and organization name (configurable secret name and key names, matching the chart's `secrets` block), so that credentials never appear in the CR and my current Secret works unchanged. +11. As a database operator, I want to configure PVC size, access mode, and storage class for lib and log storage per role, so that storage matches my cluster's capabilities. +12. As a database operator, I want to choose whether PVCs are retained or deleted when the CR is deleted (default: retained), so that production data survives accidental deletion while dev clusters clean up after themselves. +13. As a database operator, I want to configure resource requests and limits per role, so that pods are scheduled and bounded appropriately. +14. As a database operator, I want to tune probe timings per role (startup, readiness, liveness), so that large snapshot restores don't get killed mid-load. +15. As a database operator, I want to set custom labels on pods, StatefulSets, and Services per role, so that the resources integrate with my organization's selectors and policies. +16. As a database operator, I want to configure the internal ports (bolt, management, replication, coordinator), so that I can resolve port conflicts with other workloads or policies. +17. As a database operator, I want to set the cluster domain used in advertised FQDNs, so that the operator works on clusters with a non-default DNS domain. +18. As a database operator, I want to pass additional non-secret Memgraph flags and environment variables per role, so that I can use any Memgraph flag without waiting for a typed CRD field. +19. As a database operator, I want attempts to change the coordinator or data-instance count on a live cluster to be rejected at admission time with a clear message, so that I cannot accidentally trigger an unsupported topology change in v1. +20. As a database operator, I want the operator's registration logic to be idempotent (read cluster state before issuing commands), so that an operator crash or restart mid-reconcile causes no harm. +21. As a database operator, I want to install the operator itself with a Helm chart from the existing `memgraph.github.io/helm-charts` repository, so that I use the same helm repo I already have configured. +22. As a database operator, I want the operator to run namespaced with least-privilege RBAC generated by the install chart, so that it passes my cluster's security review. +23. As a platform engineer, I want the operator to run as a non-root user with a restricted security context (matching Memgraph's uid 101 / gid 103 conventions for workload pods), so that it complies with restricted Pod Security Standards. +24. As a platform engineer, I want the CR to be safe to store in git (no secret material in spec or status), so that GitOps workflows need no redaction. +25. As a developer evaluating Memgraph, I want a minimal `MemgraphCluster` example that boots a working cluster with only image, counts, and a license secret reference, so that first contact takes minutes. +26. As a Memgraph chart user planning migration, I want the operator's secret block and knob names to mirror the chart's vocabulary where concepts carry over, so that translating my values file is mechanical. +27. As a contributor, I want the resource-building and registration-planning logic to be pure and unit-testable without a cluster, so that I can develop and verify changes quickly. +28. As a maintainer, I want every pull request gated on unit, envtest, and KinD end-to-end suites, so that broken registration logic never merges. + +## Implementation Decisions + +### Strategy and repository layout + +- The operator ultimately **replaces the HA Helm chart**: build to functional parity, publish a migration guide, then freeze the chart (security fixes only) with a deprecation timeline. Standalone and Lab charts continue independently. +- Two repositories: `helm-charts` stays untouched (its GitHub Pages URL is load-bearing for existing users); the operator lives in the existing `memgraph/kubernetes-operator` repository. The prior contents are a discarded attempt: parked on an archive branch, with a fresh kubebuilder scaffold force-pushed to `main`. +- The **operator install chart lives in the operator repository** (next to the generated CRDs so they can never drift), and the release workflow cross-publishes the packaged chart into the existing `memgraph.github.io/helm-charts` index. +- Implementation is **Go with kubebuilder** — the operator's value is state-aware reconciliation over Bolt, which helm/ansible-based operators cannot express. + +### API + +- Kind `MemgraphCluster`, group `memgraph.com`, version `v1alpha1`, short name `mgc`. +- v1alpha1 supports **HA topology only**, but topology fields are shaped so a standalone (single-instance) mode can be added later without a breaking API change — no structurally mandatory HA-only fields. +- Topology is declared as **two integer replica counts** (coordinators, data instances). Both are **immutable after creation, enforced by CEL validation** — no webhook needed for this in v1. +- All pods within a role are **uniform**; identity-dependent flags (coordinator ID, advertised addresses) are derived from the StatefulSet pod ordinal. +- Spec knobs in v1: image (repository, tag, pull policy), cluster domain, internal ports, lib/log PVC configuration per role, storage retention policy, probe timings per role, resources per role, labels per role, and a freeform non-secret env/args passthrough per role. Knob vocabulary mirrors the HA chart where the concept carries over. +- License and organization are consumed via a **secret reference block identical in shape to the chart's** (`secrets.name`, `secrets.licenseKey`, `secrets.organizationKey`). A Bolt-auth secret reference joins this block in a later version. +- **No Memgraph version enforcement**: the image tag is plain user input; the operator assumes the HA query surface (`SHOW INSTANCES`, `REGISTER INSTANCE`, `ADD COORDINATOR`, `SET INSTANCE TO MAIN`) is stable across versions. +- Storage: `storage.retentionPolicy` (`Retain` | `Delete`, default `Retain`) maps directly onto the StatefulSet PVC retention policy (`whenDeleted`). The operator carries **no finalizer-based storage cleanup** — no destructive code paths in v1. + +### Workload architecture + +- **One StatefulSet for all coordinators and one for all data instances**, each backed by a headless Service — a deliberate redesign away from the chart's StatefulSet-per-instance model. +- Workload pods keep the established Memgraph security conventions: non-root (uid 101 / gid 103), seccomp RuntimeDefault, all capabilities dropped. + +### Reconciliation semantics + +- The reconcile loop does **continuous registration reconciliation, hands-off leadership**: each reconcile queries `SHOW INSTANCES` on the coordinator leader, diffs against declared topology, and issues only the missing `ADD COORDINATOR` / `REGISTER INSTANCE` commands. Registration state that a pod loses is restored automatically. +- The operator issues `SET INSTANCE TO MAIN` **exactly once, at bootstrap** (when no MAIN exists). After that, failover belongs to the Raft coordinators; the operator only observes and reports MAIN in status. +- Reconciliation is **idempotent and read-before-write**: cluster state is always queried before commands are issued, so operator crashes or restarts mid-reconcile are harmless. +- The CR status carries the observed MAIN instance and convergence/readiness conditions. + +### Module structure + +Seven modules, with two deep pure cores and one mock seam: + +1. **API types** — `MemgraphCluster` spec/status types with CEL markers; generated CRD manifests. The public contract. +2. **Resource builders** — pure functions from spec to desired Kubernetes objects (StatefulSets, headless Services, PVC templates, probes, ordinal-derived args, secret/env wiring). No API calls, no side effects. +3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. All higher layers depend on the interface, never the driver — this is the mock seam for testing. +4. **Registration planner** — pure diff logic: declared topology plus observed instances in, ordered registration commands out (empty when converged). The reconciliation semantics above live here. +5. **Controller** — reconciler wiring: fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status and conditions. +6. **Operator install chart** — CRDs, RBAC, controller Deployment; cross-published to the existing helm repo index at release time. +7. **E2E harness** — multi-node KinD suite booting a real licensed Memgraph cluster. + +## Testing Decisions + +- A good test asserts **external behavior, not implementation**: given a spec, the right Kubernetes objects exist; given a topology and an observed cluster state, the right registration commands (and only those) are planned; given a degraded real cluster, it converges. Tests never assert on internal call ordering or private state. +- **Resource builders**: golden-style unit tests — spec in, expected objects out — covering defaults, overrides, and ordinal-derived identity. +- **Registration planner**: pure unit tests over topology-diff cases — fresh cluster bootstrap, single lost registration, fully converged no-op, coordinator missing, MAIN already present (no promotion issued). +- **Controller**: envtest (real kube-apiserver, no kubelet) with the HA client mocked behind its interface — asserting created resources, status updates, and that immutability violations are rejected. +- **E2E**: KinD multi-node cluster, real Memgraph images, enterprise license supplied via repository secrets (established practice from the HA chart's CI); asserts a `MemgraphCluster` reaches a registered, MAIN-elected state, and that deleting a registered pod's state leads to automatic re-registration. +- **CI gate**: unit, envtest, and e2e suites all run on every pull request. Chaos/soak testing is explicitly deferred to the separate HA chaos-testing project. +- Prior art: the HA chart's CI already boots licensed multi-node clusters (Minikube) with the license as a repository secret; the e2e suite follows that pattern on KinD. + +## Out of Scope + +- **Day-2 operations**: rolling/no-downtime upgrades, scaling (both counts are immutable in v1), backup/restore orchestration, storage-mode changes. +- **External access** of any kind (LoadBalancer, NodePort, ingress, gateway) — deliberately deferred because the approach is expected to change; v1 is in-cluster access only. +- **Embedded ingress-nginx controller installation** — permanently dropped, not just deferred; users bring their own ingress controller. +- **TLS** (bolt and intra-cluster) — later version. +- **Monitoring** (Prometheus exporter, ServiceMonitor, Grafana dashboards, vmagent/Vector integrations) — later version. +- **Bolt authentication support** (and the operator authenticating its own connections) — later version. +- **Standalone (non-HA) topology** — designed-for but not implemented in v1alpha1. +- **In-place adoption of chart-deployed clusters** — never; migration is fresh-cluster only (backup/restore or replication cutover), documented when parity is reached. +- **Affinity strategies, tolerations, init containers, sidecar/user containers, core-dump handling, snapshot-restore fields** from the chart — parity roadmap, not MVP. +- **Memgraph version compatibility logic** — no version parsing, gating, or branching. +- **Coordinator or data-instance removal** (`REMOVE COORDINATOR`, `UNREGISTER INSTANCE`) — arrives with mutable counts post-v1. + +## Further Notes + +- Parity with the HA chart is the milestone for *deprecating the chart*, not for the first release; the MVP ships early as an alpha to validate the reconciliation core while parity features land incrementally. +- Post-v1 roadmap order (indicative): data-instance scale-up (registration on grow), then scale-down with MAIN-safety, then external access, TLS, monitoring, orchestrated upgrades. +- The chaos-testing project (ArgoCD, ChaosMesh, VictoriaMetrics on EKS) is the natural proving ground for the operator's re-registration behavior once both exist. +- The full decision log behind this PRD was produced in a structured design interview on 2026-07-23 (15 resolved decisions). diff --git a/specs/operator-mvp/issues/01-repo-reset-scaffold.md b/specs/operator-mvp/issues/01-repo-reset-scaffold.md new file mode 100644 index 0000000..fa14ef3 --- /dev/null +++ b/specs/operator-mvp/issues/01-repo-reset-scaffold.md @@ -0,0 +1,25 @@ +# Repo reset + kubebuilder scaffold + CI skeleton + +**Type**: HITL — involves a destructive force-push and archiving the old attempt; a human must bless and execute the push. + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Reset the `memgraph/kubernetes-operator` repository for the fresh operator effort. Park the existing contents (a discarded prior attempt) on an archive branch, then force-push a clean kubebuilder scaffold to `main`: Go module, `MemgraphCluster` API skeleton in group `memgraph.com/v1alpha1` (short name `mgc`), a hello-world reconciler, and the PRD plus these issues carried into the new history. + +Stand up the CI skeleton alongside: lint, unit tests, and an envtest run against the placeholder reconciler, all triggered on every pull request. The suites may be near-empty — the point is that the pipeline exists and is green before feature work starts, so every later slice lands PR-gated. + +## Acceptance criteria + +- [ ] Old repository contents preserved on an `archive/`-prefixed branch +- [ ] `main` holds a fresh kubebuilder scaffold with kind `MemgraphCluster`, group `memgraph.com`, version `v1alpha1`, short name `mgc` +- [ ] `specs/operator-mvp/` (PRD + issues) committed as part of the new history +- [ ] CI runs lint, unit, and envtest suites on every pull request and is green +- [ ] Generated CRD manifests install cleanly on a local cluster and `kubectl get mgc` resolves + +## Blocked by + +None - can start immediately diff --git a/specs/operator-mvp/issues/02-provisioning-walking-skeleton.md b/specs/operator-mvp/issues/02-provisioning-walking-skeleton.md new file mode 100644 index 0000000..06e642f --- /dev/null +++ b/specs/operator-mvp/issues/02-provisioning-walking-skeleton.md @@ -0,0 +1,26 @@ +# Provisioning walking skeleton + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The first real vertical slice: applying a minimal `MemgraphCluster` — coordinator count, data-instance count, image (repository/tag/pullPolicy), and the chart-compatible secrets block (`secrets.name`, `secrets.licenseKey`, `secrets.organizationKey`) — makes the controller provision one StatefulSet for all coordinators and one for all data instances, each backed by a headless Service. Pods boot licensed Memgraph in HA roles with identity-dependent flags (coordinator ID, advertised FQDN addresses) derived from the pod ordinal. No registration yet — the demo is "apply one CR, watch licensed coordinator and data pods reach ready." + +Structure the code along the module boundaries from the PRD: pure resource builders (spec in, desired objects out — no API calls) invoked by the controller via server-side apply. Workload pods follow Memgraph security conventions: non-root uid 101 / gid 103, seccomp RuntimeDefault, all capabilities dropped. + +## Acceptance criteria + +- [ ] Applying a minimal `MemgraphCluster` produces exactly two StatefulSets (coordinators, data) with matching headless Services +- [ ] All pods reach ready with the enterprise license consumed from the referenced Secret using the configured key names +- [ ] Coordinator IDs and advertised addresses are derived from pod ordinals; all pods within a role are uniform +- [ ] Resource builders are pure and covered by golden-style unit tests (defaults and ordinal-derived identity) +- [ ] Controller behavior covered by envtest: CR in, expected objects created; re-reconcile is idempotent +- [ ] Deleting the CR removes the workloads (PVC handling comes in a later slice) + +## Blocked by + +- `01-repo-reset-scaffold.md` diff --git a/specs/operator-mvp/issues/03-bootstrap-registration.md b/specs/operator-mvp/issues/03-bootstrap-registration.md new file mode 100644 index 0000000..d9869af --- /dev/null +++ b/specs/operator-mvp/issues/03-bootstrap-registration.md @@ -0,0 +1,26 @@ +# Bootstrap registration + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Make a freshly provisioned cluster become an actual HA cluster without human action. Introduce the Memgraph HA client — a narrow Go interface (show instances, add coordinator, register instance, set main) over the Bolt driver, with all higher layers depending on the interface — and the registration planner: pure diff logic that takes declared topology plus observed `SHOW INSTANCES` output and returns the ordered commands needed (empty when converged). + +Wire both into the controller: once pods are ready, query the coordinator leader, plan, and execute — adding coordinators, registering data instances, and issuing the initial MAIN promotion exactly once (only when no MAIN exists). All interaction is idempotent and read-before-write, so an operator restart mid-bootstrap is harmless. The demo: apply a CR, then `SHOW INSTANCES` on a coordinator shows every declared instance registered with one MAIN elected. + +## Acceptance criteria + +- [ ] A fresh `MemgraphCluster` converges to fully registered: all coordinators added, all data instances registered, one MAIN promoted +- [ ] MAIN promotion is issued only when no MAIN exists; an existing MAIN is never overridden +- [ ] Killing and restarting the operator mid-bootstrap still converges with no duplicate or failed registrations +- [ ] Planner covered by pure unit tests: fresh-cluster bootstrap, partially registered cluster, fully converged no-op, MAIN already present +- [ ] Controller registration flow covered by envtest with the HA client mocked behind its interface +- [ ] The HA client interface is the only place the Bolt driver is referenced + +## Blocked by + +- `02-provisioning-walking-skeleton.md` diff --git a/specs/operator-mvp/issues/04-kind-e2e-harness.md b/specs/operator-mvp/issues/04-kind-e2e-harness.md new file mode 100644 index 0000000..1a7c1fc --- /dev/null +++ b/specs/operator-mvp/issues/04-kind-e2e-harness.md @@ -0,0 +1,25 @@ +# KinD e2e harness, PR-gated + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +A true end-to-end suite: a multi-node KinD cluster in CI, the operator image built and deployed into it, a `MemgraphCluster` applied with real Memgraph images and the enterprise license supplied via repository secrets (the established practice from the HA chart's CI). The suite asserts the cluster reaches a registered state with a MAIN elected — the real-world proof of slices 02 and 03. + +Wire the suite into CI as a required check on every pull request, alongside the existing lint/unit/envtest gates. Keep the harness structured so later slices can add scenarios (re-registration, storage retention) as additional cases rather than new pipelines. + +## Acceptance criteria + +- [ ] CI boots a multi-node KinD cluster, builds and deploys the operator image, and applies a `MemgraphCluster` +- [ ] The suite asserts all declared instances appear registered in `SHOW INSTANCES` with exactly one MAIN +- [ ] Enterprise license flows from repository secrets; no secret material appears in logs or the repo +- [ ] The e2e job is a required PR check and passes on the current main +- [ ] Adding a new e2e scenario requires only a new test case, not pipeline changes + +## Blocked by + +- `03-bootstrap-registration.md` diff --git a/specs/operator-mvp/issues/05-continuous-re-registration.md b/specs/operator-mvp/issues/05-continuous-re-registration.md new file mode 100644 index 0000000..b46fea6 --- /dev/null +++ b/specs/operator-mvp/issues/05-continuous-re-registration.md @@ -0,0 +1,27 @@ +# Continuous re-registration + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The operator's reason to exist over the chart's one-shot setup Job: registration state that a pod loses (rescheduled onto a fresh node, wiped storage) is restored automatically. Extend the registration planner beyond bootstrap to full diff semantics — every reconcile compares declared topology against observed `SHOW INSTANCES` and issues only the missing registrations. Leadership stays hands-off: the planner never emits a MAIN promotion when a MAIN already exists; failover belongs to the Raft coordinators. + +Ensure the controller re-reconciles on relevant events (pod changes, periodic resync) so drift is detected without manual triggers. The demo: forcibly de-register or wipe a data instance, watch the operator converge the cluster back to fully registered with no human action. + +## Acceptance criteria + +- [ ] A data instance whose registration is lost is automatically re-registered on a subsequent reconcile +- [ ] A missing coordinator is automatically re-added +- [ ] A converged cluster produces zero commands on reconcile (verified no-op) +- [ ] No MAIN promotion is ever issued while a MAIN exists, including during recovery +- [ ] Planner unit tests cover: single lost registration, multiple lost, converged no-op, recovery with MAIN present +- [ ] E2E scenario: wipe one instance's registration state, assert the cluster converges back to fully registered + +## Blocked by + +- `03-bootstrap-registration.md` +- `04-kind-e2e-harness.md` diff --git a/specs/operator-mvp/issues/06-status-and-conditions.md b/specs/operator-mvp/issues/06-status-and-conditions.md new file mode 100644 index 0000000..2e30178 --- /dev/null +++ b/specs/operator-mvp/issues/06-status-and-conditions.md @@ -0,0 +1,25 @@ +# Status & conditions + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Make cluster health inspectable without querying coordinators by hand. The CR status subresource reports the observed MAIN instance and standard conditions expressing convergence (all declared instances registered) and readiness. Status is observation only — it carries no secret material and is never used as reconcile input state. + +Add printer columns so `kubectl get mgc` answers the everyday questions at a glance: coordinator count, data-instance count, current MAIN, ready/converged state, age. GitOps tooling and monitoring should be able to gate on the conditions. + +## Acceptance criteria + +- [ ] Status reports the currently observed MAIN instance and updates when failover changes it +- [ ] Conditions express convergence and readiness, transitioning correctly through bootstrap, converged, and degraded states +- [ ] `kubectl get mgc` shows counts, MAIN, readiness, and age via printer columns +- [ ] Status updates use the status subresource and never modify spec +- [ ] Envtest coverage: status reflects mocked cluster states (bootstrapping, converged, degraded) + +## Blocked by + +- `03-bootstrap-registration.md` diff --git a/specs/operator-mvp/issues/07-cel-immutability-validation.md b/specs/operator-mvp/issues/07-cel-immutability-validation.md new file mode 100644 index 0000000..c1dfa98 --- /dev/null +++ b/specs/operator-mvp/issues/07-cel-immutability-validation.md @@ -0,0 +1,23 @@ +# CEL immutability + validation + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Enforce the v1 contract that topology is fixed at creation: CEL validation rules on the CRD reject any change to the coordinator count or data-instance count on a live cluster, with a clear admission-time message telling the user scaling is not yet supported. Add creation-time validation (sane count ranges, required fields) and spec defaulting so a minimal CR is valid — all through CRD machinery, no admission webhook in v1. + +## Acceptance criteria + +- [ ] Updating either replica count on an existing `MemgraphCluster` is rejected at admission with a message stating counts are immutable in v1 +- [ ] Invalid counts and missing required fields are rejected at creation with actionable messages +- [ ] Optional fields receive documented defaults; a minimal CR (counts, image, secrets) validates +- [ ] No admission webhook is introduced; all rules live in the CRD schema +- [ ] Envtest coverage: mutation attempts rejected, valid creates accepted, defaults materialize + +## Blocked by + +- `02-provisioning-walking-skeleton.md` diff --git a/specs/operator-mvp/issues/08-storage-configuration.md b/specs/operator-mvp/issues/08-storage-configuration.md new file mode 100644 index 0000000..e78301d --- /dev/null +++ b/specs/operator-mvp/issues/08-storage-configuration.md @@ -0,0 +1,24 @@ +# Storage configuration + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Give users control over persistence, mirroring the HA chart's storage vocabulary per role: lib and log PVC size, access mode, and storage class for coordinators and data instances. Add `storage.retentionPolicy` (`Retain` | `Delete`, default `Retain`) mapped directly onto the StatefulSet PVC retention policy, so deleting the CR preserves data by default while dev clusters can opt into self-cleanup. The operator carries no finalizer-based storage cleanup of its own — the StatefulSet machinery is the only deleter. + +## Acceptance criteria + +- [ ] PVC size, access mode, and storage class are configurable per role for lib and log volumes +- [ ] Default retention: deleting the CR leaves PVCs behind +- [ ] With `Delete` retention, deleting the CR removes the PVCs via StatefulSet retention machinery +- [ ] No operator-owned finalizer performs storage deletion +- [ ] Builder golden tests cover storage defaults, overrides, and both retention policies +- [ ] E2E scenario: default-retention CR deletion leaves PVCs intact + +## Blocked by + +- `02-provisioning-walking-skeleton.md` diff --git a/specs/operator-mvp/issues/09-pod-tuning-knobs.md b/specs/operator-mvp/issues/09-pod-tuning-knobs.md new file mode 100644 index 0000000..dc25b1e --- /dev/null +++ b/specs/operator-mvp/issues/09-pod-tuning-knobs.md @@ -0,0 +1,26 @@ +# Pod-tuning knobs + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The remaining v1 configuration surface, per role, using the HA chart's vocabulary where concepts carry over: probe timings (startup, readiness, liveness — probe type stays fixed to the established TCP-socket convention), resource requests/limits, custom labels on pods/StatefulSets/Services, internal ports (bolt, management, replication, coordinator), the cluster domain used in advertised FQDNs, and a freeform non-secret env/args passthrough so any Memgraph flag is usable without a typed field. + +Ports and cluster domain are the delicate part: they feed the ordinal-derived advertised addresses, so changing them must flow consistently through builders, registration planning, and the HA client's connection targets. + +## Acceptance criteria + +- [ ] Probe timings, resources, and labels are configurable per role and land on the right objects +- [ ] Internal ports and cluster domain are configurable and propagate consistently to container ports, Services, advertised addresses, and registration commands +- [ ] Non-secret env vars and extra args pass through per role; secret material remains only in the secrets block +- [ ] A CR with all knobs defaulted behaves identically to before this slice +- [ ] Builder golden tests cover each knob's default and override, including non-default ports/domain flowing into advertised addresses +- [ ] Planner unit tests confirm registration commands use the configured ports and domain + +## Blocked by + +- `02-provisioning-walking-skeleton.md` diff --git a/specs/operator-mvp/issues/10-operator-install-chart.md b/specs/operator-mvp/issues/10-operator-install-chart.md new file mode 100644 index 0000000..c64aac1 --- /dev/null +++ b/specs/operator-mvp/issues/10-operator-install-chart.md @@ -0,0 +1,24 @@ +# Operator install chart + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The Helm chart users install the operator with, living in this repository next to the generated CRDs so they can never drift from the controller version. The chart ships the CRDs, a least-privilege RBAC set scoped to what the controller actually touches (its CRD, StatefulSets, Services, Secrets read, events, leases), and the controller Deployment running non-root with a restricted security context. `helm install` from the local chart on a clean cluster must be the complete install story. + +## Acceptance criteria + +- [ ] `helm install` from the local chart on a clean cluster yields a running operator that reconciles a `MemgraphCluster` +- [ ] CRDs in the chart are generated from the Go types in the same commit — no hand-edited copies +- [ ] RBAC grants only the verbs/resources the controller uses; the e2e suite passes under that RBAC +- [ ] Controller pod runs non-root with a restricted security context +- [ ] Chart lints clean and install/uninstall is exercised in CI +- [ ] Image tag/repository, resources, and namespace are configurable chart values + +## Blocked by + +- `02-provisioning-walking-skeleton.md` diff --git a/specs/operator-mvp/issues/11-release-cross-publish.md b/specs/operator-mvp/issues/11-release-cross-publish.md new file mode 100644 index 0000000..ce2d96b --- /dev/null +++ b/specs/operator-mvp/issues/11-release-cross-publish.md @@ -0,0 +1,24 @@ +# Release pipeline + cross-publish + +**Type**: HITL — requires an org-level fine-grained PAT or deploy key for the helm-charts repository, which only a maintainer can create and store. + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The release path from a version tag to installable artifacts: build and push the operator container image, package the install chart, and cross-publish the packaged chart into the existing `memgraph.github.io/helm-charts` index — so users install the operator from the same helm repository they already have configured, while chart source and CRDs stay in this repository. Chart version, image tag, and git tag stay in lockstep per release. + +## Acceptance criteria + +- [ ] Pushing a version tag builds and publishes the operator image with that version +- [ ] The same pipeline packages the install chart and publishes it into the `memgraph.github.io/helm-charts` index +- [ ] `helm repo update && helm install` from the existing Memgraph helm repo installs the tagged operator version end-to-end +- [ ] Chart version, appVersion, and image tag agree for every release +- [ ] The cross-repo credential is a scoped fine-grained PAT or deploy key stored as a repository secret, documented for rotation +- [ ] A dry-run/prerelease path exists to validate the pipeline without polluting the public index + +## Blocked by + +- `10-operator-install-chart.md` diff --git a/specs/operator-mvp/issues/12-quickstart-docs.md b/specs/operator-mvp/issues/12-quickstart-docs.md new file mode 100644 index 0000000..fd498f3 --- /dev/null +++ b/specs/operator-mvp/issues/12-quickstart-docs.md @@ -0,0 +1,24 @@ +# Quickstart example + README + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +The minutes-to-cluster story for a developer evaluating Memgraph: a README walkthrough covering install (operator chart), a minimal `MemgraphCluster` example manifest (counts, image, license secret reference — everything else defaulted), and how to verify the cluster (`kubectl get mgc`, connecting over Bolt). Document the v1 contract honestly: counts are immutable, day-2 operations / external access / TLS / monitoring are not yet supported, and the operator never interferes with coordinator-driven failover. State the relationship to the HA Helm chart (operator is its successor; migration is fresh-cluster only, guide to come at parity). + +## Acceptance criteria + +- [ ] A newcomer can go from empty cluster to a registered, MAIN-elected Memgraph HA cluster following only the README +- [ ] The minimal example manifest works verbatim with only the license Secret substituted +- [ ] v1 limitations (immutable counts, deferred features) are stated explicitly +- [ ] Verification steps show expected `kubectl get mgc` output and a Bolt connection +- [ ] The example manifest is exercised in CI so it cannot rot + +## Blocked by + +- `04-kind-e2e-harness.md` +- `06-status-and-conditions.md` diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 63e2784..b1b0424 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,5 +1,8 @@ +//go:build e2e +// +build e2e + /* -Copyright 2024 Memgraph Ltd. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,15 +21,99 @@ package e2e import ( "fmt" + "os" + "os/exec" "testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + + "github.com/memgraph/kubernetes-operator/test/utils" ) -// Run e2e tests using the Ginkgo runner. +var ( + // managerImage is the manager image to be built and loaded for testing. + managerImage = "example.com/kubernetes-operator:v0.0.1" + // shouldCleanupCertManager tracks whether CertManager was installed by this suite. + shouldCleanupCertManager = false +) + +// TestE2E runs the e2e test suite to validate the solution in an isolated environment. +// The default setup requires Kind and CertManager. +// +// To enable kubectl kuberc (use custom kubectl configurations), set: KUBECTL_KUBERC=true +// By default, kuberc is disabled to ensure consistent test behavior across different environments. +// To skip CertManager installation, set: CERT_MANAGER_INSTALL_SKIP=true func TestE2E(t *testing.T) { RegisterFailHandler(Fail) - fmt.Fprintf(GinkgoWriter, "Starting kubernetes-operator suite\n") + _, _ = fmt.Fprintf(GinkgoWriter, "Starting kubernetes-operator e2e test suite\n") RunSpecs(t, "e2e suite") } + +var _ = BeforeSuite(func() { + By("building the manager image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") + + // TODO(user): If you want to change the e2e test vendor from Kind, + // ensure the image is built and available, then remove the following block. + By("loading the manager image on Kind") + err = utils.LoadImageToKindClusterWithName(managerImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + + configureKubectlKubeRC() + setupCertManager() +}) + +var _ = AfterSuite(func() { + teardownCertManager() +}) + +// Disable kubectl kuberc by default for test isolation. +// This prevents local kubectl configurations from affecting test behavior. +// To enable kuberc, set: KUBECTL_KUBERC=true +func configureKubectlKubeRC() { + if os.Getenv("KUBECTL_KUBERC") != "true" { + By("disabling kubectl kuberc for test isolation") + err := os.Setenv("KUBECTL_KUBERC", "false") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to disable kubectl kuberc") + _, _ = fmt.Fprintf(GinkgoWriter, + "kubectl kuberc disabled for consistent test behavior (override with KUBECTL_KUBERC=true)\n") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "kubectl kuberc enabled (KUBECTL_KUBERC=true)\n") + } +} + +// setupCertManager installs CertManager if needed for webhook tests. +// Skips installation if CERT_MANAGER_INSTALL_SKIP=true or if already present. +func setupCertManager() { + if os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true" { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager installation (CERT_MANAGER_INSTALL_SKIP=true)\n") + return + } + + By("checking if CertManager is already installed") + if utils.IsCertManagerCRDsInstalled() { + _, _ = fmt.Fprintf(GinkgoWriter, "CertManager is already installed. Skipping installation.\n") + return + } + + // Mark for cleanup before installation to handle interruptions and partial installs. + shouldCleanupCertManager = true + + By("installing CertManager") + Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager") +} + +// teardownCertManager uninstalls CertManager if it was installed by setupCertManager. +// This ensures we only remove what we installed. +func teardownCertManager() { + if !shouldCleanupCertManager { + _, _ = fmt.Fprintf(GinkgoWriter, "Skipping CertManager cleanup (not installed by this suite)\n") + return + } + + By("uninstalling CertManager") + utils.UninstallCertManager() +} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 8e55b96..d606215 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -1,5 +1,8 @@ +//go:build e2e +// +build e2e + /* -Copyright 2024 Memgraph Ltd. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,8 +20,11 @@ limitations under the License. package e2e import ( + "encoding/json" "fmt" + "os" "os/exec" + "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -27,64 +33,119 @@ import ( "github.com/memgraph/kubernetes-operator/test/utils" ) +// namespace where the project is deployed in const namespace = "kubernetes-operator-system" -var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) +// serviceAccountName created for the project +const serviceAccountName = "kubernetes-operator-controller-manager" + +// metricsServiceName is the name of the metrics service of the project +const metricsServiceName = "kubernetes-operator-controller-manager-metrics-service" + +// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data +const metricsRoleBindingName = "kubernetes-operator-metrics-binding" - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) +var _ = Describe("Manager", Ordered, func() { + var controllerPodName string + // Before running the tests, set up the environment by creating the namespace, + // enforce the restricted security policy to the namespace, installing CRDs, + // and deploying the controller. + BeforeAll(func() { By("creating manager namespace") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) + // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, + // and deleting the namespace. AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() + By("cleaning up the curl pod for metrics") + cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) + _, _ = utils.Run(cmd) - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() + By("undeploying the controller-manager") + cmd = exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) + cmd = exec.Command("kubectl", "delete", "ns", namespace) _, _ = utils.Run(cmd) }) - Context("Operator", func() { - It("should run successfully", func() { - var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "example.com/kubernetes-operator:v0.0.1" + // After each test, check for failures and collect logs, events, + // and pod descriptions for debugging. + AfterEach(func() { + specReport := CurrentSpecReport() + if specReport.Failed() { + By("Fetching controller manager pod logs") + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + controllerLogs, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err) + } - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("Fetching Kubernetes events") + cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + eventsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err) + } - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + By("Fetching curl-metrics logs") + cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := utils.Run(cmd) + if err == nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput) + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err) + } - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) + By("Fetching controller manager pod description") + cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + podDescription, err := utils.Run(cmd) + if err == nil { + fmt.Println("Pod description:\n", podDescription) + } else { + fmt.Println("Failed to describe controller pod") + } + } + }) - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + SetDefaultEventuallyTimeout(2 * time.Minute) + SetDefaultEventuallyPollingInterval(time.Second) + Context("Manager", func() { + It("should run successfully", func() { By("validating that the controller-manager pod is running as expected") - verifyControllerUp := func() error { - // Get pod name - - cmd = exec.Command("kubectl", "get", + verifyControllerUp := func(g Gomega) { + By("getting the name of the controller-manager pod") + cmd := exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ @@ -94,28 +155,185 @@ var _ = Describe("controller", Ordered, func() { ) podOutput, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - podNames := utils.GetNonEmptyLines(string(podOutput)) - if len(podNames) != 1 { - return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) - } + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := utils.GetNonEmptyLines(podOutput) + g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + g.Expect(controllerPodName).To(ContainSubstring("controller-manager")) - // Validate pod status + By("validating the pod's status") cmd = exec.Command("kubectl", "get", "pods", controllerPodName, "-o", "jsonpath={.status.phase}", "-n", namespace, ) - status, err := utils.Run(cmd) - ExpectWithOffset(2, err).NotTo(HaveOccurred()) - if string(status) != "Running" { - return fmt.Errorf("controller pod in %s status", status) - } - return nil + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status") } - EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed()) + Eventually(verifyControllerUp).Should(Succeed()) + }) + + It("should ensure the metrics endpoint is serving metrics", func() { + By("creating a ClusterRoleBinding for the service account to allow access to metrics") + cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, + "--clusterrole=kubernetes-operator-metrics-reader", + fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), + ) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") + + By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") + + By("getting the service account token") + token, err := serviceAccountToken() + Expect(err).NotTo(HaveOccurred()) + Expect(token).NotTo(BeEmpty()) + + By("ensuring the controller pod is ready") + verifyControllerPodReady := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pod", controllerPodName, "-n", namespace, + "-o", "jsonpath={.status.conditions[?(@.type=='Ready')].status}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("True"), "Controller pod not ready") + } + Eventually(verifyControllerPodReady, 3*time.Minute, time.Second).Should(Succeed()) + + By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(ContainSubstring("Serving metrics server"), + "Metrics server not yet started") + } + Eventually(verifyMetricsServerStarted, 3*time.Minute, time.Second).Should(Succeed()) + + // +kubebuilder:scaffold:e2e-metrics-webhooks-readiness + + By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:latest", + "--overrides", + fmt.Sprintf(`{ + "spec": { + "containers": [{ + "name": "curl", + "image": "curlimages/curl:latest", + "command": ["/bin/sh", "-c"], + "args": [ + "for i in $(seq 1 30); do curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics && exit 0 || sleep 2; done; exit 1" + ], + "securityContext": { + "readOnlyRootFilesystem": true, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"] + }, + "runAsNonRoot": true, + "runAsUser": 1000, + "seccompProfile": { + "type": "RuntimeDefault" + } + } + }], + "serviceAccountName": "%s" + } + }`, token, metricsServiceName, namespace, serviceAccountName)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") + + By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") + } + Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) + By("getting the metrics by checking curl-metrics logs") + verifyMetricsAvailable := func(g Gomega) { + metricsOutput, err := getMetricsOutput() + g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + g.Expect(metricsOutput).NotTo(BeEmpty()) + g.Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) + } + Eventually(verifyMetricsAvailable, 2*time.Minute).Should(Succeed()) }) + + // +kubebuilder:scaffold:e2e-webhooks-checks + + // TODO: Customize the e2e test suite with scenarios specific to your project. + // Consider applying sample/CR(s) and check their status and/or verifying + // the reconciliation by using the metrics, i.e.: + // metricsOutput, err := getMetricsOutput() + // Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") + // Expect(metricsOutput).To(ContainSubstring( + // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, + // strings.ToLower(), + // )) }) }) + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken() (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + By("creating temporary file to store the token request") + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g Gomega) { + By("executing kubectl command to create the token") + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(HaveOccurred()) + + By("parsing the JSON output to extract the token") + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(HaveOccurred()) + + out = token.Status.Token + } + Eventually(verifyTokenCreation).Should(Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput() (string, error) { + By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + return utils.Run(cmd) +} + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} diff --git a/test/utils/utils.go b/test/utils/utils.go index 2df8b9d..a408630 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2024 Memgraph Ltd. +Copyright 2026. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,62 +17,46 @@ limitations under the License. package utils import ( + "bufio" + "bytes" "fmt" "os" "os/exec" "strings" - . "github.com/onsi/ginkgo/v2" //nolint:golint,revive + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck ) const ( - prometheusOperatorVersion = "v0.68.0" - prometheusOperatorURL = "https://github.com/prometheus-operator/prometheus-operator/" + - "releases/download/%s/bundle.yaml" + certmanagerVersion = "v1.20.2" + certmanagerURLTmpl = "https://github.com/cert-manager/cert-manager/releases/download/%s/cert-manager.yaml" - certmanagerVersion = "v1.5.3" - certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml" + defaultKindBinary = "kind" + defaultKindCluster = "kind" ) func warnError(err error) { - fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) -} - -// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics. -func InstallPrometheusOperator() error { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "create", "-f", url) - _, err := Run(cmd) - return err + _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } // Run executes the provided command within this context -func Run(cmd *exec.Cmd) ([]byte, error) { +func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() cmd.Dir = dir if err := os.Chdir(cmd.Dir); err != nil { - fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %q\n", err) } cmd.Env = append(os.Environ(), "GO111MODULE=on") command := strings.Join(cmd.Args, " ") - fmt.Fprintf(GinkgoWriter, "running: %s\n", command) + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) output, err := cmd.CombinedOutput() if err != nil { - return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) } - return output, nil -} - -// UninstallPrometheusOperator uninstalls the prometheus -func UninstallPrometheusOperator() { - url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion) - cmd := exec.Command("kubectl", "delete", "-f", url) - if _, err := Run(cmd); err != nil { - warnError(err) - } + return string(output), nil } // UninstallCertManager uninstalls the cert manager @@ -82,6 +66,19 @@ func UninstallCertManager() { if _, err := Run(cmd); err != nil { warnError(err) } + + // Delete leftover leases in kube-system (not cleaned by default) + kubeSystemLeases := []string{ + "cert-manager-cainjector-leader-election", + "cert-manager-controller", + } + for _, lease := range kubeSystemLeases { + cmd = exec.Command("kubectl", "delete", "lease", lease, + "-n", "kube-system", "--ignore-not-found", "--force", "--grace-period=0") + if _, err := Run(cmd); err != nil { + warnError(err) + } + } } // InstallCertManager installs the cert manager bundle. @@ -103,14 +100,51 @@ func InstallCertManager() error { return err } -// LoadImageToKindCluster loads a local docker image to the kind cluster +// IsCertManagerCRDsInstalled checks if any Cert Manager CRDs are installed +// by verifying the existence of key CRDs related to Cert Manager. +func IsCertManagerCRDsInstalled() bool { + // List of common Cert Manager CRDs + certManagerCRDs := []string{ + "certificates.cert-manager.io", + "issuers.cert-manager.io", + "clusterissuers.cert-manager.io", + "certificaterequests.cert-manager.io", + "orders.acme.cert-manager.io", + "challenges.acme.cert-manager.io", + } + + // Execute the kubectl command to get all CRDs + cmd := exec.Command("kubectl", "get", "crds") + output, err := Run(cmd) + if err != nil { + return false + } + + // Check if any of the Cert Manager CRDs are present + crdList := GetNonEmptyLines(output) + for _, crd := range certManagerCRDs { + for _, line := range crdList { + if strings.Contains(line, crd) { + return true + } + } + } + + return false +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { - cluster := "kind" + cluster := defaultKindCluster if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { cluster = v } kindOptions := []string{"load", "docker-image", name, "--name", cluster} - cmd := exec.Command("kind", kindOptions...) + kindBinary := defaultKindBinary + if v, ok := os.LookupEnv("KIND"); ok { + kindBinary = v + } + cmd := exec.Command(kindBinary, kindOptions...) _, err := Run(cmd) return err } @@ -119,8 +153,8 @@ func LoadImageToKindClusterWithName(name string) error { // according to line breakers, and ignores the empty elements in it. func GetNonEmptyLines(output string) []string { var res []string - elements := strings.Split(output, "\n") - for _, element := range elements { + elements := strings.SplitSeq(output, "\n") + for element := range elements { if element != "" { res = append(res, element) } @@ -133,8 +167,60 @@ func GetNonEmptyLines(output string) []string { func GetProjectDir() (string, error) { wd, err := os.Getwd() if err != nil { - return wd, err + return wd, fmt.Errorf("failed to get current working directory: %w", err) } - wd = strings.Replace(wd, "/test/e2e", "", -1) + wd = strings.ReplaceAll(wd, "/test/e2e", "") return wd, nil } + +// UncommentCode searches for target in the file and remove the comment prefix +// of the target content. The target content may span multiple lines. +func UncommentCode(filename, target, prefix string) error { + // false positive + // nolint:gosec + content, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file %q: %w", filename, err) + } + strContent := string(content) + + idx := strings.Index(strContent, target) + if idx < 0 { + return fmt.Errorf("unable to find the code %q to be uncommented", target) + } + + out := new(bytes.Buffer) + _, err = out.Write(content[:idx]) + if err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + scanner := bufio.NewScanner(bytes.NewBufferString(target)) + if !scanner.Scan() { + return nil + } + for { + if _, err = out.WriteString(strings.TrimPrefix(scanner.Text(), prefix)); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + // Avoid writing a newline in case the previous line was the last in target. + if !scanner.Scan() { + break + } + if _, err = out.WriteString("\n"); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + } + + if _, err = out.Write(content[idx+len(target):]); err != nil { + return fmt.Errorf("failed to write to output: %w", err) + } + + // false positive + // nolint:gosec + if err = os.WriteFile(filename, out.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to write file %q: %w", filename, err) + } + + return nil +} From fe8d990f68f08c8ac14de9df19f96566bc7b70e7 Mon Sep 17 00:00:00 2001 From: as51340 Date: Thu, 23 Jul 2026 12:26:56 +0200 Subject: [PATCH 02/34] testing: Make test coverage opt-in so make test works under Go toolchain auto-switching go test -cover needs the covdata tool; Go 1.26 builds it on demand, and that build fails when the go command auto-switched toolchains (base Go older than go.mod's version), breaking local runs on distro Go installs. Nothing consumes the coverage profiles yet; pass COVER_FLAGS to opt in. --- Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d61b31c..26c7f24 100644 --- a/Makefile +++ b/Makefile @@ -59,13 +59,19 @@ fmt: ## Run go fmt against code. vet: ## Run go vet against code. go vet ./... +# Coverage is opt-in (e.g. make test COVER_FLAGS="-coverprofile cover.out"): +# `go test -cover` needs the covdata tool, whose on-demand build fails when the +# go command auto-switches toolchains (base Go older than go.mod's version), +# which would break `make test` on stock distro Go installs. +COVER_FLAGS ?= + .PHONY: test-unit test-unit: manifests generate fmt vet ## Run unit tests (pure packages, no envtest binaries required). - go test $$(go list ./... | grep -v /e2e | grep -v /internal/controller) -coverprofile cover-unit.out + go test $$(go list ./... | grep -v /e2e | grep -v /internal/controller) $(COVER_FLAGS) .PHONY: test test: manifests generate fmt vet setup-envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) $(COVER_FLAGS) # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. From 4c1e86e2f578076fcec06b24ca9225eb6b940655 Mon Sep 17 00:00:00 2001 From: as51340 Date: Thu, 23 Jul 2026 12:34:20 +0200 Subject: [PATCH 03/34] testing: Pin golangci-lint build to the project's Go toolchain go install pkg@version ignores go.mod, so under GOTOOLCHAIN=auto golangci-lint gets built with its own older minimum toolchain (go1.25) and then refuses to lint a project targeting go 1.26. Export the project's active toolchain version for the install/custom-build recipe. --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 26c7f24..5f2f6f7 100644 --- a/Makefile +++ b/Makefile @@ -240,6 +240,11 @@ $(ENVTEST): $(LOCALBIN) .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. +# Build golangci-lint with the project's Go toolchain: `go install pkg@version` +# ignores go.mod, so under GOTOOLCHAIN=auto it picks golangci-lint's own (older) +# minimum toolchain, and a golangci-lint built with a lower Go version refuses +# to lint a project targeting a higher one. +$(GOLANGCI_LINT): export GOTOOLCHAIN := $(shell go env GOVERSION) $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) @test -f .custom-gcl.yml && { \ From 18b9987beb16b520bac1d6b7d75a0ac3a5ecb24b Mon Sep 17 00:00:00 2001 From: as51340 Date: Thu, 23 Jul 2026 12:52:27 +0200 Subject: [PATCH 04/34] feat: Add CLAUDE.md --- CLAUDE.md | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..8399247 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,62 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +A kubebuilder-based Go operator for Memgraph high-availability clusters. It exposes one CRD, `MemgraphCluster` (group `memgraph.com`, version `v1alpha1`, short name `mgc`), and replaces the `memgraph-high-availability` Helm chart's fire-and-forget registration Job with continuous, state-aware reconciliation over Bolt. + +The repository was reset for this effort (prior attempt is on `archive/pre-operator-mvp`). Work is driven by `specs/operator-mvp/PRD.md` and sliced into PR-gated issues in `specs/operator-mvp/issues/` — read the PRD before making design decisions; it records what is in scope (provision, bootstrap, observe) and what is deliberately out (day-2 ops, scaling, TLS, external access, monitoring). + +`AGENTS.md` contains the generic kubebuilder agent guide (scaffolding commands, marker reference, never-edit rules). Follow it, especially: never hand-edit `config/crd/bases/*`, `config/rbac/role.yaml`, `zz_generated.*.go`, or `PROJECT`; never delete `// +kubebuilder:scaffold:*` markers. + +## Commands + +```sh +make test-unit # unit tests only (pure packages; excludes e2e and internal/controller) +make test # unit + envtest (downloads envtest binaries into bin/ on first run) +make lint # golangci-lint (lint-fix to auto-fix, lint-config to verify config) +make manifests generate # regenerate CRDs/RBAC + DeepCopy after editing *_types.go or markers +make build # build manager binary +make run # run controller locally against current kubeconfig +make test-e2e # KinD e2e suite — creates/deletes a dedicated Kind cluster; never run against a real cluster +``` + +Run a single test (Ginkgo suites): + +```sh +go test ./internal/controller/ -ginkgo.focus="" # needs KUBEBUILDER_ASSETS for envtest, see below +go test ./api/... -run TestName +``` + +Envtest packages need `KUBEBUILDER_ASSETS`; outside of `make test` set it with: +`KUBEBUILDER_ASSETS=$(bin/setup-envtest use --bin-dir bin -p path)` + +CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, and `make test` on every PR — all must be green. + +### Toolchain quirks (do not "fix" these) + +- Coverage is opt-in (`make test COVER_FLAGS="-coverprofile cover.out"`). Plain `go test -cover` breaks under Go toolchain auto-switching (covdata tool build fails), so `make test` must stay coverage-free by default. +- The golangci-lint Makefile install pins `GOTOOLCHAIN` to the project's Go version; `go install` otherwise builds it with an older toolchain that refuses to lint this project. + +## Architecture + +The PRD defines seven modules with two pure cores and one mock seam. Keep this separation — it's what makes the logic testable without a cluster: + +1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **immutable after creation, enforced by CEL** in the CRD, not a webhook. Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). +2. **Resource builders** — pure functions: spec in, desired Kubernetes objects out (one StatefulSet per role + headless Services; per-pod identity such as coordinator ID and advertised addresses derived from pod ordinals). No API calls, no side effects. Tested golden-style. +3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. +4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. `SET INSTANCE TO MAIN` is issued exactly once at bootstrap (when no MAIN exists); after that, failover belongs to the Raft coordinators — the operator only observes. +5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. +6. **Operator install chart** — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. +7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. + +Test philosophy (from the PRD): assert external behavior, never internal call ordering or private state. Builders get golden tests, planner gets pure topology-diff cases, controller gets envtest with the HA client mocked. + +## Conventions + +- Spec knob names mirror the HA Helm chart's vocabulary where the concept carries over (e.g. the `secrets.name` / `secrets.licenseKey` / `secrets.organizationKey` block) — check the chart before inventing a name. +- No secret material in spec or status; secrets are consumed by reference only. +- No destructive code paths in v1: no finalizer-based storage cleanup, no instance unregistration; PVC retention maps to the StatefulSet PVC retention policy (default `Retain`). +- Workload pods: non-root uid 101 / gid 103, seccomp RuntimeDefault, all capabilities dropped. +- Log messages follow Kubernetes style: capital first letter, no trailing period, past tense, object type named (see AGENTS.md for examples). From d7c264dccc1ffb2f67c5e68faccecd91a2c16f43 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Thu, 23 Jul 2026 13:20:56 +0200 Subject: [PATCH 05/34] feat: CR in, per-role workloads out (#21) Applying a minimal MemgraphCluster now provisions one StatefulSet per role (coordinators, data instances), each backed by a headless Service: - API types grow the walking-skeleton spec: coordinator and data-instance counts, image (repository/tag/pullPolicy), and the chart-compatible secrets block. All fields carry CRD schema defaults, mirrored as Go constants so builders behave on specs that never passed admission. - internal/resources holds pure builders (spec in, desired objects out). Coordinator identity (--coordinator-id, advertised --coordinator-hostname FQDN) is derived from the pod ordinal at startup via a uniform shell wrapper, so all pods within a role share one template. Flags, ports, probes, and the uid 101 / gid 103 restricted security context mirror the HA Helm chart. Storage is emptyDir for now; PVC templates land with the storage slice. - The controller server-side-applies builder output with controller owner references (CR deletion cascades to the workloads) and watches the owned StatefulSets and Services. - Builders are covered by golden-style unit tests; the controller by envtest (objects created, spec propagation, schema defaults and validation, idempotent re-reconcile). Slice 02 of specs/operator-mvp (02-provisioning-walking-skeleton.md). --- api/v1alpha1/memgraphcluster_types.go | 85 ++++- api/v1alpha1/zz_generated.deepcopy.go | 44 ++- .../bases/memgraph.com_memgraphclusters.yaml | 60 ++++ config/rbac/role.yaml | 24 ++ config/samples/v1alpha1_memgraphcluster.yaml | 13 +- go.mod | 6 +- .../controller/memgraphcluster_controller.go | 60 +++- .../memgraphcluster_controller_test.go | 248 +++++++++++--- internal/resources/resources.go | 142 ++++++++ internal/resources/service.go | 69 ++++ internal/resources/service_test.go | 79 +++++ internal/resources/statefulset.go | 227 +++++++++++++ internal/resources/statefulset_test.go | 315 ++++++++++++++++++ 13 files changed, 1317 insertions(+), 55 deletions(-) create mode 100644 internal/resources/resources.go create mode 100644 internal/resources/service.go create mode 100644 internal/resources/service_test.go create mode 100644 internal/resources/statefulset.go create mode 100644 internal/resources/statefulset_test.go diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 2deacb7..d4bee34 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -24,11 +25,91 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. +// Defaults for optional spec fields. They are declared as CRD schema defaults +// on the field markers below and mirrored here so resource builders behave +// correctly on specs that never passed admission (e.g. in unit tests). +const ( + DefaultCoordinatorCount int32 = 3 + DefaultDataInstanceCount int32 = 2 + + DefaultImageRepository = "docker.io/memgraph/memgraph" + DefaultImageTag = "3.12.0-relwithdebinfo" + DefaultImagePullPolicy = corev1.PullIfNotPresent + + DefaultSecretName = "memgraph-secrets" + DefaultLicenseSecretKey = "MEMGRAPH_ENTERPRISE_LICENSE" + DefaultOrganizationSecretKey = "MEMGRAPH_ORGANIZATION_NAME" +) + +// ImageSpec selects the Memgraph container image run by all cluster pods. +type ImageSpec struct { + // repository is the Memgraph container image repository. + // +kubebuilder:default="docker.io/memgraph/memgraph" + // +optional + Repository string `json:"repository,omitempty"` + + // tag is the Memgraph container image tag. Prefer pinning a specific + // Memgraph version over mutable tags such as "latest". + // +kubebuilder:default="3.12.0-relwithdebinfo" + // +optional + Tag string `json:"tag,omitempty"` + + // pullPolicy is the image pull policy applied to all cluster pods. + // +kubebuilder:validation:Enum=Always;IfNotPresent;Never + // +kubebuilder:default=IfNotPresent + // +optional + PullPolicy corev1.PullPolicy `json:"pullPolicy,omitempty"` +} + +// SecretsSpec references an existing Kubernetes Secret holding the Memgraph +// enterprise license and organization name. The block mirrors the +// memgraph-high-availability Helm chart's secrets vocabulary; secret material +// is consumed by reference only and never appears in the CR. +type SecretsSpec struct { + // name is the name of the Secret in the cluster's namespace. + // +kubebuilder:default="memgraph-secrets" + // +optional + Name string `json:"name,omitempty"` + + // licenseKey is the key within the Secret holding the enterprise license. + // +kubebuilder:default="MEMGRAPH_ENTERPRISE_LICENSE" + // +optional + LicenseKey string `json:"licenseKey,omitempty"` + + // organizationKey is the key within the Secret holding the organization + // name the license was issued to. + // +kubebuilder:default="MEMGRAPH_ORGANIZATION_NAME" + // +optional + OrganizationKey string `json:"organizationKey,omitempty"` +} + // MemgraphClusterSpec defines the desired state of MemgraphCluster. // -// Topology, image, storage, and pod-tuning fields land in subsequent -// slices of the operator MVP (see specs/operator-mvp/PRD.md). +// Storage, port, and pod-tuning fields land in subsequent slices of the +// operator MVP (see specs/operator-mvp/PRD.md). type MemgraphClusterSpec struct { + // coordinators is the number of Raft coordinator instances. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=3 + // +optional + Coordinators *int32 `json:"coordinators,omitempty"` + + // dataInstances is the number of data instances. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:default=2 + // +optional + DataInstances *int32 `json:"dataInstances,omitempty"` + + // image selects the Memgraph container image run by all cluster pods. + // +kubebuilder:default={} + // +optional + Image ImageSpec `json:"image,omitzero"` + + // secrets references the Secret holding the enterprise license and + // organization name. + // +kubebuilder:default={} + // +optional + Secrets SecretsSpec `json:"secrets,omitzero"` } // MemgraphClusterStatus defines the observed state of MemgraphCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6aea644..1ba4413 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,12 +25,27 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageSpec. +func (in *ImageSpec) DeepCopy() *ImageSpec { + if in == nil { + return nil + } + out := new(ImageSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemgraphCluster) DeepCopyInto(out *MemgraphCluster) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -87,6 +102,18 @@ func (in *MemgraphClusterList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { *out = *in + if in.Coordinators != nil { + in, out := &in.Coordinators, &out.Coordinators + *out = new(int32) + **out = **in + } + if in.DataInstances != nil { + in, out := &in.DataInstances, &out.DataInstances + *out = new(int32) + **out = **in + } + out.Image = in.Image + out.Secrets = in.Secrets } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterSpec. @@ -120,3 +147,18 @@ func (in *MemgraphClusterStatus) DeepCopy() *MemgraphClusterStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecretsSpec) DeepCopyInto(out *SecretsSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretsSpec. +func (in *SecretsSpec) DeepCopy() *SecretsSpec { + if in == nil { + return nil + } + out := new(SecretsSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index eb56325..0d05254 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -40,6 +40,66 @@ spec: type: object spec: description: spec defines the desired state of MemgraphCluster + properties: + coordinators: + default: 3 + description: coordinators is the number of Raft coordinator instances. + format: int32 + minimum: 1 + type: integer + dataInstances: + default: 2 + description: dataInstances is the number of data instances. + format: int32 + minimum: 1 + type: integer + image: + default: {} + description: image selects the Memgraph container image run by all + cluster pods. + properties: + pullPolicy: + default: IfNotPresent + description: pullPolicy is the image pull policy applied to all + cluster pods. + enum: + - Always + - IfNotPresent + - Never + type: string + repository: + default: docker.io/memgraph/memgraph + description: repository is the Memgraph container image repository. + type: string + tag: + default: 3.12.0-relwithdebinfo + description: |- + tag is the Memgraph container image tag. Prefer pinning a specific + Memgraph version over mutable tags such as "latest". + type: string + type: object + secrets: + default: {} + description: |- + secrets references the Secret holding the enterprise license and + organization name. + properties: + licenseKey: + default: MEMGRAPH_ENTERPRISE_LICENSE + description: licenseKey is the key within the Secret holding the + enterprise license. + type: string + name: + default: memgraph-secrets + description: name is the name of the Secret in the cluster's namespace. + type: string + organizationKey: + default: MEMGRAPH_ORGANIZATION_NAME + description: |- + organizationKey is the key within the Secret holding the organization + name the license was issued to. + type: string + type: object type: object status: description: status defines the observed state of MemgraphCluster diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 7496194..2172b4d 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -4,6 +4,30 @@ kind: ClusterRole metadata: name: manager-role rules: +- apiGroups: + - "" + resources: + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - memgraph.com resources: diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index a59e619..4658833 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -5,4 +5,15 @@ metadata: app.kubernetes.io/name: kubernetes-operator app.kubernetes.io/managed-by: kustomize name: memgraphcluster-sample -spec: {} +spec: + coordinators: 3 + dataInstances: 2 + image: + repository: docker.io/memgraph/memgraph + tag: 3.12.0-relwithdebinfo + # References an existing Secret holding the enterprise license; the CR + # carries no secret material itself. + secrets: + name: memgraph-secrets + licenseKey: MEMGRAPH_ENTERPRISE_LICENSE + organizationKey: MEMGRAPH_ORGANIZATION_NAME diff --git a/go.mod b/go.mod index 3448f3d..2dcf65d 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/memgraph/kubernetes-operator go 1.26.0 require ( + github.com/google/go-cmp v0.7.0 github.com/onsi/ginkgo/v2 v2.27.4 github.com/onsi/gomega v1.39.0 + k8s.io/api v0.36.0 k8s.io/apimachinery v0.36.0 k8s.io/client-go v0.36.0 + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/controller-runtime v0.24.1 ) @@ -33,7 +36,6 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/cel-go v0.26.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect - github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 // indirect @@ -84,14 +86,12 @@ require ( gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.36.0 // indirect k8s.io/apiextensions-apiserver v0.36.0 // indirect k8s.io/apiserver v0.36.0 // indirect k8s.io/component-base v0.36.0 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect k8s.io/streaming v0.36.0 // indirect - k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.34.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 2a68548..6de579a 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -18,15 +18,25 @@ package controller import ( "context" + "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + "github.com/memgraph/kubernetes-operator/internal/resources" ) +// fieldOwner identifies this controller as the server-side-apply field +// manager of the workload objects it provisions. +const fieldOwner = "memgraph-operator" + // MemgraphClusterReconciler reconciles a MemgraphCluster object type MemgraphClusterReconciler struct { client.Client @@ -36,13 +46,14 @@ type MemgraphClusterReconciler struct { // +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/finalizers,verbs=update +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -// Reconcile is part of the main kubernetes reconciliation loop which aims to -// move the current state of the cluster closer to the desired state. -// -// Placeholder implementation: fetches the MemgraphCluster and logs a -// greeting. Provisioning arrives with the walking-skeleton slice -// (specs/operator-mvp/issues/02). +// Reconcile drives the cluster toward the declared MemgraphCluster spec by +// server-side-applying the builders' desired objects: one StatefulSet per +// role (coordinators, data instances), each backed by a headless Service. +// Deletion needs no handling here — every object carries a controller owner +// reference, so garbage collection removes the workloads with the CR. func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -51,15 +62,50 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, client.IgnoreNotFound(err) } - log.Info("hello from the MemgraphCluster reconciler", "memgraphcluster", req.NamespacedName) + desired := []client.Object{ + resources.CoordinatorHeadlessService(&cluster), + resources.DataHeadlessService(&cluster), + resources.CoordinatorStatefulSet(&cluster), + resources.DataStatefulSet(&cluster), + } + for _, obj := range desired { + if err := controllerutil.SetControllerReference(&cluster, obj, r.Scheme); err != nil { + return ctrl.Result{}, fmt.Errorf("setting owner reference on %T %s: %w", obj, obj.GetName(), err) + } + if err := r.apply(ctx, obj); err != nil { + return ctrl.Result{}, fmt.Errorf("applying %T %s: %w", obj, obj.GetName(), err) + } + } + + log.Info("Applied desired workload objects for MemgraphCluster", "memgraphcluster", req.NamespacedName) return ctrl.Result{}, nil } +// apply server-side-applies a desired object built by the resource builders. +// Builders set only the fields the operator owns, so the converted apply +// configuration claims exactly those fields for this controller. +func (r *MemgraphClusterReconciler) apply(ctx context.Context, obj client.Object) error { + content, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return fmt.Errorf("converting to unstructured: %w", err) + } + u := &unstructured.Unstructured{Object: content} + // Zero-valued struct fields survive the conversion; drop them so the + // applied configuration only claims fields the builders actually set. + unstructured.RemoveNestedField(u.Object, "status") + unstructured.RemoveNestedField(u.Object, "metadata", "creationTimestamp") + unstructured.RemoveNestedField(u.Object, "spec", "template", "metadata", "creationTimestamp") + + return r.Apply(ctx, client.ApplyConfigurationFromUnstructured(u), client.FieldOwner(fieldOwner), client.ForceOwnership) +} + // SetupWithManager sets up the controller with the Manager. func (r *MemgraphClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&memgraphcomv1alpha1.MemgraphCluster{}). + Owns(&appsv1.StatefulSet{}). + Owns(&corev1.Service{}). Named("memgraphcluster"). Complete(r) } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 293ed64..56f39b8 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -18,70 +18,236 @@ package controller import ( "context" + "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/reconcile" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" ) +// Name suffixes of the per-role workload objects a reconcile creates. +const ( + coordinatorSuffix = "-coordinator" + dataSuffix = "-data" +) + var _ = Describe("MemgraphCluster Controller", func() { - Context("When reconciling a resource", func() { - const ( - resourceName = "test-resource" - resourceNamespace = "default" - ) + const resourceNamespace = "default" + + ctx := context.Background() - ctx := context.Background() + var reconciler *MemgraphClusterReconciler - typeNamespacedName := types.NamespacedName{ - Name: resourceName, - Namespace: resourceNamespace, + BeforeEach(func() { + reconciler = &MemgraphClusterReconciler{ + Client: k8sClient, + Scheme: k8sClient.Scheme(), } - memgraphcluster := &memgraphcomv1alpha1.MemgraphCluster{} + }) + + reconcileCluster := func(name string) { + GinkgoHelper() + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: name, Namespace: resourceNamespace}, + }) + Expect(err).NotTo(HaveOccurred()) + } + + get := func(name string, obj client.Object) { + GinkgoHelper() + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: resourceNamespace}, obj)).To(Succeed()) + } + + // deleteOwned removes the workload objects a reconcile created for the + // given cluster: envtest runs no garbage collector, so owner-reference + // cascade deletion never fires and each spec must clean up explicitly. + deleteOwned := func(clusterName string) { + GinkgoHelper() + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + suffix, Namespace: resourceNamespace, + }} + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, sts))).To(Succeed()) + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{ + Name: clusterName + suffix, Namespace: resourceNamespace, + }} + Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, svc))).To(Succeed()) + } + } + + expectControlledBy := func(obj client.Object, cluster *memgraphcomv1alpha1.MemgraphCluster) { + GinkgoHelper() + ref := metav1.GetControllerOf(obj) + Expect(ref).NotTo(BeNil(), "expected %s to carry a controller owner reference", obj.GetName()) + Expect(ref.Kind).To(Equal("MemgraphCluster")) + Expect(ref.Name).To(Equal(cluster.Name)) + Expect(ref.UID).To(Equal(cluster.UID)) + Expect(ref.Controller).To(HaveValue(BeTrue())) + } + + Context("when reconciling a minimal MemgraphCluster", func() { + const resourceName = "mgc-minimal" + + cluster := &memgraphcomv1alpha1.MemgraphCluster{} BeforeEach(func() { - By("creating the custom resource for the Kind MemgraphCluster") - err := k8sClient.Get(ctx, typeNamespacedName, memgraphcluster) - if err != nil && errors.IsNotFound(err) { - resource := &memgraphcomv1alpha1.MemgraphCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: resourceNamespace, - }, - // TODO(user): Specify other spec details if needed. - } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + get(resourceName, cluster) }) AfterEach(func() { - // TODO(user): Cleanup logic after each test, like removing the resource instance. - resource := &memgraphcomv1alpha1.MemgraphCluster{} - err := k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient.Delete(ctx, cluster)).To(Succeed()) + deleteOwned(resourceName) + }) - By("Cleanup the specific resource instance MemgraphCluster") - Expect(k8sClient.Delete(ctx, resource)).To(Succeed()) + It("should apply CRD schema defaults on admission", func() { + Expect(cluster.Spec.Coordinators).To(HaveValue(Equal(int32(3)))) + Expect(cluster.Spec.DataInstances).To(HaveValue(Equal(int32(2)))) + Expect(cluster.Spec.Image.Repository).To(Equal(memgraphcomv1alpha1.DefaultImageRepository)) + Expect(cluster.Spec.Image.Tag).To(Equal(memgraphcomv1alpha1.DefaultImageTag)) + Expect(cluster.Spec.Image.PullPolicy).To(Equal(memgraphcomv1alpha1.DefaultImagePullPolicy)) + Expect(cluster.Spec.Secrets.Name).To(Equal(memgraphcomv1alpha1.DefaultSecretName)) + Expect(cluster.Spec.Secrets.LicenseKey).To(Equal(memgraphcomv1alpha1.DefaultLicenseSecretKey)) + Expect(cluster.Spec.Secrets.OrganizationKey).To(Equal(memgraphcomv1alpha1.DefaultOrganizationSecretKey)) }) - It("should successfully reconcile the resource", func() { - By("Reconciling the created resource") - controllerReconciler := &MemgraphClusterReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + + It("should provision one StatefulSet and one headless Service per role", func() { + reconcileCluster(resourceName) + + coordinatorSts := &appsv1.StatefulSet{} + get(resourceName+coordinatorSuffix, coordinatorSts) + Expect(coordinatorSts.Spec.Replicas).To(HaveValue(Equal(int32(3)))) + Expect(coordinatorSts.Spec.ServiceName).To(Equal(resourceName + coordinatorSuffix)) + expectControlledBy(coordinatorSts, cluster) + + dataSts := &appsv1.StatefulSet{} + get(resourceName+dataSuffix, dataSts) + Expect(dataSts.Spec.Replicas).To(HaveValue(Equal(int32(2)))) + Expect(dataSts.Spec.ServiceName).To(Equal(resourceName + dataSuffix)) + expectControlledBy(dataSts, cluster) + + for _, sts := range []*appsv1.StatefulSet{coordinatorSts, dataSts} { + podSpec := sts.Spec.Template.Spec + Expect(podSpec.Containers).To(HaveLen(1)) + container := podSpec.Containers[0] + Expect(container.Image).To(Equal("docker.io/memgraph/memgraph:3.12.0-relwithdebinfo")) + Expect(podSpec.SecurityContext.RunAsUser).To(HaveValue(Equal(int64(101)))) + Expect(podSpec.SecurityContext.RunAsGroup).To(HaveValue(Equal(int64(103)))) + + licenseRef := container.Env[len(container.Env)-2].ValueFrom.SecretKeyRef + Expect(licenseRef.Name).To(Equal("memgraph-secrets")) + Expect(licenseRef.Key).To(Equal("MEMGRAPH_ENTERPRISE_LICENSE")) + } + + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + svc := &corev1.Service{} + get(resourceName+suffix, svc) + Expect(svc.Spec.ClusterIP).To(Equal(corev1.ClusterIPNone)) + Expect(svc.Spec.PublishNotReadyAddresses).To(BeTrue()) + expectControlledBy(svc, cluster) } + }) + + It("should be idempotent when reconciling an unchanged resource", func() { + reconcileCluster(resourceName) - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - // TODO(user): Add more specific assertions depending on your controller's reconciliation logic. - // Example: If you expect a certain status condition after reconciliation, verify it here. + versions := map[string]string{} + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{} + get(resourceName+suffix, sts) + versions["sts"+suffix] = sts.ResourceVersion + svc := &corev1.Service{} + get(resourceName+suffix, svc) + versions["svc"+suffix] = svc.ResourceVersion + } + + reconcileCluster(resourceName) + + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{} + get(resourceName+suffix, sts) + Expect(sts.ResourceVersion).To(Equal(versions["sts"+suffix]), + fmt.Sprintf("StatefulSet %s%s changed on a no-op reconcile", resourceName, suffix)) + svc := &corev1.Service{} + get(resourceName+suffix, svc) + Expect(svc.ResourceVersion).To(Equal(versions["svc"+suffix]), + fmt.Sprintf("Service %s%s changed on a no-op reconcile", resourceName, suffix)) + } + }) + }) + + Context("when reconciling a fully specified MemgraphCluster", func() { + const resourceName = "mgc-custom" + + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + + BeforeEach(func() { + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, + Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(1)), + DataInstances: ptr.To(int32(1)), + Image: memgraphcomv1alpha1.ImageSpec{ + Repository: "registry.example.com/memgraph", + Tag: "3.13.0", + PullPolicy: corev1.PullAlways, + }, + Secrets: memgraphcomv1alpha1.SecretsSpec{ + Name: "my-license", + LicenseKey: "license", + OrganizationKey: "organization", + }, + }, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + get(resourceName, cluster) + }) + + AfterEach(func() { + Expect(k8sClient.Delete(ctx, cluster)).To(Succeed()) + deleteOwned(resourceName) + }) + + It("should propagate spec values into the workload objects", func() { + reconcileCluster(resourceName) + + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{} + get(resourceName+suffix, sts) + Expect(sts.Spec.Replicas).To(HaveValue(Equal(int32(1)))) + + container := sts.Spec.Template.Spec.Containers[0] + Expect(container.Image).To(Equal("registry.example.com/memgraph:3.13.0")) + Expect(container.ImagePullPolicy).To(Equal(corev1.PullAlways)) + + licenseRef := container.Env[len(container.Env)-2].ValueFrom.SecretKeyRef + Expect(licenseRef.Name).To(Equal("my-license")) + Expect(licenseRef.Key).To(Equal("license")) + organizationRef := container.Env[len(container.Env)-1].ValueFrom.SecretKeyRef + Expect(organizationRef.Name).To(Equal("my-license")) + Expect(organizationRef.Key).To(Equal("organization")) + } + }) + + It("should reject a spec violating the schema", func() { + invalid := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "mgc-invalid", Namespace: resourceNamespace}, + Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(0)), + }, + } + Expect(k8sClient.Create(ctx, invalid)).NotTo(Succeed()) }) }) }) diff --git a/internal/resources/resources.go b/internal/resources/resources.go new file mode 100644 index 0000000..3e6da3e --- /dev/null +++ b/internal/resources/resources.go @@ -0,0 +1,142 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package resources contains pure builders from a MemgraphCluster spec to the +// desired Kubernetes objects. Builders make no API calls and have no side +// effects; the controller server-side-applies their output. +package resources + +import ( + corev1 "k8s.io/api/core/v1" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +// Internal Memgraph ports. These mirror the memgraph-high-availability Helm +// chart's defaults and become spec knobs in a later slice. +const ( + BoltPort int32 = 7687 + ManagementPort int32 = 10000 + ReplicationPort int32 = 20000 + CoordinatorPort int32 = 12000 +) + +const ( + // clusterDomain is the Kubernetes cluster domain used in advertised FQDN + // addresses. It becomes a spec knob in a later slice. + clusterDomain = "cluster.local" + + // Memgraph workload pods run as the non-root memgraph user baked into the + // official images. + memgraphUserID int64 = 101 + memgraphGroupID int64 = 103 + + coordinatorComponent = "coordinator" + dataComponent = "data" +) + +// Named container and Service port names shared by both roles. +const ( + boltPortName = "bolt" + managementPortName = "management" + coordinatorPortName = "coordinator" + replicationPortName = "replication" +) + +// CoordinatorName is the name shared by the coordinator StatefulSet and its +// headless Service. +func CoordinatorName(cluster *memgraphcomv1alpha1.MemgraphCluster) string { + return cluster.Name + "-" + coordinatorComponent +} + +// DataName is the name shared by the data-instance StatefulSet and its +// headless Service. +func DataName(cluster *memgraphcomv1alpha1.MemgraphCluster) string { + return cluster.Name + "-" + dataComponent +} + +// labels returns the full label set stamped on all objects of a role. +func labels(cluster *memgraphcomv1alpha1.MemgraphCluster, component string) map[string]string { + l := selectorLabels(cluster, component) + l["app.kubernetes.io/managed-by"] = "memgraph-operator" + return l +} + +// selectorLabels returns the immutable subset of labels used as StatefulSet +// and Service selectors. +func selectorLabels(cluster *memgraphcomv1alpha1.MemgraphCluster, component string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": "memgraph", + "app.kubernetes.io/instance": cluster.Name, + "app.kubernetes.io/component": component, + } +} + +// normalizedSpec is a MemgraphClusterSpec with every optional field resolved +// to its CRD schema default, so builders behave correctly on specs that never +// passed admission. +type normalizedSpec struct { + coordinators int32 + dataInstances int32 + image string + pullPolicy corev1.PullPolicy + secretName string + licenseKey string + organizationKey string +} + +func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { + n := normalizedSpec{ + coordinators: memgraphcomv1alpha1.DefaultCoordinatorCount, + dataInstances: memgraphcomv1alpha1.DefaultDataInstanceCount, + image: imageRef(spec.Image), + pullPolicy: spec.Image.PullPolicy, + secretName: spec.Secrets.Name, + licenseKey: spec.Secrets.LicenseKey, + organizationKey: spec.Secrets.OrganizationKey, + } + if spec.Coordinators != nil { + n.coordinators = *spec.Coordinators + } + if spec.DataInstances != nil { + n.dataInstances = *spec.DataInstances + } + if n.pullPolicy == "" { + n.pullPolicy = memgraphcomv1alpha1.DefaultImagePullPolicy + } + if n.secretName == "" { + n.secretName = memgraphcomv1alpha1.DefaultSecretName + } + if n.licenseKey == "" { + n.licenseKey = memgraphcomv1alpha1.DefaultLicenseSecretKey + } + if n.organizationKey == "" { + n.organizationKey = memgraphcomv1alpha1.DefaultOrganizationSecretKey + } + return n +} + +func imageRef(image memgraphcomv1alpha1.ImageSpec) string { + repository := image.Repository + if repository == "" { + repository = memgraphcomv1alpha1.DefaultImageRepository + } + tag := image.Tag + if tag == "" { + tag = memgraphcomv1alpha1.DefaultImageTag + } + return repository + ":" + tag +} diff --git a/internal/resources/service.go b/internal/resources/service.go new file mode 100644 index 0000000..cb9a393 --- /dev/null +++ b/internal/resources/service.go @@ -0,0 +1,69 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +// CoordinatorHeadlessService builds the headless Service backing the +// coordinator StatefulSet's per-pod DNS identities. +func CoordinatorHeadlessService(cluster *memgraphcomv1alpha1.MemgraphCluster) *corev1.Service { + return headlessService(cluster, coordinatorComponent, CoordinatorName(cluster), []corev1.ServicePort{ + {Name: boltPortName, Port: BoltPort}, + {Name: managementPortName, Port: ManagementPort}, + {Name: coordinatorPortName, Port: CoordinatorPort}, + }) +} + +// DataHeadlessService builds the headless Service backing the data-instance +// StatefulSet's per-pod DNS identities. +func DataHeadlessService(cluster *memgraphcomv1alpha1.MemgraphCluster) *corev1.Service { + return headlessService(cluster, dataComponent, DataName(cluster), []corev1.ServicePort{ + {Name: boltPortName, Port: BoltPort}, + {Name: managementPortName, Port: ManagementPort}, + {Name: replicationPortName, Port: ReplicationPort}, + }) +} + +func headlessService( + cluster *memgraphcomv1alpha1.MemgraphCluster, + component, name string, + ports []corev1.ServicePort, +) *corev1.Service { + return &corev1.Service{ + // TypeMeta is set explicitly because the controller server-side + // applies builder output, and apply patches must carry the GVK. + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: cluster.Namespace, + Labels: labels(cluster, component), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + Selector: selectorLabels(cluster, component), + // Pods must resolve each other's DNS names before they are ready, + // otherwise coordinators could never form a cluster. + PublishNotReadyAddresses: true, + Ports: ports, + }, + } +} diff --git a/internal/resources/service_test.go b/internal/resources/service_test.go new file mode 100644 index 0000000..ac7d278 --- /dev/null +++ b/internal/resources/service_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/memgraph/kubernetes-operator/internal/resources" +) + +func TestCoordinatorHeadlessService(t *testing.T) { + want := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: coordinatorName, + Namespace: testNamespace, + Labels: expectedLabels(coordinatorComponent), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + Selector: expectedSelectorLabels(coordinatorComponent), + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + {Name: boltPortName, Port: 7687}, + {Name: managementPortName, Port: 10000}, + {Name: coordinatorComponent, Port: 12000}, + }, + }, + } + + got := resources.CoordinatorHeadlessService(minimalCluster()) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("CoordinatorHeadlessService() mismatch (-want +got):\n%s", diff) + } +} + +func TestDataHeadlessService(t *testing.T) { + want := &corev1.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: testNamespace, + Labels: expectedLabels(dataComponent), + }, + Spec: corev1.ServiceSpec{ + ClusterIP: corev1.ClusterIPNone, + Selector: expectedSelectorLabels(dataComponent), + PublishNotReadyAddresses: true, + Ports: []corev1.ServicePort{ + {Name: boltPortName, Port: 7687}, + {Name: managementPortName, Port: 10000}, + {Name: replicationPortName, Port: 20000}, + }, + }, + } + + got := resources.DataHeadlessService(minimalCluster()) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("DataHeadlessService() mismatch (-want +got):\n%s", diff) + } +} diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go new file mode 100644 index 0000000..8e713fb --- /dev/null +++ b/internal/resources/statefulset.go @@ -0,0 +1,227 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +const ( + memgraphBinary = "/usr/lib/memgraph/memgraph" + dataDirectory = "/var/lib/memgraph/mg_data" + logFile = "/var/log/memgraph/memgraph.log" + + libMountPath = "/var/lib/memgraph" + logMountPath = "/var/log/memgraph" + tmpMountPath = "/tmp" +) + +// CoordinatorStatefulSet builds the single StatefulSet running all +// coordinator instances. Per-pod identity (coordinator ID, advertised FQDN) +// is derived from the pod ordinal at startup, so the pod template stays +// uniform across replicas. +func CoordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { + spec := normalize(cluster.Spec) + + container := memgraphContainer(spec) + // The coordinator ID and advertised FQDN depend on the pod ordinal, which + // only the pod itself knows; a shell wrapper derives them from the pod + // name so all replicas share one template. + container.Command = []string{"/bin/sh", "-ec", coordinatorStartScript(cluster)} + container.Env = append([]corev1.EnvVar{{ + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }}, container.Env...) + container.Ports = []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: BoltPort}, + {Name: managementPortName, ContainerPort: ManagementPort}, + {Name: coordinatorPortName, ContainerPort: CoordinatorPort}, + } + container.StartupProbe = tcpProbe(CoordinatorPort, 20) + container.ReadinessProbe = tcpProbe(CoordinatorPort, 20) + container.LivenessProbe = tcpProbe(CoordinatorPort, 20) + + return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec.coordinators, container) +} + +// DataStatefulSet builds the single StatefulSet running all data instances. +func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { + spec := normalize(cluster.Spec) + + container := memgraphContainer(spec) + container.Args = dataArgs() + container.Ports = []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: BoltPort}, + {Name: managementPortName, ContainerPort: ManagementPort}, + {Name: replicationPortName, ContainerPort: ReplicationPort}, + } + // A generous startup budget so large snapshot restores are not killed + // mid-load (mirrors the HA chart's default of 1440 * 5s = 2h). + container.StartupProbe = tcpProbe(BoltPort, 1440) + container.ReadinessProbe = tcpProbe(BoltPort, 20) + container.LivenessProbe = tcpProbe(BoltPort, 20) + + return statefulSet(cluster, dataComponent, DataName(cluster), spec.dataInstances, container) +} + +// coordinatorStartScript derives the coordinator's identity from its pod +// ordinal (the numeric suffix of the pod name): ordinal N becomes coordinator +// ID N+1 (Raft IDs start at 1) advertised at the pod's stable DNS name within +// the headless Service. +func coordinatorStartScript(cluster *memgraphcomv1alpha1.MemgraphCluster) string { + fqdnSuffix := fmt.Sprintf("%s.%s.svc.%s", CoordinatorName(cluster), cluster.Namespace, clusterDomain) + return fmt.Sprintf(`ordinal="${POD_NAME##*-}" +exec %s \ + --coordinator-id="$((ordinal + 1))" \ + --coordinator-hostname="${POD_NAME}.%s" \ + --coordinator-port=%d \ + %s`, memgraphBinary, fqdnSuffix, CoordinatorPort, shellJoin(commonArgs())) +} + +func dataArgs() []string { + return commonArgs() +} + +// commonArgs are the Memgraph flags shared by both roles, mirroring the HA +// chart's auto-appended and default logging arguments. +func commonArgs() []string { + return []string{ + fmt.Sprintf("--bolt-port=%d", BoltPort), + fmt.Sprintf("--management-port=%d", ManagementPort), + "--data-directory=" + dataDirectory, + "--log-level=TRACE", + "--also-log-to-stderr", + "--log-file=" + logFile, + "--log-retention-days=35", + } +} + +func shellJoin(args []string) string { + return strings.Join(args, " \\\n ") +} + +// memgraphContainer builds the parts of the Memgraph container shared by both +// roles: image, license env wiring, storage mounts, and the restricted +// security context. +func memgraphContainer(spec normalizedSpec) corev1.Container { + return corev1.Container{ + Name: "memgraph", + Image: spec.image, + ImagePullPolicy: spec.pullPolicy, + Env: []corev1.EnvVar{ + { + Name: "MEMGRAPH_ENTERPRISE_LICENSE", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: spec.secretName}, + Key: spec.licenseKey, + }, + }, + }, + { + Name: "MEMGRAPH_ORGANIZATION_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: spec.secretName}, + Key: spec.organizationKey, + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: "lib-storage", MountPath: libMountPath}, + {Name: "log-storage", MountPath: logMountPath}, + {Name: "tmp", MountPath: tmpMountPath}, + }, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + ReadOnlyRootFilesystem: ptr.To(true), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + } +} + +func statefulSet( + cluster *memgraphcomv1alpha1.MemgraphCluster, + component, name string, + replicas int32, + container corev1.Container, +) *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + // TypeMeta is set explicitly because the controller server-side + // applies builder output, and apply patches must carry the GVK. + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "StatefulSet"}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: cluster.Namespace, + Labels: labels(cluster, component), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(replicas), + ServiceName: name, + PodManagementPolicy: appsv1.ParallelPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(cluster, component)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels(cluster, component), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{container}, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To(memgraphUserID), + RunAsGroup: ptr.To(memgraphGroupID), + FSGroup: ptr.To(memgraphGroupID), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + // Storage is ephemeral in this slice; PVC templates and + // retention policy land with the storage-configuration + // slice (specs/operator-mvp/issues/08). + Volumes: []corev1.Volume{ + {Name: "lib-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "log-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + }, + }, + }, + } +} + +func tcpProbe(port, failureThreshold int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, + }, + FailureThreshold: failureThreshold, + TimeoutSeconds: 10, + PeriodSeconds: 5, + } +} diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go new file mode 100644 index 0000000..d07a8bf --- /dev/null +++ b/internal/resources/statefulset_test.go @@ -0,0 +1,315 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + "github.com/memgraph/kubernetes-operator/internal/resources" +) + +// Shared fixture names for the golden tests in this package. +const ( + testNamespace = "memgraph-test" + clusterName = "example" + + coordinatorComponent = "coordinator" + dataComponent = "data" + + coordinatorName = clusterName + "-" + coordinatorComponent + dataName = clusterName + "-" + dataComponent + + memgraphName = "memgraph" + + boltPortName = "bolt" + managementPortName = "management" + replicationPortName = "replication" +) + +// minimalCluster returns a MemgraphCluster as a client would minimally create +// it, deliberately without CRD schema defaults applied: builders must resolve +// defaults themselves on specs that never passed admission. +func minimalCluster() *memgraphcomv1alpha1.MemgraphCluster { + return &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: testNamespace}, + } +} + +func specifiedCluster() *memgraphcomv1alpha1.MemgraphCluster { + return &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: testNamespace}, + Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(5)), + DataInstances: ptr.To(int32(3)), + Image: memgraphcomv1alpha1.ImageSpec{ + Repository: "registry.example.com/memgraph", + Tag: "3.13.0", + PullPolicy: corev1.PullAlways, + }, + Secrets: memgraphcomv1alpha1.SecretsSpec{ + Name: "my-license", + LicenseKey: "license", + OrganizationKey: "organization", + }, + }, + } +} + +func licenseEnv(secretName, licenseKey, organizationKey string) []corev1.EnvVar { + return []corev1.EnvVar{ + { + Name: "MEMGRAPH_ENTERPRISE_LICENSE", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: licenseKey, + }, + }, + }, + { + Name: "MEMGRAPH_ORGANIZATION_NAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: secretName}, + Key: organizationKey, + }, + }, + }, + } +} + +func tcpProbe(port, failureThreshold int32) *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, + }, + FailureThreshold: failureThreshold, + TimeoutSeconds: 10, + PeriodSeconds: 5, + } +} + +func expectedPodSecurityContext() *corev1.PodSecurityContext { + return &corev1.PodSecurityContext{ + RunAsUser: ptr.To(int64(101)), + RunAsGroup: ptr.To(int64(103)), + FSGroup: ptr.To(int64(103)), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +func expectedContainerSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + ReadOnlyRootFilesystem: ptr.To(true), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +func expectedVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: "lib-storage", MountPath: "/var/lib/memgraph"}, + {Name: "log-storage", MountPath: "/var/log/memgraph"}, + {Name: "tmp", MountPath: "/tmp"}, + } +} + +func expectedVolumes() []corev1.Volume { + return []corev1.Volume{ + {Name: "lib-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "log-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + } +} + +func expectedLabels(component string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": memgraphName, + "app.kubernetes.io/instance": clusterName, + "app.kubernetes.io/component": component, + "app.kubernetes.io/managed-by": "memgraph-operator", + } +} + +func expectedSelectorLabels(component string) map[string]string { + return map[string]string{ + "app.kubernetes.io/name": memgraphName, + "app.kubernetes.io/instance": clusterName, + "app.kubernetes.io/component": component, + } +} + +const expectedCoordinatorScript = `ordinal="${POD_NAME##*-}" +exec /usr/lib/memgraph/memgraph \ + --coordinator-id="$((ordinal + 1))" \ + --coordinator-hostname="${POD_NAME}.example-coordinator.memgraph-test.svc.cluster.local" \ + --coordinator-port=12000 \ + --bolt-port=7687 \ + --management-port=10000 \ + --data-directory=/var/lib/memgraph/mg_data \ + --log-level=TRACE \ + --also-log-to-stderr \ + --log-file=/var/log/memgraph/memgraph.log \ + --log-retention-days=35` + +func TestCoordinatorStatefulSetDefaults(t *testing.T) { + want := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "StatefulSet"}, + ObjectMeta: metav1.ObjectMeta{ + Name: coordinatorName, + Namespace: testNamespace, + Labels: expectedLabels(coordinatorComponent), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(3)), + ServiceName: coordinatorName, + PodManagementPolicy: appsv1.ParallelPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: expectedSelectorLabels(coordinatorComponent)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: expectedLabels(coordinatorComponent)}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: memgraphName, + Image: "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/bin/sh", "-ec", expectedCoordinatorScript}, + Env: append([]corev1.EnvVar{{ + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }}, licenseEnv("memgraph-secrets", "MEMGRAPH_ENTERPRISE_LICENSE", "MEMGRAPH_ORGANIZATION_NAME")...), + Ports: []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: 7687}, + {Name: managementPortName, ContainerPort: 10000}, + {Name: coordinatorComponent, ContainerPort: 12000}, + }, + StartupProbe: tcpProbe(12000, 20), + ReadinessProbe: tcpProbe(12000, 20), + LivenessProbe: tcpProbe(12000, 20), + VolumeMounts: expectedVolumeMounts(), + SecurityContext: expectedContainerSecurityContext(), + }}, + SecurityContext: expectedPodSecurityContext(), + Volumes: expectedVolumes(), + }, + }, + }, + } + + got := resources.CoordinatorStatefulSet(minimalCluster()) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("CoordinatorStatefulSet() mismatch (-want +got):\n%s", diff) + } +} + +func TestDataStatefulSetDefaults(t *testing.T) { + want := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "StatefulSet"}, + ObjectMeta: metav1.ObjectMeta{ + Name: dataName, + Namespace: testNamespace, + Labels: expectedLabels(dataComponent), + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: ptr.To(int32(2)), + ServiceName: dataName, + PodManagementPolicy: appsv1.ParallelPodManagement, + Selector: &metav1.LabelSelector{MatchLabels: expectedSelectorLabels(dataComponent)}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: expectedLabels(dataComponent)}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: memgraphName, + Image: "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "--bolt-port=7687", + "--management-port=10000", + "--data-directory=/var/lib/memgraph/mg_data", + "--log-level=TRACE", + "--also-log-to-stderr", + "--log-file=/var/log/memgraph/memgraph.log", + "--log-retention-days=35", + }, + Env: licenseEnv("memgraph-secrets", "MEMGRAPH_ENTERPRISE_LICENSE", "MEMGRAPH_ORGANIZATION_NAME"), + Ports: []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: 7687}, + {Name: managementPortName, ContainerPort: 10000}, + {Name: replicationPortName, ContainerPort: 20000}, + }, + StartupProbe: tcpProbe(7687, 1440), + ReadinessProbe: tcpProbe(7687, 20), + LivenessProbe: tcpProbe(7687, 20), + VolumeMounts: expectedVolumeMounts(), + SecurityContext: expectedContainerSecurityContext(), + }}, + SecurityContext: expectedPodSecurityContext(), + Volumes: expectedVolumes(), + }, + }, + }, + } + + got := resources.DataStatefulSet(minimalCluster()) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("DataStatefulSet() mismatch (-want +got):\n%s", diff) + } +} + +func TestStatefulSetSpecOverrides(t *testing.T) { + cluster := specifiedCluster() + + tests := []struct { + name string + sts *appsv1.StatefulSet + replicas int32 + }{ + {name: coordinatorComponent, sts: resources.CoordinatorStatefulSet(cluster), replicas: 5}, + {name: dataComponent, sts: resources.DataStatefulSet(cluster), replicas: 3}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := *tc.sts.Spec.Replicas; got != tc.replicas { + t.Errorf("replicas = %d, want %d", got, tc.replicas) + } + container := tc.sts.Spec.Template.Spec.Containers[0] + if container.Image != "registry.example.com/memgraph:3.13.0" { + t.Errorf("image = %q, want %q", container.Image, "registry.example.com/memgraph:3.13.0") + } + if container.ImagePullPolicy != corev1.PullAlways { + t.Errorf("pull policy = %q, want %q", container.ImagePullPolicy, corev1.PullAlways) + } + wantEnv := licenseEnv("my-license", "license", "organization") + gotEnv := container.Env[len(container.Env)-2:] + if diff := cmp.Diff(wantEnv, gotEnv); diff != "" { + t.Errorf("license env mismatch (-want +got):\n%s", diff) + } + }) + } +} From d07655246d773d44249b18a1acf04de8fbdd9311 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Thu, 23 Jul 2026 15:18:31 +0200 Subject: [PATCH 06/34] feat: HA client, planner, controller wiring (#22) * Bootstrap registration: HA client, planner, controller wiring Make a freshly provisioned cluster become an actual HA cluster without human action (specs/operator-mvp/issues/03-bootstrap-registration.md): - internal/memgraph: narrow HA client interface (show instances, add coordinator, register instance, set main) over the Bolt driver; the only place the driver is referenced, and the mock seam for tests. - internal/planner: pure diff of declared topology against observed SHOW INSTANCES output into ordered registration commands, empty when converged. SET INSTANCE TO MAIN is planned only when no MAIN exists. - internal/resources: DeclaredTopology derives per-pod registration identity (coordinator IDs, instance names, advertised FQDNs) from pod ordinals, sharing the FQDN derivation with the coordinator start script so advertised and registered addresses cannot drift. - internal/controller: once both role StatefulSets report all pods ready, find the coordinator leader (follower views redirect to the leader they report; a fresh cluster bootstraps against the first reachable coordinator), plan, and execute. All interaction is read-before-write, so operator restarts mid-bootstrap are harmless. Planner and topology derivation are covered by pure unit tests; the controller registration flow by envtest with an in-memory fake cluster that rejects duplicate registrations and second MAIN promotions. * feat: 0-based indexing for data instances --- cmd/main.go | 6 +- config/manager/kustomization.yaml | 6 + go.mod | 1 + go.sum | 2 + internal/controller/fake_memgraph_test.go | 163 ++++++++++++++ .../controller/memgraphcluster_controller.go | 185 +++++++++++++++- .../memgraphcluster_controller_test.go | 147 +++++++++++- internal/memgraph/bolt.go | 123 +++++++++++ internal/memgraph/client.go | 98 ++++++++ internal/memgraph/queries.go | 50 +++++ internal/memgraph/queries_test.go | 98 ++++++++ internal/planner/planner.go | 120 ++++++++++ internal/planner/planner_test.go | 209 ++++++++++++++++++ internal/resources/statefulset.go | 7 +- internal/resources/topology.go | 75 +++++++ internal/resources/topology_test.go | 115 ++++++++++ 16 files changed, 1389 insertions(+), 16 deletions(-) create mode 100644 internal/controller/fake_memgraph_test.go create mode 100644 internal/memgraph/bolt.go create mode 100644 internal/memgraph/client.go create mode 100644 internal/memgraph/queries.go create mode 100644 internal/memgraph/queries_test.go create mode 100644 internal/planner/planner.go create mode 100644 internal/planner/planner_test.go create mode 100644 internal/resources/topology.go create mode 100644 internal/resources/topology_test.go diff --git a/cmd/main.go b/cmd/main.go index 035ca0c..bddce0c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -37,6 +37,7 @@ import ( memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" "github.com/memgraph/kubernetes-operator/internal/controller" + "github.com/memgraph/kubernetes-operator/internal/memgraph" // +kubebuilder:scaffold:imports ) @@ -179,8 +180,9 @@ func main() { } if err := (&controller.MemgraphClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Memgraph: memgraph.NewBoltConnector(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "Failed to create controller", "controller", "memgraphcluster") os.Exit(1) diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..a7b129f 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/kubernetes-operator + newTag: v0.0.1 diff --git a/go.mod b/go.mod index 2dcf65d..7ff709f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.0 require ( github.com/google/go-cmp v0.7.0 + github.com/neo4j/neo4j-go-driver/v5 v5.28.4 github.com/onsi/ginkgo/v2 v2.27.4 github.com/onsi/gomega v1.39.0 k8s.io/api v0.36.0 diff --git a/go.sum b/go.sum index 690c70d..ff062ed 100644 --- a/go.sum +++ b/go.sum @@ -105,6 +105,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/neo4j/neo4j-go-driver/v5 v5.28.4 h1:7toxehVcYkZbyxV4W3Ib9VcnyRBQPucF+VwNNmtSXi4= +github.com/neo4j/neo4j-go-driver/v5 v5.28.4/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= github.com/onsi/ginkgo/v2 v2.27.4 h1:fcEcQW/A++6aZAZQNUmNjvA9PSOzefMJBerHJ4t8v8Y= github.com/onsi/ginkgo/v2 v2.27.4/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.39.0 h1:y2ROC3hKFmQZJNFeGAMeHZKkjBL65mIZcvrLQBF9k6Q= diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go new file mode 100644 index 0000000..e66eafa --- /dev/null +++ b/internal/controller/fake_memgraph_test.go @@ -0,0 +1,163 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/memgraph/kubernetes-operator/internal/memgraph" +) + +// fakeMemgraph is an in-memory Memgraph HA cluster behind the +// memgraph.Connector seam. It keeps one shared SHOW INSTANCES view, applies +// registration commands to it, and — like the real thing — rejects duplicate +// registrations and second MAIN promotions, so any controller behavior that +// is not read-before-write fails the suite loudly. +type fakeMemgraph struct { + mu sync.Mutex + + // instances is the cluster view every coordinator serves. + instances []memgraph.Instance + connectAttempts int + // executed records every mutating command as ": ". + executed []string +} + +func newFakeMemgraph() *fakeMemgraph { + return &fakeMemgraph{} +} + +func (f *fakeMemgraph) Connect(_ context.Context, address string) (memgraph.Client, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.connectAttempts++ + return &fakeClient{cluster: f, address: address}, nil +} + +func (f *fakeMemgraph) connects() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.connectAttempts +} + +func (f *fakeMemgraph) executedCommands() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.executed) +} + +func (f *fakeMemgraph) setInstances(instances []memgraph.Instance) { + f.mu.Lock() + defer f.mu.Unlock() + f.instances = slices.Clone(instances) +} + +type fakeClient struct { + cluster *fakeMemgraph + address string + closed bool +} + +func (c *fakeClient) ShowInstances(context.Context) ([]memgraph.Instance, error) { + c.cluster.mu.Lock() + defer c.cluster.mu.Unlock() + if c.closed { + return nil, fmt.Errorf("fake memgraph: connection to %s already closed", c.address) + } + return slices.Clone(c.cluster.instances), nil +} + +func (c *fakeClient) AddCoordinator(_ context.Context, coordinator memgraph.CoordinatorSpec) error { + return c.execute(fmt.Sprintf("ADD COORDINATOR %d", coordinator.ID), func() error { + if c.cluster.hasInstance(coordinator.Name()) { + return fmt.Errorf("fake memgraph: coordinator %s already exists", coordinator.Name()) + } + c.cluster.instances = append(c.cluster.instances, memgraph.Instance{ + Name: coordinator.Name(), + BoltServer: coordinator.BoltServer, + CoordinatorServer: coordinator.CoordinatorServer, + ManagementServer: coordinator.ManagementServer, + Health: "up", + Role: memgraph.RoleFollower, + }) + return nil + }) +} + +func (c *fakeClient) RegisterInstance(_ context.Context, instance memgraph.DataInstanceSpec) error { + return c.execute("REGISTER INSTANCE "+instance.Name, func() error { + if c.cluster.hasInstance(instance.Name) { + return fmt.Errorf("fake memgraph: instance %s already registered", instance.Name) + } + c.cluster.instances = append(c.cluster.instances, memgraph.Instance{ + Name: instance.Name, + BoltServer: instance.BoltServer, + ManagementServer: instance.ManagementServer, + Health: "up", + Role: memgraph.RoleReplica, + }) + return nil + }) +} + +func (c *fakeClient) SetInstanceToMain(_ context.Context, name string) error { + return c.execute(fmt.Sprintf("SET INSTANCE %s TO MAIN", name), func() error { + for _, instance := range c.cluster.instances { + if instance.IsMain() { + return fmt.Errorf("fake memgraph: %s is already MAIN", instance.Name) + } + } + for i, instance := range c.cluster.instances { + if instance.Name == name { + c.cluster.instances[i].Role = memgraph.RoleMain + return nil + } + } + return fmt.Errorf("fake memgraph: instance %s is not registered", name) + }) +} + +func (c *fakeClient) Close(context.Context) error { + c.cluster.mu.Lock() + defer c.cluster.mu.Unlock() + c.closed = true + return nil +} + +// execute records the command and applies it to the shared cluster view. +func (c *fakeClient) execute(command string, apply func() error) error { + c.cluster.mu.Lock() + defer c.cluster.mu.Unlock() + if c.closed { + return fmt.Errorf("fake memgraph: connection to %s already closed", c.address) + } + if err := apply(); err != nil { + return err + } + c.cluster.executed = append(c.cluster.executed, c.address+": "+command) + return nil +} + +// hasInstance must be called with the cluster lock held. +func (f *fakeMemgraph) hasInstance(name string) bool { + return slices.ContainsFunc(f.instances, func(instance memgraph.Instance) bool { + return instance.Name == name + }) +} diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 6de579a..fa5e011 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -18,18 +18,23 @@ package controller import ( "context" + "errors" "fmt" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + "github.com/memgraph/kubernetes-operator/internal/memgraph" + "github.com/memgraph/kubernetes-operator/internal/planner" "github.com/memgraph/kubernetes-operator/internal/resources" ) @@ -37,10 +42,26 @@ import ( // manager of the workload objects it provisions. const fieldOwner = "memgraph-operator" +const ( + // requeueWhilePending is how long to wait before retrying when the + // cluster cannot be registered yet — pods not ready, or coordinators not + // answering Bolt queries. Both are expected while the cluster starts up. + requeueWhilePending = 10 * time.Second + + // requeueAfterRegistration schedules the follow-up reconcile that + // verifies issued registration commands actually converged the cluster. + requeueAfterRegistration = 10 * time.Second +) + // MemgraphClusterReconciler reconciles a MemgraphCluster object type MemgraphClusterReconciler struct { client.Client Scheme *runtime.Scheme + + // Memgraph opens Bolt connections to coordinators. Tests substitute a + // fake; everything above the memgraph.Client interface never touches the + // Bolt driver. + Memgraph memgraph.Connector } // +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters,verbs=get;list;watch;create;update;patch;delete @@ -49,11 +70,16 @@ type MemgraphClusterReconciler struct { // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -// Reconcile drives the cluster toward the declared MemgraphCluster spec by -// server-side-applying the builders' desired objects: one StatefulSet per -// role (coordinators, data instances), each backed by a headless Service. -// Deletion needs no handling here — every object carries a controller owner -// reference, so garbage collection removes the workloads with the CR. +// Reconcile drives the cluster toward the declared MemgraphCluster spec in +// two stages. First it server-side-applies the builders' desired objects: one +// StatefulSet per role (coordinators, data instances), each backed by a +// headless Service. Then, once every pod is ready, it reconciles cluster +// registration: observe SHOW INSTANCES on the coordinator leader, diff +// against the declared topology, and issue only the missing commands. All +// interaction is read-before-write and idempotent, so an operator restart +// mid-bootstrap is harmless. Deletion needs no handling here — every object +// carries a controller owner reference, so garbage collection removes the +// workloads with the CR. func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -79,7 +105,154 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Info("Applied desired workload objects for MemgraphCluster", "memgraphcluster", req.NamespacedName) - return ctrl.Result{}, nil + return r.reconcileRegistration(ctx, &cluster) +} + +// reconcileRegistration converges cluster registration once the workloads are +// ready: find the coordinator leader, plan against its SHOW INSTANCES view, +// and execute the missing commands. Unreachable coordinators are retried on a +// delay rather than surfaced as errors — Bolt endpoints lagging pod readiness +// is a normal startup phase, not a failure. +func (r *MemgraphClusterReconciler) reconcileRegistration( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, +) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + ready, err := r.workloadsReady(ctx, cluster) + if err != nil { + return ctrl.Result{}, err + } + if !ready { + log.Info("Waited for workload pods to become ready before registration") + return ctrl.Result{RequeueAfter: requeueWhilePending}, nil + } + + topology := resources.DeclaredTopology(cluster) + leader, observed, err := r.observeCluster(ctx, topology) + if err != nil { + log.Info("Deferred registration because no coordinator answered", "reason", err.Error()) + return ctrl.Result{RequeueAfter: requeueWhilePending}, nil + } + defer func() { + if err := leader.Close(ctx); err != nil { + log.Error(err, "Failed to close coordinator connection") + } + }() + + commands := planner.Plan(topology, observed) + if len(commands) == 0 { + log.Info("Confirmed cluster registration is converged") + return ctrl.Result{}, nil + } + for _, command := range commands { + if err := command.Run(ctx, leader); err != nil { + return ctrl.Result{}, fmt.Errorf("executing registration command %q: %w", command, err) + } + log.Info("Executed registration command", "command", command.String()) + } + + // Registration was issued, not yet observed back; verify convergence on a + // follow-up reconcile instead of assuming success. + return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil +} + +// workloadsReady reports whether both role StatefulSets have all their pods +// ready. Registration waits for the full topology: coordinators cannot form a +// Raft cluster and data instances cannot be registered until every advertised +// address resolves to a running pod. +func (r *MemgraphClusterReconciler) workloadsReady( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, +) (bool, error) { + for _, name := range []string{resources.CoordinatorName(cluster), resources.DataName(cluster)} { + var sts appsv1.StatefulSet + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: cluster.Namespace}, &sts); err != nil { + return false, fmt.Errorf("getting StatefulSet %s: %w", name, err) + } + if sts.Spec.Replicas == nil || sts.Status.ReadyReplicas < *sts.Spec.Replicas { + return false, nil + } + } + return true, nil +} + +// observeCluster connects to the coordinator leader and returns its client +// together with the SHOW INSTANCES view the planner diffs against. +// Coordinators are tried in ordinal order: one reporting itself leader is +// used directly, a follower redirects to the leader it reports, and when no +// leader exists yet (fresh cluster, Raft not formed) the first reachable +// coordinator is used — adding coordinators to it makes it the leader, +// mirroring the HA chart's bootstrap against its first coordinator. +func (r *MemgraphClusterReconciler) observeCluster( + ctx context.Context, + topology planner.Topology, +) (memgraph.Client, []memgraph.Instance, error) { + var errs []error + for _, coordinator := range topology.Coordinators { + leader, observed, err := r.showInstances(ctx, coordinator) + if err != nil { + errs = append(errs, err) + continue + } + + leaderName := "" + for _, instance := range observed { + if instance.IsLeader() { + leaderName = instance.Name + break + } + } + if leaderName == "" || leaderName == coordinator.Name() { + return leader, observed, nil + } + + // This coordinator is a follower; redirect to the leader it reports. + if err := leader.Close(ctx); err != nil { + errs = append(errs, err) + } + candidate, found := coordinatorByName(topology, leaderName) + if !found { + errs = append(errs, fmt.Errorf("%s reported leader %s, which is not declared", coordinator.Name(), leaderName)) + continue + } + leader, observed, err = r.showInstances(ctx, candidate) + if err != nil { + errs = append(errs, err) + continue + } + return leader, observed, nil + } + return nil, nil, fmt.Errorf("no coordinator leader reachable: %w", errors.Join(errs...)) +} + +func coordinatorByName(topology planner.Topology, name string) (memgraph.CoordinatorSpec, bool) { + for _, coordinator := range topology.Coordinators { + if coordinator.Name() == name { + return coordinator, true + } + } + return memgraph.CoordinatorSpec{}, false +} + +// showInstances connects to one coordinator and fetches its cluster view, +// closing the connection again on query failure. +func (r *MemgraphClusterReconciler) showInstances( + ctx context.Context, + coordinator memgraph.CoordinatorSpec, +) (memgraph.Client, []memgraph.Instance, error) { + c, err := r.Memgraph.Connect(ctx, coordinator.BoltServer) + if err != nil { + return nil, nil, err + } + observed, err := c.ShowInstances(ctx) + if err != nil { + if closeErr := c.Close(ctx); closeErr != nil { + return nil, nil, errors.Join(err, closeErr) + } + return nil, nil, err + } + return c, observed, nil } // apply server-side-applies a desired object built by the resource builders. diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 56f39b8..344d497 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + "github.com/memgraph/kubernetes-operator/internal/memgraph" ) // Name suffixes of the per-role workload objects a reconcile creates. @@ -44,21 +45,27 @@ var _ = Describe("MemgraphCluster Controller", func() { ctx := context.Background() - var reconciler *MemgraphClusterReconciler + var ( + reconciler *MemgraphClusterReconciler + fake *fakeMemgraph + ) BeforeEach(func() { + fake = newFakeMemgraph() reconciler = &MemgraphClusterReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), + Client: k8sClient, + Scheme: k8sClient.Scheme(), + Memgraph: fake, } }) - reconcileCluster := func(name string) { + reconcileCluster := func(name string) reconcile.Result { GinkgoHelper() - _, err := reconciler.Reconcile(ctx, reconcile.Request{ + result, err := reconciler.Reconcile(ctx, reconcile.Request{ NamespacedName: types.NamespacedName{Name: name, Namespace: resourceNamespace}, }) Expect(err).NotTo(HaveOccurred()) + return result } get := func(name string, obj client.Object) { @@ -83,6 +90,22 @@ var _ = Describe("MemgraphCluster Controller", func() { } } + // markWorkloadsReady simulates the kubelet envtest does not run: it + // reports every replica of both role StatefulSets as ready, which is what + // gates the registration flow. + markWorkloadsReady := func(clusterName string) { + GinkgoHelper() + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{} + get(clusterName+suffix, sts) + sts.Status.Replicas = *sts.Spec.Replicas + sts.Status.ReadyReplicas = *sts.Spec.Replicas + sts.Status.AvailableReplicas = *sts.Spec.Replicas + sts.Status.ObservedGeneration = sts.Generation + Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) + } + } + expectControlledBy := func(obj client.Object, cluster *memgraphcomv1alpha1.MemgraphCluster) { GinkgoHelper() ref := metav1.GetControllerOf(obj) @@ -250,4 +273,118 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(k8sClient.Create(ctx, invalid)).NotTo(Succeed()) }) }) + + Context("when bootstrapping cluster registration", func() { + const resourceName = "mgc-bootstrap" + + coordinatorAddress := func(ordinal int) string { + return fmt.Sprintf("%s-coordinator-%d.%s-coordinator.%s.svc.cluster.local:7687", + resourceName, ordinal, resourceName, resourceNamespace) + } + + // observedCoordinator reports the coordinator with the given 1-based + // Raft ID, which runs on the pod with ordinal ID-1. + observedCoordinator := func(id int, role string) memgraph.Instance { + return memgraph.Instance{ + Name: fmt.Sprintf("coordinator_%d", id), + BoltServer: coordinatorAddress(id - 1), + Health: "up", + Role: role, + } + } + + observedDataInstance := func(i int, role string) memgraph.Instance { + return memgraph.Instance{ + Name: fmt.Sprintf("instance_%d", i), + Health: "up", + Role: role, + } + } + + BeforeEach(func() { + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + }) + + AfterEach(func() { + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + Expect(k8sClient.Delete(ctx, cluster)).To(Succeed()) + deleteOwned(resourceName) + }) + + It("should not touch Memgraph before every pod is ready", func() { + result := reconcileCluster(resourceName) + + Expect(result.RequeueAfter).To(BeNumerically(">", 0)) + Expect(fake.connects()).To(BeZero()) + }) + + It("should bootstrap a fresh cluster to fully registered with one MAIN", func() { + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + + result := reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": ADD COORDINATOR 1", + leader + ": ADD COORDINATOR 2", + leader + ": ADD COORDINATOR 3", + leader + ": REGISTER INSTANCE instance_0", + leader + ": REGISTER INSTANCE instance_1", + leader + ": SET INSTANCE instance_0 TO MAIN", + })) + Expect(result.RequeueAfter).To(BeNumerically(">", 0), + "registration was issued, so a follow-up reconcile must verify convergence") + + result = reconcileCluster(resourceName) + Expect(fake.executedCommands()).To(HaveLen(6), + "a converged cluster must not receive further commands") + Expect(result.RequeueAfter).To(BeZero()) + }) + + It("should resume a partial bootstrap without duplicate registrations or a second MAIN", func() { + // The state a crash mid-bootstrap leaves behind: two coordinators + // formed, the first data instance registered and promoted. The + // fake rejects duplicate registrations and second promotions, so + // re-issuing anything fails this test loudly. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }) + + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": ADD COORDINATOR 3", + leader + ": REGISTER INSTANCE instance_1", + })) + }) + + It("should execute registration on the leader a follower reports", func() { + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleLeader), + observedCoordinator(3, memgraph.RoleFollower), + }) + + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + leader := coordinatorAddress(1) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": REGISTER INSTANCE instance_0", + leader + ": REGISTER INSTANCE instance_1", + leader + ": SET INSTANCE instance_0 TO MAIN", + })) + }) + }) }) diff --git a/internal/memgraph/bolt.go b/internal/memgraph/bolt.go new file mode 100644 index 0000000..cbec9ef --- /dev/null +++ b/internal/memgraph/bolt.go @@ -0,0 +1,123 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memgraph + +import ( + "context" + "fmt" + + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "github.com/neo4j/neo4j-go-driver/v5/neo4j/db" +) + +// NewBoltConnector returns the production Connector dialing coordinators over +// unauthenticated Bolt (Bolt auth is out of scope for v1alpha1). +func NewBoltConnector() Connector { + return boltConnector{} +} + +type boltConnector struct{} + +func (boltConnector) Connect(ctx context.Context, address string) (Client, error) { + driver, err := neo4j.NewDriverWithContext("bolt://"+address, neo4j.NoAuth()) + if err != nil { + return nil, fmt.Errorf("creating bolt driver for %s: %w", address, err) + } + if err := driver.VerifyConnectivity(ctx); err != nil { + _ = driver.Close(ctx) + return nil, fmt.Errorf("connecting to %s: %w", address, err) + } + return &boltClient{driver: driver}, nil +} + +type boltClient struct { + driver neo4j.DriverWithContext +} + +func (c *boltClient) ShowInstances(ctx context.Context) ([]Instance, error) { + records, err := c.run(ctx, showInstancesQuery) + if err != nil { + return nil, err + } + instances := make([]Instance, 0, len(records)) + for _, record := range records { + instances = append(instances, instanceFromRecord(record)) + } + return instances, nil +} + +func (c *boltClient) AddCoordinator(ctx context.Context, coordinator CoordinatorSpec) error { + _, err := c.run(ctx, addCoordinatorQuery(coordinator)) + return err +} + +func (c *boltClient) RegisterInstance(ctx context.Context, instance DataInstanceSpec) error { + _, err := c.run(ctx, registerInstanceQuery(instance)) + return err +} + +func (c *boltClient) SetInstanceToMain(ctx context.Context, name string) error { + _, err := c.run(ctx, setInstanceToMainQuery(name)) + return err +} + +func (c *boltClient) Close(ctx context.Context) error { + return c.driver.Close(ctx) +} + +// run executes one query in an autocommit session; Memgraph's coordinator +// queries cannot run inside explicit transactions. +func (c *boltClient) run(ctx context.Context, query string) ([]*db.Record, error) { + session := c.driver.NewSession(ctx, neo4j.SessionConfig{}) + defer func() { _ = session.Close(ctx) }() + + result, err := session.Run(ctx, query, nil) + if err != nil { + return nil, fmt.Errorf("running %q: %w", query, err) + } + records, err := result.Collect(ctx) + if err != nil { + return nil, fmt.Errorf("collecting results of %q: %w", query, err) + } + return records, nil +} + +// instanceFromRecord maps one SHOW INSTANCES row to an Instance. Columns are +// looked up by name so the parsing survives added or reordered columns +// (last_succ_resp_ms is deliberately ignored). +func instanceFromRecord(record *db.Record) Instance { + return Instance{ + Name: stringColumn(record, "name"), + BoltServer: stringColumn(record, "bolt_server"), + CoordinatorServer: stringColumn(record, "coordinator_server"), + ManagementServer: stringColumn(record, "management_server"), + Health: stringColumn(record, "health"), + Role: stringColumn(record, "role"), + } +} + +func stringColumn(record *db.Record, key string) string { + value, ok := record.Get(key) + if !ok { + return "" + } + s, ok := value.(string) + if !ok { + return "" + } + return s +} diff --git a/internal/memgraph/client.go b/internal/memgraph/client.go new file mode 100644 index 0000000..f9a3fda --- /dev/null +++ b/internal/memgraph/client.go @@ -0,0 +1,98 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package memgraph provides the narrow client surface the operator uses to +// drive a Memgraph high-availability cluster over Bolt: show instances, add +// coordinator, register instance, set main. All higher layers depend on the +// Client and Connector interfaces, never on the Bolt driver — this package is +// the mock seam for testing and the only place the driver is referenced. +package memgraph + +import ( + "context" + "fmt" + "strings" +) + +// Roles reported in the SHOW INSTANCES role column: coordinators are +// leader/follower, data instances are main/replica. +const ( + RoleLeader = "leader" + RoleFollower = "follower" + RoleMain = "main" + RoleReplica = "replica" +) + +// Instance is one row of SHOW INSTANCES: a coordinator or data instance the +// cluster currently knows about. +type Instance struct { + Name string + BoltServer string + CoordinatorServer string + ManagementServer string + Health string + Role string +} + +// IsLeader reports whether the instance is the current coordinator leader. +func (i Instance) IsLeader() bool { + return strings.EqualFold(i.Role, RoleLeader) +} + +// IsMain reports whether the instance is the current MAIN data instance. +func (i Instance) IsMain() bool { + return strings.EqualFold(i.Role, RoleMain) +} + +// CoordinatorSpec declares one coordinator to add to the cluster. Servers are +// "host:port" addresses the rest of the cluster reaches the coordinator at. +type CoordinatorSpec struct { + // ID is the Raft coordinator ID (1-based; Memgraph treats ID 0 as unset). + ID int32 + BoltServer string + CoordinatorServer string + ManagementServer string +} + +// Name returns the instance name Memgraph derives from the coordinator ID and +// reports in SHOW INSTANCES. +func (c CoordinatorSpec) Name() string { + return fmt.Sprintf("coordinator_%d", c.ID) +} + +// DataInstanceSpec declares one data instance to register with the cluster. +type DataInstanceSpec struct { + Name string + BoltServer string + ManagementServer string + ReplicationServer string +} + +// Client is the narrow surface of a single coordinator's Bolt endpoint. Every +// method issues exactly one HA management query. +type Client interface { + ShowInstances(ctx context.Context) ([]Instance, error) + AddCoordinator(ctx context.Context, coordinator CoordinatorSpec) error + RegisterInstance(ctx context.Context, instance DataInstanceSpec) error + SetInstanceToMain(ctx context.Context, name string) error + Close(ctx context.Context) error +} + +// Connector opens a Client to a coordinator's "host:port" Bolt address. The +// controller depends on this interface so tests can substitute a fake cluster. +type Connector interface { + Connect(ctx context.Context, address string) (Client, error) +} diff --git a/internal/memgraph/queries.go b/internal/memgraph/queries.go new file mode 100644 index 0000000..f19b9a9 --- /dev/null +++ b/internal/memgraph/queries.go @@ -0,0 +1,50 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memgraph + +import "fmt" + +// The HA management query grammar, mirroring what the memgraph-high-availability +// Helm chart's registration job issues. Config values are rendered inline +// because Memgraph's coordinator queries do not accept Bolt parameters; all +// inputs are operator-derived names and "host:port" addresses, never user text. + +const showInstancesQuery = "SHOW INSTANCES" + +func addCoordinatorQuery(coordinator CoordinatorSpec) string { + return fmt.Sprintf( + `ADD COORDINATOR %d WITH CONFIG {"bolt_server": %q, "coordinator_server": %q, "management_server": %q}`, + coordinator.ID, + coordinator.BoltServer, + coordinator.CoordinatorServer, + coordinator.ManagementServer, + ) +} + +func registerInstanceQuery(instance DataInstanceSpec) string { + return fmt.Sprintf( + `REGISTER INSTANCE %s WITH CONFIG {"bolt_server": %q, "management_server": %q, "replication_server": %q}`, + instance.Name, + instance.BoltServer, + instance.ManagementServer, + instance.ReplicationServer, + ) +} + +func setInstanceToMainQuery(name string) string { + return fmt.Sprintf("SET INSTANCE %s TO MAIN", name) +} diff --git a/internal/memgraph/queries_test.go b/internal/memgraph/queries_test.go new file mode 100644 index 0000000..572d64e --- /dev/null +++ b/internal/memgraph/queries_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memgraph + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/neo4j/neo4j-go-driver/v5/neo4j/db" +) + +const testInstanceName = "instance_1" + +func TestAddCoordinatorQuery(t *testing.T) { + got := addCoordinatorQuery(CoordinatorSpec{ + ID: 2, + BoltServer: "example-coordinator-1.example-coordinator.default.svc.cluster.local:7687", + CoordinatorServer: "example-coordinator-1.example-coordinator.default.svc.cluster.local:12000", + ManagementServer: "example-coordinator-1.example-coordinator.default.svc.cluster.local:10000", + }) + want := `ADD COORDINATOR 2 WITH CONFIG {` + + `"bolt_server": "example-coordinator-1.example-coordinator.default.svc.cluster.local:7687", ` + + `"coordinator_server": "example-coordinator-1.example-coordinator.default.svc.cluster.local:12000", ` + + `"management_server": "example-coordinator-1.example-coordinator.default.svc.cluster.local:10000"}` + if got != want { + t.Errorf("addCoordinatorQuery() = %q, want %q", got, want) + } +} + +func TestRegisterInstanceQuery(t *testing.T) { + got := registerInstanceQuery(DataInstanceSpec{ + Name: testInstanceName, + BoltServer: "example-data-0.example-data.default.svc.cluster.local:7687", + ManagementServer: "example-data-0.example-data.default.svc.cluster.local:10000", + ReplicationServer: "example-data-0.example-data.default.svc.cluster.local:20000", + }) + want := `REGISTER INSTANCE instance_1 WITH CONFIG {` + + `"bolt_server": "example-data-0.example-data.default.svc.cluster.local:7687", ` + + `"management_server": "example-data-0.example-data.default.svc.cluster.local:10000", ` + + `"replication_server": "example-data-0.example-data.default.svc.cluster.local:20000"}` + if got != want { + t.Errorf("registerInstanceQuery() = %q, want %q", got, want) + } +} + +func TestSetInstanceToMainQuery(t *testing.T) { + got := setInstanceToMainQuery(testInstanceName) + if want := "SET INSTANCE instance_1 TO MAIN"; got != want { + t.Errorf("setInstanceToMainQuery() = %q, want %q", got, want) + } +} + +func TestInstanceFromRecord(t *testing.T) { + record := &db.Record{ + Keys: []string{ + "name", "bolt_server", "coordinator_server", "management_server", "health", "role", "last_succ_resp_ms", + }, + Values: []any{ + "coordinator_1", "localhost:7687", "localhost:12000", "localhost:10000", "up", "leader", int64(12), + }, + } + want := Instance{ + Name: "coordinator_1", + BoltServer: "localhost:7687", + CoordinatorServer: "localhost:12000", + ManagementServer: "localhost:10000", + Health: "up", + Role: "leader", + } + if diff := cmp.Diff(want, instanceFromRecord(record)); diff != "" { + t.Errorf("instanceFromRecord() mismatch (-want +got):\n%s", diff) + } +} + +func TestInstanceFromRecordToleratesMissingColumns(t *testing.T) { + record := &db.Record{ + Keys: []string{"name", "role"}, + Values: []any{testInstanceName, "main"}, + } + want := Instance{Name: testInstanceName, Role: "main"} + if diff := cmp.Diff(want, instanceFromRecord(record)); diff != "" { + t.Errorf("instanceFromRecord() mismatch (-want +got):\n%s", diff) + } +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go new file mode 100644 index 0000000..336efbb --- /dev/null +++ b/internal/planner/planner.go @@ -0,0 +1,120 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package planner computes the ordered registration commands that drive an +// observed Memgraph HA cluster toward its declared topology. The logic is a +// pure diff — declared topology plus observed SHOW INSTANCES output in, +// commands out (empty when converged) — so reconciliation stays idempotent +// and read-before-write: only missing registrations are re-issued, and the +// initial MAIN promotion happens exactly once, when no MAIN exists. After +// bootstrap, failover belongs to the Raft coordinators; the planner never +// overrides an existing MAIN. +package planner + +import ( + "context" + "fmt" + + "github.com/memgraph/kubernetes-operator/internal/memgraph" +) + +// Topology is the declared cluster registration state: every coordinator and +// data instance the CR says must exist, with the addresses each advertises. +type Topology struct { + Coordinators []memgraph.CoordinatorSpec + DataInstances []memgraph.DataInstanceSpec +} + +// Command is one registration step to execute against the coordinator leader. +type Command interface { + Run(ctx context.Context, client memgraph.Client) error + fmt.Stringer +} + +// AddCoordinator adds one declared coordinator to the Raft cluster. +type AddCoordinator struct { + Coordinator memgraph.CoordinatorSpec +} + +// Run implements Command. +func (c AddCoordinator) Run(ctx context.Context, client memgraph.Client) error { + return client.AddCoordinator(ctx, c.Coordinator) +} + +func (c AddCoordinator) String() string { + return fmt.Sprintf("ADD COORDINATOR %d", c.Coordinator.ID) +} + +// RegisterInstance registers one declared data instance with the cluster. +type RegisterInstance struct { + Instance memgraph.DataInstanceSpec +} + +// Run implements Command. +func (c RegisterInstance) Run(ctx context.Context, client memgraph.Client) error { + return client.RegisterInstance(ctx, c.Instance) +} + +func (c RegisterInstance) String() string { + return "REGISTER INSTANCE " + c.Instance.Name +} + +// SetInstanceToMain promotes the named data instance to MAIN at bootstrap. +type SetInstanceToMain struct { + Name string +} + +// Run implements Command. +func (c SetInstanceToMain) Run(ctx context.Context, client memgraph.Client) error { + return client.SetInstanceToMain(ctx, c.Name) +} + +func (c SetInstanceToMain) String() string { + return fmt.Sprintf("SET INSTANCE %s TO MAIN", c.Name) +} + +// Plan diffs the declared topology against the observed instances and returns +// the commands still needed, in execution order: coordinators before data +// instances (registration requires a formed Raft cluster), the initial MAIN +// promotion last. Instances the cluster knows but the topology does not +// declare are left untouched — unregistration is out of scope for v1. +func Plan(declared Topology, observed []memgraph.Instance) []Command { + registered := make(map[string]memgraph.Instance, len(observed)) + hasMain := false + for _, instance := range observed { + registered[instance.Name] = instance + hasMain = hasMain || instance.IsMain() + } + + var commands []Command + for _, coordinator := range declared.Coordinators { + // A coordinator reports itself in SHOW INSTANCES with an empty + // bolt_server until ADD COORDINATOR is issued for its ID, so presence + // alone does not prove registration. + if observed, ok := registered[coordinator.Name()]; !ok || observed.BoltServer == "" { + commands = append(commands, AddCoordinator{Coordinator: coordinator}) + } + } + for _, instance := range declared.DataInstances { + if _, ok := registered[instance.Name]; !ok { + commands = append(commands, RegisterInstance{Instance: instance}) + } + } + if !hasMain && len(declared.DataInstances) > 0 { + commands = append(commands, SetInstanceToMain{Name: declared.DataInstances[0].Name}) + } + return commands +} diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go new file mode 100644 index 0000000..d3e8bfe --- /dev/null +++ b/internal/planner/planner_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package planner_test + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/memgraph/kubernetes-operator/internal/memgraph" + "github.com/memgraph/kubernetes-operator/internal/planner" +) + +// declaredTopology is the canonical 3-coordinator, 2-data-instance fixture +// the cases below diff observed cluster states against. +func declaredTopology() planner.Topology { + topology := planner.Topology{} + for id := int32(1); id <= 3; id++ { + topology.Coordinators = append(topology.Coordinators, coordinatorSpec(id)) + } + for i := range 2 { + topology.DataInstances = append(topology.DataInstances, dataInstanceSpec(i)) + } + return topology +} + +// coordinatorSpec builds the declared coordinator with the given 1-based Raft +// ID (Memgraph treats ID 0 as unset), hosted on the pod with ordinal ID-1. +func coordinatorSpec(id int32) memgraph.CoordinatorSpec { + host := fmt.Sprintf("example-coordinator-%d.example-coordinator.default.svc.cluster.local", id-1) + return memgraph.CoordinatorSpec{ + ID: id, + BoltServer: host + ":7687", + CoordinatorServer: host + ":12000", + ManagementServer: host + ":10000", + } +} + +func dataInstanceSpec(i int) memgraph.DataInstanceSpec { + host := fmt.Sprintf("example-data-%d.example-data.default.svc.cluster.local", i) + return memgraph.DataInstanceSpec{ + Name: fmt.Sprintf("instance_%d", i), + BoltServer: host + ":7687", + ManagementServer: host + ":10000", + ReplicationServer: host + ":20000", + } +} + +func observedCoordinator(id int32, role string) memgraph.Instance { + spec := coordinatorSpec(id) + return memgraph.Instance{ + Name: spec.Name(), + BoltServer: spec.BoltServer, + CoordinatorServer: spec.CoordinatorServer, + ManagementServer: spec.ManagementServer, + Health: "up", + Role: role, + } +} + +func observedDataInstance(i int, role string) memgraph.Instance { + spec := dataInstanceSpec(i) + return memgraph.Instance{ + Name: spec.Name, + BoltServer: spec.BoltServer, + ManagementServer: spec.ManagementServer, + Health: "up", + Role: role, + } +} + +func TestPlan(t *testing.T) { + declared := declaredTopology() + + cases := []struct { + name string + observed []memgraph.Instance + want []planner.Command + }{ + { + name: "fresh cluster bootstraps everything and promotes one MAIN", + observed: nil, + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(1)}, + planner.AddCoordinator{Coordinator: coordinatorSpec(2)}, + planner.AddCoordinator{Coordinator: coordinatorSpec(3)}, + planner.RegisterInstance{Instance: dataInstanceSpec(0)}, + planner.RegisterInstance{Instance: dataInstanceSpec(1)}, + planner.SetInstanceToMain{Name: "instance_0"}, + }, + }, + { + name: "partially registered cluster gets only the missing registrations", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }, + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(2)}, + planner.RegisterInstance{Instance: dataInstanceSpec(1)}, + }, + }, + { + name: "self-reporting coordinator with empty bolt server is still added", + observed: []memgraph.Instance{ + // The coordinator the client is connected to lists itself in + // SHOW INSTANCES with an empty bolt_server until explicitly + // added. + func() memgraph.Instance { + instance := observedCoordinator(2, memgraph.RoleLeader) + instance.BoltServer = "" + return instance + }(), + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(2)}, + }, + }, + { + name: "fully converged cluster is a no-op", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: nil, + }, + { + name: "an existing MAIN is never overridden, even on another instance", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleMain), + }, + want: nil, + }, + { + name: "registered but leaderless data plane still gets the one MAIN promotion", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: []planner.Command{ + planner.SetInstanceToMain{Name: "instance_0"}, + }, + }, + { + name: "missing instance registers without MAIN promotion when a MAIN exists", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(1, memgraph.RoleMain), + }, + want: []planner.Command{ + planner.RegisterInstance{Instance: dataInstanceSpec(0)}, + }, + }, + { + name: "instances the topology does not declare are left untouched", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedCoordinator(4, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + want: nil, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := planner.Plan(declared, tc.observed) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("Plan() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index 8e713fb..c784c2a 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -91,10 +91,11 @@ func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.State // coordinatorStartScript derives the coordinator's identity from its pod // ordinal (the numeric suffix of the pod name): ordinal N becomes coordinator -// ID N+1 (Raft IDs start at 1) advertised at the pod's stable DNS name within -// the headless Service. +// ID N+1 (Memgraph treats coordinator ID 0 as unset and refuses to start, so +// IDs stay 1-based) advertised at the pod's stable DNS name within the +// headless Service. func coordinatorStartScript(cluster *memgraphcomv1alpha1.MemgraphCluster) string { - fqdnSuffix := fmt.Sprintf("%s.%s.svc.%s", CoordinatorName(cluster), cluster.Namespace, clusterDomain) + fqdnSuffix := podFQDNSuffix(cluster, CoordinatorName(cluster)) return fmt.Sprintf(`ordinal="${POD_NAME##*-}" exec %s \ --coordinator-id="$((ordinal + 1))" \ diff --git a/internal/resources/topology.go b/internal/resources/topology.go new file mode 100644 index 0000000..9c5370b --- /dev/null +++ b/internal/resources/topology.go @@ -0,0 +1,75 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources + +import ( + "fmt" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" + "github.com/memgraph/kubernetes-operator/internal/memgraph" + "github.com/memgraph/kubernetes-operator/internal/planner" +) + +// DeclaredTopology derives the registration topology the planner drives the +// cluster toward. Identity follows the pod ordinal exactly as the workload +// pods advertise it: coordinator ordinal N is Raft coordinator N+1 (Memgraph +// treats coordinator ID 0 as unset, so IDs stay 1-based), data ordinal N +// registers as instance_N, and every address is the pod's stable DNS name +// within its headless Service. +func DeclaredTopology(cluster *memgraphcomv1alpha1.MemgraphCluster) planner.Topology { + spec := normalize(cluster.Spec) + + topology := planner.Topology{ + Coordinators: make([]memgraph.CoordinatorSpec, 0, spec.coordinators), + DataInstances: make([]memgraph.DataInstanceSpec, 0, spec.dataInstances), + } + for ordinal := range spec.coordinators { + fqdn := podFQDN(cluster, CoordinatorName(cluster), ordinal) + topology.Coordinators = append(topology.Coordinators, memgraph.CoordinatorSpec{ + ID: ordinal + 1, + BoltServer: hostPort(fqdn, BoltPort), + CoordinatorServer: hostPort(fqdn, CoordinatorPort), + ManagementServer: hostPort(fqdn, ManagementPort), + }) + } + for ordinal := range spec.dataInstances { + fqdn := podFQDN(cluster, DataName(cluster), ordinal) + topology.DataInstances = append(topology.DataInstances, memgraph.DataInstanceSpec{ + Name: fmt.Sprintf("instance_%d", ordinal), + BoltServer: hostPort(fqdn, BoltPort), + ManagementServer: hostPort(fqdn, ManagementPort), + ReplicationServer: hostPort(fqdn, ReplicationPort), + }) + } + return topology +} + +// podFQDNSuffix returns the DNS suffix a pod name is appended to for pods of +// the given headless Service: "..svc.". +func podFQDNSuffix(cluster *memgraphcomv1alpha1.MemgraphCluster, serviceName string) string { + return fmt.Sprintf("%s.%s.svc.%s", serviceName, cluster.Namespace, clusterDomain) +} + +// podFQDN returns the stable DNS name of the pod with the given ordinal in +// the StatefulSet backed by the given headless Service (both share one name). +func podFQDN(cluster *memgraphcomv1alpha1.MemgraphCluster, serviceName string, ordinal int32) string { + return fmt.Sprintf("%s-%d.%s", serviceName, ordinal, podFQDNSuffix(cluster, serviceName)) +} + +func hostPort(host string, port int32) string { + return fmt.Sprintf("%s:%d", host, port) +} diff --git a/internal/resources/topology_test.go b/internal/resources/topology_test.go new file mode 100644 index 0000000..0ee9d6a --- /dev/null +++ b/internal/resources/topology_test.go @@ -0,0 +1,115 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resources_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/memgraph/kubernetes-operator/internal/memgraph" + "github.com/memgraph/kubernetes-operator/internal/planner" + "github.com/memgraph/kubernetes-operator/internal/resources" +) + +func TestDeclaredTopologyDefaults(t *testing.T) { + got := resources.DeclaredTopology(minimalCluster()) + + coordinatorFQDN := func(ordinal int) string { + return fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local", coordinatorName, ordinal, coordinatorName, testNamespace) + } + dataFQDN := func(ordinal int) string { + return fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local", dataName, ordinal, dataName, testNamespace) + } + + want := planner.Topology{ + Coordinators: []memgraph.CoordinatorSpec{ + { + ID: 1, + BoltServer: coordinatorFQDN(0) + ":7687", + CoordinatorServer: coordinatorFQDN(0) + ":12000", + ManagementServer: coordinatorFQDN(0) + ":10000", + }, + { + ID: 2, + BoltServer: coordinatorFQDN(1) + ":7687", + CoordinatorServer: coordinatorFQDN(1) + ":12000", + ManagementServer: coordinatorFQDN(1) + ":10000", + }, + { + ID: 3, + BoltServer: coordinatorFQDN(2) + ":7687", + CoordinatorServer: coordinatorFQDN(2) + ":12000", + ManagementServer: coordinatorFQDN(2) + ":10000", + }, + }, + DataInstances: []memgraph.DataInstanceSpec{ + { + Name: "instance_0", + BoltServer: dataFQDN(0) + ":7687", + ManagementServer: dataFQDN(0) + ":10000", + ReplicationServer: dataFQDN(0) + ":20000", + }, + { + Name: "instance_1", + BoltServer: dataFQDN(1) + ":7687", + ManagementServer: dataFQDN(1) + ":10000", + ReplicationServer: dataFQDN(1) + ":20000", + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("DeclaredTopology() mismatch (-want +got):\n%s", diff) + } +} + +func TestDeclaredTopologyFollowsReplicaCounts(t *testing.T) { + got := resources.DeclaredTopology(specifiedCluster()) + + if len(got.Coordinators) != 5 { + t.Errorf("DeclaredTopology() declared %d coordinators, want 5", len(got.Coordinators)) + } + if len(got.DataInstances) != 3 { + t.Errorf("DeclaredTopology() declared %d data instances, want 3", len(got.DataInstances)) + } +} + +// The registration topology must advertise exactly the identity the +// coordinator pods derive for themselves at startup, otherwise the Raft +// cluster and the registrations disagree about who is who. +func TestDeclaredTopologyMatchesCoordinatorStartScript(t *testing.T) { + cluster := minimalCluster() + topology := resources.DeclaredTopology(cluster) + sts := resources.CoordinatorStatefulSet(cluster) + script := strings.Join(sts.Spec.Template.Spec.Containers[0].Command, "\n") + + // The script derives '.' from POD_NAME; every declared + // coordinator_server must be a pod FQDN under that same suffix. + suffix := fmt.Sprintf("%s.%s.svc.cluster.local", coordinatorName, testNamespace) + if !strings.Contains(script, `--coordinator-hostname="${POD_NAME}.`+suffix+`"`) { + t.Errorf("coordinator start script does not advertise the headless-service pod FQDN:\n%s", script) + } + for i, coordinator := range topology.Coordinators { + wantHost := fmt.Sprintf("%s-%d.%s:12000", coordinatorName, i, suffix) + if coordinator.CoordinatorServer != wantHost { + t.Errorf("coordinator %d advertises %q, want %q", coordinator.ID, coordinator.CoordinatorServer, wantHost) + } + } +} From 9cd11634b814ad9982e261885b685a6f8a187a3a Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Fri, 24 Jul 2026 15:13:31 +0200 Subject: [PATCH 07/34] testing: licensed MemgraphCluster bootstrap test #23 * KinD e2e harness: licensed MemgraphCluster bootstrap proof, PR-gated A true end-to-end suite for the provisioning and bootstrap slices: CI boots a multi-node Kind cluster, builds and deploys the operator image into it, applies a MemgraphCluster running real Memgraph images, and asserts every declared instance appears registered in SHOW INSTANCES with exactly one MAIN (queried through mgconsole inside a coordinator pod, the HA chart CI's established practice). Operator deployment (namespace, CRDs, controller) moves from the Manager container into BeforeSuite/AfterSuite, so scenario containers only exercise MemgraphCluster behavior and later slices (re-registration, storage retention) add test cases, not pipeline. The enterprise license flows from the MEMGRAPH_ENTERPRISE_LICENSE / MEMGRAPH_ORGANIZATION_NAME repository secrets into a Kubernetes Secret applied over stdin, so no secret material reaches logged command lines or the repo. Part of specs/operator-mvp/issues/04-kind-e2e-harness.md. * testing: Fix checking for is main * refactor: Move role main under the constants block --- .github/workflows/test.yml | 29 +++ CLAUDE.md | 2 +- Makefile | 9 +- test/e2e/e2e_suite_test.go | 38 ++++ test/e2e/e2e_test.go | 43 +---- test/e2e/kind-config.yaml | 10 + test/e2e/memgraphcluster_test.go | 317 +++++++++++++++++++++++++++++++ test/utils/utils.go | 8 + 8 files changed, 413 insertions(+), 43 deletions(-) create mode 100644 test/e2e/kind-config.yaml create mode 100644 test/e2e/memgraphcluster_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04f5442..af348f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,3 +46,32 @@ jobs: - name: Run tests against envtest run: make test + + e2e: + permissions: + contents: read + name: E2E tests + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Install Kind + run: go install sigs.k8s.io/kind@v0.32.0 + + - name: Run e2e tests against Kind + env: + # Enterprise license for the Memgraph HA cluster the suite boots, + # following the HA chart CI's repository-secret practice. + MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }} + MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }} + # The operator has no webhooks in v1, so the suite needs no CertManager. + CERT_MANAGER_INSTALL_SKIP: "true" + run: make test-e2e diff --git a/CLAUDE.md b/CLAUDE.md index 8399247..fa4ce27 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ go test ./api/... -run TestName Envtest packages need `KUBEBUILDER_ASSETS`; outside of `make test` set it with: `KUBEBUILDER_ASSETS=$(bin/setup-envtest use --bin-dir bin -p path)` -CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, and `make test` on every PR — all must be green. +CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, `make test`, and `make test-e2e` on every PR — all must be green. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license flowing from the `MEMGRAPH_ENTERPRISE_LICENSE` / `MEMGRAPH_ORGANIZATION_NAME` repository secrets (set the same env vars to run it locally). ### Toolchain quirks (do not "fix" these) diff --git a/Makefile b/Makefile index 5f2f6f7..1f1ce45 100644 --- a/Makefile +++ b/Makefile @@ -80,9 +80,10 @@ test: manifests generate fmt vet setup-envtest ## Run tests. # CertManager is installed by default; skip with: # - CERT_MANAGER_INSTALL_SKIP=true KIND_CLUSTER ?= kubernetes-operator-test-e2e +KIND_CONFIG ?= test/e2e/kind-config.yaml .PHONY: setup-test-e2e -setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist +setup-test-e2e: ## Set up a multi-node Kind cluster for e2e tests if it does not exist @command -v $(KIND) >/dev/null 2>&1 || { \ echo "Kind is not installed. Please install Kind manually."; \ exit 1; \ @@ -92,12 +93,14 @@ setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ *) \ echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ + $(KIND) create cluster --name $(KIND_CLUSTER) --config $(KIND_CONFIG) ;; \ esac +# The generous timeout covers the whole suite end to end: building the manager +# image, pulling real Memgraph images, and bootstrapping an HA cluster in Kind. .PHONY: test-e2e test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. - KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v -timeout 40m $(MAKE) cleanup-test-e2e .PHONY: cleanup-test-e2e diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b1b0424..ec5d6bd 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -50,6 +50,11 @@ func TestE2E(t *testing.T) { RunSpecs(t, "e2e suite") } +// The suite deploys the operator once, before any scenario runs: build and +// load the manager image, install the CRDs, and deploy the controller into its +// namespace. Scenario containers (Describe blocks) then only exercise +// MemgraphCluster behavior, so a new scenario is a new test case, never new +// pipeline or deployment plumbing. var _ = BeforeSuite(func() { By("building the manager image") cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) @@ -64,9 +69,42 @@ var _ = BeforeSuite(func() { configureKubectlKubeRC() setupCertManager() + + By("creating manager namespace") + cmd = exec.Command("kubectl", "create", "ns", namespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("installing CRDs") + cmd = exec.Command("make", "install") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") + + By("deploying the controller-manager") + cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) var _ = AfterSuite(func() { + By("undeploying the controller-manager") + cmd := exec.Command("make", "undeploy") + _, _ = utils.Run(cmd) + + By("uninstalling CRDs") + cmd = exec.Command("make", "uninstall") + _, _ = utils.Run(cmd) + + By("removing manager namespace") + cmd = exec.Command("kubectl", "delete", "ns", namespace) + _, _ = utils.Run(cmd) + teardownCertManager() }) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index d606215..dbd3573 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -48,50 +48,15 @@ const metricsRoleBindingName = "kubernetes-operator-metrics-binding" var _ = Describe("Manager", Ordered, func() { var controllerPodName string - // Before running the tests, set up the environment by creating the namespace, - // enforce the restricted security policy to the namespace, installing CRDs, - // and deploying the controller. - BeforeAll(func() { - By("creating manager namespace") - cmd := exec.Command("kubectl", "create", "ns", namespace) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") - - By("labeling the namespace to enforce the restricted security policy") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") - }) + // The operator itself (namespace, CRDs, controller Deployment) is set up + // once for the whole suite in BeforeSuite; this container only validates + // the already-deployed manager. - // After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs, - // and deleting the namespace. + // After all tests have been executed, clean up resources created by this container. AfterAll(func() { By("cleaning up the curl pod for metrics") cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace) _, _ = utils.Run(cmd) - - By("undeploying the controller-manager") - cmd = exec.Command("make", "undeploy") - _, _ = utils.Run(cmd) - - By("uninstalling CRDs") - cmd = exec.Command("make", "uninstall") - _, _ = utils.Run(cmd) - - By("removing manager namespace") - cmd = exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) }) // After each test, check for failures and collect logs, events, diff --git a/test/e2e/kind-config.yaml b/test/e2e/kind-config.yaml new file mode 100644 index 0000000..9e0d2b6 --- /dev/null +++ b/test/e2e/kind-config.yaml @@ -0,0 +1,10 @@ +# Multi-node Kind cluster for the e2e suite. A Memgraph HA cluster is only a +# meaningful end-to-end proof when its coordinators and data instances spread +# across several nodes, mirroring the HA chart's multi-node CI. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + - role: worker + - role: worker diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go new file mode 100644 index 0000000..d2035e2 --- /dev/null +++ b/test/e2e/memgraphcluster_test.go @@ -0,0 +1,317 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/memgraph/kubernetes-operator/test/utils" +) + +// The declared topology of the e2e cluster and the identities the operator +// derives from it: coordinator ordinal N registers as coordinator_N+1, data +// ordinal N as instance_N. +const ( + clusterNamespace = "memgraph-e2e" + clusterName = "memgraph" + + coordinatorCount = 3 + dataInstanceCount = 2 + + memgraphImageRepository = "docker.io/memgraph/memgraph" + memgraphImageTag = "3.12.0" + + // licenseSecretName and the env var names below follow the HA Helm chart's + // CI convention: repository secrets of the same names are exported into the + // job environment and materialize as one Kubernetes Secret the CR + // references. + licenseSecretName = "memgraph-secrets" + licenseEnvVar = "MEMGRAPH_ENTERPRISE_LICENSE" + organizationEnvVar = "MEMGRAPH_ORGANIZATION_NAME" + + // roleMain is the MAIN data-instance role reported in the SHOW INSTANCES role + // column. + roleMain = "main" +) + +// declaredInstances returns the instance names every coordinator and data +// instance must appear under in SHOW INSTANCES once the operator has converged +// registration. +func declaredInstances() []string { + names := make([]string, 0, coordinatorCount+dataInstanceCount) + for ordinal := range coordinatorCount { + names = append(names, fmt.Sprintf("coordinator_%d", ordinal+1)) + } + for ordinal := range dataInstanceCount { + names = append(names, fmt.Sprintf("instance_%d", ordinal)) + } + return names +} + +// MemgraphCluster is the end-to-end proof of the provisioning and bootstrap +// slices: a real multi-node Kind cluster, the deployed operator, real licensed +// Memgraph images, and assertions over SHOW INSTANCES. +// +// The container owns one cluster for all its specs: BeforeAll provisions the +// namespace, license Secret, and CR, so a new scenario against the same +// cluster is just another It block (with later Its observing earlier +// mutations, e.g. a deliberately wiped pod). A scenario needing a +// differently-shaped cluster gets its own Ordered container following this +// same pattern — no pipeline changes. +var _ = Describe("MemgraphCluster", Ordered, func() { + BeforeAll(func() { + license := os.Getenv(licenseEnvVar) + organization := os.Getenv(organizationEnvVar) + Expect(license).NotTo(BeEmpty(), + "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", licenseEnvVar) + Expect(organization).NotTo(BeEmpty(), + "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", organizationEnvVar) + + By("preloading the Memgraph image into the Kind cluster") + memgraphImage := memgraphImageRepository + ":" + memgraphImageTag + cmd := exec.Command("docker", "pull", memgraphImage) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to pull the Memgraph image") + Expect(utils.LoadImageToKindClusterWithName(memgraphImage)).To(Succeed(), + "Failed to load the Memgraph image into Kind") + + By("creating the cluster namespace") + cmd = exec.Command("kubectl", "create", "ns", clusterNamespace) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", clusterNamespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + + By("creating the enterprise license Secret") + createLicenseSecret(license, organization) + + By("applying the MemgraphCluster") + applyMemgraphCluster() + }) + + AfterAll(func() { + By("removing the cluster namespace") + cmd := exec.Command("kubectl", "delete", "ns", clusterNamespace, + "--ignore-not-found", "--wait=false") + _, _ = utils.Run(cmd) + }) + + // On failure, dump everything needed to debug a broken bootstrap from CI + // logs alone. + AfterEach(func() { + if !CurrentSpecReport().Failed() { + return + } + for _, args := range [][]string{ + {"get", "pods", "-n", clusterNamespace, "-o", "wide"}, + {"get", "memgraphclusters", "-n", clusterNamespace, "-o", "yaml"}, + {"get", "events", "-n", clusterNamespace, "--sort-by=.lastTimestamp"}, + {"logs", "deploy/kubernetes-operator-controller-manager", "-n", namespace}, + } { + cmd := exec.Command("kubectl", args...) + output, err := utils.Run(cmd) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to collect diagnostics %v: %s\n", args, err) + continue + } + _, _ = fmt.Fprintf(GinkgoWriter, "Diagnostics kubectl %v:\n%s\n", args, output) + } + }) + + It("bootstraps every declared instance registered with exactly one MAIN", func() { + verifyClusterRegistered := func(g Gomega) { + view, err := leaderView() + g.Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0, len(view)) + mains := make([]string, 0, 1) + for _, instance := range view { + names = append(names, instance.name) + g.Expect(instance.health).To(Equal("up"), + "instance %s is registered but unhealthy", instance.name) + if instance.role == roleMain { + mains = append(mains, instance.name) + } + } + g.Expect(names).To(ConsistOf(declaredInstances())) + g.Expect(mains).To(HaveLen(1), "expected exactly one MAIN, got %v", mains) + } + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) +}) + +// createLicenseSecret applies the Secret the MemgraphCluster references. The +// manifest is piped over stdin so no secret material ever reaches the logged +// command line. +func createLicenseSecret(license, organization string) { + secret := map[string]any{ + "apiVersion": "v1", + "kind": "Secret", + "metadata": map[string]any{ + "name": licenseSecretName, + "namespace": clusterNamespace, + }, + "stringData": map[string]string{ + licenseEnvVar: license, + organizationEnvVar: organization, + }, + } + manifest, err := json.Marshal(secret) + Expect(err).NotTo(HaveOccurred(), "Failed to marshal the license Secret") + + cmd := exec.Command("kubectl", "apply", "-f", "-") + _, err = utils.RunWithInput(cmd, string(manifest)) + Expect(err).NotTo(HaveOccurred(), "Failed to apply the license Secret") +} + +// applyMemgraphCluster applies the CR under test: the minimal spec of the PRD's +// first-contact story — image, counts, and a license secret reference. +func applyMemgraphCluster() { + manifest := fmt.Sprintf(`apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + name: %s + namespace: %s +spec: + coordinators: %d + dataInstances: %d + image: + repository: %s + tag: %s + secrets: + name: %s + licenseKey: %s + organizationKey: %s +`, clusterName, clusterNamespace, coordinatorCount, dataInstanceCount, + memgraphImageRepository, memgraphImageTag, + licenseSecretName, licenseEnvVar, organizationEnvVar) + + cmd := exec.Command("kubectl", "apply", "-f", "-") + _, err := utils.RunWithInput(cmd, manifest) + Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") +} + +// instanceRow is one parsed row of SHOW INSTANCES. +type instanceRow struct { + name string + health string + role string +} + +// leaderView returns the SHOW INSTANCES view of the first coordinator that +// reports a MAIN. Only the coordinator leader health-checks data instances and +// reports their roles (followers show them as unknown), so a view containing a +// MAIN is the leader's authoritative view. +func leaderView() ([]instanceRow, error) { + var errs []error + for ordinal := range coordinatorCount { + pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) + view, err := showInstances(pod) + if err != nil { + errs = append(errs, err) + continue + } + for _, instance := range view { + if instance.role == roleMain { + return view, nil + } + } + errs = append(errs, fmt.Errorf("%s reports no MAIN among %d instances", pod, len(view))) + } + return nil, errors.Join(errs...) +} + +// showInstances runs SHOW INSTANCES through mgconsole inside the given +// coordinator pod (the Memgraph image ships the client) and parses the CSV +// output. +func showInstances(pod string) ([]instanceRow, error) { + cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", + "bash", "-c", "echo 'SHOW INSTANCES;' | mgconsole --output-format=csv") + output, err := utils.Run(cmd) + if err != nil { + return nil, err + } + return parseInstances(output) +} + +// parseInstances parses mgconsole CSV output into rows keyed by the header +// columns, so the assertion survives added or reordered columns across +// Memgraph versions. +func parseInstances(output string) ([]instanceRow, error) { + lines := utils.GetNonEmptyLines(output) + header := -1 + for i, line := range lines { + if strings.Contains(line, "name") && strings.Contains(line, "bolt_server") { + header = i + break + } + } + if header == -1 { + return nil, fmt.Errorf("no SHOW INSTANCES header in mgconsole output: %q", output) + } + + reader := csv.NewReader(strings.NewReader(strings.Join(lines[header:], "\n"))) + records, err := reader.ReadAll() + if err != nil { + return nil, fmt.Errorf("parsing mgconsole CSV output: %w", err) + } + + columns := map[string]int{} + for i, column := range records[0] { + columns[strings.TrimSpace(column)] = i + } + for _, column := range []string{"name", "health", "role"} { + if _, ok := columns[column]; !ok { + return nil, fmt.Errorf("SHOW INSTANCES output has no %q column: %q", column, records[0]) + } + } + + instances := make([]instanceRow, 0, len(records)-1) + for _, record := range records[1:] { + instances = append(instances, instanceRow{ + name: unquoteCell(record[columns["name"]]), + health: unquoteCell(record[columns["health"]]), + role: unquoteCell(record[columns["role"]]), + }) + } + return instances, nil +} + +// unquoteCell strips the residual double quotes mgconsole wraps around string +// cells in CSV output. mgconsole emits string values already double-quoted, so +// after the CSV reader unwraps its own layer a value like main still arrives as +// "main"; the operator and assertions compare against the bare token. +func unquoteCell(cell string) string { + return strings.Trim(strings.TrimSpace(cell), `"`) +} diff --git a/test/utils/utils.go b/test/utils/utils.go index a408630..3e9fb17 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -39,6 +39,14 @@ func warnError(err error) { _, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err) } +// RunWithInput executes the provided command with the given string fed to its +// standard input. Only the command line is logged, never the input, so it is +// safe for manifests carrying secret material (e.g. a license Secret). +func RunWithInput(cmd *exec.Cmd, input string) (string, error) { + cmd.Stdin = strings.NewReader(input) + return Run(cmd) +} + // Run executes the provided command within this context func Run(cmd *exec.Cmd) (string, error) { dir, _ := GetProjectDir() From 03065ba65110c52337fe257f9adc1d4cd5a28d6e Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Fri, 24 Jul 2026 15:41:43 +0200 Subject: [PATCH 08/34] feat: continuous re-registration reconciliation (#24) * feat: continuous re-registration reconciliation (#24) Extend registration reconciliation from one-shot bootstrap to continuous drift recovery: a converged cluster now reschedules a periodic resync so a registration a pod loses (rescheduled onto a fresh node, wiped storage) is re-issued without human action. A lost registration on a still-running pod produces no watch event, so this resync is the mechanism that detects drift. The planner already computed full-diff semantics; the gap was the controller returning no requeue once converged. Add a resyncInterval and requeue on it from the converged branch. Tests: - planner: multiple lost registrations re-issued without a second MAIN - controller envtest: re-register a lost data instance (MAIN untouched), re-add a lost coordinator, no-op across repeated resyncs - e2e: wipe a data instance's registration on the leader, assert the operator converges the cluster back to fully registered * testing: Add test for removal of coordinators --- .../controller/memgraphcluster_controller.go | 17 +- .../memgraphcluster_controller_test.go | 79 +++++++- internal/planner/planner_test.go | 15 ++ test/e2e/memgraphcluster_test.go | 170 ++++++++++++++++-- 4 files changed, 262 insertions(+), 19 deletions(-) diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index fa5e011..012c4ec 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -51,6 +51,14 @@ const ( // requeueAfterRegistration schedules the follow-up reconcile that // verifies issued registration commands actually converged the cluster. requeueAfterRegistration = 10 * time.Second + + // resyncInterval is how often a converged cluster is re-observed to catch + // registration drift. A pod that loses its registration (rescheduled onto + // a fresh node, wiped storage) while still running produces no watch event + // — its StatefulSet is unchanged — so a lost registration would otherwise + // go undetected until an unrelated reconcile. This periodic resync is what + // makes re-registration continuous rather than one-shot. + resyncInterval = 30 * time.Second ) // MemgraphClusterReconciler reconciles a MemgraphCluster object @@ -77,7 +85,10 @@ type MemgraphClusterReconciler struct { // registration: observe SHOW INSTANCES on the coordinator leader, diff // against the declared topology, and issue only the missing commands. All // interaction is read-before-write and idempotent, so an operator restart -// mid-bootstrap is harmless. Deletion needs no handling here — every object +// mid-bootstrap is harmless. Registration reconciliation is continuous, not +// one-shot: a converged cluster is re-observed on a periodic resync, so a +// registration a pod loses (rescheduled, wiped storage) is re-issued without +// human action. Deletion needs no handling here — every object // carries a controller owner reference, so garbage collection removes the // workloads with the CR. func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -142,8 +153,10 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( commands := planner.Plan(topology, observed) if len(commands) == 0 { + // Converged, but keep re-observing: a registration a pod loses later + // produces no watch event, so drift is only caught by resyncing. log.Info("Confirmed cluster registration is converged") - return ctrl.Result{}, nil + return ctrl.Result{RequeueAfter: resyncInterval}, nil } for _, command := range commands { if err := command.Run(ctx, leader); err != nil { diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 344d497..cc0af55 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -343,7 +343,8 @@ var _ = Describe("MemgraphCluster Controller", func() { result = reconcileCluster(resourceName) Expect(fake.executedCommands()).To(HaveLen(6), "a converged cluster must not receive further commands") - Expect(result.RequeueAfter).To(BeZero()) + Expect(result.RequeueAfter).To(Equal(resyncInterval), + "a converged cluster must still reschedule a resync to catch registration drift") }) It("should resume a partial bootstrap without duplicate registrations or a second MAIN", func() { @@ -386,5 +387,81 @@ var _ = Describe("MemgraphCluster Controller", func() { leader + ": SET INSTANCE instance_0 TO MAIN", })) }) + + // convergedCluster is the fully registered view of the default + // 3-coordinator, 2-data topology with instance_0 elected MAIN — the + // steady state drift is introduced against below. + convergedCluster := func() []memgraph.Instance { + return []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + } + } + + It("should re-register a data instance whose registration was lost, leaving MAIN untouched", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + Expect(fake.executedCommands()).To(BeEmpty(), "the cluster started converged") + + // instance_1 loses its registration (pod rescheduled onto a fresh + // node): drop it from the observed view and reconcile again. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }) + + result := reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": REGISTER INSTANCE instance_1", + }), "only the lost registration is re-issued; the existing MAIN is not re-promoted") + Expect(result.RequeueAfter).To(BeNumerically(">", 0), + "re-registration was issued, so a follow-up reconcile must verify convergence") + }) + + It("should re-add a coordinator whose registration was lost", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + Expect(fake.executedCommands()).To(BeEmpty(), "the cluster started converged") + + // coordinator_3 disappears from the Raft cluster view. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }) + + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": ADD COORDINATOR 3", + }), "only the missing coordinator is re-added") + }) + + It("should stay a no-op on a converged cluster across repeated resyncs", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + + for range 3 { + result := reconcileCluster(resourceName) + Expect(fake.executedCommands()).To(BeEmpty(), + "a converged cluster must never receive commands, however often it is resynced") + Expect(result.RequeueAfter).To(Equal(resyncInterval), + "each converged reconcile reschedules the drift-detection resync") + } + }) }) }) diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index d3e8bfe..5433f3c 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -183,6 +183,21 @@ func TestPlan(t *testing.T) { planner.RegisterInstance{Instance: dataInstanceSpec(0)}, }, }, + { + name: "multiple lost registrations are all re-issued without a second MAIN", + // A coordinator and a data instance both lost their registration + // while instance_1 remained MAIN: every missing registration is + // re-issued, and no promotion is planned because a MAIN exists. + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(1, memgraph.RoleMain), + }, + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(2)}, + planner.RegisterInstance{Instance: dataInstanceSpec(0)}, + }, + }, { name: "instances the topology does not declare are left untouched", observed: []memgraph.Instance{ diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index d2035e2..9dab002 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -150,27 +150,165 @@ var _ = Describe("MemgraphCluster", Ordered, func() { }) It("bootstraps every declared instance registered with exactly one MAIN", func() { - verifyClusterRegistered := func(g Gomega) { - view, err := leaderView() - g.Expect(err).NotTo(HaveOccurred()) - - names := make([]string, 0, len(view)) - mains := make([]string, 0, 1) - for _, instance := range view { - names = append(names, instance.name) - g.Expect(instance.health).To(Equal("up"), - "instance %s is registered but unhealthy", instance.name) - if instance.role == roleMain { - mains = append(mains, instance.name) - } - } - g.Expect(names).To(ConsistOf(declaredInstances())) - g.Expect(mains).To(HaveLen(1), "expected exactly one MAIN, got %v", mains) + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) + + // The operator's reason to exist over the chart's one-shot Job: a data + // instance that loses its registration is re-registered with no human + // action. This runs after the bootstrap spec (Ordered) against the same + // converged cluster. + It("re-registers a data instance whose registration was wiped", func() { + const wiped = "instance_1" + + By("confirming the cluster is converged before wiping a registration") + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + + By("unregistering a data instance on the coordinator leader") + Expect(wipeInstanceRegistration(wiped)).To(Succeed()) + + By("confirming the instance really left the cluster view") + view, err := leaderView() + Expect(err).NotTo(HaveOccurred()) + names := make([]string, 0, len(view)) + for _, instance := range view { + names = append(names, instance.name) + } + Expect(names).NotTo(ContainElement(wiped), + "the wipe must actually remove the registration for the test to be meaningful") + + By("waiting for the operator to converge the cluster back to fully registered") + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) + + // The coordinator analogue of the data-instance re-registration: a + // coordinator removed from the Raft cluster is re-added by the operator's + // continuous ADD COORDINATOR reconciliation, with no human action. This + // proves the re-registration loop covers coordinators, not just data + // instances. Runs after the preceding specs (Ordered) against the same + // converged cluster. + It("re-adds a coordinator that was removed from the cluster", func() { + By("confirming the cluster is converged before removing a coordinator") + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + + By("removing a follower coordinator on the coordinator leader") + removed, err := removeCoordinatorRegistration() + Expect(err).NotTo(HaveOccurred()) + + By("confirming the coordinator really left the cluster view") + view, err := leaderView() + Expect(err).NotTo(HaveOccurred()) + names := make([]string, 0, len(view)) + for _, instance := range view { + names = append(names, instance.name) } + Expect(names).NotTo(ContainElement(removed), + "the removal must actually drop the coordinator for the test to be meaningful") + + By("waiting for the operator to converge the cluster back to fully registered") Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) }) }) +// verifyClusterRegistered asserts the coordinator leader reports every declared +// instance registered and healthy with exactly one MAIN — the converged steady +// state both the bootstrap and re-registration specs check for. +func verifyClusterRegistered(g Gomega) { + view, err := leaderView() + g.Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0, len(view)) + mains := make([]string, 0, 1) + for _, instance := range view { + names = append(names, instance.name) + g.Expect(instance.health).To(Equal("up"), + "instance %s is registered but unhealthy", instance.name) + if instance.role == roleMain { + mains = append(mains, instance.name) + } + } + g.Expect(names).To(ConsistOf(declaredInstances())) + g.Expect(mains).To(HaveLen(1), "expected exactly one MAIN, got %v", mains) +} + +// wipeInstanceRegistration unregisters the named data instance on the +// coordinator leader, simulating registration state a pod loses when it is +// rescheduled onto a fresh node. UNREGISTER INSTANCE must run on the leader — +// only it holds the authoritative cluster view — so the leader is located the +// same way leaderView does: the coordinator that reports a MAIN. +func wipeInstanceRegistration(name string) error { + var errs []error + for ordinal := range coordinatorCount { + pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) + view, err := showInstances(pod) + if err != nil { + errs = append(errs, err) + continue + } + isLeader := false + for _, instance := range view { + if instance.role == roleMain { + isLeader = true + break + } + } + if !isLeader { + continue + } + cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", + "bash", "-c", fmt.Sprintf("echo 'UNREGISTER INSTANCE %s;' | mgconsole", name)) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("unregistering %s on %s: %w", name, pod, err) + } + return nil + } + return fmt.Errorf("no coordinator leader found to unregister %s: %w", name, errors.Join(errs...)) +} + +// removeCoordinatorRegistration removes a follower coordinator from the Raft +// cluster on the coordinator leader, simulating a coordinator that fell out of +// the cluster view (e.g. rescheduled onto a fresh node). REMOVE COORDINATOR +// mutates Raft membership, so it must run on the leader — located the same way +// leaderView does: the coordinator that reports a MAIN. A follower is chosen +// (never the leader itself) so the leader keeps the authoritative view it needs +// to accept the removal and observe the operator's re-ADD. It returns the +// instance name of the coordinator it removed. +func removeCoordinatorRegistration() (string, error) { + var errs []error + for ordinal := range coordinatorCount { + pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) + view, err := showInstances(pod) + if err != nil { + errs = append(errs, err) + continue + } + isLeader := false + for _, instance := range view { + if instance.role == roleMain { + isLeader = true + break + } + } + if !isLeader { + continue + } + // The leader hosts coordinator_ordinal+1; remove a different + // coordinator so the leader keeps quorum and its authoritative view. + leaderID := ordinal + 1 + removeID := 1 + if leaderID == 1 { + removeID = 2 + } + name := fmt.Sprintf("coordinator_%d", removeID) + cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", + "bash", "-c", fmt.Sprintf("echo 'REMOVE COORDINATOR %d;' | mgconsole", removeID)) + if _, err := utils.Run(cmd); err != nil { + return "", fmt.Errorf("removing coordinator %d on %s: %w", removeID, pod, err) + } + return name, nil + } + return "", fmt.Errorf("no coordinator leader found to remove a coordinator: %w", errors.Join(errs...)) +} + // createLicenseSecret applies the Secret the MemgraphCluster references. The // manifest is piped over stdin so no secret material ever reaches the logged // command line. From 271e6151e416e1cf2cf694b1b42855f4fa40a1be Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 09:23:50 +0200 Subject: [PATCH 09/34] feat: status subresource with MAIN, conditions, printer columns (#25) --- api/v1alpha1/memgraphcluster_types.go | 63 ++++++- .../bases/memgraph.com_memgraphclusters.yaml | 28 ++- internal/controller/fake_memgraph_test.go | 12 ++ .../controller/memgraphcluster_controller.go | 97 +++++++++++ .../memgraphcluster_controller_test.go | 164 ++++++++++++++++++ 5 files changed, 358 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index d4bee34..a7a6d1a 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -41,6 +41,49 @@ const ( DefaultOrganizationSecretKey = "MEMGRAPH_ORGANIZATION_NAME" ) +// Condition types reported on MemgraphCluster status. Both use normal-True +// polarity: True is the healthy state. Ready answers "is the cluster serving" +// (a MAIN is elected and reachable); Converged answers "does registration +// match the declared topology" (every coordinator and data instance is +// registered). A cluster can be Ready but not Converged — a MAIN still serves +// while a lost replica registration is being restored. +const ( + // ConditionReady is True when a MAIN data instance is elected and the + // coordinator leader is reachable. + ConditionReady = "Ready" + + // ConditionConverged is True when the observed cluster matches the declared + // topology and no registration commands are pending. + ConditionConverged = "Converged" +) + +// Condition reasons reported on MemgraphCluster status. Reasons are CamelCase +// per Kubernetes API conventions and are stable enough for tooling to gate on. +const ( + // ReasonWorkloadsNotReady is set while not every workload pod is ready, so + // registration has not been attempted. + ReasonWorkloadsNotReady = "WorkloadsNotReady" + + // ReasonCoordinatorUnreachable is set when no coordinator answered + // SHOW INSTANCES, so the cluster state cannot be observed. + ReasonCoordinatorUnreachable = "CoordinatorUnreachable" + + // ReasonRegistrationInProgress is set while registration commands are being + // issued to converge the cluster toward the declared topology. + ReasonRegistrationInProgress = "RegistrationInProgress" + + // ReasonAllInstancesRegistered is set when the observed cluster matches the + // declared topology. + ReasonAllInstancesRegistered = "AllInstancesRegistered" + + // ReasonMainElected is set when a data instance is observed as MAIN. + ReasonMainElected = "MainElected" + + // ReasonNoMainElected is set when the cluster is reachable but no data + // instance has yet been promoted to MAIN. + ReasonNoMainElected = "NoMainElected" +) + // ImageSpec selects the Memgraph container image run by all cluster pods. type ImageSpec struct { // repository is the Memgraph container image repository. @@ -113,12 +156,16 @@ type MemgraphClusterSpec struct { } // MemgraphClusterStatus defines the observed state of MemgraphCluster. +// +// Status is observation only: it carries no secret material and is never read +// back as reconcile input state. type MemgraphClusterStatus struct { - // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster - // Important: Run "make" to regenerate code after modifying this file - - // For Kubernetes API conventions, see: - // https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties + // main is the name of the data instance currently observed as MAIN, as + // reported by SHOW INSTANCES on the coordinator leader. It is empty before + // the initial MAIN is elected and updates when the Raft coordinators fail + // over to a different instance. + // +optional + Main string `json:"main,omitempty"` // conditions represent the current state of the MemgraphCluster resource. // Each condition has a unique type and reflects the status of a specific aspect of the resource. @@ -138,6 +185,12 @@ type MemgraphClusterStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:shortName=mgc +// +kubebuilder:printcolumn:name="Coordinators",type=integer,JSONPath=`.spec.coordinators` +// +kubebuilder:printcolumn:name="Data",type=integer,JSONPath=`.spec.dataInstances` +// +kubebuilder:printcolumn:name="Main",type=string,JSONPath=`.status.main` +// +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` +// +kubebuilder:printcolumn:name="Converged",type=string,JSONPath=`.status.conditions[?(@.type=="Converged")].status` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` // MemgraphCluster is the Schema for the memgraphclusters API type MemgraphCluster struct { diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 0d05254..446c349 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -16,7 +16,26 @@ spec: singular: memgraphcluster scope: Namespaced versions: - - name: v1alpha1 + - additionalPrinterColumns: + - jsonPath: .spec.coordinators + name: Coordinators + type: integer + - jsonPath: .spec.dataInstances + name: Data + type: integer + - jsonPath: .status.main + name: Main + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Converged")].status + name: Converged + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 schema: openAPIV3Schema: description: MemgraphCluster is the Schema for the memgraphclusters API @@ -173,6 +192,13 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + main: + description: |- + main is the name of the data instance currently observed as MAIN, as + reported by SHOW INSTANCES on the coordinator leader. It is empty before + the initial MAIN is elected and updates when the Raft coordinators fail + over to a different instance. + type: string type: object required: - spec diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go index e66eafa..4fd7b30 100644 --- a/internal/controller/fake_memgraph_test.go +++ b/internal/controller/fake_memgraph_test.go @@ -36,6 +36,9 @@ type fakeMemgraph struct { // instances is the cluster view every coordinator serves. instances []memgraph.Instance connectAttempts int + // connectErr, when set, makes every Connect fail — the operator's view of a + // cluster whose coordinators do not yet answer Bolt. + connectErr error // executed records every mutating command as ": ". executed []string } @@ -48,9 +51,18 @@ func (f *fakeMemgraph) Connect(_ context.Context, address string) (memgraph.Clie f.mu.Lock() defer f.mu.Unlock() f.connectAttempts++ + if f.connectErr != nil { + return nil, f.connectErr + } return &fakeClient{cluster: f, address: address}, nil } +func (f *fakeMemgraph) setConnectErr(err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.connectErr = err +} + func (f *fakeMemgraph) connects() int { f.mu.Lock() defer f.mu.Unlock() diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 012c4ec..4794ffa 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -24,6 +24,9 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -136,6 +139,13 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( } if !ready { log.Info("Waited for workload pods to become ready before registration") + msg := "Waiting for all workload pods to become ready" + if statusErr := r.writeStatus(ctx, cluster, cluster.Status.Main, + notReadyCondition(memgraphcomv1alpha1.ReasonWorkloadsNotReady, msg), + notConvergedCondition(memgraphcomv1alpha1.ReasonWorkloadsNotReady, msg), + ); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: requeueWhilePending}, nil } @@ -143,6 +153,13 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( leader, observed, err := r.observeCluster(ctx, topology) if err != nil { log.Info("Deferred registration because no coordinator answered", "reason", err.Error()) + msg := "No coordinator answered SHOW INSTANCES" + if statusErr := r.writeStatus(ctx, cluster, cluster.Status.Main, + notReadyCondition(memgraphcomv1alpha1.ReasonCoordinatorUnreachable, msg), + notConvergedCondition(memgraphcomv1alpha1.ReasonCoordinatorUnreachable, msg), + ); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: requeueWhilePending}, nil } defer func() { @@ -151,13 +168,30 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( } }() + main := observedMain(observed) commands := planner.Plan(topology, observed) if len(commands) == 0 { // Converged, but keep re-observing: a registration a pod loses later // produces no watch event, so drift is only caught by resyncing. log.Info("Confirmed cluster registration is converged") + converged := trueCondition(memgraphcomv1alpha1.ConditionConverged, + memgraphcomv1alpha1.ReasonAllInstancesRegistered, + fmt.Sprintf("All %d declared instances are registered", len(topology.Coordinators)+len(topology.DataInstances))) + if statusErr := r.writeStatus(ctx, cluster, main, readyOrNot(main), converged); statusErr != nil { + return ctrl.Result{}, statusErr + } return ctrl.Result{RequeueAfter: resyncInterval}, nil } + + // Report the in-progress state before mutating the cluster: a MAIN already + // serving stays Ready while a lost registration is restored; a fresh + // bootstrap has no MAIN yet, so Ready is False until one is elected. + inProgress := notConvergedCondition(memgraphcomv1alpha1.ReasonRegistrationInProgress, + fmt.Sprintf("Issuing %d registration command(s) to converge the cluster", len(commands))) + if statusErr := r.writeStatus(ctx, cluster, main, readyOrNot(main), inProgress); statusErr != nil { + return ctrl.Result{}, statusErr + } + for _, command := range commands { if err := command.Run(ctx, leader); err != nil { return ctrl.Result{}, fmt.Errorf("executing registration command %q: %w", command, err) @@ -170,6 +204,69 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil } +// observedMain returns the name of the data instance reported as MAIN, or the +// empty string when none is elected yet. +func observedMain(observed []memgraph.Instance) string { + for _, instance := range observed { + if instance.IsMain() { + return instance.Name + } + } + return "" +} + +// readyOrNot builds the Ready condition from whether a MAIN is elected: the +// cluster serves writes exactly when a data instance is MAIN. +func readyOrNot(main string) metav1.Condition { + if main == "" { + return notReadyCondition(memgraphcomv1alpha1.ReasonNoMainElected, + "No data instance has been promoted to MAIN yet") + } + return trueCondition(memgraphcomv1alpha1.ConditionReady, + memgraphcomv1alpha1.ReasonMainElected, "Data instance "+main+" is MAIN") +} + +func trueCondition(condType, reason, message string) metav1.Condition { + return metav1.Condition{Type: condType, Status: metav1.ConditionTrue, Reason: reason, Message: message} +} + +func notReadyCondition(reason, message string) metav1.Condition { + return metav1.Condition{ + Type: memgraphcomv1alpha1.ConditionReady, Status: metav1.ConditionFalse, Reason: reason, Message: message, + } +} + +func notConvergedCondition(reason, message string) metav1.Condition { + return metav1.Condition{ + Type: memgraphcomv1alpha1.ConditionConverged, Status: metav1.ConditionFalse, Reason: reason, Message: message, + } +} + +// writeStatus patches the status subresource with the observed MAIN and the +// given conditions. It uses the status subresource exclusively — spec is never +// touched — and skips the patch when nothing changed, so a converged cluster +// re-observed on every resync does not churn the resource version. +func (r *MemgraphClusterReconciler) writeStatus( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, + main string, + conditions ...metav1.Condition, +) error { + base := cluster.DeepCopy() + cluster.Status.Main = main + for _, condition := range conditions { + condition.ObservedGeneration = cluster.Generation + apimeta.SetStatusCondition(&cluster.Status.Conditions, condition) + } + if equality.Semantic.DeepEqual(base.Status, cluster.Status) { + return nil + } + if err := r.Status().Patch(ctx, cluster, client.MergeFrom(base)); err != nil { + return fmt.Errorf("patching MemgraphCluster status: %w", err) + } + return nil +} + // workloadsReady reports whether both role StatefulSets have all their pods // ready. Registration waits for the full topology: coordinators cannot form a // Raft cluster and data instances cannot be registered until every advertised diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index cc0af55..25bd8fb 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -18,12 +18,14 @@ package controller import ( "context" + "errors" "fmt" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -464,4 +466,166 @@ var _ = Describe("MemgraphCluster Controller", func() { } }) }) + + Context("when reporting status and conditions", func() { + const resourceName = "mgc-status" + + observedCoordinator := func(id int, role string) memgraph.Instance { + return memgraph.Instance{ + Name: fmt.Sprintf("coordinator_%d", id), + BoltServer: fmt.Sprintf("%s-coordinator-%d.%s-coordinator.%s.svc.cluster.local:7687", + resourceName, id-1, resourceName, resourceNamespace), + Health: "up", + Role: role, + } + } + observedDataInstance := func(i int, role string) memgraph.Instance { + return memgraph.Instance{Name: fmt.Sprintf("instance_%d", i), Health: "up", Role: role} + } + convergedCluster := func() []memgraph.Instance { + return []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + } + } + + status := func() memgraphcomv1alpha1.MemgraphClusterStatus { + GinkgoHelper() + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + return cluster.Status + } + condition := func(condType string) *metav1.Condition { + GinkgoHelper() + s := status() + return apimeta.FindStatusCondition(s.Conditions, condType) + } + + BeforeEach(func() { + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + }) + + AfterEach(func() { + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + Expect(k8sClient.Delete(ctx, cluster)).To(Succeed()) + deleteOwned(resourceName) + }) + + It("should report bootstrapping while workload pods are not ready", func() { + reconcileCluster(resourceName) + + s := status() + Expect(s.Main).To(BeEmpty(), "no MAIN is known before the cluster is observed") + ready := condition(memgraphcomv1alpha1.ConditionReady) + Expect(ready).NotTo(BeNil()) + Expect(ready.Status).To(Equal(metav1.ConditionFalse)) + Expect(ready.Reason).To(Equal(memgraphcomv1alpha1.ReasonWorkloadsNotReady)) + converged := condition(memgraphcomv1alpha1.ConditionConverged) + Expect(converged).NotTo(BeNil()) + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonWorkloadsNotReady)) + }) + + It("should report degraded when no coordinator is reachable", func() { + fake.setConnectErr(errors.New("connection refused")) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + ready := condition(memgraphcomv1alpha1.ConditionReady) + Expect(ready.Status).To(Equal(metav1.ConditionFalse)) + Expect(ready.Reason).To(Equal(memgraphcomv1alpha1.ReasonCoordinatorUnreachable)) + converged := condition(memgraphcomv1alpha1.ConditionConverged) + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonCoordinatorUnreachable)) + }) + + It("should report ready and converged once the cluster is bootstrapped", func() { + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + // Second reconcile observes the registrations issued by the first. + reconcileCluster(resourceName) + + s := status() + Expect(s.Main).To(Equal("instance_0")) + ready := condition(memgraphcomv1alpha1.ConditionReady) + Expect(ready.Status).To(Equal(metav1.ConditionTrue)) + Expect(ready.Reason).To(Equal(memgraphcomv1alpha1.ReasonMainElected)) + converged := condition(memgraphcomv1alpha1.ConditionConverged) + Expect(converged.Status).To(Equal(metav1.ConditionTrue)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonAllInstancesRegistered)) + }) + + It("should stay ready but drop convergence while a lost registration is restored", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + Expect(condition(memgraphcomv1alpha1.ConditionConverged).Status).To(Equal(metav1.ConditionTrue)) + + // A replica loses its registration; the MAIN keeps serving. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }) + reconcileCluster(resourceName) + + s := status() + Expect(s.Main).To(Equal("instance_0"), "the serving MAIN is unchanged") + ready := condition(memgraphcomv1alpha1.ConditionReady) + Expect(ready.Status).To(Equal(metav1.ConditionTrue), + "a cluster with a MAIN still serves while a replica is re-registered") + converged := condition(memgraphcomv1alpha1.ConditionConverged) + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonRegistrationInProgress)) + }) + + It("should track MAIN across a coordinator-driven failover", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + Expect(status().Main).To(Equal("instance_0")) + + // The Raft coordinators fail over to instance_1; the operator only + // observes the new MAIN, it never promotes one. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleMain), + }) + reconcileCluster(resourceName) + + Expect(status().Main).To(Equal("instance_1")) + Expect(fake.executedCommands()).To(BeEmpty(), + "failover belongs to the coordinators; the operator issues no promotion") + }) + + It("should update status through the subresource without modifying spec", func() { + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + specBefore := cluster.Spec.DeepCopy() + + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + get(resourceName, cluster) + Expect(&cluster.Spec).To(Equal(specBefore), "status updates must never mutate spec") + Expect(cluster.Status.Conditions).NotTo(BeEmpty()) + }) + }) }) From 5183afcb62e5977eacb623fde93992aac04a87cd Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 10:18:57 +0200 Subject: [PATCH 10/34] feat: CEL immutability + creation-time validation (#26) * feat: CEL immutability and creation-time validation on the CRD Pin the v1alpha1 topology contract in the CRD schema, with no admission webhook: CEL transition rules reject any change to spec.coordinators or spec.dataInstances on a live MemgraphCluster, and creation-time rules bound the counts and reject unusable image and secret references. - coordinators: 1-7 and odd, so the Raft quorum cannot split; an even count is never more fault tolerant than the odd count below it, and the field cannot be corrected once the cluster exists - dataInstances: 1-15, a typo guard on an equally immutable field - image.repository/tag and secrets.name/licenseKey/organizationKey get length and format rules; a tag or digest smuggled into repository is rejected with a message pointing at image.tag - secrets.licenseKey and secrets.organizationKey must name different keys of the Secret Envtest coverage lands in a dedicated validation suite: rejected mutations, accepted valid creates and updates that leave the counts alone, and the defaults a minimal CR materializes checked against the Go constants the resource builders fall back to. * feat: Remove upper bounds --- api/v1alpha1/memgraphcluster_types.go | 35 ++- .../bases/memgraph.com_memgraphclusters.yaml | 50 ++- config/samples/v1alpha1_memgraphcluster.yaml | 3 + .../memgraphcluster_controller_test.go | 25 +- .../memgraphcluster_validation_test.go | 297 ++++++++++++++++++ 5 files changed, 390 insertions(+), 20 deletions(-) create mode 100644 internal/controller/memgraphcluster_validation_test.go diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index a7a6d1a..e3b15fe 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -86,13 +86,22 @@ const ( // ImageSpec selects the Memgraph container image run by all cluster pods. type ImageSpec struct { - // repository is the Memgraph container image repository. + // repository is the Memgraph container image repository. It carries the + // optional registry host and the image path only; the version belongs in + // tag. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=255 + // +kubebuilder:validation:XValidation:rule="!self.contains('@')",message="repository must not contain a digest; pin the image with tag instead" + // +kubebuilder:validation:XValidation:rule="!self.substring(self.lastIndexOf('/') + 1).contains(':')",message="repository must not contain a tag; set image.tag instead" // +kubebuilder:default="docker.io/memgraph/memgraph" // +optional Repository string `json:"repository,omitempty"` // tag is the Memgraph container image tag. Prefer pinning a specific // Memgraph version over mutable tags such as "latest". + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9_][a-zA-Z0-9._-]*$` // +kubebuilder:default="3.12.0-relwithdebinfo" // +optional Tag string `json:"tag,omitempty"` @@ -108,19 +117,33 @@ type ImageSpec struct { // enterprise license and organization name. The block mirrors the // memgraph-high-availability Helm chart's secrets vocabulary; secret material // is consumed by reference only and never appears in the CR. +// +// The has() guards keep the rule evaluable against the block's empty object +// default, which the API server checks before nested field defaults apply. +// +// +kubebuilder:validation:XValidation:rule="!has(self.licenseKey) || !has(self.organizationKey) || self.licenseKey != self.organizationKey",message="licenseKey and organizationKey must name different keys of the Secret" type SecretsSpec struct { // name is the name of the Secret in the cluster's namespace. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` // +kubebuilder:default="memgraph-secrets" // +optional Name string `json:"name,omitempty"` // licenseKey is the key within the Secret holding the enterprise license. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[-._a-zA-Z0-9]+$` // +kubebuilder:default="MEMGRAPH_ENTERPRISE_LICENSE" // +optional LicenseKey string `json:"licenseKey,omitempty"` // organizationKey is the key within the Secret holding the organization // name the license was issued to. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[-._a-zA-Z0-9]+$` // +kubebuilder:default="MEMGRAPH_ORGANIZATION_NAME" // +optional OrganizationKey string `json:"organizationKey,omitempty"` @@ -131,14 +154,20 @@ type SecretsSpec struct { // Storage, port, and pod-tuning fields land in subsequent slices of the // operator MVP (see specs/operator-mvp/PRD.md). type MemgraphClusterSpec struct { - // coordinators is the number of Raft coordinator instances. + // coordinators is the number of Raft coordinator instances. It must be odd + // so the Raft quorum cannot split, and it is immutable: scaling is not + // supported in v1alpha1. // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:XValidation:rule="self % 2 == 1",message="coordinators must be an odd number so the Raft quorum cannot split" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="coordinators is immutable: changing the coordinator count of an existing MemgraphCluster is not supported in v1alpha1" // +kubebuilder:default=3 // +optional Coordinators *int32 `json:"coordinators,omitempty"` - // dataInstances is the number of data instances. + // dataInstances is the number of data instances. It is immutable: scaling + // is not supported in v1alpha1. // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="dataInstances is immutable: changing the data instance count of an existing MemgraphCluster is not supported in v1alpha1" // +kubebuilder:default=2 // +optional DataInstances *int32 `json:"dataInstances,omitempty"` diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 446c349..d5acf94 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -62,16 +62,32 @@ spec: properties: coordinators: default: 3 - description: coordinators is the number of Raft coordinator instances. + description: |- + coordinators is the number of Raft coordinator instances. It must be odd + so the Raft quorum cannot split, and it is immutable: scaling is not + supported in v1alpha1. format: int32 minimum: 1 type: integer + x-kubernetes-validations: + - message: coordinators must be an odd number so the Raft quorum cannot + split + rule: self % 2 == 1 + - message: 'coordinators is immutable: changing the coordinator count + of an existing MemgraphCluster is not supported in v1alpha1' + rule: self == oldSelf dataInstances: default: 2 - description: dataInstances is the number of data instances. + description: |- + dataInstances is the number of data instances. It is immutable: scaling + is not supported in v1alpha1. format: int32 minimum: 1 type: integer + x-kubernetes-validations: + - message: 'dataInstances is immutable: changing the data instance + count of an existing MemgraphCluster is not supported in v1alpha1' + rule: self == oldSelf image: default: {} description: image selects the Memgraph container image run by all @@ -88,13 +104,27 @@ spec: type: string repository: default: docker.io/memgraph/memgraph - description: repository is the Memgraph container image repository. + description: |- + repository is the Memgraph container image repository. It carries the + optional registry host and the image path only; the version belongs in + tag. + maxLength: 255 + minLength: 1 type: string + x-kubernetes-validations: + - message: repository must not contain a digest; pin the image + with tag instead + rule: '!self.contains(''@'')' + - message: repository must not contain a tag; set image.tag instead + rule: '!self.substring(self.lastIndexOf(''/'') + 1).contains('':'')' tag: default: 3.12.0-relwithdebinfo description: |- tag is the Memgraph container image tag. Prefer pinning a specific Memgraph version over mutable tags such as "latest". + maxLength: 128 + minLength: 1 + pattern: ^[a-zA-Z0-9_][a-zA-Z0-9._-]*$ type: string type: object secrets: @@ -107,18 +137,32 @@ spec: default: MEMGRAPH_ENTERPRISE_LICENSE description: licenseKey is the key within the Secret holding the enterprise license. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ type: string name: default: memgraph-secrets description: name is the name of the Secret in the cluster's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ type: string organizationKey: default: MEMGRAPH_ORGANIZATION_NAME description: |- organizationKey is the key within the Secret holding the organization name the license was issued to. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ type: string type: object + x-kubernetes-validations: + - message: licenseKey and organizationKey must name different keys + of the Secret + rule: '!has(self.licenseKey) || !has(self.organizationKey) || self.licenseKey + != self.organizationKey' type: object status: description: status defines the observed state of MemgraphCluster diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 4658833..46dde40 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -6,6 +6,9 @@ metadata: app.kubernetes.io/managed-by: kustomize name: memgraphcluster-sample spec: + # Both counts are immutable: v1alpha1 provisions and bootstraps a fixed + # topology, and admission rejects any later change. The coordinator count + # must be odd so the Raft quorum cannot split. coordinators: 3 dataInstances: 2 image: diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 25bd8fb..2c34841 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -42,6 +42,13 @@ const ( dataSuffix = "-data" ) +// Non-default spec values the specs in this package override with, chosen so +// that they cannot be confused with the CRD schema defaults. +const ( + customImageTag = "3.13.0" + customSecretName = "my-license" +) + var _ = Describe("MemgraphCluster Controller", func() { const resourceNamespace = "default" @@ -225,11 +232,11 @@ var _ = Describe("MemgraphCluster Controller", func() { DataInstances: ptr.To(int32(1)), Image: memgraphcomv1alpha1.ImageSpec{ Repository: "registry.example.com/memgraph", - Tag: "3.13.0", + Tag: customImageTag, PullPolicy: corev1.PullAlways, }, Secrets: memgraphcomv1alpha1.SecretsSpec{ - Name: "my-license", + Name: customSecretName, LicenseKey: "license", OrganizationKey: "organization", }, @@ -257,23 +264,13 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(container.ImagePullPolicy).To(Equal(corev1.PullAlways)) licenseRef := container.Env[len(container.Env)-2].ValueFrom.SecretKeyRef - Expect(licenseRef.Name).To(Equal("my-license")) + Expect(licenseRef.Name).To(Equal(customSecretName)) Expect(licenseRef.Key).To(Equal("license")) organizationRef := container.Env[len(container.Env)-1].ValueFrom.SecretKeyRef - Expect(organizationRef.Name).To(Equal("my-license")) + Expect(organizationRef.Name).To(Equal(customSecretName)) Expect(organizationRef.Key).To(Equal("organization")) } }) - - It("should reject a spec violating the schema", func() { - invalid := &memgraphcomv1alpha1.MemgraphCluster{ - ObjectMeta: metav1.ObjectMeta{Name: "mgc-invalid", Namespace: resourceNamespace}, - Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ - Coordinators: ptr.To(int32(0)), - }, - } - Expect(k8sClient.Create(ctx, invalid)).NotTo(Succeed()) - }) }) Context("when bootstrapping cluster registration", func() { diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go new file mode 100644 index 0000000..2978e78 --- /dev/null +++ b/internal/controller/memgraphcluster_validation_test.go @@ -0,0 +1,297 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" +) + +// These specs exercise the CRD schema itself — defaults, creation-time +// validation and the CEL transition rules that pin the topology counts. They +// never reconcile: the API server is the unit under test, which is exactly the +// v1 contract that no admission webhook is involved. +var _ = Describe("MemgraphCluster CRD validation", func() { + const resourceNamespace = "default" + + ctx := context.Background() + + // create posts a cluster and returns the admission error, if any. + create := func(name string, spec memgraphcomv1alpha1.MemgraphClusterSpec) error { + return k8sClient.Create(ctx, &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: resourceNamespace}, + Spec: spec, + }) + } + + // createAccepted posts a cluster, asserts admission accepted it, registers + // its cleanup and returns the stored object with defaults applied. + createAccepted := func(name string, spec memgraphcomv1alpha1.MemgraphClusterSpec) *memgraphcomv1alpha1.MemgraphCluster { + GinkgoHelper() + Expect(create(name, spec)).To(Succeed()) + + stored := &memgraphcomv1alpha1.MemgraphCluster{} + Expect(k8sClient.Get(ctx, + types.NamespacedName{Name: name, Namespace: resourceNamespace}, stored)).To(Succeed()) + DeferCleanup(func() { + Expect(k8sClient.Delete(ctx, stored)).To(Succeed()) + }) + return stored + } + + // expectRejected asserts that admission rejected the spec as invalid and + // that the message actually tells the user what to change. + expectRejected := func(name string, spec memgraphcomv1alpha1.MemgraphClusterSpec, wantMessage string) { + GinkgoHelper() + err := create(name, spec) + Expect(err).To(HaveOccurred(), "expected admission to reject the spec") + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected an Invalid error, got %v", err) + Expect(err.Error()).To(ContainSubstring(wantMessage)) + } + + Context("when creating a cluster", func() { + It("should accept the minimal spec of counts, image and secrets", func() { + stored := createAccepted("valid-minimal-explicit", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(3)), + DataInstances: ptr.To(int32(2)), + Image: memgraphcomv1alpha1.ImageSpec{ + Repository: "docker.io/memgraph/memgraph", + Tag: "3.12.0-relwithdebinfo", + }, + Secrets: memgraphcomv1alpha1.SecretsSpec{Name: customSecretName}, + }) + + Expect(stored.Spec.Secrets.Name).To(Equal(customSecretName)) + }) + + It("should accept an empty spec and materialize every documented default", func() { + stored := createAccepted("valid-empty-spec", memgraphcomv1alpha1.MemgraphClusterSpec{}) + + Expect(stored.Spec).To(Equal(memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(memgraphcomv1alpha1.DefaultCoordinatorCount), + DataInstances: ptr.To(memgraphcomv1alpha1.DefaultDataInstanceCount), + Image: memgraphcomv1alpha1.ImageSpec{ + Repository: memgraphcomv1alpha1.DefaultImageRepository, + Tag: memgraphcomv1alpha1.DefaultImageTag, + PullPolicy: memgraphcomv1alpha1.DefaultImagePullPolicy, + }, + Secrets: memgraphcomv1alpha1.SecretsSpec{ + Name: memgraphcomv1alpha1.DefaultSecretName, + LicenseKey: memgraphcomv1alpha1.DefaultLicenseSecretKey, + OrganizationKey: memgraphcomv1alpha1.DefaultOrganizationSecretKey, + }, + }), "the CRD schema defaults must match the Go constants the builders fall back to") + }) + + It("should default the fields a partially specified block leaves out", func() { + stored := createAccepted("valid-partial-blocks", memgraphcomv1alpha1.MemgraphClusterSpec{ + Image: memgraphcomv1alpha1.ImageSpec{Tag: customImageTag}, + Secrets: memgraphcomv1alpha1.SecretsSpec{Name: customSecretName}, + }) + + Expect(stored.Spec.Image.Tag).To(Equal(customImageTag)) + Expect(stored.Spec.Image.Repository).To(Equal(memgraphcomv1alpha1.DefaultImageRepository)) + Expect(stored.Spec.Secrets.LicenseKey).To(Equal(memgraphcomv1alpha1.DefaultLicenseSecretKey)) + Expect(stored.Spec.Secrets.OrganizationKey).To(Equal(memgraphcomv1alpha1.DefaultOrganizationSecretKey)) + }) + + DescribeTable("should accept any odd coordinator count and any positive data instance count", + func(name string, coordinators, dataInstances int32) { + createAccepted(name, memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(coordinators), + DataInstances: ptr.To(dataInstances), + }) + }, + Entry("single coordinator, single data instance", "valid-topology-min", int32(1), int32(1)), + Entry("a quorum and replica count beyond the former upper bounds", "valid-topology-large", + int32(9), int32(16)), + ) + + It("should accept a registry host carrying a port", func() { + stored := createAccepted("valid-registry-port", memgraphcomv1alpha1.MemgraphClusterSpec{ + Image: memgraphcomv1alpha1.ImageSpec{Repository: "registry.example.com:5000/memgraph"}, + }) + + Expect(stored.Spec.Image.Repository).To(Equal("registry.example.com:5000/memgraph")) + }) + + DescribeTable("should reject an invalid spec with an actionable message", + func(name string, spec memgraphcomv1alpha1.MemgraphClusterSpec, wantMessage string) { + expectRejected(name, spec, wantMessage) + }, + Entry("zero coordinators", "invalid-coordinators-zero", + memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(0))}, + "should be greater than or equal to 1"), + Entry("an even coordinator count", "invalid-coordinators-even", + memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(2))}, + "coordinators must be an odd number"), + Entry("zero data instances", "invalid-data-zero", + memgraphcomv1alpha1.MemgraphClusterSpec{DataInstances: ptr.To(int32(0))}, + "should be greater than or equal to 1"), + Entry("a tag smuggled into the repository", "invalid-image-repository-tagged", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Image: memgraphcomv1alpha1.ImageSpec{Repository: "memgraph/memgraph:3.12.0"}, + }, + "repository must not contain a tag; set image.tag instead"), + Entry("a digest smuggled into the repository", "invalid-image-repository-digest", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Image: memgraphcomv1alpha1.ImageSpec{ + Repository: "memgraph/memgraph@sha256:0000000000000000000000000000000000000000000000000000000000000000", + }, + }, + "repository must not contain a digest"), + Entry("an image tag that is not a valid OCI tag", "invalid-image-tag-chars", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Image: memgraphcomv1alpha1.ImageSpec{Tag: "3.12.0 relwithdebinfo"}, + }, + "in body should match"), + Entry("a secret name that is not a DNS subdomain", "invalid-secret-name", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Secrets: memgraphcomv1alpha1.SecretsSpec{Name: "My_Secret"}, + }, + "in body should match"), + Entry("one secret key serving both values", "invalid-secret-keys-collide", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Secrets: memgraphcomv1alpha1.SecretsSpec{ + LicenseKey: "MEMGRAPH_LICENSE", + OrganizationKey: "MEMGRAPH_LICENSE", + }, + }, + "licenseKey and organizationKey must name different keys of the Secret"), + ) + + // The typed client drops empty strings before they reach the API + // server (omitempty), so the fields a user can only blank out from + // YAML are submitted as a raw manifest instead. + DescribeTable("should reject a blanked-out field of a hand-written manifest", + func(name, block, field string) { + raw := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": memgraphcomv1alpha1.SchemeGroupVersion.String(), + "kind": "MemgraphCluster", + "metadata": map[string]any{"name": name, "namespace": resourceNamespace}, + "spec": map[string]any{block: map[string]any{field: ""}}, + }} + + err := k8sClient.Create(ctx, raw) + Expect(err).To(HaveOccurred(), "expected admission to reject the manifest") + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected an Invalid error, got %v", err) + Expect(err.Error()).To(ContainSubstring("should be at least 1 chars long")) + }, + Entry("an empty image repository", "invalid-raw-repository", "image", "repository"), + Entry("an empty image tag", "invalid-raw-tag", "image", "tag"), + Entry("an empty secret name", "invalid-raw-secret-name", "secrets", "name"), + Entry("an empty license key", "invalid-raw-license-key", "secrets", "licenseKey"), + Entry("an empty organization key", "invalid-raw-organization-key", "secrets", "organizationKey"), + ) + }) + + Context("when updating a live cluster", func() { + // live creates a cluster with the default topology and returns a + // mutate-and-update helper over the freshest stored copy. + update := func(name string, mutate func(*memgraphcomv1alpha1.MemgraphCluster)) error { + GinkgoHelper() + stored := &memgraphcomv1alpha1.MemgraphCluster{} + Expect(k8sClient.Get(ctx, + types.NamespacedName{Name: name, Namespace: resourceNamespace}, stored)).To(Succeed()) + mutate(stored) + return k8sClient.Update(ctx, stored) + } + + expectImmutable := func(name string, mutate func(*memgraphcomv1alpha1.MemgraphCluster), field string) { + GinkgoHelper() + err := update(name, mutate) + Expect(err).To(HaveOccurred(), "expected admission to reject the topology change") + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected an Invalid error, got %v", err) + Expect(err.Error()).To(ContainSubstring(field + " is immutable")) + Expect(err.Error()).To(ContainSubstring("not supported in v1alpha1")) + } + + It("should reject growing or shrinking the coordinator count", func() { + createAccepted("immutable-coordinators", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(3)), + }) + + expectImmutable("immutable-coordinators", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(5)) + }, "coordinators") + expectImmutable("immutable-coordinators", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(1)) + }, "coordinators") + }) + + It("should reject growing or shrinking the data instance count", func() { + createAccepted("immutable-data", memgraphcomv1alpha1.MemgraphClusterSpec{ + DataInstances: ptr.To(int32(2)), + }) + + expectImmutable("immutable-data", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.DataInstances = ptr.To(int32(3)) + }, "dataInstances") + expectImmutable("immutable-data", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.DataInstances = ptr.To(int32(1)) + }, "dataInstances") + }) + + It("should reject a count change that arrives as a field removal", func() { + // Dropping a non-default count from the manifest re-defaults it, + // which is a topology change dressed up as a deletion. + createAccepted("immutable-omitted", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(5)), + }) + + expectImmutable("immutable-omitted", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = nil + }, "coordinators") + }) + + It("should accept an update that leaves the counts alone", func() { + createAccepted("immutable-unchanged", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(3)), + DataInstances: ptr.To(int32(2)), + }) + + Expect(update("immutable-unchanged", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Image.Tag = customImageTag + c.Spec.Secrets.Name = "another-license" + })).To(Succeed()) + + Expect(update("immutable-unchanged", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(3)) + c.Spec.DataInstances = ptr.To(int32(2)) + })).To(Succeed(), "re-applying the same counts is not a topology change") + }) + + It("should accept an update that omits a count matching the default", func() { + createAccepted("immutable-omitted-default", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(memgraphcomv1alpha1.DefaultCoordinatorCount), + }) + + Expect(update("immutable-omitted-default", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = nil + })).To(Succeed(), "defaulting restores the same count, so the topology is unchanged") + }) + }) +}) From b2377a42f795aacd196b4b0dacefa42321f65a8f Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 11:38:10 +0200 Subject: [PATCH 11/34] feat: storage configuration with PVC retention policy (#27) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the ephemeral lib and log emptyDirs with StatefulSet volumeClaimTemplates, configurable per role with the HA chart's storage vocabulary (libPVCSize / libStorageAccessMode / libStorageClassName and the log counterparts). Add storage.retentionPolicy (Retain | Delete, default Retain) mapped straight onto the StatefulSets' persistentVolumeClaimRetentionPolicy, so deleting the CR keeps the data by default while dev clusters can opt into self-cleanup. whenScaled stays Retain: both replica counts are immutable in v1alpha1, so nothing ever scales down. The StatefulSet machinery is the only deleter — the operator owns no finalizer. Builder golden tests cover storage defaults, per-role overrides and both retention policies; envtest pins the schema defaults against the Go constants and asserts no operator finalizer; e2e covers both the Retain (PVCs survive CR deletion) and Delete (PVCs go with it) paths. --- api/v1alpha1/memgraphcluster_types.go | 110 ++++++++++++- api/v1alpha1/zz_generated.deepcopy.go | 53 +++++++ .../bases/memgraph.com_memgraphclusters.yaml | 149 ++++++++++++++++++ config/samples/v1alpha1_memgraphcluster.yaml | 18 +++ .../memgraphcluster_controller_test.go | 94 ++++++++++- .../memgraphcluster_validation_test.go | 72 +++++++++ internal/resources/resources.go | 75 +++++++-- internal/resources/statefulset.go | 80 ++++++++-- internal/resources/statefulset_test.go | 142 ++++++++++++++++- test/e2e/memgraphcluster_test.go | 134 ++++++++++++++++ 10 files changed, 894 insertions(+), 33 deletions(-) diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index e3b15fe..75a4bbf 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -18,6 +18,7 @@ package v1alpha1 import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -39,6 +40,26 @@ const ( DefaultSecretName = "memgraph-secrets" DefaultLicenseSecretKey = "MEMGRAPH_ENTERPRISE_LICENSE" DefaultOrganizationSecretKey = "MEMGRAPH_ORGANIZATION_NAME" + + DefaultLibPVCSize = "1Gi" + DefaultLogPVCSize = "1Gi" + DefaultStorageAccessMode = corev1.ReadWriteOnce + DefaultStorageRetention = RetentionPolicyRetain +) + +// StorageRetentionPolicy decides what happens to the cluster's +// PersistentVolumeClaims when the MemgraphCluster is deleted. +// +kubebuilder:validation:Enum=Retain;Delete +type StorageRetentionPolicy string + +const ( + // RetentionPolicyRetain leaves the PVCs behind when the MemgraphCluster is + // deleted, so the data survives an accidental deletion. + RetentionPolicyRetain StorageRetentionPolicy = "Retain" + + // RetentionPolicyDelete lets the StatefulSet controller garbage-collect the + // PVCs together with the MemgraphCluster. + RetentionPolicyDelete StorageRetentionPolicy = "Delete" ) // Condition types reported on MemgraphCluster status. Both use normal-True @@ -149,10 +170,89 @@ type SecretsSpec struct { OrganizationKey string `json:"organizationKey,omitempty"` } +// RoleStorageSpec configures the two PersistentVolumeClaims every pod of a +// role gets: lib storage backing Memgraph's data directory, and log storage +// backing its log file. The knob names mirror the +// memgraph-high-availability Helm chart's storage. block so translating +// a values file is mechanical. +// +// The fields below become StatefulSet volumeClaimTemplates, which Kubernetes +// treats as immutable: changing them on a live MemgraphCluster is rejected by +// the StatefulSet controller, not silently applied. Storage changes are a +// day-2 operation and out of scope for v1alpha1. +type RoleStorageSpec struct { + // libPVCSize is the requested size of the lib storage claim, which backs + // Memgraph's data directory (snapshots, WAL, and durability metadata). + // +kubebuilder:default="1Gi" + // +optional + LibPVCSize *resource.Quantity `json:"libPVCSize,omitempty"` + + // libStorageAccessMode is the access mode requested for the lib storage + // claim. + // +kubebuilder:validation:Enum=ReadWriteOnce;ReadOnlyMany;ReadWriteMany;ReadWriteOncePod + // +kubebuilder:default=ReadWriteOnce + // +optional + LibStorageAccessMode corev1.PersistentVolumeAccessMode `json:"libStorageAccessMode,omitempty"` + + // libStorageClassName is the StorageClass backing the lib storage claim. + // Leave it unset to use the cluster's default StorageClass; set it to the + // empty string to disable dynamic provisioning and bind a pre-created + // PersistentVolume. + // +kubebuilder:validation:MaxLength=253 + // +optional + LibStorageClassName *string `json:"libStorageClassName,omitempty"` + + // logPVCSize is the requested size of the log storage claim, which backs + // Memgraph's log file. + // +kubebuilder:default="1Gi" + // +optional + LogPVCSize *resource.Quantity `json:"logPVCSize,omitempty"` + + // logStorageAccessMode is the access mode requested for the log storage + // claim. + // +kubebuilder:validation:Enum=ReadWriteOnce;ReadOnlyMany;ReadWriteMany;ReadWriteOncePod + // +kubebuilder:default=ReadWriteOnce + // +optional + LogStorageAccessMode corev1.PersistentVolumeAccessMode `json:"logStorageAccessMode,omitempty"` + + // logStorageClassName is the StorageClass backing the log storage claim. + // Leave it unset to use the cluster's default StorageClass; set it to the + // empty string to disable dynamic provisioning and bind a pre-created + // PersistentVolume. + // +kubebuilder:validation:MaxLength=253 + // +optional + LogStorageClassName *string `json:"logStorageClassName,omitempty"` +} + +// StorageSpec configures persistence for both roles plus what happens to the +// claims when the MemgraphCluster goes away. +type StorageSpec struct { + // retentionPolicy decides whether the cluster's PersistentVolumeClaims + // survive deletion of the MemgraphCluster. It maps directly onto the + // StatefulSets' persistentVolumeClaimRetentionPolicy.whenDeleted, so the + // StatefulSet controller is the only thing that ever deletes storage — the + // operator owns no finalizer and runs no cleanup of its own. The default + // keeps production data safe from an accidental delete; dev clusters can + // opt into self-cleanup. + // +kubebuilder:default=Retain + // +optional + RetentionPolicy StorageRetentionPolicy `json:"retentionPolicy,omitempty"` + + // coordinators configures the storage of every coordinator pod. + // +kubebuilder:default={} + // +optional + Coordinators RoleStorageSpec `json:"coordinators,omitzero"` + + // data configures the storage of every data instance pod. + // +kubebuilder:default={} + // +optional + Data RoleStorageSpec `json:"data,omitzero"` +} + // MemgraphClusterSpec defines the desired state of MemgraphCluster. // -// Storage, port, and pod-tuning fields land in subsequent slices of the -// operator MVP (see specs/operator-mvp/PRD.md). +// Port and pod-tuning fields land in subsequent slices of the operator MVP +// (see specs/operator-mvp/PRD.md). type MemgraphClusterSpec struct { // coordinators is the number of Raft coordinator instances. It must be odd // so the Raft quorum cannot split, and it is immutable: scaling is not @@ -182,6 +282,12 @@ type MemgraphClusterSpec struct { // +kubebuilder:default={} // +optional Secrets SecretsSpec `json:"secrets,omitzero"` + + // storage configures the persistent volumes backing both roles and their + // retention on cluster deletion. + // +kubebuilder:default={} + // +optional + Storage StorageSpec `json:"storage,omitzero"` } // MemgraphClusterStatus defines the observed state of MemgraphCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 1ba4413..d17e80b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -114,6 +114,7 @@ func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { } out.Image = in.Image out.Secrets = in.Secrets + in.Storage.DeepCopyInto(&out.Storage) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterSpec. @@ -148,6 +149,41 @@ func (in *MemgraphClusterStatus) DeepCopy() *MemgraphClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RoleStorageSpec) DeepCopyInto(out *RoleStorageSpec) { + *out = *in + if in.LibPVCSize != nil { + in, out := &in.LibPVCSize, &out.LibPVCSize + x := (*in).DeepCopy() + *out = &x + } + if in.LibStorageClassName != nil { + in, out := &in.LibStorageClassName, &out.LibStorageClassName + *out = new(string) + **out = **in + } + if in.LogPVCSize != nil { + in, out := &in.LogPVCSize, &out.LogPVCSize + x := (*in).DeepCopy() + *out = &x + } + if in.LogStorageClassName != nil { + in, out := &in.LogStorageClassName, &out.LogStorageClassName + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleStorageSpec. +func (in *RoleStorageSpec) DeepCopy() *RoleStorageSpec { + if in == nil { + return nil + } + out := new(RoleStorageSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretsSpec) DeepCopyInto(out *SecretsSpec) { *out = *in @@ -162,3 +198,20 @@ func (in *SecretsSpec) DeepCopy() *SecretsSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StorageSpec) DeepCopyInto(out *StorageSpec) { + *out = *in + in.Coordinators.DeepCopyInto(&out.Coordinators) + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageSpec. +func (in *StorageSpec) DeepCopy() *StorageSpec { + if in == nil { + return nil + } + out := new(StorageSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index d5acf94..d8cca12 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -163,6 +163,155 @@ spec: of the Secret rule: '!has(self.licenseKey) || !has(self.organizationKey) || self.licenseKey != self.organizationKey' + storage: + default: {} + description: |- + storage configures the persistent volumes backing both roles and their + retention on cluster deletion. + properties: + coordinators: + default: {} + description: coordinators configures the storage of every coordinator + pod. + properties: + libPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + libPVCSize is the requested size of the lib storage claim, which backs + Memgraph's data directory (snapshots, WAL, and durability metadata). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + libStorageAccessMode: + default: ReadWriteOnce + description: |- + libStorageAccessMode is the access mode requested for the lib storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + libStorageClassName: + description: |- + libStorageClassName is the StorageClass backing the lib storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + logPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + logPVCSize is the requested size of the log storage claim, which backs + Memgraph's log file. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + logStorageAccessMode: + default: ReadWriteOnce + description: |- + logStorageAccessMode is the access mode requested for the log storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + logStorageClassName: + description: |- + logStorageClassName is the StorageClass backing the log storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + type: object + data: + default: {} + description: data configures the storage of every data instance + pod. + properties: + libPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + libPVCSize is the requested size of the lib storage claim, which backs + Memgraph's data directory (snapshots, WAL, and durability metadata). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + libStorageAccessMode: + default: ReadWriteOnce + description: |- + libStorageAccessMode is the access mode requested for the lib storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + libStorageClassName: + description: |- + libStorageClassName is the StorageClass backing the lib storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + logPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + logPVCSize is the requested size of the log storage claim, which backs + Memgraph's log file. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + logStorageAccessMode: + default: ReadWriteOnce + description: |- + logStorageAccessMode is the access mode requested for the log storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + logStorageClassName: + description: |- + logStorageClassName is the StorageClass backing the log storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + type: object + retentionPolicy: + default: Retain + description: |- + retentionPolicy decides whether the cluster's PersistentVolumeClaims + survive deletion of the MemgraphCluster. It maps directly onto the + StatefulSets' persistentVolumeClaimRetentionPolicy.whenDeleted, so the + StatefulSet controller is the only thing that ever deletes storage — the + operator owns no finalizer and runs no cleanup of its own. The default + keeps production data safe from an accidental delete; dev clusters can + opt into self-cleanup. + enum: + - Retain + - Delete + type: string + type: object type: object status: description: status defines the observed state of MemgraphCluster diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 46dde40..f27aae5 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -20,3 +20,21 @@ spec: name: memgraph-secrets licenseKey: MEMGRAPH_ENTERPRISE_LICENSE organizationKey: MEMGRAPH_ORGANIZATION_NAME + storage: + # Retain (the default) leaves the PersistentVolumeClaims behind when this + # resource is deleted, so the data survives an accidental delete. Switch to + # Delete on dev clusters that should clean up after themselves. + retentionPolicy: Retain + # Each pod of a role gets a lib claim (Memgraph's data directory) and a log + # claim. Leave the storage class names out to use the cluster's default + # StorageClass. Claim sizes and classes cannot be changed after creation. + coordinators: + libPVCSize: 1Gi + libStorageAccessMode: ReadWriteOnce + logPVCSize: 1Gi + logStorageAccessMode: ReadWriteOnce + data: + libPVCSize: 1Gi + libStorageAccessMode: ReadWriteOnce + logPVCSize: 1Gi + logStorageAccessMode: ReadWriteOnce diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 2c34841..4e19875 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -26,6 +26,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" @@ -45,10 +46,24 @@ const ( // Non-default spec values the specs in this package override with, chosen so // that they cannot be confused with the CRD schema defaults. const ( - customImageTag = "3.13.0" - customSecretName = "my-license" + customImageTag = "3.13.0" + customSecretName = "my-license" + customStorageClassName = "fast-ssd" ) +// libClaim returns the lib storage claim template of a provisioned +// StatefulSet, failing the spec if the builder stopped emitting it. +func libClaim(sts *appsv1.StatefulSet) corev1.PersistentVolumeClaimSpec { + GinkgoHelper() + for _, claim := range sts.Spec.VolumeClaimTemplates { + if claim.Name == "lib-storage" { + return claim.Spec + } + } + Fail("StatefulSet " + sts.Name + " has no lib-storage claim template") + return corev1.PersistentVolumeClaimSpec{} +} + var _ = Describe("MemgraphCluster Controller", func() { const resourceNamespace = "default" @@ -191,6 +206,48 @@ var _ = Describe("MemgraphCluster Controller", func() { } }) + It("should back both roles with retained lib and log claims", func() { + reconcileCluster(resourceName) + + for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + sts := &appsv1.StatefulSet{} + get(resourceName+suffix, sts) + + claims := map[string]corev1.PersistentVolumeClaimSpec{} + for _, claim := range sts.Spec.VolumeClaimTemplates { + claims[claim.Name] = claim.Spec + } + Expect(claims).To(HaveKey("lib-storage")) + Expect(claims).To(HaveKey("log-storage")) + for name, claim := range claims { + Expect(claim.AccessModes).To(ConsistOf(corev1.ReadWriteOnce), "claim %s", name) + Expect(claim.Resources.Requests.Storage()).To(HaveValue(Equal(resource.MustParse("1Gi"))), + "claim %s", name) + Expect(claim.StorageClassName).To(BeNil(), + "claim %s must fall back to the cluster's default StorageClass", name) + } + + // The default keeps data safe from an accidental CR delete, and + // nothing ever scales down because both counts are immutable. + Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy).To(HaveValue(Equal( + appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }))) + } + }) + + // Deleting storage is the StatefulSet machinery's job alone. An + // operator-owned finalizer would be a second, undeclared deleter — and + // one that can wedge a deletion when the operator is down. + It("should claim no finalizer on the MemgraphCluster", func() { + reconcileCluster(resourceName) + + stored := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, stored) + Expect(stored.Finalizers).To(BeEmpty()) + }) + It("should be idempotent when reconciling an unchanged resource", func() { reconcileCluster(resourceName) @@ -225,7 +282,7 @@ var _ = Describe("MemgraphCluster Controller", func() { cluster := &memgraphcomv1alpha1.MemgraphCluster{} BeforeEach(func() { - resource := &memgraphcomv1alpha1.MemgraphCluster{ + cr := &memgraphcomv1alpha1.MemgraphCluster{ ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ Coordinators: ptr.To(int32(1)), @@ -240,9 +297,19 @@ var _ = Describe("MemgraphCluster Controller", func() { LicenseKey: "license", OrganizationKey: "organization", }, + Storage: memgraphcomv1alpha1.StorageSpec{ + RetentionPolicy: memgraphcomv1alpha1.RetentionPolicyDelete, + Coordinators: memgraphcomv1alpha1.RoleStorageSpec{ + LibPVCSize: ptr.To(resource.MustParse("2Gi")), + }, + Data: memgraphcomv1alpha1.RoleStorageSpec{ + LibPVCSize: ptr.To(resource.MustParse("100Gi")), + LibStorageClassName: ptr.To(customStorageClassName), + }, + }, }, } - Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + Expect(k8sClient.Create(ctx, cr)).To(Succeed()) get(resourceName, cluster) }) @@ -269,8 +336,27 @@ var _ = Describe("MemgraphCluster Controller", func() { organizationRef := container.Env[len(container.Env)-1].ValueFrom.SecretKeyRef Expect(organizationRef.Name).To(Equal(customSecretName)) Expect(organizationRef.Key).To(Equal("organization")) + + Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted).To( + Equal(appsv1.DeletePersistentVolumeClaimRetentionPolicyType)) } }) + + It("should size each role's claims from that role's storage block", func() { + reconcileCluster(resourceName) + + coordinatorSts := &appsv1.StatefulSet{} + get(resourceName+coordinatorSuffix, coordinatorSts) + coordinatorLib := libClaim(coordinatorSts) + Expect(coordinatorLib.Resources.Requests.Storage()).To(HaveValue(Equal(resource.MustParse("2Gi")))) + Expect(coordinatorLib.StorageClassName).To(BeNil()) + + dataSts := &appsv1.StatefulSet{} + get(resourceName+dataSuffix, dataSts) + dataLib := libClaim(dataSts) + Expect(dataLib.Resources.Requests.Storage()).To(HaveValue(Equal(resource.MustParse("100Gi")))) + Expect(dataLib.StorageClassName).To(HaveValue(Equal(customStorageClassName))) + }) }) Context("when bootstrapping cluster registration", func() { diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index 2978e78..a038f09 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -22,6 +22,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" @@ -34,6 +35,17 @@ import ( // validation and the CEL transition rules that pin the topology counts. They // never reconcile: the API server is the unit under test, which is exactly the // v1 contract that no admission webhook is involved. +// defaultRoleStorage is one role's storage block as the CRD schema defaults +// materialize it. +func defaultRoleStorage() memgraphcomv1alpha1.RoleStorageSpec { + return memgraphcomv1alpha1.RoleStorageSpec{ + LibPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize)), + LibStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, + LogPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize)), + LogStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, + } +} + var _ = Describe("MemgraphCluster CRD validation", func() { const resourceNamespace = "default" @@ -103,6 +115,11 @@ var _ = Describe("MemgraphCluster CRD validation", func() { LicenseKey: memgraphcomv1alpha1.DefaultLicenseSecretKey, OrganizationKey: memgraphcomv1alpha1.DefaultOrganizationSecretKey, }, + Storage: memgraphcomv1alpha1.StorageSpec{ + RetentionPolicy: memgraphcomv1alpha1.DefaultStorageRetention, + Coordinators: defaultRoleStorage(), + Data: defaultRoleStorage(), + }, }), "the CRD schema defaults must match the Go constants the builders fall back to") }) @@ -110,12 +127,55 @@ var _ = Describe("MemgraphCluster CRD validation", func() { stored := createAccepted("valid-partial-blocks", memgraphcomv1alpha1.MemgraphClusterSpec{ Image: memgraphcomv1alpha1.ImageSpec{Tag: customImageTag}, Secrets: memgraphcomv1alpha1.SecretsSpec{Name: customSecretName}, + Storage: memgraphcomv1alpha1.StorageSpec{ + Data: memgraphcomv1alpha1.RoleStorageSpec{LibPVCSize: ptr.To(resource.MustParse("100Gi"))}, + }, }) Expect(stored.Spec.Image.Tag).To(Equal(customImageTag)) Expect(stored.Spec.Image.Repository).To(Equal(memgraphcomv1alpha1.DefaultImageRepository)) Expect(stored.Spec.Secrets.LicenseKey).To(Equal(memgraphcomv1alpha1.DefaultLicenseSecretKey)) Expect(stored.Spec.Secrets.OrganizationKey).To(Equal(memgraphcomv1alpha1.DefaultOrganizationSecretKey)) + Expect(stored.Spec.Storage.Data.LibPVCSize).To(Equal(ptr.To(resource.MustParse("100Gi")))) + Expect(stored.Spec.Storage.Data.LogPVCSize).To( + Equal(ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize)))) + Expect(stored.Spec.Storage.RetentionPolicy).To(Equal(memgraphcomv1alpha1.DefaultStorageRetention)) + Expect(stored.Spec.Storage.Coordinators).To(Equal(defaultRoleStorage())) + }) + + It("should accept an explicit Delete retention policy", func() { + stored := createAccepted("valid-retention-delete", memgraphcomv1alpha1.MemgraphClusterSpec{ + Storage: memgraphcomv1alpha1.StorageSpec{ + RetentionPolicy: memgraphcomv1alpha1.RetentionPolicyDelete, + }, + }) + + Expect(stored.Spec.Storage.RetentionPolicy).To(Equal(memgraphcomv1alpha1.RetentionPolicyDelete)) + }) + + // An unset storage class means "cluster default" and an empty one means + // "no dynamic provisioning"; both must survive a round trip through the + // API server as distinct values. + It("should preserve the difference between an unset and an empty storage class", func() { + raw := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": memgraphcomv1alpha1.SchemeGroupVersion.String(), + "kind": "MemgraphCluster", + "metadata": map[string]any{"name": "valid-empty-storage-class", "namespace": resourceNamespace}, + "spec": map[string]any{ + "storage": map[string]any{ + "data": map[string]any{"libStorageClassName": ""}, + }, + }, + }} + Expect(k8sClient.Create(ctx, raw)).To(Succeed()) + DeferCleanup(func() { Expect(k8sClient.Delete(ctx, raw)).To(Succeed()) }) + + stored := &memgraphcomv1alpha1.MemgraphCluster{} + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Name: "valid-empty-storage-class", Namespace: resourceNamespace}, stored)).To(Succeed()) + + Expect(stored.Spec.Storage.Data.LibStorageClassName).To(Equal(ptr.To(""))) + Expect(stored.Spec.Storage.Data.LogStorageClassName).To(BeNil()) }) DescribeTable("should accept any odd coordinator count and any positive data instance count", @@ -181,6 +241,18 @@ var _ = Describe("MemgraphCluster CRD validation", func() { }, }, "licenseKey and organizationKey must name different keys of the Secret"), + Entry("a retention policy outside the enum", "invalid-retention-policy", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Storage: memgraphcomv1alpha1.StorageSpec{RetentionPolicy: "Purge"}, + }, + `Unsupported value: "Purge"`), + Entry("an access mode outside the enum", "invalid-access-mode", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Storage: memgraphcomv1alpha1.StorageSpec{ + Data: memgraphcomv1alpha1.RoleStorageSpec{LibStorageAccessMode: "ReadWriteSometimes"}, + }, + }, + `Unsupported value: "ReadWriteSometimes"`), ) // The typed client drops empty strings before they reach the API diff --git a/internal/resources/resources.go b/internal/resources/resources.go index 3e6da3e..2b9447a 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -21,6 +21,7 @@ package resources import ( corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" ) @@ -89,24 +90,43 @@ func selectorLabels(cluster *memgraphcomv1alpha1.MemgraphCluster, component stri // to its CRD schema default, so builders behave correctly on specs that never // passed admission. type normalizedSpec struct { - coordinators int32 - dataInstances int32 - image string - pullPolicy corev1.PullPolicy - secretName string - licenseKey string - organizationKey string + coordinators int32 + dataInstances int32 + image string + pullPolicy corev1.PullPolicy + secretName string + licenseKey string + organizationKey string + retentionPolicy memgraphcomv1alpha1.StorageRetentionPolicy + coordinatorStorage normalizedStorage + dataStorage normalizedStorage +} + +// normalizedStorage is one role's lib and log claim configuration with every +// optional field resolved to its CRD schema default. A nil storage class means +// "use the cluster default" and is passed through as nil, which is distinct +// from the empty string (no dynamic provisioning). +type normalizedStorage struct { + libSize resource.Quantity + libAccessMode corev1.PersistentVolumeAccessMode + libClass *string + logSize resource.Quantity + logAccessMode corev1.PersistentVolumeAccessMode + logClass *string } func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { n := normalizedSpec{ - coordinators: memgraphcomv1alpha1.DefaultCoordinatorCount, - dataInstances: memgraphcomv1alpha1.DefaultDataInstanceCount, - image: imageRef(spec.Image), - pullPolicy: spec.Image.PullPolicy, - secretName: spec.Secrets.Name, - licenseKey: spec.Secrets.LicenseKey, - organizationKey: spec.Secrets.OrganizationKey, + coordinators: memgraphcomv1alpha1.DefaultCoordinatorCount, + dataInstances: memgraphcomv1alpha1.DefaultDataInstanceCount, + image: imageRef(spec.Image), + pullPolicy: spec.Image.PullPolicy, + secretName: spec.Secrets.Name, + licenseKey: spec.Secrets.LicenseKey, + organizationKey: spec.Secrets.OrganizationKey, + retentionPolicy: spec.Storage.RetentionPolicy, + coordinatorStorage: normalizeStorage(spec.Storage.Coordinators), + dataStorage: normalizeStorage(spec.Storage.Data), } if spec.Coordinators != nil { n.coordinators = *spec.Coordinators @@ -126,6 +146,33 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { if n.organizationKey == "" { n.organizationKey = memgraphcomv1alpha1.DefaultOrganizationSecretKey } + if n.retentionPolicy == "" { + n.retentionPolicy = memgraphcomv1alpha1.DefaultStorageRetention + } + return n +} + +func normalizeStorage(spec memgraphcomv1alpha1.RoleStorageSpec) normalizedStorage { + n := normalizedStorage{ + libSize: resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize), + libAccessMode: spec.LibStorageAccessMode, + libClass: spec.LibStorageClassName, + logSize: resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize), + logAccessMode: spec.LogStorageAccessMode, + logClass: spec.LogStorageClassName, + } + if spec.LibPVCSize != nil { + n.libSize = *spec.LibPVCSize + } + if spec.LogPVCSize != nil { + n.logSize = *spec.LogPVCSize + } + if n.libAccessMode == "" { + n.libAccessMode = memgraphcomv1alpha1.DefaultStorageAccessMode + } + if n.logAccessMode == "" { + n.logAccessMode = memgraphcomv1alpha1.DefaultStorageAccessMode + } return n } diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index c784c2a..bd661a8 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -22,6 +22,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -37,6 +38,12 @@ const ( libMountPath = "/var/lib/memgraph" logMountPath = "/var/log/memgraph" tmpMountPath = "/tmp" + + // Volume names double as the StatefulSet volumeClaimTemplate names, so the + // provisioned claims are -, e.g. lib-storage-example-data-0. + libVolumeName = "lib-storage" + logVolumeName = "log-storage" + tmpVolumeName = "tmp" ) // CoordinatorStatefulSet builds the single StatefulSet running all @@ -66,7 +73,7 @@ func CoordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv container.ReadinessProbe = tcpProbe(CoordinatorPort, 20) container.LivenessProbe = tcpProbe(CoordinatorPort, 20) - return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec.coordinators, container) + return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec, spec.coordinators, container) } // DataStatefulSet builds the single StatefulSet running all data instances. @@ -86,7 +93,7 @@ func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.State container.ReadinessProbe = tcpProbe(BoltPort, 20) container.LivenessProbe = tcpProbe(BoltPort, 20) - return statefulSet(cluster, dataComponent, DataName(cluster), spec.dataInstances, container) + return statefulSet(cluster, dataComponent, DataName(cluster), spec, spec.dataInstances, container) } // coordinatorStartScript derives the coordinator's identity from its pod @@ -155,9 +162,9 @@ func memgraphContainer(spec normalizedSpec) corev1.Container { }, }, VolumeMounts: []corev1.VolumeMount{ - {Name: "lib-storage", MountPath: libMountPath}, - {Name: "log-storage", MountPath: logMountPath}, - {Name: "tmp", MountPath: tmpMountPath}, + {Name: libVolumeName, MountPath: libMountPath}, + {Name: logVolumeName, MountPath: logMountPath}, + {Name: tmpVolumeName, MountPath: tmpMountPath}, }, SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), @@ -172,9 +179,15 @@ func memgraphContainer(spec normalizedSpec) corev1.Container { func statefulSet( cluster *memgraphcomv1alpha1.MemgraphCluster, component, name string, + spec normalizedSpec, replicas int32, container corev1.Container, ) *appsv1.StatefulSet { + storage := spec.dataStorage + if component == coordinatorComponent { + storage = spec.coordinatorStorage + } + return &appsv1.StatefulSet{ // TypeMeta is set explicitly because the controller server-side // applies builder output, and apply patches must carry the GVK. @@ -189,6 +202,19 @@ func statefulSet( ServiceName: name, PodManagementPolicy: appsv1.ParallelPodManagement, Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(cluster, component)}, + // The StatefulSet controller is the only thing that ever deletes + // this cluster's storage; the operator owns no finalizer and runs + // no cleanup of its own. whenScaled is always Retain because both + // replica counts are immutable in v1alpha1 — nothing scales down, + // so no claim is ever orphaned by scaling. + PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: retentionType(spec.retentionPolicy), + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ + volumeClaimTemplate(libVolumeName, storage.libSize, storage.libAccessMode, storage.libClass), + volumeClaimTemplate(logVolumeName, storage.logSize, storage.logAccessMode, storage.logClass), + }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels(cluster, component), @@ -202,13 +228,11 @@ func statefulSet( RunAsNonRoot: ptr.To(true), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, - // Storage is ephemeral in this slice; PVC templates and - // retention policy land with the storage-configuration - // slice (specs/operator-mvp/issues/08). + // Lib and log storage come from the volumeClaimTemplates + // above; only the scratch directory the read-only root + // filesystem still needs is ephemeral. Volumes: []corev1.Volume{ - {Name: "lib-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "log-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: tmpVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, }, }, }, @@ -216,6 +240,40 @@ func statefulSet( } } +// volumeClaimTemplate builds one StatefulSet volumeClaimTemplate. A nil class +// is left unset so the cluster's default StorageClass applies; the empty +// string is passed through as-is, which disables dynamic provisioning. +func volumeClaimTemplate( + name string, + size resource.Quantity, + accessMode corev1.PersistentVolumeAccessMode, + class *string, +) corev1.PersistentVolumeClaim { + return corev1.PersistentVolumeClaim{ + // TypeMeta is set explicitly for the same reason the StatefulSet sets + // it: the controller server-side applies the builder output. + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{accessMode}, + StorageClassName: class, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: size}, + }, + }, + } +} + +// retentionType maps the spec's retention policy onto the StatefulSet's +// whenDeleted policy. The two vocabularies coincide, so the mapping is a +// rename rather than a decision. +func retentionType(policy memgraphcomv1alpha1.StorageRetentionPolicy) appsv1.PersistentVolumeClaimRetentionPolicyType { + if policy == memgraphcomv1alpha1.RetentionPolicyDelete { + return appsv1.DeletePersistentVolumeClaimRetentionPolicyType + } + return appsv1.RetainPersistentVolumeClaimRetentionPolicyType +} + func tcpProbe(port, failureThreshold int32) *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index d07a8bf..4516049 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-cmp/cmp" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" @@ -139,14 +140,47 @@ func expectedVolumeMounts() []corev1.VolumeMount { } } +// expectedVolumes covers only the ephemeral scratch volume: lib and log +// storage are provisioned through volumeClaimTemplates. func expectedVolumes() []corev1.Volume { return []corev1.Volume{ - {Name: "lib-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - {Name: "log-storage", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } } +func expectedClaimTemplate(name, size string, accessMode corev1.PersistentVolumeAccessMode, + class *string) corev1.PersistentVolumeClaim { + return corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{accessMode}, + StorageClassName: class, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse(size)}, + }, + }, + } +} + +// expectedClaimTemplates are the claims a spec that never set storage gets: +// 1Gi ReadWriteOnce on the cluster's default StorageClass for both volumes. +func expectedClaimTemplates() []corev1.PersistentVolumeClaim { + return []corev1.PersistentVolumeClaim{ + expectedClaimTemplate("lib-storage", "1Gi", corev1.ReadWriteOnce, nil), + expectedClaimTemplate("log-storage", "1Gi", corev1.ReadWriteOnce, nil), + } +} + +func expectedRetentionPolicy( + whenDeleted appsv1.PersistentVolumeClaimRetentionPolicyType, +) *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy { + return &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: whenDeleted, + WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + } +} + func expectedLabels(component string) map[string]string { return map[string]string{ "app.kubernetes.io/name": memgraphName, @@ -190,6 +224,9 @@ func TestCoordinatorStatefulSetDefaults(t *testing.T) { ServiceName: coordinatorName, PodManagementPolicy: appsv1.ParallelPodManagement, Selector: &metav1.LabelSelector{MatchLabels: expectedSelectorLabels(coordinatorComponent)}, + PersistentVolumeClaimRetentionPolicy: expectedRetentionPolicy( + appsv1.RetainPersistentVolumeClaimRetentionPolicyType), + VolumeClaimTemplates: expectedClaimTemplates(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: expectedLabels(coordinatorComponent)}, Spec: corev1.PodSpec{ @@ -241,6 +278,9 @@ func TestDataStatefulSetDefaults(t *testing.T) { ServiceName: dataName, PodManagementPolicy: appsv1.ParallelPodManagement, Selector: &metav1.LabelSelector{MatchLabels: expectedSelectorLabels(dataComponent)}, + PersistentVolumeClaimRetentionPolicy: expectedRetentionPolicy( + appsv1.RetainPersistentVolumeClaimRetentionPolicyType), + VolumeClaimTemplates: expectedClaimTemplates(), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: expectedLabels(dataComponent)}, Spec: corev1.PodSpec{ @@ -313,3 +353,101 @@ func TestStatefulSetSpecOverrides(t *testing.T) { }) } } + +// TestStatefulSetStorageOverrides asserts each role's claim templates follow +// that role's storage block only, so sizing coordinators and data instances +// differently really produces differently sized claims. +func TestStatefulSetStorageOverrides(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.Storage = memgraphcomv1alpha1.StorageSpec{ + Coordinators: memgraphcomv1alpha1.RoleStorageSpec{ + LibPVCSize: ptr.To(resource.MustParse("4Gi")), + LibStorageAccessMode: corev1.ReadWriteOncePod, + LibStorageClassName: ptr.To("fast-ssd"), + LogPVCSize: ptr.To(resource.MustParse("512Mi")), + LogStorageClassName: ptr.To(""), + }, + Data: memgraphcomv1alpha1.RoleStorageSpec{ + LibPVCSize: ptr.To(resource.MustParse("100Gi")), + LibStorageClassName: ptr.To("gp3"), + }, + } + + tests := []struct { + name string + sts *appsv1.StatefulSet + want []corev1.PersistentVolumeClaim + }{ + { + name: coordinatorComponent, + sts: resources.CoordinatorStatefulSet(cluster), + want: []corev1.PersistentVolumeClaim{ + expectedClaimTemplate("lib-storage", "4Gi", corev1.ReadWriteOncePod, ptr.To("fast-ssd")), + // An empty storage class is passed through verbatim: it means + // "no dynamic provisioning", not "cluster default". + expectedClaimTemplate("log-storage", "512Mi", corev1.ReadWriteOnce, ptr.To("")), + }, + }, + { + name: dataComponent, + sts: resources.DataStatefulSet(cluster), + want: []corev1.PersistentVolumeClaim{ + expectedClaimTemplate("lib-storage", "100Gi", corev1.ReadWriteOnce, ptr.To("gp3")), + // Untouched by the spec, so it keeps every schema default. + expectedClaimTemplate("log-storage", "1Gi", corev1.ReadWriteOnce, nil), + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if diff := cmp.Diff(tc.want, tc.sts.Spec.VolumeClaimTemplates); diff != "" { + t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestStatefulSetRetentionPolicy pins the mapping from the spec's retention +// policy onto the StatefulSet machinery that is the only deleter of this +// cluster's storage. whenScaled stays Retain regardless: both replica counts +// are immutable, so nothing ever scales down. +func TestStatefulSetRetentionPolicy(t *testing.T) { + tests := []struct { + name string + policy memgraphcomv1alpha1.StorageRetentionPolicy + whenDeleted appsv1.PersistentVolumeClaimRetentionPolicyType + }{ + { + name: "unset defaults to retain", + policy: "", + whenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + name: "retain", + policy: memgraphcomv1alpha1.RetentionPolicyRetain, + whenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + }, + { + name: "delete", + policy: memgraphcomv1alpha1.RetentionPolicyDelete, + whenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.Storage.RetentionPolicy = tc.policy + + want := expectedRetentionPolicy(tc.whenDeleted) + for _, sts := range []*appsv1.StatefulSet{ + resources.CoordinatorStatefulSet(cluster), + resources.DataStatefulSet(cluster), + } { + got := sts.Spec.PersistentVolumeClaimRetentionPolicy + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("%s retention policy mismatch (-want +got):\n%s", sts.Name, diff) + } + } + }) + } +} diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index 9dab002..b945d56 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -207,8 +207,142 @@ var _ = Describe("MemgraphCluster", Ordered, func() { By("waiting for the operator to converge the cluster back to fully registered") Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) }) + + // Storage survives the cluster under the default retention policy: an + // accidental `kubectl delete mgc` must not take a production database with + // it. This deletes the CR, so it runs last in this Ordered container. + It("leaves the PVCs behind when the default-retention CR is deleted", func() { + By("confirming the cluster is converged before deleting it") + Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + + By("recording the provisioned PVCs") + before, err := listPVCs(clusterNamespace) + Expect(err).NotTo(HaveOccurred()) + // Two claims (lib and log) per coordinator and data instance pod. + Expect(before).To(HaveLen(2 * (coordinatorCount + dataInstanceCount))) + + By("deleting the MemgraphCluster") + cmd := exec.Command("kubectl", "delete", "memgraphcluster", clusterName, + "-n", clusterNamespace, "--wait=true") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete the MemgraphCluster") + + By("waiting for garbage collection to remove the workloads") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "statefulsets", "-n", clusterNamespace, + "-o", "jsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(BeEmpty()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + By("confirming every PVC is still there") + // Consistently, not Eventually: the failure mode is a delayed deletion, + // which a single post-condition check would race straight past. + Consistently(func(g Gomega) { + after, err := listPVCs(clusterNamespace) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(after).To(ConsistOf(before)) + }, 30*time.Second, 5*time.Second).Should(Succeed()) + }) }) +// The Delete retention policy is the dev-cluster counterpart of the spec +// above: the StatefulSet machinery takes the claims down with the CR. It gets +// its own container and namespace because it needs a differently-configured +// cluster, and it never waits for registration to converge — the StatefulSet +// controller provisions the claims as soon as the pods are created, so the +// retention behavior is observable long before Memgraph is. +var _ = Describe("MemgraphCluster with Delete storage retention", Ordered, func() { + const retentionNamespace = "memgraph-e2e-retention" + const retentionCluster = "retention" + + BeforeAll(func() { + By("creating the cluster namespace") + cmd := exec.Command("kubectl", "create", "ns", retentionNamespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + }) + + AfterAll(func() { + By("removing the cluster namespace") + cmd := exec.Command("kubectl", "delete", "ns", retentionNamespace, + "--ignore-not-found", "--wait=false") + _, _ = utils.Run(cmd) + }) + + It("removes the PVCs when the CR is deleted", func() { + By("applying a MemgraphCluster with Delete retention") + manifest := fmt.Sprintf(`apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + name: %s + namespace: %s +spec: + coordinators: 1 + dataInstances: 1 + image: + repository: %s + tag: %s + storage: + retentionPolicy: Delete +`, retentionCluster, retentionNamespace, memgraphImageRepository, memgraphImageTag) + cmd := exec.Command("kubectl", "apply", "-f", "-") + _, err := utils.RunWithInput(cmd, manifest) + Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") + + By("waiting for the claims to be provisioned") + // One lib and one log claim for the single coordinator and the single + // data instance. + Eventually(func(g Gomega) { + claims, err := listPVCs(retentionNamespace) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(claims).To(HaveLen(4)) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + By("deleting the MemgraphCluster") + cmd = exec.Command("kubectl", "delete", "memgraphcluster", retentionCluster, + "-n", retentionNamespace, "--wait=true") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete the MemgraphCluster") + + By("waiting for the StatefulSet machinery to take the claims down with it") + Eventually(func(g Gomega) { + claims, err := listPVCs(retentionNamespace) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(claims).To(BeEmpty()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + }) +}) + +// listPVCs returns the names of the PersistentVolumeClaims in a namespace, +// excluding any already marked for deletion — a claim with a deletion +// timestamp is gone as far as the retention contract is concerned, even while +// a finalizer keeps the object around. +func listPVCs(namespace string) ([]string, error) { + cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, "-o", + `jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.deletionTimestamp}{"\n"}{end}`) + output, err := utils.Run(cmd) + if err != nil { + return nil, err + } + + // A claim with no deletion timestamp yields a line of "\t"; the + // separator is always emitted, so the split is total. + names := []string{} + for _, line := range utils.GetNonEmptyLines(output) { + name, deletionTimestamp, found := strings.Cut(line, "\t") + if !found { + return nil, fmt.Errorf("unexpected kubectl get pvc output line: %q", line) + } + if strings.TrimSpace(deletionTimestamp) != "" { + continue + } + names = append(names, strings.TrimSpace(name)) + } + return names, nil +} + // verifyClusterRegistered asserts the coordinator leader reports every declared // instance registered and healthy with exactly one MAIN — the converged steady // state both the bootstrap and re-registration specs check for. From 67b94149b8b72f56e9ddc9f7541816a4c2ba681f Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 12:31:13 +0200 Subject: [PATCH 12/34] =?UTF-8?q?feat:=20pod-tuning=20knobs=20=E2=80=94=20?= =?UTF-8?q?probes,=20resources,=20labels,=20ports,=20cluster=20domain,=20e?= =?UTF-8?q?nv/args=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: pod-tuning knobs (probes, resources, labels, ports, env/args) Add the remaining v1 configuration surface to the MemgraphCluster spec, mirroring the memgraph-high-availability Helm chart's vocabulary: - clusterDomain and ports (bolt, management, replication, coordinator), which propagate to container ports, Services, the coordinator start script and every advertised address the operator registers - probes per role and per probe (timings only; the probe stays a TCP-socket check against the role's own port) - resources per role - labels per role for pods, StatefulSets and Services - extraEnv and extraArgs per role, non-secret by construction The two ports the registration commands are built from cannot be overridden through extraArgs, and extraEnv cannot shadow the license or the pod's own identity: admission rejects both. A CR with every knob defaulted produces the same objects as before this slice. * fix: Improve sample config file --- api/v1alpha1/memgraphcluster_types.go | 290 +++++++++- api/v1alpha1/zz_generated.deepcopy.go | 241 ++++++++ .../bases/memgraph.com_memgraphclusters.yaml | 518 ++++++++++++++++++ config/samples/v1alpha1_memgraphcluster.yaml | 112 +++- .../memgraphcluster_validation_test.go | 151 +++++ internal/planner/planner_test.go | 96 +++- internal/resources/resources.go | 209 +++++-- internal/resources/service.go | 29 +- internal/resources/service_test.go | 55 +- internal/resources/statefulset.go | 109 ++-- internal/resources/statefulset_test.go | 421 +++++++++++++- internal/resources/topology.go | 36 +- internal/resources/topology_test.go | 79 +++ 13 files changed, 2214 insertions(+), 132 deletions(-) diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 75a4bbf..632f4e5 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -45,6 +45,46 @@ const ( DefaultLogPVCSize = "1Gi" DefaultStorageAccessMode = corev1.ReadWriteOnce DefaultStorageRetention = RetentionPolicyRetain + + DefaultClusterDomain = "cluster.local" + + DefaultBoltPort int32 = 7687 + DefaultManagementPort int32 = 10000 + DefaultReplicationPort int32 = 20000 + DefaultCoordinatorPort int32 = 12000 +) + +// Probe timing defaults. Unlike the other defaults these are Go constants only: +// a CRD schema default is per-field, and the data instances' startup budget +// deliberately differs from every other probe's, which one shared schema +// default cannot express. The doc comments on ProbeSpec name them. +const ( + // DefaultProbeFailureThreshold is the failure budget of every probe except + // the data instances' startup probe. + DefaultProbeFailureThreshold int32 = 20 + + // DefaultDataStartupProbeFailureThreshold gives data instances a generous + // startup budget so a large snapshot restore is not killed mid-load: 1440 + // failures at the default 5s period is 2h, mirroring the + // memgraph-high-availability Helm chart's default. + DefaultDataStartupProbeFailureThreshold int32 = 1440 + + DefaultProbeTimeoutSeconds int32 = 10 + DefaultProbePeriodSeconds int32 = 5 +) + +// Names of the environment variables the operator itself sets on the Memgraph +// container, which extraEnv therefore may not carry. +const ( + // EnvLicense holds the enterprise license, wired from the secrets block. + EnvLicense = "MEMGRAPH_ENTERPRISE_LICENSE" + + // EnvOrganization holds the organization name, wired from the secrets block. + EnvOrganization = "MEMGRAPH_ORGANIZATION_NAME" + + // EnvPodName carries the pod's own name, from which a coordinator derives + // its ordinal-dependent identity at startup. + EnvPodName = "POD_NAME" ) // StorageRetentionPolicy decides what happens to the cluster's @@ -249,10 +289,218 @@ type StorageSpec struct { Data RoleStorageSpec `json:"data,omitzero"` } -// MemgraphClusterSpec defines the desired state of MemgraphCluster. +// PortsSpec configures the internal ports Memgraph listens on. The knob names +// mirror the memgraph-high-availability Helm chart's ports block. +// +// These ports are load-bearing beyond the container: they are part of every +// advertised address the operator registers with the cluster (bolt_server, +// coordinator_server, management_server, replication_server), so a change +// reaches container ports, Services, and registration commands together. +// Changing a port on a live cluster is a day-2 operation and out of scope for +// v1alpha1: the pods restart on the new ports while the coordinators keep the +// addresses they were registered with. +// +// The has() guards keep the rule evaluable against the block's empty object +// default, which the API server checks before nested field defaults apply. +// +// +kubebuilder:validation:XValidation:rule="!(has(self.boltPort) && has(self.managementPort) && has(self.replicationPort) && has(self.coordinatorPort)) || [self.boltPort, self.managementPort, self.replicationPort, self.coordinatorPort].all(p, [self.boltPort, self.managementPort, self.replicationPort, self.coordinatorPort].exists_one(q, q == p))",message="boltPort, managementPort, replicationPort and coordinatorPort must all be different ports" +type PortsSpec struct { + // boltPort is the port Memgraph serves the Bolt protocol on. Clients and + // the operator's own management queries both use it. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=7687 + // +optional + BoltPort *int32 `json:"boltPort,omitempty"` + + // managementPort is the port instances exchange HA management traffic on. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=10000 + // +optional + ManagementPort *int32 `json:"managementPort,omitempty"` + + // replicationPort is the port data instances replicate over. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=20000 + // +optional + ReplicationPort *int32 `json:"replicationPort,omitempty"` + + // coordinatorPort is the port coordinators run their Raft protocol on. + // +kubebuilder:validation:Minimum=1 + // +kubebuilder:validation:Maximum=65535 + // +kubebuilder:default=12000 + // +optional + CoordinatorPort *int32 `json:"coordinatorPort,omitempty"` +} + +// ProbeSpec tunes the timings of one probe. The probe type itself is not +// configurable: every probe is a TCP-socket check against the role's own port +// (the coordinator port for coordinators, the Bolt port for data instances), +// which is the memgraph-high-availability Helm chart's established convention. // -// Port and pod-tuning fields land in subsequent slices of the operator MVP -// (see specs/operator-mvp/PRD.md). +// Every field defaults to the value named in its doc comment. +type ProbeSpec struct { + // failureThreshold is how many consecutive failures the probe tolerates + // before acting. Defaults to 1440 for the data instances' startup probe — + // 2h at the default period, so a large snapshot restore is not killed + // mid-load — and to 20 for every other probe. + // +kubebuilder:validation:Minimum=1 + // +optional + FailureThreshold *int32 `json:"failureThreshold,omitempty"` + + // timeoutSeconds is how long a single probe attempt may take. Defaults to + // 10. + // +kubebuilder:validation:Minimum=1 + // +optional + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + + // periodSeconds is how often the probe runs. Defaults to 5. + // +kubebuilder:validation:Minimum=1 + // +optional + PeriodSeconds *int32 `json:"periodSeconds,omitempty"` +} + +// RoleProbesSpec tunes all three probes of one role. +type RoleProbesSpec struct { + // startupProbe gates the other two probes until the instance has started. + // +optional + StartupProbe ProbeSpec `json:"startupProbe,omitzero"` + + // readinessProbe decides whether the pod receives traffic and whether the + // operator considers the workloads ready to register. + // +optional + ReadinessProbe ProbeSpec `json:"readinessProbe,omitzero"` + + // livenessProbe decides whether the container is restarted. + // +optional + LivenessProbe ProbeSpec `json:"livenessProbe,omitzero"` +} + +// ProbesSpec tunes probe timings per role. +type ProbesSpec struct { + // coordinators tunes the probes of every coordinator pod. + // +optional + Coordinators RoleProbesSpec `json:"coordinators,omitzero"` + + // data tunes the probes of every data instance pod. + // +optional + Data RoleProbesSpec `json:"data,omitzero"` +} + +// ResourcesSpec sets the compute resources of the Memgraph container per role. +// When setting Memgraph's own --memory-limit through extraArgs, keep it below +// the pod's memory limit: Memgraph must hit its own limit and raise a query +// exception before the kubelet evicts the pod. +type ResourcesSpec struct { + // coordinators are the resource requests and limits of every coordinator + // pod's Memgraph container. + // +optional + Coordinators corev1.ResourceRequirements `json:"coordinators,omitzero"` + + // data are the resource requests and limits of every data instance pod's + // Memgraph container. + // +optional + Data corev1.ResourceRequirements `json:"data,omitzero"` +} + +// RoleLabelsSpec adds custom labels to one role's objects. The operator's own +// identity labels (app.kubernetes.io/name, /instance, /component, /managed-by) +// always win a key collision: they are what the StatefulSets and Services +// select on, so a custom label can never detach a pod from its cluster. +type RoleLabelsSpec struct { + // podLabels are added to the role's pods. + // +optional + PodLabels map[string]string `json:"podLabels,omitempty"` + + // statefulSetLabels are added to the role's StatefulSet. + // +optional + StatefulSetLabels map[string]string `json:"statefulSetLabels,omitempty"` + + // serviceLabels are added to the role's headless Service. + // +optional + ServiceLabels map[string]string `json:"serviceLabels,omitempty"` +} + +// LabelsSpec adds custom labels per role, mirroring the +// memgraph-high-availability Helm chart's labels block. +type LabelsSpec struct { + // coordinators labels the coordinator objects. + // +optional + Coordinators RoleLabelsSpec `json:"coordinators,omitzero"` + + // data labels the data instance objects. + // +optional + Data RoleLabelsSpec `json:"data,omitzero"` +} + +// EnvVar is one non-secret environment variable set on a role's Memgraph +// container. Only literal values are supported — there is deliberately no +// valueFrom — so secret material stays confined to the secrets block and the CR +// remains safe to commit. +type EnvVar struct { + // name is the environment variable's name. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[A-Za-z_][A-Za-z0-9_]*$` + // +required + Name string `json:"name"` + + // value is the literal, non-secret value. + // +kubebuilder:validation:MaxLength=4096 + // +optional + Value string `json:"value,omitempty"` +} + +// ExtraEnvSpec passes additional non-secret environment variables to a role's +// Memgraph container, mirroring the memgraph-high-availability Helm chart's +// extraEnv block. +type ExtraEnvSpec struct { + // coordinators are added to every coordinator pod's Memgraph container. + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME']))",message="extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) or POD_NAME (it carries the pod's own identity)" + // +optional + Coordinators []EnvVar `json:"coordinators,omitempty"` + + // data are added to every data instance pod's Memgraph container. + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:XValidation:rule="self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME']))",message="extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) or POD_NAME (it carries the pod's own identity)" + // +optional + Data []EnvVar `json:"data,omitempty"` +} + +// ExtraArgsSpec passes additional Memgraph flags to a role, so any flag is +// usable without waiting for a typed field. The flags are appended after the +// ones the operator derives, and Memgraph takes the last occurrence of a +// repeated flag, so a flag set here overrides the operator's value. +// +// The ports and the coordinator identity are excluded from that override: they +// must stay consistent with the advertised addresses the operator registers +// with the cluster. Configure ports through spec.ports instead. +type ExtraArgsSpec struct { + // coordinators are appended to every coordinator pod's Memgraph flags. + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=4096 + // +kubebuilder:validation:XValidation:rule="self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', '--coordinator-hostname', '--coordinator-port'].exists(f, a.startsWith(f)))",message="extraArgs must not set a port or the coordinator identity the operator derives (--bolt-port, --management-port, --coordinator-id, --coordinator-hostname, --coordinator-port); configure ports through spec.ports" + // +optional + Coordinators []string `json:"coordinators,omitempty"` + + // data are appended to every data instance pod's Memgraph flags. + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:items:MinLength=1 + // +kubebuilder:validation:items:MaxLength=4096 + // +kubebuilder:validation:XValidation:rule="self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', '--coordinator-hostname', '--coordinator-port'].exists(f, a.startsWith(f)))",message="extraArgs must not set a port or the coordinator identity the operator derives (--bolt-port, --management-port, --coordinator-id, --coordinator-hostname, --coordinator-port); configure ports through spec.ports" + // +optional + Data []string `json:"data,omitempty"` +} + +// MemgraphClusterSpec defines the desired state of MemgraphCluster. type MemgraphClusterSpec struct { // coordinators is the number of Raft coordinator instances. It must be odd // so the Raft quorum cannot split, and it is immutable: scaling is not @@ -288,6 +536,42 @@ type MemgraphClusterSpec struct { // +kubebuilder:default={} // +optional Storage StorageSpec `json:"storage,omitzero"` + + // clusterDomain is the Kubernetes cluster domain the advertised FQDN + // addresses are built from: ...svc.. + // Override it on clusters configured with a domain other than the default. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$` + // +kubebuilder:default="cluster.local" + // +optional + ClusterDomain string `json:"clusterDomain,omitempty"` + + // ports configures the internal ports Memgraph listens on. + // +kubebuilder:default={} + // +optional + Ports PortsSpec `json:"ports,omitzero"` + + // probes tunes the probe timings of both roles. + // +optional + Probes ProbesSpec `json:"probes,omitzero"` + + // resources sets the compute resources of both roles' Memgraph containers. + // +optional + Resources ResourcesSpec `json:"resources,omitzero"` + + // labels adds custom labels to both roles' pods, StatefulSets and Services. + // +optional + Labels LabelsSpec `json:"labels,omitzero"` + + // extraEnv passes additional non-secret environment variables to both + // roles' Memgraph containers. + // +optional + ExtraEnv ExtraEnvSpec `json:"extraEnv,omitzero"` + + // extraArgs passes additional Memgraph flags to both roles. + // +optional + ExtraArgs ExtraArgsSpec `json:"extraArgs,omitzero"` } // MemgraphClusterStatus defines the observed state of MemgraphCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d17e80b..64b52ed 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,71 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvVar) DeepCopyInto(out *EnvVar) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvVar. +func (in *EnvVar) DeepCopy() *EnvVar { + if in == nil { + return nil + } + out := new(EnvVar) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtraArgsSpec) DeepCopyInto(out *ExtraArgsSpec) { + *out = *in + if in.Coordinators != nil { + in, out := &in.Coordinators, &out.Coordinators + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraArgsSpec. +func (in *ExtraArgsSpec) DeepCopy() *ExtraArgsSpec { + if in == nil { + return nil + } + out := new(ExtraArgsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtraEnvSpec) DeepCopyInto(out *ExtraEnvSpec) { + *out = *in + if in.Coordinators != nil { + in, out := &in.Coordinators, &out.Coordinators + *out = make([]EnvVar, len(*in)) + copy(*out, *in) + } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make([]EnvVar, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraEnvSpec. +func (in *ExtraEnvSpec) DeepCopy() *ExtraEnvSpec { + if in == nil { + return nil + } + out := new(ExtraEnvSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { *out = *in @@ -40,6 +105,23 @@ func (in *ImageSpec) DeepCopy() *ImageSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LabelsSpec) DeepCopyInto(out *LabelsSpec) { + *out = *in + in.Coordinators.DeepCopyInto(&out.Coordinators) + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LabelsSpec. +func (in *LabelsSpec) DeepCopy() *LabelsSpec { + if in == nil { + return nil + } + out := new(LabelsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemgraphCluster) DeepCopyInto(out *MemgraphCluster) { *out = *in @@ -115,6 +197,12 @@ func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { out.Image = in.Image out.Secrets = in.Secrets in.Storage.DeepCopyInto(&out.Storage) + in.Ports.DeepCopyInto(&out.Ports) + in.Probes.DeepCopyInto(&out.Probes) + in.Resources.DeepCopyInto(&out.Resources) + in.Labels.DeepCopyInto(&out.Labels) + in.ExtraEnv.DeepCopyInto(&out.ExtraEnv) + in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterSpec. @@ -149,6 +237,159 @@ func (in *MemgraphClusterStatus) DeepCopy() *MemgraphClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PortsSpec) DeepCopyInto(out *PortsSpec) { + *out = *in + if in.BoltPort != nil { + in, out := &in.BoltPort, &out.BoltPort + *out = new(int32) + **out = **in + } + if in.ManagementPort != nil { + in, out := &in.ManagementPort, &out.ManagementPort + *out = new(int32) + **out = **in + } + if in.ReplicationPort != nil { + in, out := &in.ReplicationPort, &out.ReplicationPort + *out = new(int32) + **out = **in + } + if in.CoordinatorPort != nil { + in, out := &in.CoordinatorPort, &out.CoordinatorPort + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PortsSpec. +func (in *PortsSpec) DeepCopy() *PortsSpec { + if in == nil { + return nil + } + out := new(PortsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProbeSpec) DeepCopyInto(out *ProbeSpec) { + *out = *in + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(int32) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.PeriodSeconds != nil { + in, out := &in.PeriodSeconds, &out.PeriodSeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbeSpec. +func (in *ProbeSpec) DeepCopy() *ProbeSpec { + if in == nil { + return nil + } + out := new(ProbeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProbesSpec) DeepCopyInto(out *ProbesSpec) { + *out = *in + in.Coordinators.DeepCopyInto(&out.Coordinators) + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProbesSpec. +func (in *ProbesSpec) DeepCopy() *ProbesSpec { + if in == nil { + return nil + } + out := new(ProbesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourcesSpec) DeepCopyInto(out *ResourcesSpec) { + *out = *in + in.Coordinators.DeepCopyInto(&out.Coordinators) + in.Data.DeepCopyInto(&out.Data) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourcesSpec. +func (in *ResourcesSpec) DeepCopy() *ResourcesSpec { + if in == nil { + return nil + } + out := new(ResourcesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RoleLabelsSpec) DeepCopyInto(out *RoleLabelsSpec) { + *out = *in + if in.PodLabels != nil { + in, out := &in.PodLabels, &out.PodLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.StatefulSetLabels != nil { + in, out := &in.StatefulSetLabels, &out.StatefulSetLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ServiceLabels != nil { + in, out := &in.ServiceLabels, &out.ServiceLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleLabelsSpec. +func (in *RoleLabelsSpec) DeepCopy() *RoleLabelsSpec { + if in == nil { + return nil + } + out := new(RoleLabelsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RoleProbesSpec) DeepCopyInto(out *RoleProbesSpec) { + *out = *in + in.StartupProbe.DeepCopyInto(&out.StartupProbe) + in.ReadinessProbe.DeepCopyInto(&out.ReadinessProbe) + in.LivenessProbe.DeepCopyInto(&out.LivenessProbe) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleProbesSpec. +func (in *RoleProbesSpec) DeepCopy() *RoleProbesSpec { + if in == nil { + return nil + } + out := new(RoleProbesSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleStorageSpec) DeepCopyInto(out *RoleStorageSpec) { *out = *in diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index d8cca12..6193b44 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -60,6 +60,16 @@ spec: spec: description: spec defines the desired state of MemgraphCluster properties: + clusterDomain: + default: cluster.local + description: |- + clusterDomain is the Kubernetes cluster domain the advertised FQDN + addresses are built from: ...svc.. + Override it on clusters configured with a domain other than the default. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string coordinators: default: 3 description: |- @@ -88,6 +98,118 @@ spec: - message: 'dataInstances is immutable: changing the data instance count of an existing MemgraphCluster is not supported in v1alpha1' rule: self == oldSelf + extraArgs: + description: extraArgs passes additional Memgraph flags to both roles. + properties: + coordinators: + description: coordinators are appended to every coordinator pod's + Memgraph flags. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array + x-kubernetes-validations: + - message: extraArgs must not set a port or the coordinator identity + the operator derives (--bolt-port, --management-port, --coordinator-id, + --coordinator-hostname, --coordinator-port); configure ports + through spec.ports + rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', + '--coordinator-hostname', '--coordinator-port'].exists(f, + a.startsWith(f))) + data: + description: data are appended to every data instance pod's Memgraph + flags. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array + x-kubernetes-validations: + - message: extraArgs must not set a port or the coordinator identity + the operator derives (--bolt-port, --management-port, --coordinator-id, + --coordinator-hostname, --coordinator-port); configure ports + through spec.ports + rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', + '--coordinator-hostname', '--coordinator-port'].exists(f, + a.startsWith(f))) + type: object + extraEnv: + description: |- + extraEnv passes additional non-secret environment variables to both + roles' Memgraph containers. + properties: + coordinators: + description: coordinators are added to every coordinator pod's + Memgraph container. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or + MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) + or POD_NAME (it carries the pod's own identity) + rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', + 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) + data: + description: data are added to every data instance pod's Memgraph + container. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or + MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) + or POD_NAME (it carries the pod's own identity) + rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', + 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) + type: object image: default: {} description: image selects the Memgraph container image run by all @@ -127,6 +249,402 @@ spec: pattern: ^[a-zA-Z0-9_][a-zA-Z0-9._-]*$ type: string type: object + labels: + description: labels adds custom labels to both roles' pods, StatefulSets + and Services. + properties: + coordinators: + description: coordinators labels the coordinator objects. + properties: + podLabels: + additionalProperties: + type: string + description: podLabels are added to the role's pods. + type: object + serviceLabels: + additionalProperties: + type: string + description: serviceLabels are added to the role's headless + Service. + type: object + statefulSetLabels: + additionalProperties: + type: string + description: statefulSetLabels are added to the role's StatefulSet. + type: object + type: object + data: + description: data labels the data instance objects. + properties: + podLabels: + additionalProperties: + type: string + description: podLabels are added to the role's pods. + type: object + serviceLabels: + additionalProperties: + type: string + description: serviceLabels are added to the role's headless + Service. + type: object + statefulSetLabels: + additionalProperties: + type: string + description: statefulSetLabels are added to the role's StatefulSet. + type: object + type: object + type: object + ports: + default: {} + description: ports configures the internal ports Memgraph listens + on. + properties: + boltPort: + default: 7687 + description: |- + boltPort is the port Memgraph serves the Bolt protocol on. Clients and + the operator's own management queries both use it. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + coordinatorPort: + default: 12000 + description: coordinatorPort is the port coordinators run their + Raft protocol on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + managementPort: + default: 10000 + description: managementPort is the port instances exchange HA + management traffic on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + replicationPort: + default: 20000 + description: replicationPort is the port data instances replicate + over. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: boltPort, managementPort, replicationPort and coordinatorPort + must all be different ports + rule: '!(has(self.boltPort) && has(self.managementPort) && has(self.replicationPort) + && has(self.coordinatorPort)) || [self.boltPort, self.managementPort, + self.replicationPort, self.coordinatorPort].all(p, [self.boltPort, + self.managementPort, self.replicationPort, self.coordinatorPort].exists_one(q, + q == p))' + probes: + description: probes tunes the probe timings of both roles. + properties: + coordinators: + description: coordinators tunes the probes of every coordinator + pod. + properties: + livenessProbe: + description: livenessProbe decides whether the container is + restarted. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + readinessProbe decides whether the pod receives traffic and whether the + operator considers the workloads ready to register. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: startupProbe gates the other two probes until + the instance has started. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + type: object + data: + description: data tunes the probes of every data instance pod. + properties: + livenessProbe: + description: livenessProbe decides whether the container is + restarted. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + readinessProbe decides whether the pod receives traffic and whether the + operator considers the workloads ready to register. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: startupProbe gates the other two probes until + the instance has started. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object + resources: + description: resources sets the compute resources of both roles' Memgraph + containers. + properties: + coordinators: + description: |- + coordinators are the resource requests and limits of every coordinator + pod's Memgraph container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + data: + description: |- + data are the resource requests and limits of every data instance pod's + Memgraph container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object secrets: default: {} description: |- diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index f27aae5..0293f77 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -11,9 +11,13 @@ spec: # must be odd so the Raft quorum cannot split. coordinators: 3 dataInstances: 2 + # repository carries the registry host and image path only — the version + # belongs in tag, and admission rejects a digest or a tag smuggled into the + # repository. image: repository: docker.io/memgraph/memgraph tag: 3.12.0-relwithdebinfo + pullPolicy: IfNotPresent # References an existing Secret holding the enterprise license; the CR # carries no secret material itself. secrets: @@ -26,15 +30,119 @@ spec: # Delete on dev clusters that should clean up after themselves. retentionPolicy: Retain # Each pod of a role gets a lib claim (Memgraph's data directory) and a log - # claim. Leave the storage class names out to use the cluster's default - # StorageClass. Claim sizes and classes cannot be changed after creation. + # claim. The storage class names have no default and are commented out on + # purpose: left unset, each claim uses the cluster's default StorageClass, + # while the empty string turns dynamic provisioning off and binds a + # pre-created PersistentVolume. Sizes, access modes and classes become + # StatefulSet volumeClaimTemplates, which Kubernetes treats as immutable. coordinators: libPVCSize: 1Gi libStorageAccessMode: ReadWriteOnce + # libStorageClassName: standard logPVCSize: 1Gi logStorageAccessMode: ReadWriteOnce + # logStorageClassName: standard data: libPVCSize: 1Gi libStorageAccessMode: ReadWriteOnce + # libStorageClassName: standard logPVCSize: 1Gi logStorageAccessMode: ReadWriteOnce + # logStorageClassName: standard + # The cluster domain and the internal ports are part of every advertised + # address the operator registers with the cluster, so they reach the + # container ports, the Services and the registration commands together. + # Changing them on a live cluster is not supported in v1alpha1. + clusterDomain: cluster.local + ports: + boltPort: 7687 + managementPort: 10000 + replicationPort: 20000 + coordinatorPort: 12000 + # Every probe is a TCP-socket check against the role's own port (the + # coordinator port for coordinators, the Bolt port for data instances); only + # the timings are configurable. Timings left out keep their defaults: + # failureThreshold 20 (1440 for the data instances' startup probe, which is + # 2h at the default period, so a large snapshot restore is not killed + # mid-load), timeoutSeconds 10, periodSeconds 5. + probes: + coordinators: + # Gates the other two probes until the coordinator has started. + startupProbe: + failureThreshold: 20 + timeoutSeconds: 10 + periodSeconds: 5 + # Decides whether the pod receives traffic, and whether the operator + # considers the workloads ready to register. + readinessProbe: + failureThreshold: 20 + timeoutSeconds: 10 + periodSeconds: 5 + # Decides whether the container is restarted. + livenessProbe: + failureThreshold: 20 + timeoutSeconds: 10 + periodSeconds: 5 + data: + startupProbe: + failureThreshold: 1440 + timeoutSeconds: 10 + periodSeconds: 5 + readinessProbe: + failureThreshold: 20 + timeoutSeconds: 10 + periodSeconds: 5 + livenessProbe: + failureThreshold: 20 + timeoutSeconds: 10 + periodSeconds: 5 + # Requests and limits of the Memgraph container, per role, unset by default so + # the pods schedule anywhere. When setting Memgraph's own --memory-limit + # below, keep it under the pod's memory limit: Memgraph must hit its own limit + # and fail the query before the kubelet evicts the pod. (The block is the core + # ResourceRequirements type, so it also accepts claims — leave that one alone, + # it needs pod-level resourceClaims the operator does not set.) + resources: + coordinators: {} + # requests: + # cpu: 100m + # memory: 256Mi + # limits: + # memory: 1Gi + data: {} + # requests: + # cpu: 500m + # memory: 2Gi + # limits: + # memory: 4Gi + # Custom labels per object. The operator's own app.kubernetes.io identity + # labels always win a collision: they are what the StatefulSets and Services + # select on. + labels: + coordinators: + podLabels: {} + # team: platform + statefulSetLabels: {} + serviceLabels: {} + data: + podLabels: {} + statefulSetLabels: {} + serviceLabels: {} + # Literal, non-secret environment variables only: there is no valueFrom, so + # the CR stays safe to commit. The license and the organization name come from + # the secrets block above, and admission rejects an entry that tries to shadow + # either of them or POD_NAME, which carries the pod's own identity. + extraEnv: + coordinators: [] + # - name: MEMGRAPH_LOG_LEVEL + # value: TRACE + data: [] + # Any Memgraph flag, without waiting for a typed field. These are appended + # after the flags the operator derives, and Memgraph takes the last + # occurrence of a repeated flag — except for the ports and the coordinator + # identity, which admission rejects here because they must stay consistent + # with the registered addresses. + extraArgs: + coordinators: [] + data: + - --storage-snapshot-on-exit=false diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index a038f09..f466541 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,6 +47,17 @@ func defaultRoleStorage() memgraphcomv1alpha1.RoleStorageSpec { } } +// defaultPorts are the internal ports as the CRD schema defaults materialize +// them. +func defaultPorts() memgraphcomv1alpha1.PortsSpec { + return memgraphcomv1alpha1.PortsSpec{ + BoltPort: ptr.To(memgraphcomv1alpha1.DefaultBoltPort), + ManagementPort: ptr.To(memgraphcomv1alpha1.DefaultManagementPort), + ReplicationPort: ptr.To(memgraphcomv1alpha1.DefaultReplicationPort), + CoordinatorPort: ptr.To(memgraphcomv1alpha1.DefaultCoordinatorPort), + } +} + var _ = Describe("MemgraphCluster CRD validation", func() { const resourceNamespace = "default" @@ -120,6 +132,11 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Coordinators: defaultRoleStorage(), Data: defaultRoleStorage(), }, + ClusterDomain: memgraphcomv1alpha1.DefaultClusterDomain, + Ports: defaultPorts(), + // Probes, resources, labels and the env/args passthrough have no + // schema defaults: the probe timings' defaults depend on the role + // and the rest default to "nothing added". }), "the CRD schema defaults must match the Go constants the builders fall back to") }) @@ -190,6 +207,56 @@ var _ = Describe("MemgraphCluster CRD validation", func() { int32(9), int32(16)), ) + It("should accept a fully tuned pod configuration", func() { + stored := createAccepted("valid-pod-tuning", memgraphcomv1alpha1.MemgraphClusterSpec{ + ClusterDomain: "k8s.example.com", + Ports: memgraphcomv1alpha1.PortsSpec{ + BoltPort: ptr.To(int32(7777)), + ManagementPort: ptr.To(int32(10001)), + ReplicationPort: ptr.To(int32(20001)), + CoordinatorPort: ptr.To(int32(12001)), + }, + Probes: memgraphcomv1alpha1.ProbesSpec{ + Data: memgraphcomv1alpha1.RoleProbesSpec{ + StartupProbe: memgraphcomv1alpha1.ProbeSpec{FailureThreshold: ptr.To(int32(4320))}, + }, + }, + Resources: memgraphcomv1alpha1.ResourcesSpec{ + Data: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + Labels: memgraphcomv1alpha1.LabelsSpec{ + Data: memgraphcomv1alpha1.RoleLabelsSpec{PodLabels: map[string]string{"team": "data"}}, + }, + ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ + Data: []memgraphcomv1alpha1.EnvVar{{Name: "DATA_LABEL_ONE", Value: "one"}}, + }, + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{ + Data: []string{"--storage-snapshot-on-exit=true"}, + }, + }) + + Expect(stored.Spec.ClusterDomain).To(Equal("k8s.example.com")) + Expect(stored.Spec.Ports.BoltPort).To(HaveValue(Equal(int32(7777)))) + Expect(stored.Spec.Probes.Data.StartupProbe.FailureThreshold).To(HaveValue(Equal(int32(4320)))) + Expect(stored.Spec.Probes.Data.ReadinessProbe).To(Equal(memgraphcomv1alpha1.ProbeSpec{}), + "an unset probe stays unset; its defaults are resolved by the builders, not the schema") + Expect(stored.Spec.ExtraEnv.Data).To(HaveLen(1)) + Expect(stored.Spec.ExtraArgs.Data).To(ConsistOf("--storage-snapshot-on-exit=true")) + }) + + It("should default the ports a partially specified block leaves out", func() { + stored := createAccepted("valid-partial-ports", memgraphcomv1alpha1.MemgraphClusterSpec{ + Ports: memgraphcomv1alpha1.PortsSpec{BoltPort: ptr.To(int32(7777))}, + }) + + Expect(stored.Spec.Ports.BoltPort).To(HaveValue(Equal(int32(7777)))) + Expect(stored.Spec.Ports.ManagementPort).To(HaveValue(Equal(memgraphcomv1alpha1.DefaultManagementPort))) + Expect(stored.Spec.Ports.ReplicationPort).To(HaveValue(Equal(memgraphcomv1alpha1.DefaultReplicationPort))) + Expect(stored.Spec.Ports.CoordinatorPort).To(HaveValue(Equal(memgraphcomv1alpha1.DefaultCoordinatorPort))) + }) + It("should accept a registry host carrying a port", func() { stored := createAccepted("valid-registry-port", memgraphcomv1alpha1.MemgraphClusterSpec{ Image: memgraphcomv1alpha1.ImageSpec{Repository: "registry.example.com:5000/memgraph"}, @@ -253,8 +320,92 @@ var _ = Describe("MemgraphCluster CRD validation", func() { }, }, `Unsupported value: "ReadWriteSometimes"`), + Entry("a port outside the valid range", "invalid-port-range", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Ports: memgraphcomv1alpha1.PortsSpec{BoltPort: ptr.To(int32(70000))}, + }, + "should be less than or equal to 65535"), + Entry("a port of zero", "invalid-port-zero", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Ports: memgraphcomv1alpha1.PortsSpec{ManagementPort: ptr.To(int32(0))}, + }, + "should be greater than or equal to 1"), + // Two roles sharing a port number would make the advertised + // addresses ambiguous, so it is rejected instead of half-working. + Entry("two ports colliding", "invalid-ports-collide", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Ports: memgraphcomv1alpha1.PortsSpec{ManagementPort: ptr.To(memgraphcomv1alpha1.DefaultBoltPort)}, + }, + "must all be different ports"), + Entry("a cluster domain that is not a DNS name", "invalid-cluster-domain", + memgraphcomv1alpha1.MemgraphClusterSpec{ClusterDomain: "Cluster_Local"}, + "in body should match"), + Entry("an env var name that is not a shell identifier", "invalid-env-name", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ + Data: []memgraphcomv1alpha1.EnvVar{{Name: "not-an-identifier", Value: "x"}}, + }, + }, + "in body should match"), + Entry("an env var shadowing the license the secrets block owns", "invalid-env-license", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ + Data: []memgraphcomv1alpha1.EnvVar{{ + Name: memgraphcomv1alpha1.EnvLicense, + Value: "smuggled-license", + }}, + }, + }, + "they come from the secrets block"), + Entry("an env var shadowing the pod's own identity", "invalid-env-pod-name", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ + Coordinators: []memgraphcomv1alpha1.EnvVar{{ + Name: memgraphcomv1alpha1.EnvPodName, + Value: "not-my-name", + }}, + }, + }, + "it carries the pod's own identity"), + // A port set through extraArgs would leave the pods listening + // somewhere the registered addresses do not point. + Entry("an extra arg overriding a port", "invalid-args-port", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Data: []string{"--bolt-port=7777"}}, + }, + "configure ports through spec.ports"), + Entry("an extra arg overriding the coordinator identity", "invalid-args-coordinator-id", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Coordinators: []string{"--coordinator-id=9"}}, + }, + "the coordinator identity the operator derives"), + Entry("a probe timing below one", "invalid-probe-period", + memgraphcomv1alpha1.MemgraphClusterSpec{ + Probes: memgraphcomv1alpha1.ProbesSpec{ + Coordinators: memgraphcomv1alpha1.RoleProbesSpec{ + ReadinessProbe: memgraphcomv1alpha1.ProbeSpec{PeriodSeconds: ptr.To(int32(0))}, + }, + }, + }, + "should be greater than or equal to 1"), ) + // Two extra env vars of the same name would be an ambiguous + // configuration, so the list is keyed by name. + It("should reject a repeated env var name", func() { + err := create("invalid-env-duplicate", memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ + Data: []memgraphcomv1alpha1.EnvVar{ + {Name: "DATA_LABEL", Value: "one"}, + {Name: "DATA_LABEL", Value: "two"}, + }, + }, + }) + + Expect(err).To(HaveOccurred(), "expected admission to reject the duplicate") + Expect(err.Error()).To(ContainSubstring("Duplicate value")) + }) + // The typed client drops empty strings before they reach the API // server (omitempty), so the fields a user can only blank out from // YAML are submitted as a raw manifest instead. diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 5433f3c..eacd3d8 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -21,11 +21,19 @@ import ( "testing" "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" "github.com/memgraph/kubernetes-operator/internal/memgraph" "github.com/memgraph/kubernetes-operator/internal/planner" + "github.com/memgraph/kubernetes-operator/internal/resources" ) +// firstInstance is the data instance the planner promotes at bootstrap: the +// first one the topology declares. +const firstInstance = "instance_0" + // declaredTopology is the canonical 3-coordinator, 2-data-instance fixture // the cases below diff observed cluster states against. func declaredTopology() planner.Topology { @@ -101,7 +109,7 @@ func TestPlan(t *testing.T) { planner.AddCoordinator{Coordinator: coordinatorSpec(3)}, planner.RegisterInstance{Instance: dataInstanceSpec(0)}, planner.RegisterInstance{Instance: dataInstanceSpec(1)}, - planner.SetInstanceToMain{Name: "instance_0"}, + planner.SetInstanceToMain{Name: firstInstance}, }, }, { @@ -168,7 +176,7 @@ func TestPlan(t *testing.T) { observedDataInstance(1, memgraph.RoleReplica), }, want: []planner.Command{ - planner.SetInstanceToMain{Name: "instance_0"}, + planner.SetInstanceToMain{Name: firstInstance}, }, }, { @@ -222,3 +230,87 @@ func TestPlan(t *testing.T) { }) } } + +// TestPlanUsesConfiguredPortsAndClusterDomain plans a fresh bootstrap over a +// topology derived from a CR with non-default ports and cluster domain: the +// registration commands must carry exactly those addresses, because they are +// what the coordinators will use to reach every instance. +func TestPlanUsesConfiguredPortsAndClusterDomain(t *testing.T) { + cluster := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "memgraph-test"}, + Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(1)), + DataInstances: ptr.To(int32(1)), + ClusterDomain: "k8s.example.com", + Ports: memgraphcomv1alpha1.PortsSpec{ + BoltPort: ptr.To(int32(7777)), + ManagementPort: ptr.To(int32(10001)), + ReplicationPort: ptr.To(int32(20001)), + CoordinatorPort: ptr.To(int32(12001)), + }, + }, + } + + coordinatorHost := "example-coordinator-0.example-coordinator.memgraph-test.svc.k8s.example.com" + dataHost := "example-data-0.example-data.memgraph-test.svc.k8s.example.com" + want := []planner.Command{ + planner.AddCoordinator{Coordinator: memgraph.CoordinatorSpec{ + ID: 1, + BoltServer: coordinatorHost + ":7777", + CoordinatorServer: coordinatorHost + ":12001", + ManagementServer: coordinatorHost + ":10001", + }}, + planner.RegisterInstance{Instance: memgraph.DataInstanceSpec{ + Name: firstInstance, + BoltServer: dataHost + ":7777", + ManagementServer: dataHost + ":10001", + ReplicationServer: dataHost + ":20001", + }}, + planner.SetInstanceToMain{Name: firstInstance}, + } + + got := planner.Plan(resources.DeclaredTopology(cluster), nil) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Plan() mismatch (-want +got):\n%s", diff) + } +} + +// A cluster already registered on the configured addresses is converged: the +// planner must not re-issue registrations just because the ports are not the +// defaults. +func TestPlanConvergedOnConfiguredPorts(t *testing.T) { + cluster := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "example", Namespace: "memgraph-test"}, + Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(1)), + DataInstances: ptr.To(int32(1)), + ClusterDomain: "k8s.example.com", + Ports: memgraphcomv1alpha1.PortsSpec{BoltPort: ptr.To(int32(7777))}, + }, + } + declared := resources.DeclaredTopology(cluster) + + coordinator := declared.Coordinators[0] + instance := declared.DataInstances[0] + observed := []memgraph.Instance{ + { + Name: coordinator.Name(), + BoltServer: coordinator.BoltServer, + CoordinatorServer: coordinator.CoordinatorServer, + ManagementServer: coordinator.ManagementServer, + Health: "up", + Role: memgraph.RoleLeader, + }, + { + Name: instance.Name, + BoltServer: instance.BoltServer, + ManagementServer: instance.ManagementServer, + Health: "up", + Role: memgraph.RoleMain, + }, + } + + if got := planner.Plan(declared, observed); got != nil { + t.Errorf("Plan() = %v, want no commands", got) + } +} diff --git a/internal/resources/resources.go b/internal/resources/resources.go index 2b9447a..f5fdfe1 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -20,26 +20,15 @@ limitations under the License. package resources import ( + "maps" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" ) -// Internal Memgraph ports. These mirror the memgraph-high-availability Helm -// chart's defaults and become spec knobs in a later slice. const ( - BoltPort int32 = 7687 - ManagementPort int32 = 10000 - ReplicationPort int32 = 20000 - CoordinatorPort int32 = 12000 -) - -const ( - // clusterDomain is the Kubernetes cluster domain used in advertised FQDN - // addresses. It becomes a spec knob in a later slice. - clusterDomain = "cluster.local" - // Memgraph workload pods run as the non-root memgraph user baked into the // official images. memgraphUserID int64 = 101 @@ -69,9 +58,18 @@ func DataName(cluster *memgraphcomv1alpha1.MemgraphCluster) string { return cluster.Name + "-" + dataComponent } -// labels returns the full label set stamped on all objects of a role. -func labels(cluster *memgraphcomv1alpha1.MemgraphCluster, component string) map[string]string { - l := selectorLabels(cluster, component) +// labels returns the full label set stamped on all objects of a role, with the +// role's custom labels merged underneath: the operator's own identity labels +// always win a key collision, so a custom label can never detach an object +// from its cluster. +func labels( + cluster *memgraphcomv1alpha1.MemgraphCluster, + component string, + custom map[string]string, +) map[string]string { + l := make(map[string]string, len(custom)+4) + maps.Copy(l, custom) + maps.Copy(l, selectorLabels(cluster, component)) l["app.kubernetes.io/managed-by"] = "memgraph-operator" return l } @@ -87,19 +85,53 @@ func selectorLabels(cluster *memgraphcomv1alpha1.MemgraphCluster, component stri } // normalizedSpec is a MemgraphClusterSpec with every optional field resolved -// to its CRD schema default, so builders behave correctly on specs that never -// passed admission. +// to its default, so builders behave correctly on specs that never passed +// admission. Most defaults are CRD schema defaults mirrored as Go constants; +// the probe failure thresholds are Go-only, because they depend on the role. type normalizedSpec struct { - coordinators int32 - dataInstances int32 - image string - pullPolicy corev1.PullPolicy - secretName string - licenseKey string - organizationKey string - retentionPolicy memgraphcomv1alpha1.StorageRetentionPolicy - coordinatorStorage normalizedStorage - dataStorage normalizedStorage + coordinators int32 + dataInstances int32 + image string + pullPolicy corev1.PullPolicy + secretName string + licenseKey string + organizationKey string + clusterDomain string + ports normalizedPorts + retentionPolicy memgraphcomv1alpha1.StorageRetentionPolicy + coordinatorRole normalizedRole + dataRole normalizedRole +} + +// normalizedPorts are the internal ports every advertised address, container +// port and Service port is built from. +type normalizedPorts struct { + bolt int32 + management int32 + replication int32 + coordinator int32 +} + +// normalizedRole is everything the builders need that is configured per role. +type normalizedRole struct { + storage normalizedStorage + startupProbe normalizedProbe + readinessProbe normalizedProbe + livenessProbe normalizedProbe + resources corev1.ResourceRequirements + podLabels map[string]string + statefulSetLabels map[string]string + serviceLabels map[string]string + env []corev1.EnvVar + extraArgs []string +} + +// normalizedProbe is one probe's timings; the probe type is always a TCP-socket +// check against the role's own port. +type normalizedProbe struct { + failureThreshold int32 + timeoutSeconds int32 + periodSeconds int32 } // normalizedStorage is one role's lib and log claim configuration with every @@ -117,16 +149,38 @@ type normalizedStorage struct { func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { n := normalizedSpec{ - coordinators: memgraphcomv1alpha1.DefaultCoordinatorCount, - dataInstances: memgraphcomv1alpha1.DefaultDataInstanceCount, - image: imageRef(spec.Image), - pullPolicy: spec.Image.PullPolicy, - secretName: spec.Secrets.Name, - licenseKey: spec.Secrets.LicenseKey, - organizationKey: spec.Secrets.OrganizationKey, - retentionPolicy: spec.Storage.RetentionPolicy, - coordinatorStorage: normalizeStorage(spec.Storage.Coordinators), - dataStorage: normalizeStorage(spec.Storage.Data), + coordinators: memgraphcomv1alpha1.DefaultCoordinatorCount, + dataInstances: memgraphcomv1alpha1.DefaultDataInstanceCount, + image: imageRef(spec.Image), + pullPolicy: spec.Image.PullPolicy, + secretName: spec.Secrets.Name, + licenseKey: spec.Secrets.LicenseKey, + organizationKey: spec.Secrets.OrganizationKey, + clusterDomain: spec.ClusterDomain, + ports: normalizePorts(spec.Ports), + retentionPolicy: spec.Storage.RetentionPolicy, + coordinatorRole: normalizeRole(roleSpec{ + storage: spec.Storage.Coordinators, + probes: spec.Probes.Coordinators, + resources: spec.Resources.Coordinators, + labels: spec.Labels.Coordinators, + env: spec.ExtraEnv.Coordinators, + extraArgs: spec.ExtraArgs.Coordinators, + + startupFailureThreshold: memgraphcomv1alpha1.DefaultProbeFailureThreshold, + }), + dataRole: normalizeRole(roleSpec{ + storage: spec.Storage.Data, + probes: spec.Probes.Data, + resources: spec.Resources.Data, + labels: spec.Labels.Data, + env: spec.ExtraEnv.Data, + extraArgs: spec.ExtraArgs.Data, + + // Data instances get the long startup budget: only they load + // snapshots, and a large restore must not be killed mid-load. + startupFailureThreshold: memgraphcomv1alpha1.DefaultDataStartupProbeFailureThreshold, + }), } if spec.Coordinators != nil { n.coordinators = *spec.Coordinators @@ -134,6 +188,9 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { if spec.DataInstances != nil { n.dataInstances = *spec.DataInstances } + if n.clusterDomain == "" { + n.clusterDomain = memgraphcomv1alpha1.DefaultClusterDomain + } if n.pullPolicy == "" { n.pullPolicy = memgraphcomv1alpha1.DefaultImagePullPolicy } @@ -152,6 +209,82 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { return n } +// roleSpec gathers the per-role pieces the spec's concern-first blocks +// (storage, probes, resources, labels, extraEnv, extraArgs) scatter across the +// CR, so normalization is written once and both roles resolve their defaults +// the same way. +type roleSpec struct { + storage memgraphcomv1alpha1.RoleStorageSpec + probes memgraphcomv1alpha1.RoleProbesSpec + resources corev1.ResourceRequirements + labels memgraphcomv1alpha1.RoleLabelsSpec + env []memgraphcomv1alpha1.EnvVar + extraArgs []string + + // startupFailureThreshold is this role's default startup probe failure + // budget — the one default that differs between the roles. + startupFailureThreshold int32 +} + +func normalizeRole(role roleSpec) normalizedRole { + return normalizedRole{ + storage: normalizeStorage(role.storage), + startupProbe: normalizeProbe(role.probes.StartupProbe, role.startupFailureThreshold), + readinessProbe: normalizeProbe(role.probes.ReadinessProbe, memgraphcomv1alpha1.DefaultProbeFailureThreshold), + livenessProbe: normalizeProbe(role.probes.LivenessProbe, memgraphcomv1alpha1.DefaultProbeFailureThreshold), + resources: role.resources, + podLabels: role.labels.PodLabels, + statefulSetLabels: role.labels.StatefulSetLabels, + serviceLabels: role.labels.ServiceLabels, + env: normalizeEnv(role.env), + extraArgs: role.extraArgs, + } +} + +// normalizePorts resolves the internal ports, whose defaults mirror the +// memgraph-high-availability Helm chart's. +func normalizePorts(spec memgraphcomv1alpha1.PortsSpec) normalizedPorts { + return normalizedPorts{ + bolt: intOrDefault(spec.BoltPort, memgraphcomv1alpha1.DefaultBoltPort), + management: intOrDefault(spec.ManagementPort, memgraphcomv1alpha1.DefaultManagementPort), + replication: intOrDefault(spec.ReplicationPort, memgraphcomv1alpha1.DefaultReplicationPort), + coordinator: intOrDefault(spec.CoordinatorPort, memgraphcomv1alpha1.DefaultCoordinatorPort), + } +} + +// intOrDefault resolves an optional numeric knob against its default. +func intOrDefault(configured *int32, fallback int32) int32 { + if configured == nil { + return fallback + } + return *configured +} + +// normalizeProbe resolves one probe's timings. Only the failure threshold's +// default depends on the role — the probe that guards a snapshot restore needs +// a far larger budget than the rest. +func normalizeProbe(spec memgraphcomv1alpha1.ProbeSpec, defaultFailureThreshold int32) normalizedProbe { + return normalizedProbe{ + failureThreshold: intOrDefault(spec.FailureThreshold, defaultFailureThreshold), + timeoutSeconds: intOrDefault(spec.TimeoutSeconds, memgraphcomv1alpha1.DefaultProbeTimeoutSeconds), + periodSeconds: intOrDefault(spec.PeriodSeconds, memgraphcomv1alpha1.DefaultProbePeriodSeconds), + } +} + +// normalizeEnv converts the spec's non-secret name/value pairs into container +// environment variables. Nothing is defaulted: an unset list means no extra +// environment. +func normalizeEnv(spec []memgraphcomv1alpha1.EnvVar) []corev1.EnvVar { + if len(spec) == 0 { + return nil + } + env := make([]corev1.EnvVar, 0, len(spec)) + for _, variable := range spec { + env = append(env, corev1.EnvVar{Name: variable.Name, Value: variable.Value}) + } + return env +} + func normalizeStorage(spec memgraphcomv1alpha1.RoleStorageSpec) normalizedStorage { n := normalizedStorage{ libSize: resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize), diff --git a/internal/resources/service.go b/internal/resources/service.go index cb9a393..4354ed5 100644 --- a/internal/resources/service.go +++ b/internal/resources/service.go @@ -26,26 +26,33 @@ import ( // CoordinatorHeadlessService builds the headless Service backing the // coordinator StatefulSet's per-pod DNS identities. func CoordinatorHeadlessService(cluster *memgraphcomv1alpha1.MemgraphCluster) *corev1.Service { - return headlessService(cluster, coordinatorComponent, CoordinatorName(cluster), []corev1.ServicePort{ - {Name: boltPortName, Port: BoltPort}, - {Name: managementPortName, Port: ManagementPort}, - {Name: coordinatorPortName, Port: CoordinatorPort}, - }) + spec := normalize(cluster.Spec) + + return headlessService(cluster, coordinatorComponent, CoordinatorName(cluster), + spec.coordinatorRole.serviceLabels, []corev1.ServicePort{ + {Name: boltPortName, Port: spec.ports.bolt}, + {Name: managementPortName, Port: spec.ports.management}, + {Name: coordinatorPortName, Port: spec.ports.coordinator}, + }) } // DataHeadlessService builds the headless Service backing the data-instance // StatefulSet's per-pod DNS identities. func DataHeadlessService(cluster *memgraphcomv1alpha1.MemgraphCluster) *corev1.Service { - return headlessService(cluster, dataComponent, DataName(cluster), []corev1.ServicePort{ - {Name: boltPortName, Port: BoltPort}, - {Name: managementPortName, Port: ManagementPort}, - {Name: replicationPortName, Port: ReplicationPort}, - }) + spec := normalize(cluster.Spec) + + return headlessService(cluster, dataComponent, DataName(cluster), + spec.dataRole.serviceLabels, []corev1.ServicePort{ + {Name: boltPortName, Port: spec.ports.bolt}, + {Name: managementPortName, Port: spec.ports.management}, + {Name: replicationPortName, Port: spec.ports.replication}, + }) } func headlessService( cluster *memgraphcomv1alpha1.MemgraphCluster, component, name string, + customLabels map[string]string, ports []corev1.ServicePort, ) *corev1.Service { return &corev1.Service{ @@ -55,7 +62,7 @@ func headlessService( ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: cluster.Namespace, - Labels: labels(cluster, component), + Labels: labels(cluster, component, customLabels), }, Spec: corev1.ServiceSpec{ ClusterIP: corev1.ClusterIPNone, diff --git a/internal/resources/service_test.go b/internal/resources/service_test.go index ac7d278..dc607c7 100644 --- a/internal/resources/service_test.go +++ b/internal/resources/service_test.go @@ -28,7 +28,7 @@ import ( func TestCoordinatorHeadlessService(t *testing.T) { want := &corev1.Service{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: serviceKind}, ObjectMeta: metav1.ObjectMeta{ Name: coordinatorName, Namespace: testNamespace, @@ -54,7 +54,7 @@ func TestCoordinatorHeadlessService(t *testing.T) { func TestDataHeadlessService(t *testing.T) { want := &corev1.Service{ - TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Service"}, + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: serviceKind}, ObjectMeta: metav1.ObjectMeta{ Name: dataName, Namespace: testNamespace, @@ -77,3 +77,54 @@ func TestDataHeadlessService(t *testing.T) { t.Errorf("DataHeadlessService() mismatch (-want +got):\n%s", diff) } } + +// TestHeadlessServicePortsAndLabels asserts the Services publish the configured +// ports — the pods listen on nothing else — and carry the role's custom Service +// labels while keeping the operator-owned selector. +func TestHeadlessServicePortsAndLabels(t *testing.T) { + cluster := tunedCluster() + + tests := []struct { + name string + service *corev1.Service + component string + labels map[string]string + ports []corev1.ServicePort + }{ + { + name: coordinatorComponent, + service: resources.CoordinatorHeadlessService(cluster), + component: coordinatorComponent, + labels: map[string]string{exposeLabel: "internal"}, + ports: []corev1.ServicePort{ + {Name: boltPortName, Port: customBoltPort}, + {Name: managementPortName, Port: customManagementPort}, + {Name: coordinatorComponent, Port: customCoordinatorPort}, + }, + }, + { + name: dataComponent, + service: resources.DataHeadlessService(cluster), + component: dataComponent, + labels: map[string]string{exposeLabel: "bolt"}, + ports: []corev1.ServicePort{ + {Name: boltPortName, Port: customBoltPort}, + {Name: managementPortName, Port: customManagementPort}, + {Name: replicationPortName, Port: customReplicationPort}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if diff := cmp.Diff(tc.ports, tc.service.Spec.Ports); diff != "" { + t.Errorf("Service ports mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedLabelsWith(tc.component, tc.labels), tc.service.Labels); diff != "" { + t.Errorf("Service labels mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedSelectorLabels(tc.component), tc.service.Spec.Selector); diff != "" { + t.Errorf("Service selector mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index bd661a8..8dae5bc 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -52,48 +52,50 @@ const ( // uniform across replicas. func CoordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { spec := normalize(cluster.Spec) + role := spec.coordinatorRole - container := memgraphContainer(spec) + container := memgraphContainer(spec, role) // The coordinator ID and advertised FQDN depend on the pod ordinal, which // only the pod itself knows; a shell wrapper derives them from the pod // name so all replicas share one template. - container.Command = []string{"/bin/sh", "-ec", coordinatorStartScript(cluster)} + container.Command = []string{"/bin/sh", "-ec", coordinatorStartScript(cluster, spec, role)} container.Env = append([]corev1.EnvVar{{ - Name: "POD_NAME", + Name: memgraphcomv1alpha1.EnvPodName, ValueFrom: &corev1.EnvVarSource{ FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, }, }}, container.Env...) container.Ports = []corev1.ContainerPort{ - {Name: boltPortName, ContainerPort: BoltPort}, - {Name: managementPortName, ContainerPort: ManagementPort}, - {Name: coordinatorPortName, ContainerPort: CoordinatorPort}, + {Name: boltPortName, ContainerPort: spec.ports.bolt}, + {Name: managementPortName, ContainerPort: spec.ports.management}, + {Name: coordinatorPortName, ContainerPort: spec.ports.coordinator}, } - container.StartupProbe = tcpProbe(CoordinatorPort, 20) - container.ReadinessProbe = tcpProbe(CoordinatorPort, 20) - container.LivenessProbe = tcpProbe(CoordinatorPort, 20) + // Coordinators are probed on their Raft port: it is the one they serve + // even before the Raft cluster has been formed. + container.StartupProbe = tcpProbe(spec.ports.coordinator, role.startupProbe) + container.ReadinessProbe = tcpProbe(spec.ports.coordinator, role.readinessProbe) + container.LivenessProbe = tcpProbe(spec.ports.coordinator, role.livenessProbe) - return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec, spec.coordinators, container) + return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec, role, spec.coordinators, container) } // DataStatefulSet builds the single StatefulSet running all data instances. func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { spec := normalize(cluster.Spec) + role := spec.dataRole - container := memgraphContainer(spec) - container.Args = dataArgs() + container := memgraphContainer(spec, role) + container.Args = append(commonArgs(spec), role.extraArgs...) container.Ports = []corev1.ContainerPort{ - {Name: boltPortName, ContainerPort: BoltPort}, - {Name: managementPortName, ContainerPort: ManagementPort}, - {Name: replicationPortName, ContainerPort: ReplicationPort}, + {Name: boltPortName, ContainerPort: spec.ports.bolt}, + {Name: managementPortName, ContainerPort: spec.ports.management}, + {Name: replicationPortName, ContainerPort: spec.ports.replication}, } - // A generous startup budget so large snapshot restores are not killed - // mid-load (mirrors the HA chart's default of 1440 * 5s = 2h). - container.StartupProbe = tcpProbe(BoltPort, 1440) - container.ReadinessProbe = tcpProbe(BoltPort, 20) - container.LivenessProbe = tcpProbe(BoltPort, 20) + container.StartupProbe = tcpProbe(spec.ports.bolt, role.startupProbe) + container.ReadinessProbe = tcpProbe(spec.ports.bolt, role.readinessProbe) + container.LivenessProbe = tcpProbe(spec.ports.bolt, role.livenessProbe) - return statefulSet(cluster, dataComponent, DataName(cluster), spec, spec.dataInstances, container) + return statefulSet(cluster, dataComponent, DataName(cluster), spec, role, spec.dataInstances, container) } // coordinatorStartScript derives the coordinator's identity from its pod @@ -101,26 +103,29 @@ func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.State // ID N+1 (Memgraph treats coordinator ID 0 as unset and refuses to start, so // IDs stay 1-based) advertised at the pod's stable DNS name within the // headless Service. -func coordinatorStartScript(cluster *memgraphcomv1alpha1.MemgraphCluster) string { - fqdnSuffix := podFQDNSuffix(cluster, CoordinatorName(cluster)) +func coordinatorStartScript( + cluster *memgraphcomv1alpha1.MemgraphCluster, + spec normalizedSpec, + role normalizedRole, +) string { + fqdnSuffix := podFQDNSuffix(cluster, CoordinatorName(cluster), spec) + args := append(commonArgs(spec), role.extraArgs...) return fmt.Sprintf(`ordinal="${POD_NAME##*-}" exec %s \ --coordinator-id="$((ordinal + 1))" \ --coordinator-hostname="${POD_NAME}.%s" \ --coordinator-port=%d \ - %s`, memgraphBinary, fqdnSuffix, CoordinatorPort, shellJoin(commonArgs())) -} - -func dataArgs() []string { - return commonArgs() + %s`, memgraphBinary, fqdnSuffix, spec.ports.coordinator, shellJoin(args)) } // commonArgs are the Memgraph flags shared by both roles, mirroring the HA -// chart's auto-appended and default logging arguments. -func commonArgs() []string { +// chart's auto-appended and default logging arguments. A role's extra args are +// appended after these, and Memgraph takes the last occurrence of a repeated +// flag, so a user-supplied flag wins. +func commonArgs(spec normalizedSpec) []string { return []string{ - fmt.Sprintf("--bolt-port=%d", BoltPort), - fmt.Sprintf("--management-port=%d", ManagementPort), + fmt.Sprintf("--bolt-port=%d", spec.ports.bolt), + fmt.Sprintf("--management-port=%d", spec.ports.management), "--data-directory=" + dataDirectory, "--log-level=TRACE", "--also-log-to-stderr", @@ -134,16 +139,20 @@ func shellJoin(args []string) string { } // memgraphContainer builds the parts of the Memgraph container shared by both -// roles: image, license env wiring, storage mounts, and the restricted -// security context. -func memgraphContainer(spec normalizedSpec) corev1.Container { +// roles: image, license env wiring, the role's extra environment and resources, +// storage mounts, and the restricted security context. +func memgraphContainer(spec normalizedSpec, role normalizedRole) corev1.Container { return corev1.Container{ Name: "memgraph", Image: spec.image, ImagePullPolicy: spec.pullPolicy, - Env: []corev1.EnvVar{ + Resources: role.resources, + // The license variables come first and the role's extra environment + // last; admission rejects an extra variable that repeats one of the + // names the operator owns, so the two can never collide. + Env: append([]corev1.EnvVar{ { - Name: "MEMGRAPH_ENTERPRISE_LICENSE", + Name: memgraphcomv1alpha1.EnvLicense, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: spec.secretName}, @@ -152,7 +161,7 @@ func memgraphContainer(spec normalizedSpec) corev1.Container { }, }, { - Name: "MEMGRAPH_ORGANIZATION_NAME", + Name: memgraphcomv1alpha1.EnvOrganization, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: spec.secretName}, @@ -160,7 +169,7 @@ func memgraphContainer(spec normalizedSpec) corev1.Container { }, }, }, - }, + }, role.env...), VolumeMounts: []corev1.VolumeMount{ {Name: libVolumeName, MountPath: libMountPath}, {Name: logVolumeName, MountPath: logMountPath}, @@ -180,14 +189,11 @@ func statefulSet( cluster *memgraphcomv1alpha1.MemgraphCluster, component, name string, spec normalizedSpec, + role normalizedRole, replicas int32, container corev1.Container, ) *appsv1.StatefulSet { - storage := spec.dataStorage - if component == coordinatorComponent { - storage = spec.coordinatorStorage - } - + storage := role.storage return &appsv1.StatefulSet{ // TypeMeta is set explicitly because the controller server-side // applies builder output, and apply patches must carry the GVK. @@ -195,7 +201,7 @@ func statefulSet( ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: cluster.Namespace, - Labels: labels(cluster, component), + Labels: labels(cluster, component, role.statefulSetLabels), }, Spec: appsv1.StatefulSetSpec{ Replicas: ptr.To(replicas), @@ -217,7 +223,7 @@ func statefulSet( }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels(cluster, component), + Labels: labels(cluster, component, role.podLabels), }, Spec: corev1.PodSpec{ Containers: []corev1.Container{container}, @@ -274,13 +280,16 @@ func retentionType(policy memgraphcomv1alpha1.StorageRetentionPolicy) appsv1.Per return appsv1.RetainPersistentVolumeClaimRetentionPolicyType } -func tcpProbe(port, failureThreshold int32) *corev1.Probe { +// tcpProbe builds one probe. The handler is always a TCP-socket check against +// the role's own port — the probe type is deliberately not configurable — so +// only the timings come from the spec. +func tcpProbe(port int32, timings normalizedProbe) *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, }, - FailureThreshold: failureThreshold, - TimeoutSeconds: 10, - PeriodSeconds: 5, + FailureThreshold: timings.failureThreshold, + TimeoutSeconds: timings.timeoutSeconds, + PeriodSeconds: timings.periodSeconds, } } diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index 4516049..5a7797d 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -17,6 +17,7 @@ limitations under the License. package resources_test import ( + "maps" "testing" "github.com/google/go-cmp/cmp" @@ -47,6 +48,22 @@ const ( boltPortName = "bolt" managementPortName = "management" replicationPortName = "replication" + + statefulSetKind = "StatefulSet" + serviceKind = "Service" + + // The operator's identity labels, which custom labels may never override. + nameLabel = "app.kubernetes.io/name" + instanceLabel = "app.kubernetes.io/instance" + componentLabel = "app.kubernetes.io/component" + managedByLabel = "app.kubernetes.io/managed-by" + + // Custom label keys and values used by the tuning tests. + teamLabel = "team" + tierLabel = "tier" + exposeLabel = "expose" + + platformTeam = "platform" ) // minimalCluster returns a MemgraphCluster as a client would minimally create @@ -78,6 +95,81 @@ func specifiedCluster() *memgraphcomv1alpha1.MemgraphCluster { } } +// Non-default ports and cluster domain shared by the tuning tests. Every one +// differs from its default, so a knob that fails to propagate cannot hide +// behind a value that happened to be right anyway. +const ( + customBoltPort int32 = 7777 + customManagementPort int32 = 10001 + customReplicationPort int32 = 20001 + customCoordinatorPort int32 = 12001 + + customClusterDomain = "k8s.example.com" +) + +// tunedCluster returns a MemgraphCluster with every pod-tuning knob set away +// from its default, so the golden tests can pin what each one lands on. +func tunedCluster() *memgraphcomv1alpha1.MemgraphCluster { + cluster := minimalCluster() + cluster.Spec.ClusterDomain = customClusterDomain + cluster.Spec.Ports = memgraphcomv1alpha1.PortsSpec{ + BoltPort: ptr.To(customBoltPort), + ManagementPort: ptr.To(customManagementPort), + ReplicationPort: ptr.To(customReplicationPort), + CoordinatorPort: ptr.To(customCoordinatorPort), + } + cluster.Spec.Probes = memgraphcomv1alpha1.ProbesSpec{ + Coordinators: memgraphcomv1alpha1.RoleProbesSpec{ + StartupProbe: memgraphcomv1alpha1.ProbeSpec{FailureThreshold: ptr.To(int32(30))}, + ReadinessProbe: memgraphcomv1alpha1.ProbeSpec{ + TimeoutSeconds: ptr.To(int32(3)), + PeriodSeconds: ptr.To(int32(2)), + }, + }, + Data: memgraphcomv1alpha1.RoleProbesSpec{ + StartupProbe: memgraphcomv1alpha1.ProbeSpec{ + FailureThreshold: ptr.To(int32(4320)), + TimeoutSeconds: ptr.To(int32(15)), + PeriodSeconds: ptr.To(int32(10)), + }, + LivenessProbe: memgraphcomv1alpha1.ProbeSpec{FailureThreshold: ptr.To(int32(6))}, + }, + } + cluster.Spec.Resources = memgraphcomv1alpha1.ResourcesSpec{ + Coordinators: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + Data: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + } + cluster.Spec.Labels = memgraphcomv1alpha1.LabelsSpec{ + Coordinators: memgraphcomv1alpha1.RoleLabelsSpec{ + PodLabels: map[string]string{teamLabel: platformTeam}, + StatefulSetLabels: map[string]string{tierLabel: "control"}, + ServiceLabels: map[string]string{exposeLabel: "internal"}, + }, + Data: memgraphcomv1alpha1.RoleLabelsSpec{ + PodLabels: map[string]string{teamLabel: dataComponent}, + StatefulSetLabels: map[string]string{tierLabel: "storage"}, + ServiceLabels: map[string]string{exposeLabel: "bolt"}, + }, + } + cluster.Spec.ExtraEnv = memgraphcomv1alpha1.ExtraEnvSpec{ + Coordinators: []memgraphcomv1alpha1.EnvVar{{Name: "COORDINATOR_LABEL", Value: "coord"}}, + Data: []memgraphcomv1alpha1.EnvVar{ + {Name: "DATA_LABEL_ONE", Value: "one"}, + {Name: "DATA_LABEL_TWO", Value: "two"}, + }, + } + cluster.Spec.ExtraArgs = memgraphcomv1alpha1.ExtraArgsSpec{ + Coordinators: []string{"--log-level=WARNING"}, + Data: []string{"--storage-snapshot-on-exit=true", "--memory-limit=2048"}, + } + return cluster +} + func licenseEnv(secretName, licenseKey, organizationKey string) []corev1.EnvVar { return []corev1.EnvVar{ { @@ -101,14 +193,20 @@ func licenseEnv(secretName, licenseKey, organizationKey string) []corev1.EnvVar } } +// tcpProbe is a probe with the default timings, of which only the failure +// threshold differs between probes. func tcpProbe(port, failureThreshold int32) *corev1.Probe { + return tunedTCPProbe(port, failureThreshold, 10, 5) +} + +func tunedTCPProbe(port, failureThreshold, timeoutSeconds, periodSeconds int32) *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(port)}, }, FailureThreshold: failureThreshold, - TimeoutSeconds: 10, - PeriodSeconds: 5, + TimeoutSeconds: timeoutSeconds, + PeriodSeconds: periodSeconds, } } @@ -183,18 +281,26 @@ func expectedRetentionPolicy( func expectedLabels(component string) map[string]string { return map[string]string{ - "app.kubernetes.io/name": memgraphName, - "app.kubernetes.io/instance": clusterName, - "app.kubernetes.io/component": component, - "app.kubernetes.io/managed-by": "memgraph-operator", + nameLabel: memgraphName, + instanceLabel: clusterName, + componentLabel: component, + managedByLabel: "memgraph-operator", } } +// expectedLabelsWith is the full label set of a role's object once the user's +// custom labels are merged in. +func expectedLabelsWith(component string, custom map[string]string) map[string]string { + l := expectedLabels(component) + maps.Copy(l, custom) + return l +} + func expectedSelectorLabels(component string) map[string]string { return map[string]string{ - "app.kubernetes.io/name": memgraphName, - "app.kubernetes.io/instance": clusterName, - "app.kubernetes.io/component": component, + nameLabel: memgraphName, + instanceLabel: clusterName, + componentLabel: component, } } @@ -213,7 +319,7 @@ exec /usr/lib/memgraph/memgraph \ func TestCoordinatorStatefulSetDefaults(t *testing.T) { want := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "StatefulSet"}, + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: statefulSetKind}, ObjectMeta: metav1.ObjectMeta{ Name: coordinatorName, Namespace: testNamespace, @@ -267,7 +373,7 @@ func TestCoordinatorStatefulSetDefaults(t *testing.T) { func TestDataStatefulSetDefaults(t *testing.T) { want := &appsv1.StatefulSet{ - TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "StatefulSet"}, + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: statefulSetKind}, ObjectMeta: metav1.ObjectMeta{ Name: dataName, Namespace: testNamespace, @@ -451,3 +557,296 @@ func TestStatefulSetRetentionPolicy(t *testing.T) { }) } } + +// The coordinator start script derives per-pod identity at runtime, so the +// configured coordinator port and cluster domain have to be baked into it — +// this is the same identity the operator registers with the cluster. +const expectedTunedCoordinatorScript = `ordinal="${POD_NAME##*-}" +exec /usr/lib/memgraph/memgraph \ + --coordinator-id="$((ordinal + 1))" \ + --coordinator-hostname="${POD_NAME}.example-coordinator.memgraph-test.svc.k8s.example.com" \ + --coordinator-port=12001 \ + --bolt-port=7777 \ + --management-port=10001 \ + --data-directory=/var/lib/memgraph/mg_data \ + --log-level=TRACE \ + --also-log-to-stderr \ + --log-file=/var/log/memgraph/memgraph.log \ + --log-retention-days=35 \ + --log-level=WARNING` + +// TestStatefulSetPortsAndClusterDomain pins every place a configured port or +// cluster domain has to surface: the container ports, the flags Memgraph is +// started with, the ports the probes dial, and the coordinator's advertised +// hostname. +func TestStatefulSetPortsAndClusterDomain(t *testing.T) { + cluster := tunedCluster() + + t.Run(coordinatorComponent, func(t *testing.T) { + container := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec.Containers[0] + + wantPorts := []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: customBoltPort}, + {Name: managementPortName, ContainerPort: customManagementPort}, + {Name: coordinatorComponent, ContainerPort: customCoordinatorPort}, + } + if diff := cmp.Diff(wantPorts, container.Ports); diff != "" { + t.Errorf("container ports mismatch (-want +got):\n%s", diff) + } + wantCommand := []string{"/bin/sh", "-ec", expectedTunedCoordinatorScript} + if diff := cmp.Diff(wantCommand, container.Command); diff != "" { + t.Errorf("start script mismatch (-want +got):\n%s", diff) + } + for name, probe := range map[string]*corev1.Probe{ + "startup": container.StartupProbe, + "readiness": container.ReadinessProbe, + "liveness": container.LivenessProbe, + } { + if got := probe.TCPSocket.Port; got != intstr.FromInt32(customCoordinatorPort) { + t.Errorf("%s probe dials %v, want the configured coordinator port %d", + name, got, customCoordinatorPort) + } + } + }) + + t.Run(dataComponent, func(t *testing.T) { + container := resources.DataStatefulSet(cluster).Spec.Template.Spec.Containers[0] + + wantPorts := []corev1.ContainerPort{ + {Name: boltPortName, ContainerPort: customBoltPort}, + {Name: managementPortName, ContainerPort: customManagementPort}, + {Name: replicationPortName, ContainerPort: customReplicationPort}, + } + if diff := cmp.Diff(wantPorts, container.Ports); diff != "" { + t.Errorf("container ports mismatch (-want +got):\n%s", diff) + } + wantArgs := []string{ + "--bolt-port=7777", + "--management-port=10001", + "--data-directory=/var/lib/memgraph/mg_data", + "--log-level=TRACE", + "--also-log-to-stderr", + "--log-file=/var/log/memgraph/memgraph.log", + "--log-retention-days=35", + "--storage-snapshot-on-exit=true", + "--memory-limit=2048", + } + if diff := cmp.Diff(wantArgs, container.Args); diff != "" { + t.Errorf("args mismatch (-want +got):\n%s", diff) + } + for name, probe := range map[string]*corev1.Probe{ + "startup": container.StartupProbe, + "readiness": container.ReadinessProbe, + "liveness": container.LivenessProbe, + } { + if got := probe.TCPSocket.Port; got != intstr.FromInt32(customBoltPort) { + t.Errorf("%s probe dials %v, want the configured bolt port %d", name, got, customBoltPort) + } + } + }) +} + +// TestStatefulSetProbeOverrides asserts probe timings are per role and per +// probe, and that a partially specified probe keeps the defaults for the +// timings it leaves out — including the data instances' 2h startup budget. +func TestStatefulSetProbeOverrides(t *testing.T) { + cluster := tunedCluster() + + tests := []struct { + name string + sts *appsv1.StatefulSet + startup, readiness, liveness *corev1.Probe + }{ + { + name: coordinatorComponent, + sts: resources.CoordinatorStatefulSet(cluster), + // Only the failure threshold was raised, so the timings default. + startup: tunedTCPProbe(customCoordinatorPort, 30, 10, 5), + // Timings tightened, failure threshold left at its default. + readiness: tunedTCPProbe(customCoordinatorPort, 20, 3, 2), + liveness: tunedTCPProbe(customCoordinatorPort, 20, 10, 5), + }, + { + name: dataComponent, + sts: resources.DataStatefulSet(cluster), + startup: tunedTCPProbe(customBoltPort, 4320, 15, 10), + readiness: tunedTCPProbe(customBoltPort, 20, 10, 5), + liveness: tunedTCPProbe(customBoltPort, 6, 10, 5), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + container := tc.sts.Spec.Template.Spec.Containers[0] + if diff := cmp.Diff(tc.startup, container.StartupProbe); diff != "" { + t.Errorf("startup probe mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.readiness, container.ReadinessProbe); diff != "" { + t.Errorf("readiness probe mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tc.liveness, container.LivenessProbe); diff != "" { + t.Errorf("liveness probe mismatch (-want +got):\n%s", diff) + } + }) + } +} + +func TestStatefulSetResourceOverrides(t *testing.T) { + cluster := tunedCluster() + + tests := []struct { + name string + sts *appsv1.StatefulSet + want corev1.ResourceRequirements + }{ + { + name: coordinatorComponent, + sts: resources.CoordinatorStatefulSet(cluster), + want: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, + }, + }, + { + name: dataComponent, + sts: resources.DataStatefulSet(cluster), + want: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := tc.sts.Spec.Template.Spec.Containers[0].Resources + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("resources mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// TestStatefulSetLabelOverrides asserts custom labels land on the object they +// name: StatefulSet labels on the StatefulSet, pod labels on the pod template, +// and neither on the selector, which stays operator-owned. +func TestStatefulSetLabelOverrides(t *testing.T) { + cluster := tunedCluster() + + tests := []struct { + name string + sts *appsv1.StatefulSet + component string + stsLabels map[string]string + podLabels map[string]string + }{ + { + name: coordinatorComponent, + sts: resources.CoordinatorStatefulSet(cluster), + component: coordinatorComponent, + stsLabels: map[string]string{tierLabel: "control"}, + podLabels: map[string]string{teamLabel: platformTeam}, + }, + { + name: dataComponent, + sts: resources.DataStatefulSet(cluster), + component: dataComponent, + stsLabels: map[string]string{tierLabel: "storage"}, + podLabels: map[string]string{teamLabel: dataComponent}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if diff := cmp.Diff(expectedLabelsWith(tc.component, tc.stsLabels), tc.sts.Labels); diff != "" { + t.Errorf("StatefulSet labels mismatch (-want +got):\n%s", diff) + } + wantPodLabels := expectedLabelsWith(tc.component, tc.podLabels) + if diff := cmp.Diff(wantPodLabels, tc.sts.Spec.Template.Labels); diff != "" { + t.Errorf("pod labels mismatch (-want +got):\n%s", diff) + } + wantSelector := &metav1.LabelSelector{MatchLabels: expectedSelectorLabels(tc.component)} + if diff := cmp.Diff(wantSelector, tc.sts.Spec.Selector); diff != "" { + t.Errorf("selector mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// A custom label that collides with one of the operator's identity labels must +// lose: those labels are what the StatefulSet and Service select on, so a +// custom label winning would detach the pods from their cluster. +func TestStatefulSetCustomLabelsCannotOverrideIdentity(t *testing.T) { + cluster := minimalCluster() + hijack := map[string]string{ + nameLabel: "not-memgraph", + instanceLabel: "other-cluster", + componentLabel: dataComponent, + managedByLabel: "someone-else", + teamLabel: platformTeam, + } + cluster.Spec.Labels.Coordinators = memgraphcomv1alpha1.RoleLabelsSpec{ + PodLabels: hijack, + StatefulSetLabels: hijack, + ServiceLabels: hijack, + } + + want := expectedLabelsWith(coordinatorComponent, map[string]string{teamLabel: platformTeam}) + sts := resources.CoordinatorStatefulSet(cluster) + for name, got := range map[string]map[string]string{ + statefulSetKind: sts.Labels, + "pod": sts.Spec.Template.Labels, + serviceKind: resources.CoordinatorHeadlessService(cluster).Labels, + } { + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("%s labels mismatch (-want +got):\n%s", name, diff) + } + } +} + +// TestStatefulSetExtraEnv asserts the passthrough environment lands after the +// license variables the operator wires from the secrets block, and that no +// secret material can ride along with it. +func TestStatefulSetExtraEnv(t *testing.T) { + cluster := tunedCluster() + + tests := []struct { + name string + sts *appsv1.StatefulSet + want []corev1.EnvVar + }{ + { + name: coordinatorComponent, + sts: resources.CoordinatorStatefulSet(cluster), + want: append( + append([]corev1.EnvVar{{ + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }}, licenseEnv("memgraph-secrets", "MEMGRAPH_ENTERPRISE_LICENSE", "MEMGRAPH_ORGANIZATION_NAME")...), + corev1.EnvVar{Name: "COORDINATOR_LABEL", Value: "coord"}, + ), + }, + { + name: dataComponent, + sts: resources.DataStatefulSet(cluster), + want: append( + licenseEnv("memgraph-secrets", "MEMGRAPH_ENTERPRISE_LICENSE", "MEMGRAPH_ORGANIZATION_NAME"), + corev1.EnvVar{Name: "DATA_LABEL_ONE", Value: "one"}, + corev1.EnvVar{Name: "DATA_LABEL_TWO", Value: "two"}, + ), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := tc.sts.Spec.Template.Spec.Containers[0].Env + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("env mismatch (-want +got):\n%s", diff) + } + for _, variable := range got { + if variable.ValueFrom != nil && variable.ValueFrom.SecretKeyRef != nil { + if variable.Name != "MEMGRAPH_ENTERPRISE_LICENSE" && variable.Name != "MEMGRAPH_ORGANIZATION_NAME" { + t.Errorf("env %q reads a Secret; only the secrets block may", variable.Name) + } + } + } + }) + } +} diff --git a/internal/resources/topology.go b/internal/resources/topology.go index 9c5370b..8c30497 100644 --- a/internal/resources/topology.go +++ b/internal/resources/topology.go @@ -38,36 +38,46 @@ func DeclaredTopology(cluster *memgraphcomv1alpha1.MemgraphCluster) planner.Topo DataInstances: make([]memgraph.DataInstanceSpec, 0, spec.dataInstances), } for ordinal := range spec.coordinators { - fqdn := podFQDN(cluster, CoordinatorName(cluster), ordinal) + fqdn := podFQDN(cluster, CoordinatorName(cluster), spec, ordinal) topology.Coordinators = append(topology.Coordinators, memgraph.CoordinatorSpec{ ID: ordinal + 1, - BoltServer: hostPort(fqdn, BoltPort), - CoordinatorServer: hostPort(fqdn, CoordinatorPort), - ManagementServer: hostPort(fqdn, ManagementPort), + BoltServer: hostPort(fqdn, spec.ports.bolt), + CoordinatorServer: hostPort(fqdn, spec.ports.coordinator), + ManagementServer: hostPort(fqdn, spec.ports.management), }) } for ordinal := range spec.dataInstances { - fqdn := podFQDN(cluster, DataName(cluster), ordinal) + fqdn := podFQDN(cluster, DataName(cluster), spec, ordinal) topology.DataInstances = append(topology.DataInstances, memgraph.DataInstanceSpec{ Name: fmt.Sprintf("instance_%d", ordinal), - BoltServer: hostPort(fqdn, BoltPort), - ManagementServer: hostPort(fqdn, ManagementPort), - ReplicationServer: hostPort(fqdn, ReplicationPort), + BoltServer: hostPort(fqdn, spec.ports.bolt), + ManagementServer: hostPort(fqdn, spec.ports.management), + ReplicationServer: hostPort(fqdn, spec.ports.replication), }) } return topology } // podFQDNSuffix returns the DNS suffix a pod name is appended to for pods of -// the given headless Service: "..svc.". -func podFQDNSuffix(cluster *memgraphcomv1alpha1.MemgraphCluster, serviceName string) string { - return fmt.Sprintf("%s.%s.svc.%s", serviceName, cluster.Namespace, clusterDomain) +// the given headless Service: "..svc.", where the +// domain is the configured cluster domain. +func podFQDNSuffix( + cluster *memgraphcomv1alpha1.MemgraphCluster, + serviceName string, + spec normalizedSpec, +) string { + return fmt.Sprintf("%s.%s.svc.%s", serviceName, cluster.Namespace, spec.clusterDomain) } // podFQDN returns the stable DNS name of the pod with the given ordinal in // the StatefulSet backed by the given headless Service (both share one name). -func podFQDN(cluster *memgraphcomv1alpha1.MemgraphCluster, serviceName string, ordinal int32) string { - return fmt.Sprintf("%s-%d.%s", serviceName, ordinal, podFQDNSuffix(cluster, serviceName)) +func podFQDN( + cluster *memgraphcomv1alpha1.MemgraphCluster, + serviceName string, + spec normalizedSpec, + ordinal int32, +) string { + return fmt.Sprintf("%s-%d.%s", serviceName, ordinal, podFQDNSuffix(cluster, serviceName, spec)) } func hostPort(host string, port int32) string { diff --git a/internal/resources/topology_test.go b/internal/resources/topology_test.go index 0ee9d6a..deb10d7 100644 --- a/internal/resources/topology_test.go +++ b/internal/resources/topology_test.go @@ -113,3 +113,82 @@ func TestDeclaredTopologyMatchesCoordinatorStartScript(t *testing.T) { } } } + +// TestDeclaredTopologyPortsAndClusterDomain asserts the configured ports and +// cluster domain reach every advertised address, since these are exactly the +// addresses the operator registers with the cluster. +func TestDeclaredTopologyPortsAndClusterDomain(t *testing.T) { + got := resources.DeclaredTopology(tunedCluster()) + + coordinatorFQDN := func(ordinal int) string { + return fmt.Sprintf("%s-%d.%s.%s.svc.k8s.example.com", coordinatorName, ordinal, coordinatorName, testNamespace) + } + dataFQDN := func(ordinal int) string { + return fmt.Sprintf("%s-%d.%s.%s.svc.k8s.example.com", dataName, ordinal, dataName, testNamespace) + } + + want := planner.Topology{ + Coordinators: []memgraph.CoordinatorSpec{ + { + ID: 1, + BoltServer: coordinatorFQDN(0) + ":7777", + CoordinatorServer: coordinatorFQDN(0) + ":12001", + ManagementServer: coordinatorFQDN(0) + ":10001", + }, + { + ID: 2, + BoltServer: coordinatorFQDN(1) + ":7777", + CoordinatorServer: coordinatorFQDN(1) + ":12001", + ManagementServer: coordinatorFQDN(1) + ":10001", + }, + { + ID: 3, + BoltServer: coordinatorFQDN(2) + ":7777", + CoordinatorServer: coordinatorFQDN(2) + ":12001", + ManagementServer: coordinatorFQDN(2) + ":10001", + }, + }, + DataInstances: []memgraph.DataInstanceSpec{ + { + Name: "instance_0", + BoltServer: dataFQDN(0) + ":7777", + ManagementServer: dataFQDN(0) + ":10001", + ReplicationServer: dataFQDN(0) + ":20001", + }, + { + Name: "instance_1", + BoltServer: dataFQDN(1) + ":7777", + ManagementServer: dataFQDN(1) + ":10001", + ReplicationServer: dataFQDN(1) + ":20001", + }, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("DeclaredTopology() mismatch (-want +got):\n%s", diff) + } +} + +// The coordinator pods must advertise the same non-default identity the +// registration topology declares for them, otherwise the Raft cluster and the +// registrations disagree about who is who. +func TestDeclaredTopologyMatchesTunedCoordinatorStartScript(t *testing.T) { + cluster := tunedCluster() + topology := resources.DeclaredTopology(cluster) + sts := resources.CoordinatorStatefulSet(cluster) + script := strings.Join(sts.Spec.Template.Spec.Containers[0].Command, "\n") + + suffix := fmt.Sprintf("%s.%s.svc.k8s.example.com", coordinatorName, testNamespace) + if !strings.Contains(script, `--coordinator-hostname="${POD_NAME}.`+suffix+`"`) { + t.Errorf("coordinator start script does not advertise the configured cluster domain:\n%s", script) + } + if !strings.Contains(script, "--coordinator-port=12001") { + t.Errorf("coordinator start script does not listen on the configured coordinator port:\n%s", script) + } + for i, coordinator := range topology.Coordinators { + wantHost := fmt.Sprintf("%s-%d.%s:12001", coordinatorName, i, suffix) + if coordinator.CoordinatorServer != wantHost { + t.Errorf("coordinator %d advertises %q, want %q", coordinator.ID, coordinator.CoordinatorServer, wantHost) + } + } +} From d2dfb69bbf97521ab98b9f8b9c3fc665ea744c1e Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 13:16:43 +0200 Subject: [PATCH 13/34] feat: operator install chart (CRDs, least-privilege RBAC, controller Deployment) (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: operator install chart with generated CRDs and least-privilege RBAC Add charts/memgraph-operator, the chart users install the operator with: the MemgraphCluster CRD, the RBAC the controller needs, and the controller Deployment. `helm install` from the local chart on a clean cluster is the complete install story. The chart cannot drift from the controller version it ships with, because the two manifests that encode the contract are generated by `make chart-sync`: crds/ from the Go types, and rbac/manager-rules.yaml from the +kubebuilder:rbac markers, rendered into the manager ClusterRole with .Files.Get. `make chart-verify` regenerates into a scratch directory and diffs, so CI fails on a stale copy. Tighten the controller's RBAC markers to what the reconciler issues, since they are now the chart's permission surface: read MemgraphClusters, patch their status, and create plus patch (server-side apply) the StatefulSets and Services — no update, no delete, deletion belongs to garbage collection via the owner references. The finalizers subresource stays for the blockOwnerDeletion owner references. Leader election is a namespaced Role (Lease, Events) gated on the value that enables it, and no rule grants Secret access: the license Secret is referenced from the CR and mounted by the kubelet, never read by the operator. Install and uninstall are exercised twice in CI. hack/chart-install-test.sh (`make test-chart`, license-free) installs the chart on a clean Kind cluster into a namespace enforcing the restricted Pod Security Standard, asserts the operator runs non-root and provisions the workloads and status of a MemgraphCluster with no authorization failure in its log, then uninstalls the release and the CRDs. The e2e suite now installs the operator through the same chart instead of `make deploy`, so every scenario runs under exactly the RBAC users get. * fix: Improve chart configuration --- .github/workflows/test.yml | 33 + CLAUDE.md | 8 +- Makefile | 86 ++ README.md | 42 +- charts/memgraph-operator/.helmignore | 12 + charts/memgraph-operator/Chart.yaml | 23 + charts/memgraph-operator/README.md | 81 ++ .../crds/memgraph.com_memgraphclusters.yaml | 922 ++++++++++++++++++ .../memgraph-operator/rbac/manager-rules.yaml | 56 ++ charts/memgraph-operator/templates/NOTES.txt | 33 + .../memgraph-operator/templates/_helpers.tpl | 79 ++ .../templates/deployment.yaml | 97 ++ .../templates/leader-election-rbac.yaml | 46 + .../templates/manager-rbac.yaml | 32 + .../templates/metrics-rbac.yaml | 56 ++ .../templates/metrics-service.yaml | 17 + .../templates/serviceaccount.yaml | 13 + charts/memgraph-operator/values.yaml | 88 ++ config/rbac/role.yaml | 9 - hack/chart-install-test.sh | 163 ++++ .../controller/memgraphcluster_controller.go | 17 +- test/e2e/e2e_suite_test.go | 49 +- test/e2e/e2e_test.go | 28 +- test/e2e/memgraphcluster_test.go | 2 +- 24 files changed, 1936 insertions(+), 56 deletions(-) create mode 100644 charts/memgraph-operator/.helmignore create mode 100644 charts/memgraph-operator/Chart.yaml create mode 100644 charts/memgraph-operator/README.md create mode 100644 charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml create mode 100644 charts/memgraph-operator/rbac/manager-rules.yaml create mode 100644 charts/memgraph-operator/templates/NOTES.txt create mode 100644 charts/memgraph-operator/templates/_helpers.tpl create mode 100644 charts/memgraph-operator/templates/deployment.yaml create mode 100644 charts/memgraph-operator/templates/leader-election-rbac.yaml create mode 100644 charts/memgraph-operator/templates/manager-rbac.yaml create mode 100644 charts/memgraph-operator/templates/metrics-rbac.yaml create mode 100644 charts/memgraph-operator/templates/metrics-service.yaml create mode 100644 charts/memgraph-operator/templates/serviceaccount.yaml create mode 100644 charts/memgraph-operator/values.yaml create mode 100755 hack/chart-install-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af348f0..9fa8532 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,39 @@ jobs: - name: Run tests against envtest run: make test + chart: + permissions: + contents: read + name: Install chart + runs-on: ubuntu-latest + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + # The chart ships the CRDs and the manager's RBAC rules generated from + # the Go sources; this fails if the committed copies are stale. + - name: Verify the generated chart manifests + run: make chart-verify + + - name: Lint the chart + run: make helm-lint + + - name: Install Kind + run: go install sigs.k8s.io/kind@v0.32.0 + + # Install and uninstall the chart on a clean cluster. No license needed: + # this asserts the operator runs and reconciles under the chart's RBAC, + # while the e2e job below boots a real cluster on the same chart install. + - name: Install and uninstall the chart on Kind + run: make test-chart + e2e: permissions: contents: read diff --git a/CLAUDE.md b/CLAUDE.md index fa4ce27..ec42463 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,10 @@ make manifests generate # regenerate CRDs/RBAC + DeepCopy after editing *_type make build # build manager binary make run # run controller locally against current kubeconfig make test-e2e # KinD e2e suite — creates/deletes a dedicated Kind cluster; never run against a real cluster +make chart-sync # regenerate the install chart's CRDs + RBAC rules from the Go sources +make chart-verify # fail if those generated chart files are stale (CI gate) +make helm-lint # lint the install chart and render it with defaults and toggles flipped +make test-chart # helm install/uninstall the chart on a throwaway Kind cluster ``` Run a single test (Ginkgo suites): @@ -32,7 +36,7 @@ go test ./api/... -run TestName Envtest packages need `KUBEBUILDER_ASSETS`; outside of `make test` set it with: `KUBEBUILDER_ASSETS=$(bin/setup-envtest use --bin-dir bin -p path)` -CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, `make test`, and `make test-e2e` on every PR — all must be green. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license flowing from the `MEMGRAPH_ENTERPRISE_LICENSE` / `MEMGRAPH_ORGANIZATION_NAME` repository secrets (set the same env vars to run it locally). +CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, `make test`, `make chart-verify`, `make helm-lint`, `make test-chart`, and `make test-e2e` on every PR — all must be green. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license flowing from the `MEMGRAPH_ENTERPRISE_LICENSE` / `MEMGRAPH_ORGANIZATION_NAME` repository secrets (set the same env vars to run it locally). ### Toolchain quirks (do not "fix" these) @@ -48,7 +52,7 @@ The PRD defines seven modules with two pure cores and one mock seam. Keep this s 3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. 4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. `SET INSTANCE TO MAIN` is issued exactly once at bootstrap (when no MAIN exists); after that, failover belongs to the Raft coordinators — the operator only observes. 5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. -6. **Operator install chart** — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. +6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. 7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. Test philosophy (from the PRD): assert external behavior, never internal call ordering or private state. Builders get golden tests, planner gets pure topology-diff cases, controller gets envtest with the HA client mocked. diff --git a/Makefile b/Makefile index 1f1ce45..1f7c25e 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,10 @@ setup-test-e2e: ## Set up a multi-node Kind cluster for e2e tests if it does not echo "Kind is not installed. Please install Kind manually."; \ exit 1; \ } + @command -v $(HELM) >/dev/null 2>&1 || { \ + echo "Helm is not installed. Please install Helm manually: the suite installs the operator from charts/memgraph-operator."; \ + exit 1; \ + } @case "$$($(KIND) get clusters)" in \ *"$(KIND_CLUSTER)"*) \ echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ @@ -163,6 +167,87 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi cd config/manager && "$(KUSTOMIZE)" edit set image controller=${IMG} "$(KUSTOMIZE)" build config/default > dist/install.yaml +##@ Install chart + +# The operator install chart lives next to the generated manifests so it can +# never drift from the controller version it installs: its CRDs and its RBAC +# rules are generated, and chart-verify fails when they are stale. +CHART_DIR ?= charts/memgraph-operator +CHART_RELEASE ?= memgraph-operator +CHART_NAMESPACE ?= memgraph-operator-system + +.PHONY: chart-sync +chart-sync: manifests ## Regenerate the chart's CRDs and RBAC rules from the Go sources. + rm -f "$(CHART_DIR)"/crds/*.yaml + @for crd in config/crd/bases/*.yaml; do \ + out="$(CHART_DIR)/crds/$$(basename $$crd)"; \ + echo "Writing $$out"; \ + { echo "# Generated by controller-gen from the Go types in api/."; \ + echo "# Regenerate with 'make chart-sync'; do not edit by hand."; \ + cat "$$crd"; } > "$$out"; \ + done + @echo "Writing $(CHART_DIR)/rbac/manager-rules.yaml" + @{ echo "# The rule list of the operator's manager ClusterRole: every API call the"; \ + echo "# reconciler makes is authorized against exactly these rules, because the"; \ + echo "# manager Pod authenticates as the ServiceAccount they are bound to."; \ + echo "#"; \ + echo "# templates/manager-rbac.yaml inlines this file under 'rules:' with .Files.Get"; \ + echo "# (Helm cannot read config/rbac/role.yaml from outside the chart directory)."; \ + echo "#"; \ + echo "# Note what is absent: no 'delete' anywhere, so the operator cannot remove a"; \ + echo "# StatefulSet or PVC; no 'secrets', because license and auth material reaches"; \ + echo "# the workload Pods via the kubelet, never through the operator; no 'pods',"; \ + echo "# because readiness gating reads StatefulSet status instead."; \ + echo "#"; \ + echo "# Generated from the +kubebuilder:rbac markers in the controller sources."; \ + echo "# Regenerate with 'make chart-sync'; do not edit by hand. To widen or tighten"; \ + echo "# the permissions, edit the markers -- 'make chart-verify' fails if the two drift."; \ + awk 'found { print } /^rules:$$/ { found = 1 }' config/rbac/role.yaml; \ + } > "$(CHART_DIR)/rbac/manager-rules.yaml" + +# Generates into a scratch directory and diffs, so the check answers "do the +# committed chart manifests match the Go sources" regardless of what else is +# uncommitted in the working tree. +.PHONY: chart-verify +chart-verify: manifests ## Fail if the chart's generated CRDs and RBAC rules are out of date. + @tmp="$$(mktemp -d)"; trap 'rm -rf "$$tmp"' EXIT; \ + mkdir -p "$$tmp/crds" "$$tmp/rbac"; \ + $(MAKE) --no-print-directory chart-sync CHART_DIR="$$tmp" >/dev/null; \ + if ! diff -ru "$$tmp/crds" "$(CHART_DIR)/crds" \ + || ! diff -u "$$tmp/rbac/manager-rules.yaml" "$(CHART_DIR)/rbac/manager-rules.yaml"; then \ + echo "$(CHART_DIR) is out of date with the Go sources."; \ + echo "Run 'make chart-sync' and commit the result."; \ + exit 1; \ + fi; \ + echo "$(CHART_DIR) is in sync with the Go sources." + +.PHONY: helm-lint +helm-lint: ## Lint the install chart and render it with defaults and with the toggles flipped. + "$(HELM)" lint --strict "$(CHART_DIR)" + "$(HELM)" template "$(CHART_RELEASE)" "$(CHART_DIR)" --namespace "$(CHART_NAMESPACE)" > /dev/null + "$(HELM)" template "$(CHART_RELEASE)" "$(CHART_DIR)" --namespace "$(CHART_NAMESPACE)" \ + --set metrics.enabled=false --set leaderElection.enabled=false \ + --set-string namespaceOverride=elsewhere --set-string image.tag=v9.9.9 > /dev/null + +.PHONY: test-chart +test-chart: chart-sync ## Install/uninstall the chart on a throwaway Kind cluster; never run against a real cluster. + KIND=$(KIND) KUBECTL=$(KUBECTL) HELM=$(HELM) CONTAINER_TOOL=$(CONTAINER_TOOL) \ + CHART_DIR=$(CHART_DIR) CHART_RELEASE=$(CHART_RELEASE) CHART_NAMESPACE=$(CHART_NAMESPACE) \ + ./hack/chart-install-test.sh + +.PHONY: helm-install +helm-install: ## Install the operator from the local chart, running the image specified by IMG. + img="$(IMG)"; "$(HELM)" upgrade --install "$(CHART_RELEASE)" "$(CHART_DIR)" \ + --namespace "$(CHART_NAMESPACE)" --create-namespace \ + --set-string image.repository="$${img%:*}" \ + --set-string image.tag="$${img##*:}" \ + --wait --timeout 5m $(HELM_EXTRA_ARGS) + +.PHONY: helm-uninstall +helm-uninstall: ## Uninstall the operator, including the CRDs helm leaves behind. + "$(HELM)" uninstall "$(CHART_RELEASE)" --namespace "$(CHART_NAMESPACE)" --ignore-not-found + "$(KUBECTL)" delete --ignore-not-found=true -f "$(CHART_DIR)/crds" + ##@ Deployment ifndef ignore-not-found @@ -198,6 +283,7 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl KIND ?= kind +HELM ?= helm KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest diff --git a/README.md b/README.md index 660d29a..5ebec43 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,23 @@ The operator replaces the `memgraph-high-availability` Helm chart's fire-and-for - kubectl version v1.11.3+. - Access to a Kubernetes v1.11.3+ cluster. -### To Deploy on the cluster +### To install the operator + +The install chart in [`charts/memgraph-operator`](charts/memgraph-operator/README.md) is the +complete install story — it ships the `MemgraphCluster` CRD, a least-privilege RBAC set, and the +controller Deployment: + +```sh +helm install memgraph-operator ./charts/memgraph-operator \ + --namespace memgraph-operator-system --create-namespace --wait +``` + +Uninstall with `helm uninstall memgraph-operator --namespace memgraph-operator-system`. Helm never +deletes CRDs it installed, so remove the CRD explicitly (`kubectl delete crd +memgraphclusters.memgraph.com`) once no cluster needs it — see the +[chart README](charts/memgraph-operator/README.md) for the values and the upgrade caveat. + +### To deploy a development build on the cluster **Build and push your image to the location specified by `IMG`:** ```sh @@ -96,23 +112,23 @@ the project, i.e.: kubectl apply -f https://raw.githubusercontent.com//kubernetes-operator//dist/install.yaml ``` -### By providing a Helm Chart +### By providing a Helm chart -1. Build the chart using the optional helm plugin +The install chart is maintained in this repository under +[`charts/memgraph-operator`](charts/memgraph-operator/README.md), next to the manifests it ships: +its CRDs and the manager's RBAC rules are generated from the Go types and the +`+kubebuilder:rbac` markers, so the chart can never drift from the controller version it +installs. ```sh -kubebuilder edit --plugins=helm/v2-alpha +make chart-sync # regenerate the chart's CRDs and RBAC rules after changing the API or markers +make helm-lint # lint the chart and render it with defaults and with the toggles flipped +make test-chart # install/uninstall the chart on a throwaway Kind cluster ``` -2. See that a chart was generated under 'dist/chart', and users -can obtain this solution from there. - -**NOTE:** If you change the project, you need to update the Helm Chart -using the same command above to sync the latest changes. Furthermore, -if you create webhooks, you need to use the above command with -the '--force' flag and manually ensure that any custom configuration -previously added to 'dist/chart/values.yaml' or 'dist/chart/manager/manager.yaml' -is manually re-applied afterwards. +Releases cross-publish the packaged chart into the existing +[`memgraph.github.io/helm-charts`](https://memgraph.github.io/helm-charts) index, so users install +it from the helm repository they already have configured. ## Contributing diff --git a/charts/memgraph-operator/.helmignore b/charts/memgraph-operator/.helmignore new file mode 100644 index 0000000..dac0bdd --- /dev/null +++ b/charts/memgraph-operator/.helmignore @@ -0,0 +1,12 @@ +# Patterns to ignore when building packages. +.DS_Store +.git/ +.gitignore +*.swp +*.bak +*.tmp +*.orig +*~ +.project +.idea/ +.vscode/ diff --git a/charts/memgraph-operator/Chart.yaml b/charts/memgraph-operator/Chart.yaml new file mode 100644 index 0000000..984aff3 --- /dev/null +++ b/charts/memgraph-operator/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: memgraph-operator +description: Kubernetes operator for Memgraph high-availability clusters +type: application +# version is the chart version, appVersion the operator version it installs. +# Both move together with the git tag at release time. +version: 0.1.0 +appVersion: "0.1.0" +# The CustomResourceDefinition uses CEL validation rules +# (x-kubernetes-validations), available from Kubernetes 1.25. +kubeVersion: ">=1.25.0-0" +home: https://memgraph.com +icon: https://memgraph.com/images/logo-memgraph-purple.svg +sources: + - https://github.com/memgraph/kubernetes-operator +keywords: + - memgraph + - graph-database + - high-availability + - operator +maintainers: + - name: Memgraph + url: https://memgraph.com diff --git a/charts/memgraph-operator/README.md b/charts/memgraph-operator/README.md new file mode 100644 index 0000000..16c2a49 --- /dev/null +++ b/charts/memgraph-operator/README.md @@ -0,0 +1,81 @@ +# memgraph-operator + +Installs the Memgraph Kubernetes operator: the `MemgraphCluster` CustomResourceDefinition, a +least-privilege RBAC set, and the controller Deployment. Declaring a cluster is then a single +resource — see the [repository README](../../README.md) and the +[PRD](../../specs/operator-mvp/PRD.md). + +The chart lives in the operator repository next to the generated manifests: the CRDs under +`crds/` and the manager's RBAC rules under `rbac/` are generated from the Go types and the +`+kubebuilder:rbac` markers (`make chart-sync`), and CI fails if they drift from the controller +they ship with. + +## Install + +```sh +helm install memgraph-operator ./charts/memgraph-operator \ + --namespace memgraph-operator-system --create-namespace --wait +``` + +The operator watches every namespace, so one release per cluster is enough. + +## Uninstall + +```sh +helm uninstall memgraph-operator --namespace memgraph-operator-system +``` + +Helm never deletes CRDs it installed, so the `MemgraphCluster` CRD — and with it your clusters +and their PersistentVolumeClaims — survive the uninstall. Remove it explicitly when no cluster +needs it any more: + +```sh +kubectl delete crd memgraphclusters.memgraph.com +``` + +For the same reason, `helm upgrade` does not update the CRD. Apply the new one before upgrading +to a chart version that changes the API: + +```sh +kubectl apply -f charts/memgraph-operator/crds/ +``` + +## Permissions + +The manager's ClusterRole is generated from the controller's own RBAC markers, so it grants +exactly what the reconciler issues: read MemgraphClusters, patch their status, and create and +patch (server-side apply) the StatefulSets and Services it provisions. It cannot delete +workloads — deleting a `MemgraphCluster` removes them through garbage collection of the owner +references. Leader election adds a Lease and Events in the operator's own namespace, and the +metrics endpoint adds the TokenReview/SubjectAccessReview permissions it authorizes scrapes +with. Nothing grants read access to Secrets: the license Secret is referenced from the +`MemgraphCluster` and mounted by the kubelet into the Memgraph pods, never read by the operator. + +## Values + +| Key | Default | Description | +| --- | --- | --- | +| `image.repository` | `docker.io/memgraph/kubernetes-operator` | Operator image repository | +| `image.tag` | `""` | Operator image tag; defaults to the chart's `appVersion` | +| `image.pullPolicy` | `IfNotPresent` | Operator image pull policy | +| `imagePullSecrets` | `[]` | Secrets used to pull the operator image | +| `replicaCount` | `1` | Controller replicas | +| `leaderElection.enabled` | `true` | Elect a leader, so a second replica can stand by | +| `resources` | 10m/64Mi requests, 500m/128Mi limits | Controller container resources | +| `metrics.enabled` | `true` | Serve the controller-runtime metrics endpoint and create its Service | +| `metrics.port` | `8443` | Port the metrics endpoint binds to | +| `metrics.secure` | `true` | Serve metrics over HTTPS with authenticated, authorized scrapes | +| `rbac.create` | `true` | Create the operator's Roles and bindings | +| `serviceAccount.create` | `true` | Create the operator's ServiceAccount | +| `serviceAccount.name` | `""` | ServiceAccount name; required when `create` is false | +| `serviceAccount.annotations` | `{}` | Annotations for the ServiceAccount | +| `namespaceOverride` | `""` | Install the namespaced objects outside the release namespace | +| `nameOverride` / `fullnameOverride` | `""` | Override the generated object names | +| `podAnnotations` / `podLabels` | `{}` | Extra metadata on the controller pod | +| `nodeSelector` / `tolerations` / `affinity` | `{}` / `[]` / `{}` | Scheduling of the controller pod | +| `priorityClassName` | `""` | PriorityClass of the controller pod | +| `terminationGracePeriodSeconds` | `10` | Grace period of the controller pod | +| `podSecurityContext` | non-root uid/gid 65532, seccomp `RuntimeDefault` | Pod security context | +| `securityContext` | no privilege escalation, read-only root, all capabilities dropped | Container security context | +| `extraArgs` | `[]` | Additional controller command-line arguments | +| `extraEnv` | `[]` | Additional controller environment variables | diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml new file mode 100644 index 0000000..a436abd --- /dev/null +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -0,0 +1,922 @@ +# Generated by controller-gen from the Go types in api/. +# Regenerate with 'make chart-sync'; do not edit by hand. +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.21.0 + name: memgraphclusters.memgraph.com +spec: + group: memgraph.com + names: + kind: MemgraphCluster + listKind: MemgraphClusterList + plural: memgraphclusters + shortNames: + - mgc + singular: memgraphcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.coordinators + name: Coordinators + type: integer + - jsonPath: .spec.dataInstances + name: Data + type: integer + - jsonPath: .status.main + name: Main + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=="Converged")].status + name: Converged + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: MemgraphCluster is the Schema for the memgraphclusters API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of MemgraphCluster + properties: + clusterDomain: + default: cluster.local + description: |- + clusterDomain is the Kubernetes cluster domain the advertised FQDN + addresses are built from: ...svc.. + Override it on clusters configured with a domain other than the default. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + coordinators: + default: 3 + description: |- + coordinators is the number of Raft coordinator instances. It must be odd + so the Raft quorum cannot split, and it is immutable: scaling is not + supported in v1alpha1. + format: int32 + minimum: 1 + type: integer + x-kubernetes-validations: + - message: coordinators must be an odd number so the Raft quorum cannot + split + rule: self % 2 == 1 + - message: 'coordinators is immutable: changing the coordinator count + of an existing MemgraphCluster is not supported in v1alpha1' + rule: self == oldSelf + dataInstances: + default: 2 + description: |- + dataInstances is the number of data instances. It is immutable: scaling + is not supported in v1alpha1. + format: int32 + minimum: 1 + type: integer + x-kubernetes-validations: + - message: 'dataInstances is immutable: changing the data instance + count of an existing MemgraphCluster is not supported in v1alpha1' + rule: self == oldSelf + extraArgs: + description: extraArgs passes additional Memgraph flags to both roles. + properties: + coordinators: + description: coordinators are appended to every coordinator pod's + Memgraph flags. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array + x-kubernetes-validations: + - message: extraArgs must not set a port or the coordinator identity + the operator derives (--bolt-port, --management-port, --coordinator-id, + --coordinator-hostname, --coordinator-port); configure ports + through spec.ports + rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', + '--coordinator-hostname', '--coordinator-port'].exists(f, + a.startsWith(f))) + data: + description: data are appended to every data instance pod's Memgraph + flags. + items: + maxLength: 4096 + minLength: 1 + type: string + maxItems: 64 + type: array + x-kubernetes-validations: + - message: extraArgs must not set a port or the coordinator identity + the operator derives (--bolt-port, --management-port, --coordinator-id, + --coordinator-hostname, --coordinator-port); configure ports + through spec.ports + rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', + '--coordinator-hostname', '--coordinator-port'].exists(f, + a.startsWith(f))) + type: object + extraEnv: + description: |- + extraEnv passes additional non-secret environment variables to both + roles' Memgraph containers. + properties: + coordinators: + description: coordinators are added to every coordinator pod's + Memgraph container. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or + MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) + or POD_NAME (it carries the pod's own identity) + rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', + 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) + data: + description: data are added to every data instance pod's Memgraph + container. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraEnv must not set MEMGRAPH_ENTERPRISE_LICENSE or + MEMGRAPH_ORGANIZATION_NAME (they come from the secrets block) + or POD_NAME (it carries the pod's own identity) + rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', + 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) + type: object + image: + default: {} + description: image selects the Memgraph container image run by all + cluster pods. + properties: + pullPolicy: + default: IfNotPresent + description: pullPolicy is the image pull policy applied to all + cluster pods. + enum: + - Always + - IfNotPresent + - Never + type: string + repository: + default: docker.io/memgraph/memgraph + description: |- + repository is the Memgraph container image repository. It carries the + optional registry host and the image path only; the version belongs in + tag. + maxLength: 255 + minLength: 1 + type: string + x-kubernetes-validations: + - message: repository must not contain a digest; pin the image + with tag instead + rule: '!self.contains(''@'')' + - message: repository must not contain a tag; set image.tag instead + rule: '!self.substring(self.lastIndexOf(''/'') + 1).contains('':'')' + tag: + default: 3.12.0-relwithdebinfo + description: |- + tag is the Memgraph container image tag. Prefer pinning a specific + Memgraph version over mutable tags such as "latest". + maxLength: 128 + minLength: 1 + pattern: ^[a-zA-Z0-9_][a-zA-Z0-9._-]*$ + type: string + type: object + labels: + description: labels adds custom labels to both roles' pods, StatefulSets + and Services. + properties: + coordinators: + description: coordinators labels the coordinator objects. + properties: + podLabels: + additionalProperties: + type: string + description: podLabels are added to the role's pods. + type: object + serviceLabels: + additionalProperties: + type: string + description: serviceLabels are added to the role's headless + Service. + type: object + statefulSetLabels: + additionalProperties: + type: string + description: statefulSetLabels are added to the role's StatefulSet. + type: object + type: object + data: + description: data labels the data instance objects. + properties: + podLabels: + additionalProperties: + type: string + description: podLabels are added to the role's pods. + type: object + serviceLabels: + additionalProperties: + type: string + description: serviceLabels are added to the role's headless + Service. + type: object + statefulSetLabels: + additionalProperties: + type: string + description: statefulSetLabels are added to the role's StatefulSet. + type: object + type: object + type: object + ports: + default: {} + description: ports configures the internal ports Memgraph listens + on. + properties: + boltPort: + default: 7687 + description: |- + boltPort is the port Memgraph serves the Bolt protocol on. Clients and + the operator's own management queries both use it. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + coordinatorPort: + default: 12000 + description: coordinatorPort is the port coordinators run their + Raft protocol on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + managementPort: + default: 10000 + description: managementPort is the port instances exchange HA + management traffic on. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + replicationPort: + default: 20000 + description: replicationPort is the port data instances replicate + over. + format: int32 + maximum: 65535 + minimum: 1 + type: integer + type: object + x-kubernetes-validations: + - message: boltPort, managementPort, replicationPort and coordinatorPort + must all be different ports + rule: '!(has(self.boltPort) && has(self.managementPort) && has(self.replicationPort) + && has(self.coordinatorPort)) || [self.boltPort, self.managementPort, + self.replicationPort, self.coordinatorPort].all(p, [self.boltPort, + self.managementPort, self.replicationPort, self.coordinatorPort].exists_one(q, + q == p))' + probes: + description: probes tunes the probe timings of both roles. + properties: + coordinators: + description: coordinators tunes the probes of every coordinator + pod. + properties: + livenessProbe: + description: livenessProbe decides whether the container is + restarted. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + readinessProbe decides whether the pod receives traffic and whether the + operator considers the workloads ready to register. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: startupProbe gates the other two probes until + the instance has started. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + type: object + data: + description: data tunes the probes of every data instance pod. + properties: + livenessProbe: + description: livenessProbe decides whether the container is + restarted. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + readinessProbe: + description: |- + readinessProbe decides whether the pod receives traffic and whether the + operator considers the workloads ready to register. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + startupProbe: + description: startupProbe gates the other two probes until + the instance has started. + properties: + failureThreshold: + description: |- + failureThreshold is how many consecutive failures the probe tolerates + before acting. Defaults to 1440 for the data instances' startup probe — + 2h at the default period, so a large snapshot restore is not killed + mid-load — and to 20 for every other probe. + format: int32 + minimum: 1 + type: integer + periodSeconds: + description: periodSeconds is how often the probe runs. + Defaults to 5. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + timeoutSeconds is how long a single probe attempt may take. Defaults to + 10. + format: int32 + minimum: 1 + type: integer + type: object + type: object + type: object + resources: + description: resources sets the compute resources of both roles' Memgraph + containers. + properties: + coordinators: + description: |- + coordinators are the resource requests and limits of every coordinator + pod's Memgraph container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + data: + description: |- + data are the resource requests and limits of every data instance pod's + Memgraph container. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + type: object + secrets: + default: {} + description: |- + secrets references the Secret holding the enterprise license and + organization name. + properties: + licenseKey: + default: MEMGRAPH_ENTERPRISE_LICENSE + description: licenseKey is the key within the Secret holding the + enterprise license. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + name: + default: memgraph-secrets + description: name is the name of the Secret in the cluster's namespace. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + organizationKey: + default: MEMGRAPH_ORGANIZATION_NAME + description: |- + organizationKey is the key within the Secret holding the organization + name the license was issued to. + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + type: object + x-kubernetes-validations: + - message: licenseKey and organizationKey must name different keys + of the Secret + rule: '!has(self.licenseKey) || !has(self.organizationKey) || self.licenseKey + != self.organizationKey' + storage: + default: {} + description: |- + storage configures the persistent volumes backing both roles and their + retention on cluster deletion. + properties: + coordinators: + default: {} + description: coordinators configures the storage of every coordinator + pod. + properties: + libPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + libPVCSize is the requested size of the lib storage claim, which backs + Memgraph's data directory (snapshots, WAL, and durability metadata). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + libStorageAccessMode: + default: ReadWriteOnce + description: |- + libStorageAccessMode is the access mode requested for the lib storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + libStorageClassName: + description: |- + libStorageClassName is the StorageClass backing the lib storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + logPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + logPVCSize is the requested size of the log storage claim, which backs + Memgraph's log file. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + logStorageAccessMode: + default: ReadWriteOnce + description: |- + logStorageAccessMode is the access mode requested for the log storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + logStorageClassName: + description: |- + logStorageClassName is the StorageClass backing the log storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + type: object + data: + default: {} + description: data configures the storage of every data instance + pod. + properties: + libPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + libPVCSize is the requested size of the lib storage claim, which backs + Memgraph's data directory (snapshots, WAL, and durability metadata). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + libStorageAccessMode: + default: ReadWriteOnce + description: |- + libStorageAccessMode is the access mode requested for the lib storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + libStorageClassName: + description: |- + libStorageClassName is the StorageClass backing the lib storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + logPVCSize: + anyOf: + - type: integer + - type: string + default: 1Gi + description: |- + logPVCSize is the requested size of the log storage claim, which backs + Memgraph's log file. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + logStorageAccessMode: + default: ReadWriteOnce + description: |- + logStorageAccessMode is the access mode requested for the log storage + claim. + enum: + - ReadWriteOnce + - ReadOnlyMany + - ReadWriteMany + - ReadWriteOncePod + type: string + logStorageClassName: + description: |- + logStorageClassName is the StorageClass backing the log storage claim. + Leave it unset to use the cluster's default StorageClass; set it to the + empty string to disable dynamic provisioning and bind a pre-created + PersistentVolume. + maxLength: 253 + type: string + type: object + retentionPolicy: + default: Retain + description: |- + retentionPolicy decides whether the cluster's PersistentVolumeClaims + survive deletion of the MemgraphCluster. It maps directly onto the + StatefulSets' persistentVolumeClaimRetentionPolicy.whenDeleted, so the + StatefulSet controller is the only thing that ever deletes storage — the + operator owns no finalizer and runs no cleanup of its own. The default + keeps production data safe from an accidental delete; dev clusters can + opt into self-cleanup. + enum: + - Retain + - Delete + type: string + type: object + type: object + status: + description: status defines the observed state of MemgraphCluster + properties: + conditions: + description: |- + conditions represent the current state of the MemgraphCluster resource. + Each condition has a unique type and reflects the status of a specific aspect of the resource. + + Standard condition types include: + - "Available": the resource is fully functional + - "Progressing": the resource is being created or updated + - "Degraded": the resource failed to reach or maintain its desired state + + The status of each condition is one of True, False, or Unknown. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + main: + description: |- + main is the name of the data instance currently observed as MAIN, as + reported by SHOW INSTANCES on the coordinator leader. It is empty before + the initial MAIN is elected and updates when the Raft coordinators fail + over to a different instance. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/memgraph-operator/rbac/manager-rules.yaml b/charts/memgraph-operator/rbac/manager-rules.yaml new file mode 100644 index 0000000..16eda07 --- /dev/null +++ b/charts/memgraph-operator/rbac/manager-rules.yaml @@ -0,0 +1,56 @@ +# The rule list of the operator's manager ClusterRole: every API call the +# reconciler makes is authorized against exactly these rules, because the +# manager Pod authenticates as the ServiceAccount they are bound to. +# +# templates/manager-rbac.yaml inlines this file under 'rules:' with .Files.Get +# (Helm cannot read config/rbac/role.yaml from outside the chart directory). +# +# Note what is absent: no 'delete' anywhere, so the operator cannot remove a +# StatefulSet or PVC; no 'secrets', because license and auth material reaches +# the workload Pods via the kubelet, never through the operator; no 'pods', +# because readiness gating reads StatefulSet status instead. +# +# Generated from the +kubebuilder:rbac markers in the controller sources. +# Regenerate with 'make chart-sync'; do not edit by hand. To widen or tighten +# the permissions, edit the markers -- 'make chart-verify' fails if the two drift. +- apiGroups: + - "" + resources: + - services + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - create + - get + - list + - patch + - watch +- apiGroups: + - memgraph.com + resources: + - memgraphclusters + verbs: + - get + - list + - watch +- apiGroups: + - memgraph.com + resources: + - memgraphclusters/finalizers + verbs: + - update +- apiGroups: + - memgraph.com + resources: + - memgraphclusters/status + verbs: + - get + - patch diff --git a/charts/memgraph-operator/templates/NOTES.txt b/charts/memgraph-operator/templates/NOTES.txt new file mode 100644 index 0000000..4cca918 --- /dev/null +++ b/charts/memgraph-operator/templates/NOTES.txt @@ -0,0 +1,33 @@ +{{ .Chart.Name }} {{ .Chart.AppVersion }} is installed in namespace {{ include "memgraph-operator.namespace" . }}. + +Watch the operator come up: + + kubectl rollout status deployment/{{ include "memgraph-operator.managerName" . }} -n {{ include "memgraph-operator.namespace" . }} + +Then declare a cluster (3 coordinators, 2 data instances) in any namespace, +referencing a Secret that holds your Memgraph enterprise license: + + apiVersion: memgraph.com/v1alpha1 + kind: MemgraphCluster + metadata: + name: memgraph + spec: + coordinators: 3 + dataInstances: 2 + image: + repository: docker.io/memgraph/memgraph + tag: 3.12.0-relwithdebinfo + secrets: + name: memgraph-secrets + licenseKey: MEMGRAPH_ENTERPRISE_LICENSE + organizationKey: MEMGRAPH_ORGANIZATION_NAME + +Follow the cluster reaching a registered, MAIN-elected state with: + + kubectl get mgc -w + +Note that `helm uninstall` leaves the MemgraphCluster CustomResourceDefinition +(and therefore your clusters) in place, as Helm never removes CRDs it installed. +Remove it explicitly once no cluster needs it: + + kubectl delete crd memgraphclusters.memgraph.com diff --git a/charts/memgraph-operator/templates/_helpers.tpl b/charts/memgraph-operator/templates/_helpers.tpl new file mode 100644 index 0000000..e1919f7 --- /dev/null +++ b/charts/memgraph-operator/templates/_helpers.tpl @@ -0,0 +1,79 @@ +{{/* +Chart name, overridable with nameOverride. +*/}} +{{- define "memgraph-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified release name, the prefix of every object this chart creates. +*/}} +{{- define "memgraph-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Name of the controller Deployment, its ServiceAccount and its metrics Service. +*/}} +{{- define "memgraph-operator.managerName" -}} +{{- printf "%s-controller-manager" (include "memgraph-operator.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Name of the Service in front of the metrics endpoint. +*/}} +{{- define "memgraph-operator.metricsServiceName" -}} +{{- printf "%s-metrics-service" (include "memgraph-operator.fullname" .) | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Namespace the namespaced objects are installed into. +*/}} +{{- define "memgraph-operator.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride }} +{{- end }} + +{{- define "memgraph-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Selector labels. The Deployment selector is immutable, so these must stay +stable across chart versions. +*/}} +{{- define "memgraph-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "memgraph-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +control-plane: controller-manager +{{- end }} + +{{- define "memgraph-operator.labels" -}} +helm.sh/chart: {{ include "memgraph-operator.chart" . }} +{{ include "memgraph-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/component: controller +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +ServiceAccount the operator runs as. An externally managed account must be +named explicitly, because the chart cannot bind a name it does not know. +*/}} +{{- define "memgraph-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "memgraph-operator.managerName" .) .Values.serviceAccount.name }} +{{- else }} +{{- required "serviceAccount.name is required when serviceAccount.create is false" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/memgraph-operator/templates/deployment.yaml b/charts/memgraph-operator/templates/deployment.yaml new file mode 100644 index 0000000..88d17ac --- /dev/null +++ b/charts/memgraph-operator/templates/deployment.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "memgraph-operator.managerName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "memgraph-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "memgraph-operator.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "memgraph-operator.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: manager + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /manager + args: + - --health-probe-bind-address=:8081 + {{- if .Values.leaderElection.enabled }} + - --leader-elect + {{- end }} + {{- if .Values.metrics.enabled }} + - --metrics-bind-address=:{{ .Values.metrics.port }} + {{- if not .Values.metrics.secure }} + - --metrics-secure=false + {{- end }} + {{- end }} + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.extraEnv }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: health + containerPort: 8081 + protocol: TCP + {{- if .Values.metrics.enabled }} + - name: metrics + containerPort: {{ .Values.metrics.port }} + protocol: TCP + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: health + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: health + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} diff --git a/charts/memgraph-operator/templates/leader-election-rbac.yaml b/charts/memgraph-operator/templates/leader-election-rbac.yaml new file mode 100644 index 0000000..0e1bd6f --- /dev/null +++ b/charts/memgraph-operator/templates/leader-election-rbac.yaml @@ -0,0 +1,46 @@ +{{- if and .Values.rbac.create .Values.leaderElection.enabled }} +{{/* +Leader election is namespaced: one Lease in the operator's own namespace, plus +the Events the election records on it. Nothing here is needed to reconcile a +MemgraphCluster, so it is gated on leaderElection.enabled. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "memgraph-operator.fullname" . }}-leader-election-role + namespace: {{ include "memgraph-operator.namespace" . }} + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "memgraph-operator.fullname" . }}-leader-election-rolebinding + namespace: {{ include "memgraph-operator.namespace" . }} + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "memgraph-operator.fullname" . }}-leader-election-role +subjects: + - kind: ServiceAccount + name: {{ include "memgraph-operator.serviceAccountName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} +{{- end }} diff --git a/charts/memgraph-operator/templates/manager-rbac.yaml b/charts/memgraph-operator/templates/manager-rbac.yaml new file mode 100644 index 0000000..40c73d9 --- /dev/null +++ b/charts/memgraph-operator/templates/manager-rbac.yaml @@ -0,0 +1,32 @@ +{{- if .Values.rbac.create }} +{{/* +The rules come from rbac/manager-rules.yaml, which `make chart-sync` generates +from the +kubebuilder:rbac markers in the controller sources — the chart cannot +grant a permission the controller does not declare, and CI fails when the two +drift apart. A ClusterRole, because the operator watches MemgraphClusters in +every namespace. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "memgraph-operator.fullname" . }}-manager-role + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +rules: +{{- .Files.Get "rbac/manager-rules.yaml" | nindent 0 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "memgraph-operator.fullname" . }}-manager-rolebinding + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "memgraph-operator.fullname" . }}-manager-role +subjects: + - kind: ServiceAccount + name: {{ include "memgraph-operator.serviceAccountName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} +{{- end }} diff --git a/charts/memgraph-operator/templates/metrics-rbac.yaml b/charts/memgraph-operator/templates/metrics-rbac.yaml new file mode 100644 index 0000000..ea482f9 --- /dev/null +++ b/charts/memgraph-operator/templates/metrics-rbac.yaml @@ -0,0 +1,56 @@ +{{- if and .Values.rbac.create .Values.metrics.enabled }} +{{/* +The metrics endpoint authenticates and authorizes every scrape against the API +server, which is what the TokenReview and SubjectAccessReview permissions are +for. metrics-reader is the role a scraper (e.g. Prometheus) binds its own +ServiceAccount to; the operator itself never uses it. +*/}} +{{- if .Values.metrics.secure }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "memgraph-operator.fullname" . }}-metrics-auth-role + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "memgraph-operator.fullname" . }}-metrics-auth-rolebinding + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "memgraph-operator.fullname" . }}-metrics-auth-role +subjects: + - kind: ServiceAccount + name: {{ include "memgraph-operator.serviceAccountName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} +--- +{{- end }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "memgraph-operator.fullname" . }}-metrics-reader + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +rules: + - nonResourceURLs: + - /metrics + verbs: + - get +{{- end }} diff --git a/charts/memgraph-operator/templates/metrics-service.yaml b/charts/memgraph-operator/templates/metrics-service.yaml new file mode 100644 index 0000000..d654128 --- /dev/null +++ b/charts/memgraph-operator/templates/metrics-service.yaml @@ -0,0 +1,17 @@ +{{- if .Values.metrics.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "memgraph-operator.metricsServiceName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} +spec: + ports: + - name: {{ ternary "https" "http" .Values.metrics.secure }} + port: {{ .Values.metrics.port }} + protocol: TCP + targetPort: metrics + selector: + {{- include "memgraph-operator.selectorLabels" . | nindent 4 }} +{{- end }} diff --git a/charts/memgraph-operator/templates/serviceaccount.yaml b/charts/memgraph-operator/templates/serviceaccount.yaml new file mode 100644 index 0000000..a78cd6e --- /dev/null +++ b/charts/memgraph-operator/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "memgraph-operator.serviceAccountName" . }} + namespace: {{ include "memgraph-operator.namespace" . }} + labels: + {{- include "memgraph-operator.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/memgraph-operator/values.yaml b/charts/memgraph-operator/values.yaml new file mode 100644 index 0000000..3aee892 --- /dev/null +++ b/charts/memgraph-operator/values.yaml @@ -0,0 +1,88 @@ +# Default values for the memgraph-operator chart. + +# The operator container image. tag defaults to the chart's appVersion, so a +# chart version always installs the operator version it was released with. +image: + repository: docker.io/memgraph/kubernetes-operator + tag: "" + pullPolicy: IfNotPresent + +# References to Secrets used to pull the operator image from a private registry. +imagePullSecrets: [] + # - name: my-registry-credentials + +# One replica reconciles at a time; leader election lets a second replica stand +# by (and makes a rolling restart safe) by taking over the Lease. +replicaCount: 1 +leaderElection: + enabled: true + +# Requests and limits of the operator container. The operator holds a cache of +# the objects it watches, so memory scales with the number of MemgraphClusters +# and their workloads. +resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + +# The controller-runtime metrics endpoint. Serving is HTTPS with a self-signed +# certificate and every request is authenticated and authorized against the +# API server, so scraping requires a token bound to the metrics-reader +# ClusterRole this chart installs. Set secure to false to serve plain HTTP +# without authorization — do that only on a trusted network. +metrics: + enabled: true + port: 8443 + secure: true + +# RBAC: the operator gets exactly the permissions its controller declares — +# MemgraphClusters, the StatefulSets and Services it provisions, plus a Lease +# and Events in its own namespace for leader election. Set create to false to +# bind an existing role set yourself; serviceAccount.name is then required. +rbac: + create: true + +serviceAccount: + create: true + # Defaults to -controller-manager when empty. + name: "" + annotations: {} + +# Install the operator's namespaced objects into a namespace other than the +# release namespace. Empty means the release namespace. +namespaceOverride: "" + +nameOverride: "" +fullnameOverride: "" + +# Pod-level knobs. +podAnnotations: {} +podLabels: {} +nodeSelector: {} +tolerations: [] +affinity: {} +priorityClassName: "" +terminationGracePeriodSeconds: 10 + +# The operator image is distroless and runs as uid/gid 65532, which satisfies +# the restricted Pod Security Standard. Overriding these is unsupported. +podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault +securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + +# Additional command-line arguments and environment variables for the operator +# container, appended after the ones this chart derives. +extraArgs: [] +extraEnv: [] diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2172b4d..7c6c5df 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -10,11 +10,9 @@ rules: - services verbs: - create - - delete - get - list - patch - - update - watch - apiGroups: - apps @@ -22,23 +20,17 @@ rules: - statefulsets verbs: - create - - delete - get - list - patch - - update - watch - apiGroups: - memgraph.com resources: - memgraphclusters verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - memgraph.com @@ -53,4 +45,3 @@ rules: verbs: - get - patch - - update diff --git a/hack/chart-install-test.sh b/hack/chart-install-test.sh new file mode 100755 index 0000000..9242657 --- /dev/null +++ b/hack/chart-install-test.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash +# +# Install/uninstall test for the operator install chart: helm install on a +# clean Kind cluster, then assert the operator reconciles a MemgraphCluster +# under the chart's least-privilege RBAC, then uninstall everything again. +# +# This is deliberately license-free — it asserts what the chart is responsible +# for (the operator runs, its RBAC is sufficient, install and uninstall are +# clean), not what a booted Memgraph cluster does. The e2e suite covers that +# on top of the same chart install. +# +# Set CHART_TEST_KEEP=true to keep the Kind cluster for inspection. + +set -euo pipefail + +CLUSTER=${CHART_TEST_CLUSTER:-memgraph-operator-chart-test} +CHART_DIR=${CHART_DIR:-charts/memgraph-operator} +CHART_RELEASE=${CHART_RELEASE:-memgraph-operator} +CHART_NAMESPACE=${CHART_NAMESPACE:-memgraph-operator-system} +CLUSTER_NAMESPACE=${CLUSTER_NAMESPACE:-memgraph-chart-test} +IMG=${IMG:-example.com/kubernetes-operator:chart-test} +IMAGE_REPOSITORY=${IMG%:*} +IMAGE_TAG=${IMG##*:} +CRD_NAME=memgraphclusters.memgraph.com + +KIND=${KIND:-kind} +KUBECTL=${KUBECTL:-kubectl} +HELM=${HELM:-helm} +CONTAINER_TOOL=${CONTAINER_TOOL:-docker} + +# wait_for_object polls until an object exists, so the script does not depend +# on a kubectl new enough for `wait --for=create`. +wait_for_object() { + local object=$1 + for _ in $(seq 1 60); do + if "${KUBECTL}" get "${object}" -n "${CLUSTER_NAMESPACE}" >/dev/null 2>&1; then + echo "${object} exists" + return 0 + fi + sleep 2 + done + echo "FAIL: ${object} was not created in ${CLUSTER_NAMESPACE}" >&2 + "${KUBECTL}" get memgraphcluster chart-test -n "${CLUSTER_NAMESPACE}" -o yaml >&2 || true + "${KUBECTL}" logs -n "${CHART_NAMESPACE}" \ + "deployment/${CHART_RELEASE}-controller-manager" --tail=100 >&2 || true + return 1 +} + +cleanup() { + if [ "${CHART_TEST_KEEP:-false}" = "true" ]; then + echo "Keeping Kind cluster ${CLUSTER} (CHART_TEST_KEEP=true)" + return + fi + echo "==> Deleting Kind cluster ${CLUSTER}" + "${KIND}" delete cluster --name "${CLUSTER}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "==> Creating Kind cluster ${CLUSTER}" +"${KIND}" delete cluster --name "${CLUSTER}" >/dev/null 2>&1 || true +"${KIND}" create cluster --name "${CLUSTER}" + +echo "==> Building and loading the operator image ${IMG}" +"${CONTAINER_TOOL}" build -t "${IMG}" . +"${KIND}" load docker-image "${IMG}" --name "${CLUSTER}" + +# The restricted Pod Security Standard is what makes the operator's security +# context load-bearing: a pod that is not non-root is rejected outright. +echo "==> Creating namespace ${CHART_NAMESPACE} enforcing the restricted policy" +"${KUBECTL}" create namespace "${CHART_NAMESPACE}" +"${KUBECTL}" label --overwrite namespace "${CHART_NAMESPACE}" \ + pod-security.kubernetes.io/enforce=restricted + +echo "==> helm install ${CHART_RELEASE} from ${CHART_DIR}" +"${HELM}" install "${CHART_RELEASE}" "${CHART_DIR}" \ + --namespace "${CHART_NAMESPACE}" \ + --set-string "image.repository=${IMAGE_REPOSITORY}" \ + --set-string "image.tag=${IMAGE_TAG}" \ + --wait --timeout 5m + +echo "==> Asserting the operator runs as non-root" +run_as_non_root=$("${KUBECTL}" get pods -n "${CHART_NAMESPACE}" \ + -l control-plane=controller-manager \ + -o jsonpath='{.items[0].spec.securityContext.runAsNonRoot}') +if [ "${run_as_non_root}" != "true" ]; then + echo "FAIL: operator pod does not set runAsNonRoot" >&2 + exit 1 +fi + +# A MemgraphCluster with no license Secret never boots Memgraph, but the +# operator still has to provision the workloads and report status — which is +# exactly the set of API calls the chart's RBAC has to cover. +echo "==> Applying a MemgraphCluster in ${CLUSTER_NAMESPACE}" +"${KUBECTL}" create namespace "${CLUSTER_NAMESPACE}" +"${KUBECTL}" apply -f - < Waiting for the operator to provision the workloads" +for object in \ + statefulset/chart-test-coordinator statefulset/chart-test-data \ + service/chart-test-coordinator service/chart-test-data; do + wait_for_object "${object}" +done + +echo "==> Waiting for the operator to report status on the MemgraphCluster" +"${KUBECTL}" wait --for=condition=Converged=false --timeout=2m \ + memgraphcluster/chart-test -n "${CLUSTER_NAMESPACE}" + +# Any authorization failure means the generated ClusterRole is missing +# something the controller actually issues. +echo "==> Asserting the operator hit no authorization error" +logs=$("${KUBECTL}" logs -n "${CHART_NAMESPACE}" \ + "deployment/${CHART_RELEASE}-controller-manager" --tail=-1) +if grep -qi "is forbidden" <<<"${logs}"; then + echo "FAIL: the operator was denied an API call under the chart's RBAC:" >&2 + grep -i "is forbidden" <<<"${logs}" >&2 + exit 1 +fi + +echo "==> helm uninstall ${CHART_RELEASE}" +"${KUBECTL}" delete memgraphcluster chart-test -n "${CLUSTER_NAMESPACE}" --timeout=2m +"${HELM}" uninstall "${CHART_RELEASE}" --namespace "${CHART_NAMESPACE}" --wait + +# The release's own objects are gone when uninstall returns; the controller pod +# follows once it finishes terminating. +for _ in $(seq 1 30); do + remaining=$("${KUBECTL}" get all -n "${CHART_NAMESPACE}" \ + --no-headers --ignore-not-found | wc -l) + [ "${remaining}" -eq 0 ] && break + sleep 2 +done +if [ "${remaining}" -ne 0 ]; then + echo "FAIL: helm uninstall left objects behind in ${CHART_NAMESPACE}:" >&2 + "${KUBECTL}" get all -n "${CHART_NAMESPACE}" >&2 + exit 1 +fi + +# Helm leaves CRDs behind by design, so the documented uninstall removes them +# explicitly. Both halves are asserted: still present after uninstall, gone +# after the delete. +"${KUBECTL}" get crd "${CRD_NAME}" >/dev/null +"${KUBECTL}" delete -f "${CHART_DIR}/crds" +if "${KUBECTL}" get crd "${CRD_NAME}" >/dev/null 2>&1; then + echo "FAIL: ${CRD_NAME} survived the CRD deletion" >&2 + exit 1 +fi + +echo "==> Chart install/uninstall test passed" diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 4794ffa..8621020 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -75,11 +75,20 @@ type MemgraphClusterReconciler struct { Memgraph memgraph.Connector } -// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/status,verbs=get;update;patch +// The install chart's ClusterRole is generated from these markers, so they are +// the operator's permission surface: nothing broader is ever granted. The +// verbs are only the ones this reconciler issues — it reads MemgraphClusters +// and patches their status, and server-side-applies (create plus patch) the +// workloads without ever updating or deleting them, because deletion belongs +// to garbage collection via the owner references. The finalizers subresource +// is needed to set those owner references: they block owner deletion, which +// clusters running the OwnerReferencesPermissionEnforcement admission plugin +// only allow with update access to the owner's finalizers. +// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters,verbs=get;list;watch +// +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/status,verbs=get;patch // +kubebuilder:rbac:groups=memgraph.com,resources=memgraphclusters/finalizers,verbs=update -// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;patch +// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;patch // Reconcile drives the cluster toward the declared MemgraphCluster spec in // two stages. First it server-side-applies the builders' desired objects: one diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ec5d6bd..c8f0719 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -31,13 +31,19 @@ import ( "github.com/memgraph/kubernetes-operator/test/utils" ) -var ( +const ( + // managerImageRepository and managerImageTag make up the manager image that + // is built, loaded into Kind, and installed through the chart's image values. + managerImageRepository = "example.com/kubernetes-operator" + managerImageTag = "v0.0.1" + // managerImage is the manager image to be built and loaded for testing. - managerImage = "example.com/kubernetes-operator:v0.0.1" - // shouldCleanupCertManager tracks whether CertManager was installed by this suite. - shouldCleanupCertManager = false + managerImage = managerImageRepository + ":" + managerImageTag ) +// shouldCleanupCertManager tracks whether CertManager was installed by this suite. +var shouldCleanupCertManager = false + // TestE2E runs the e2e test suite to validate the solution in an isolated environment. // The default setup requires Kind and CertManager. // @@ -51,10 +57,13 @@ func TestE2E(t *testing.T) { } // The suite deploys the operator once, before any scenario runs: build and -// load the manager image, install the CRDs, and deploy the controller into its -// namespace. Scenario containers (Describe blocks) then only exercise -// MemgraphCluster behavior, so a new scenario is a new test case, never new -// pipeline or deployment plumbing. +// load the manager image, then helm install the local chart into a namespace +// that enforces the restricted Pod Security Standard. The install path is the +// users' install path — the same chart, the same CRDs, the same +// least-privilege RBAC — so every scenario below runs against exactly the +// permissions a real installation grants. Scenario containers (Describe +// blocks) then only exercise MemgraphCluster behavior, so a new scenario is a +// new test case, never new pipeline or deployment plumbing. var _ = BeforeSuite(func() { By("building the manager image") cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) @@ -81,24 +90,26 @@ var _ = BeforeSuite(func() { _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", managerImage)) + By("installing the operator from the local install chart") + cmd = exec.Command("helm", "install", releaseName, chartDir, + "--namespace", namespace, + "--set-string", "image.repository="+managerImageRepository, + "--set-string", "image.tag="+managerImageTag, + "--wait", "--timeout", "5m", + ) _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") + Expect(err).NotTo(HaveOccurred(), "Failed to install the operator chart") }) var _ = AfterSuite(func() { - By("undeploying the controller-manager") - cmd := exec.Command("make", "undeploy") + By("uninstalling the operator chart") + cmd := exec.Command("helm", "uninstall", releaseName, "--namespace", namespace, "--wait") _, _ = utils.Run(cmd) + // Helm never removes CRDs it installed, so the documented uninstall ends + // with deleting them explicitly. By("uninstalling CRDs") - cmd = exec.Command("make", "uninstall") + cmd = exec.Command("kubectl", "delete", "-f", chartDir+"/crds", "--ignore-not-found=true") _, _ = utils.Run(cmd) By("removing manager namespace") diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index dbd3573..28764b6 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -33,17 +33,29 @@ import ( "github.com/memgraph/kubernetes-operator/test/utils" ) -// namespace where the project is deployed in -const namespace = "kubernetes-operator-system" +// namespace where the operator is installed +const namespace = "memgraph-operator-system" -// serviceAccountName created for the project -const serviceAccountName = "kubernetes-operator-controller-manager" +// chartDir is the local install chart the suite installs, and releaseName the +// Helm release it installs it as. Every name below follows from that release +// name, exactly as it would for a user's installation. +const chartDir = "charts/memgraph-operator" +const releaseName = "memgraph-operator" -// metricsServiceName is the name of the metrics service of the project -const metricsServiceName = "kubernetes-operator-controller-manager-metrics-service" +// controllerDeploymentName is the name of the operator Deployment the chart creates +const controllerDeploymentName = releaseName + "-controller-manager" + +// serviceAccountName created for the operator +const serviceAccountName = releaseName + "-controller-manager" + +// metricsServiceName is the name of the metrics service of the operator +const metricsServiceName = releaseName + "-metrics-service" + +// metricsReaderRoleName is the ClusterRole the chart creates for scraping metrics +const metricsReaderRoleName = releaseName + "-metrics-reader" // metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data -const metricsRoleBindingName = "kubernetes-operator-metrics-binding" +const metricsRoleBindingName = releaseName + "-metrics-binding" var _ = Describe("Manager", Ordered, func() { var controllerPodName string @@ -141,7 +153,7 @@ var _ = Describe("Manager", Ordered, func() { It("should ensure the metrics endpoint is serving metrics", func() { By("creating a ClusterRoleBinding for the service account to allow access to metrics") cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, - "--clusterrole=kubernetes-operator-metrics-reader", + "--clusterrole="+metricsReaderRoleName, fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), ) _, err := utils.Run(cmd) diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index b945d56..8f04e8c 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -137,7 +137,7 @@ var _ = Describe("MemgraphCluster", Ordered, func() { {"get", "pods", "-n", clusterNamespace, "-o", "wide"}, {"get", "memgraphclusters", "-n", clusterNamespace, "-o", "yaml"}, {"get", "events", "-n", clusterNamespace, "--sort-by=.lastTimestamp"}, - {"logs", "deploy/kubernetes-operator-controller-manager", "-n", namespace}, + {"logs", "deploy/" + controllerDeploymentName, "-n", namespace}, } { cmd := exec.Command("kubectl", args...) output, err := utils.Run(cmd) From 0785c33d9bd5daa457a88557362ad49e18e871a3 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 14:51:34 +0200 Subject: [PATCH 14/34] =?UTF-8?q?feat:=20release=20pipeline=20=E2=80=94=20?= =?UTF-8?q?operator=20image,=20chart=20cross-published=20to=20helm-charts?= =?UTF-8?q?=20(#30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: release pipeline — operator image, chart cross-published to helm-charts Pushing a version tag now produces installable artifacts: the multi-arch operator image on Docker Hub, and the install chart packaged into the existing memgraph.github.io/helm-charts index, so users install from the helm repository they already have configured. Chart source and CRDs stay here. The chart's version and its appVersion move independently, the Helm-standard arrangement, so a chart-only fix does not force a redundant operator release. A tag therefore names the artifact it releases: v the operator (and the chart alongside it), chart- the chart alone, with no image build. The workflow refuses a tag whose version the tagged tree does not declare. Issue 11's fourth acceptance criterion is amended accordingly. Publishing writes into another repository, so hack/chart-publish.sh is exercised offline on every pull request by hack/chart-publish-test.sh, against a bare-git stand-in for helm-charts and a gh stub: dry runs publish nothing, publishing cuts the release and indexes it, re-publishing is a no-op, and prereleases are marked as such. Every publishing step skips when its result already exists, so re-running a half-finished release completes it rather than duplicating it — including the image, which is reused only when its revision label shows this same commit built it. After publishing, the run installs the chart from the public index on Kind and asserts the running Deployment is the image just released. A dry run from the Actions tab does the same against the local package with a Kind-loaded image, publishing nothing; a SemVer prerelease tag rehearses the real thing while staying hidden from helm install. Releasing needs HELM_CHARTS_TOKEN, a fine-grained PAT scoped to contents write on memgraph/helm-charts; docs/releasing.md covers its scope and rotation. The run checks the credentials exist before it builds anything. * testing: disable the Install chart job on pull requests It boots a Kind cluster on every pull request. Its chart-version-check step is also premature until the first release: it enforces a version bump against a chart version that has never been published, so any change to the chart trips it. Removing the 'if: false' brings the job back. --- .github/workflows/release.yml | 571 ++++++++++++++++++ .github/workflows/test.yml | 19 + .gitignore | 3 + CLAUDE.md | 6 +- Makefile | 34 ++ README.md | 23 +- charts/memgraph-operator/README.md | 16 +- docs/releasing.md | 200 ++++++ hack/chart-publish-test.sh | 173 ++++++ hack/chart-publish.sh | 193 ++++++ hack/chart-version-check.sh | 71 +++ .../issues/11-release-cross-publish.md | 24 +- 12 files changed, 1319 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 docs/releasing.md create mode 100755 hack/chart-publish-test.sh create mode 100755 hack/chart-publish.sh create mode 100755 hack/chart-version-check.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..e7dc7f6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,571 @@ +name: Release + +# The path from a version tag to installable artifacts: the operator image on +# Docker Hub, and the install chart in the memgraph.github.io/helm-charts index +# users already have configured. +# +# The chart version and the operator version move independently, so a tag names +# which of the two it releases: +# +# v0.2.0 an operator release -- builds and publishes the image, and +# publishes the chart alongside it. Must equal the appVersion. +# chart-0.4.2 a chart-only release -- no image is built, and the chart keeps +# pointing at the operator its appVersion already names. Must +# equal the chart version. +# +# Running this from the Actions tab instead is a dry run: everything is built, +# packaged and installed on Kind, and nothing is published. Tagging a SemVer +# prerelease (v0.2.0-rc.1) is the other rehearsal -- it publishes for real, but +# helm hides prereleases from `helm install` unless --devel is passed, so the +# public index is exercised without being disturbed. +# +# See docs/releasing.md for the procedure and the credentials this needs. + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + - 'v[0-9]+.[0-9]+.[0-9]+-*' + - 'chart-[0-9]+.[0-9]+.[0-9]+' + - 'chart-[0-9]+.[0-9]+.[0-9]+-*' + workflow_dispatch: + inputs: + publish: + description: 'Publish for real, instead of dry-running the pipeline.' + type: boolean + default: false + +permissions: {} + +# Releases are never cancelled halfway: a run stopped between pushing the image +# and indexing the chart leaves the two out of step. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +env: + IMAGE_REPOSITORY: docker.io/memgraph/kubernetes-operator + CHARTS_REPO: memgraph/helm-charts + CHARTS_INDEX_URL: https://memgraph.github.io/helm-charts + CHART_NAME: memgraph-operator + CHART_NAMESPACE: memgraph-operator-system + +jobs: + resolve: + name: Resolve the release + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + kind: ${{ steps.resolve.outputs.kind }} + chart_version: ${{ steps.resolve.outputs.chart_version }} + app_version: ${{ steps.resolve.outputs.app_version }} + version: ${{ steps.resolve.outputs.version }} + prerelease: ${{ steps.resolve.outputs.prerelease }} + publish: ${{ steps.resolve.outputs.publish }} + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + # The chart declares both versions; a tag may only name one of them, and + # has to agree with what the chart says. Publishing a version the tagged + # tree does not declare is the one way to produce artifacts nobody can + # trace back to a commit, so it fails here, before anything is built. + - name: Resolve the versions + id: resolve + env: + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_PUBLISH: ${{ inputs.publish }} + run: | + set -euo pipefail + chart_version=$(make -s chart-version) + app_version=$(make -s chart-app-version) + + if [ "${REF_TYPE}" = tag ]; then + publish=true + case "${REF_NAME}" in + chart-*) kind=chart; tag_version=${REF_NAME#chart-} ;; + *) kind=operator; tag_version=${REF_NAME#v} ;; + esac + else + # No tag to go on, so the run releases the operator the chart + # currently declares. Dry runs take this path. + publish=${INPUT_PUBLISH:-false} + kind=operator + tag_version="" + fi + + if [ "${kind}" = operator ]; then + version=${app_version} + declared="appVersion" + else + version=${chart_version} + declared="version" + fi + + if [ -n "${tag_version}" ] && [ "${tag_version}" != "${version}" ]; then + echo "::error::Tag ${REF_NAME} releases the ${kind}, but the chart's ${declared} is ${version}, not ${tag_version}. Bump the chart on a pull request first, then tag the merge commit." + exit 1 + fi + + case "${version}" in + *-*) prerelease=true ;; + *) prerelease=false ;; + esac + + { + echo "kind=${kind}" + echo "chart_version=${chart_version}" + echo "app_version=${app_version}" + echo "version=${version}" + echo "prerelease=${prerelease}" + echo "publish=${publish}" + } >>"${GITHUB_OUTPUT}" + + { + echo "### ${kind} release ${version}" + echo + echo "| | |" + echo "|---|---|" + echo "| chart version | \`${chart_version}\` |" + echo "| operator version (appVersion) | \`${app_version}\` |" + echo "| prerelease | \`${prerelease}\` |" + echo "| publishing | \`${publish}\` |" + } >>"${GITHUB_STEP_SUMMARY}" + + # Failing here beats failing after the image is already on Docker Hub. + - name: Check the release credentials are present + if: steps.resolve.outputs.publish == 'true' + env: + KIND: ${{ steps.resolve.outputs.kind }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + HELM_CHARTS_TOKEN: ${{ secrets.HELM_CHARTS_TOKEN }} + run: | + set -euo pipefail + missing=() + [ -n "${HELM_CHARTS_TOKEN}" ] || missing+=(HELM_CHARTS_TOKEN) + if [ "${KIND}" = operator ]; then + [ -n "${DOCKERHUB_USERNAME}" ] || missing+=(DOCKERHUB_USERNAME) + [ -n "${DOCKERHUB_TOKEN}" ] || missing+=(DOCKERHUB_TOKEN) + fi + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::Missing repository secrets: ${missing[*]}. See docs/releasing.md for what they are and how to create them." + exit 1 + fi + + image: + name: Operator image + needs: resolve + if: needs.resolve.outputs.kind == 'operator' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Published tags are immutable, by convention and by the trust users place + # in them -- including the 0.0.x and 1.0.0 tags in this repository, which + # belong to the operator attempt archived on archive/pre-operator-mvp. + # + # An image built from this very commit is the exception: it is this same + # release's earlier attempt, from a run that failed after the push. That + # one is left alone and the build skipped, so re-running finishes the + # release instead of being stopped by its own progress. The revision + # label is what tells the two apart. + - name: Check whether the image tag is already published + id: existing + if: needs.resolve.outputs.publish == 'true' + env: + IMAGE: ${{ env.IMAGE_REPOSITORY }}:${{ needs.resolve.outputs.version }} + SHA: ${{ github.sha }} + run: | + set -euo pipefail + if ! docker manifest inspect "${IMAGE}" >/dev/null 2>&1; then + echo "published=false" >>"${GITHUB_OUTPUT}" + echo "${IMAGE} is free." + exit 0 + fi + + # Single-platform images inspect as a config object; multi-platform + # ones as a map keyed by platform. + revision=$(docker buildx imagetools inspect "${IMAGE}" --format '{{ json .Image }}' 2>/dev/null | + jq -r 'if has("config") then . else (to_entries | .[0].value) end + | .config.Labels["org.opencontainers.image.revision"] // empty' 2>/dev/null || true) + + if [ "${revision}" = "${SHA}" ]; then + echo "published=true" >>"${GITHUB_OUTPUT}" + echo "::notice::${IMAGE} was already pushed from ${SHA}; skipping the build and carrying on." + exit 0 + fi + + echo "::error::${IMAGE} already exists and was not built from ${SHA}. Bump the chart's appVersion, or release the chart alone with a chart- tag." + exit 1 + + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Log in to Docker Hub + if: needs.resolve.outputs.publish == 'true' + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + # latest follows stable releases only, so a release candidate never + # becomes what an unqualified pull resolves to. + - name: Work out the image tags + id: tags + env: + VERSION: ${{ needs.resolve.outputs.version }} + PRERELEASE: ${{ needs.resolve.outputs.prerelease }} + run: | + set -euo pipefail + { + echo "tags<>"${GITHUB_OUTPUT}" + + - name: Build and push the operator image + if: needs.resolve.outputs.publish == 'true' && steps.existing.outputs.published != 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags.outputs.tags }} + labels: | + org.opencontainers.image.title=memgraph-operator + org.opencontainers.image.description=Kubernetes operator for Memgraph high-availability clusters + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ needs.resolve.outputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # A dry run still builds both architectures, because a broken arm64 build + # is exactly the kind of thing a rehearsal is for. + - name: Build the operator image without pushing + if: needs.resolve.outputs.publish != 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: false + tags: ${{ steps.tags.outputs.tags }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # ... and then exports the amd64 image alone, so the verify job can load + # it into Kind and install the chart with its own default image tag. The + # layers are already cached by the build above. + - name: Export the image for the verify job + if: needs.resolve.outputs.publish != 'true' + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + platforms: linux/amd64 + tags: ${{ env.IMAGE_REPOSITORY }}:${{ needs.resolve.outputs.version }} + outputs: type=docker,dest=/tmp/operator-image.tar + cache-from: type=gha + + - name: Upload the image + if: needs.resolve.outputs.publish != 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: operator-image + path: /tmp/operator-image.tar + retention-days: 1 + + chart: + name: Install chart + needs: [resolve, image] + # The image job is skipped for a chart-only release; the chart still ships. + if: always() && needs.resolve.result == 'success' && needs.image.result != 'failure' && needs.image.result != 'cancelled' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # The chart-only path compares the generated manifests against the + # tag its appVersion names, which needs the tags and their history. + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + # The chart ships CRDs and RBAC rules generated from the Go sources. At a + # release this is not a style check: it is what makes the chart's CRD the + # one this operator's types describe. + - name: Verify the generated chart manifests + run: make chart-verify + + - name: Lint the chart + run: make helm-lint + + - name: Test the cross-publish path + run: make test-chart-publish + + # A chart is only installable if the operator image its appVersion names + # is actually on Docker Hub. For an operator release the image job just + # pushed it; for a chart-only release it has to already be there. + - name: Check the operator image the chart installs exists + if: needs.resolve.outputs.kind == 'chart' && needs.resolve.outputs.publish == 'true' + env: + IMAGE: ${{ env.IMAGE_REPOSITORY }}:${{ needs.resolve.outputs.app_version }} + run: | + set -euo pipefail + if ! docker manifest inspect "${IMAGE}" >/dev/null 2>&1; then + echo "::error::The chart's appVersion names ${IMAGE}, which does not exist. Release the operator first with a v tag." + exit 1 + fi + echo "${IMAGE} exists." + + # The chart's CRDs come from the commit being released, its appVersion + # from whenever the operator was last released. Nothing forces those to + # be the same commit, so a chart-only release can ship a CRD the operator + # it installs has never seen. This reports it rather than refusing it -- + # the two versions are deliberately independent -- but it is worth a look + # before the release goes out. + - name: Report CRD drift against the operator being installed + if: needs.resolve.outputs.kind == 'chart' + env: + APP_VERSION: ${{ needs.resolve.outputs.app_version }} + run: | + set -euo pipefail + tag="v${APP_VERSION}" + if ! git rev-parse --verify --quiet "${tag}^{commit}" >/dev/null; then + echo "No ${tag} tag to compare against; skipping." >>"${GITHUB_STEP_SUMMARY}" + exit 0 + fi + if git diff --quiet "${tag}" -- charts/memgraph-operator/crds; then + echo "CRDs are unchanged since operator ${APP_VERSION}." >>"${GITHUB_STEP_SUMMARY}" + else + echo "::warning::The chart's CRDs differ from those released with operator ${APP_VERSION}, which is the operator this chart installs." + { + echo + echo "> **CRD drift**: this chart ships CRDs that changed since \`${tag}\`," + echo "> but installs operator \`${APP_VERSION}\`. Check the operator understands them." + echo + echo '```' + git diff --stat "${tag}" -- charts/memgraph-operator/crds + echo '```' + } >>"${GITHUB_STEP_SUMMARY}" + fi + + - name: Package the chart + if: needs.resolve.outputs.publish != 'true' + env: + VERSION: ${{ needs.resolve.outputs.chart_version }} + run: ./hack/chart-publish.sh --version "${VERSION}" + + - name: Package the chart and publish it to the Memgraph helm repository + if: needs.resolve.outputs.publish == 'true' + env: + VERSION: ${{ needs.resolve.outputs.chart_version }} + # Scoped to contents write on the charts repository only; see + # docs/releasing.md for its scope and rotation. + GH_TOKEN: ${{ secrets.HELM_CHARTS_TOKEN }} + run: | + set -euo pipefail + # Lets git push to the charts repository with the same token, without + # it ever appearing in a URL or a log line. + gh auth setup-git + ./hack/chart-publish.sh --version "${VERSION}" --publish + + - name: Upload the packaged chart + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: chart-package + path: dist/chart/*.tgz + retention-days: 7 + + verify: + name: Verify the install + needs: [resolve, image, chart] + if: always() && needs.chart.result == 'success' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version-file: go.mod + + - name: Install Kind + run: go install sigs.k8s.io/kind@v0.32.0 + + - name: Create a Kind cluster + run: kind create cluster --name release-verify + + # The published path: install exactly as a user would, from the helm + # repository they already have configured, pulling the image that was + # just pushed. GitHub Pages takes a minute or two to serve a new index. + - name: Wait for the chart to appear in the Memgraph helm repository + if: needs.resolve.outputs.publish == 'true' + env: + VERSION: ${{ needs.resolve.outputs.chart_version }} + run: | + set -euo pipefail + helm repo add memgraph "${CHARTS_INDEX_URL}" + for attempt in $(seq 1 30); do + helm repo update memgraph >/dev/null + if helm search repo "memgraph/${CHART_NAME}" --version "${VERSION}" --devel | + grep -q "${VERSION}"; then + echo "The index serves ${CHART_NAME} ${VERSION} (attempt ${attempt})." + exit 0 + fi + sleep 20 + done + echo "::error::${CHARTS_INDEX_URL} did not serve ${CHART_NAME} ${VERSION} within 10 minutes." + exit 1 + + # The dry-run path: the image was never pushed, so it is loaded into Kind + # under the exact tag the chart's appVersion resolves to. The chart is + # then installed with its own defaults, image reference included. + - name: Download the locally built image + if: needs.resolve.outputs.publish != 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: operator-image + path: /tmp + + - name: Load the image into Kind + if: needs.resolve.outputs.publish != 'true' + run: | + set -euo pipefail + docker load --input /tmp/operator-image.tar + kind load docker-image "${IMAGE_REPOSITORY}:${{ needs.resolve.outputs.app_version }}" \ + --name release-verify + + - name: Download the packaged chart + if: needs.resolve.outputs.publish != 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: chart-package + path: dist/chart + + - name: Install the operator + env: + PUBLISH: ${{ needs.resolve.outputs.publish }} + VERSION: ${{ needs.resolve.outputs.chart_version }} + run: | + set -euo pipefail + if [ "${PUBLISH}" = true ]; then + chart="memgraph/${CHART_NAME}" + set -- --version "${VERSION}" --devel + else + chart="dist/chart/${CHART_NAME}-${VERSION}.tgz" + set -- + fi + helm install "${CHART_NAME}" "${chart}" "$@" \ + --namespace "${CHART_NAMESPACE}" --create-namespace \ + --wait --timeout 5m + + # What the release actually promises: installing chart runs the + # operator its appVersion names. A template bug would satisfy every string + # comparison made so far and still fail here. + - name: Check the running operator is the released image + env: + EXPECTED: ${{ env.IMAGE_REPOSITORY }}:${{ needs.resolve.outputs.app_version }} + run: | + set -euo pipefail + actual=$(kubectl get deployment "${CHART_NAME}-controller-manager" \ + --namespace "${CHART_NAMESPACE}" \ + -o jsonpath='{.spec.template.spec.containers[?(@.name=="manager")].image}') + if [ "${actual}" != "${EXPECTED}" ]; then + echo "::error::The chart installed ${actual}, expected ${EXPECTED}." + exit 1 + fi + kubectl get crd memgraphclusters.memgraph.com >/dev/null + echo "Installed ${actual}, with the MemgraphCluster CRD." + + - name: Uninstall + if: always() + run: | + helm uninstall "${CHART_NAME}" --namespace "${CHART_NAMESPACE}" --ignore-not-found + kind delete cluster --name release-verify + + github-release: + name: GitHub release + needs: [resolve, chart, verify] + if: needs.resolve.outputs.publish == 'true' && github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + # Creating the release for this tag. + contents: write + steps: + - name: Clone the code + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download the packaged chart + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: chart-package + path: dist/chart + + - name: Create the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: ${{ github.ref_name }} + KIND: ${{ needs.resolve.outputs.kind }} + CHART_VERSION: ${{ needs.resolve.outputs.chart_version }} + APP_VERSION: ${{ needs.resolve.outputs.app_version }} + PRERELEASE: ${{ needs.resolve.outputs.prerelease }} + run: | + set -euo pipefail + notes=$(cat < --bin-dir bin -p path)` -CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, `make test`, `make chart-verify`, `make helm-lint`, `make test-chart`, and `make test-e2e` on every PR — all must be green. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license flowing from the `MEMGRAPH_ENTERPRISE_LICENSE` / `MEMGRAPH_ORGANIZATION_NAME` repository secrets (set the same env vars to run it locally). +CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit`, `make test`, `make chart-verify`, `make helm-lint`, `make chart-version-check`, `make test-chart-publish`, `make test-chart`, and `make test-e2e` on every PR — all must be green. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license flowing from the `MEMGRAPH_ENTERPRISE_LICENSE` / `MEMGRAPH_ORGANIZATION_NAME` repository secrets (set the same env vars to run it locally). ### Toolchain quirks (do not "fix" these) @@ -52,7 +54,7 @@ The PRD defines seven modules with two pure cores and one mock seam. Keep this s 3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. 4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. `SET INSTANCE TO MAIN` is issued exactly once at bootstrap (when no MAIN exists); after that, failover belongs to the Raft coordinators — the operator only observes. 5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. -6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. +6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. The chart's `version` and its `appVersion` (the operator image tag) move **independently**: tag `v` releases the operator, `chart-` releases the chart alone — see `docs/releasing.md`. 7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. Test philosophy (from the PRD): assert external behavior, never internal call ordering or private state. Builders get golden tests, planner gets pure topology-diff cases, controller gets envtest with the HA client mocked. diff --git a/Makefile b/Makefile index 1f7c25e..09df2f9 100644 --- a/Makefile +++ b/Makefile @@ -248,6 +248,40 @@ helm-uninstall: ## Uninstall the operator, including the CRDs helm leaves behind "$(HELM)" uninstall "$(CHART_RELEASE)" --namespace "$(CHART_NAMESPACE)" --ignore-not-found "$(KUBECTL)" delete --ignore-not-found=true -f "$(CHART_DIR)/crds" +##@ Release + +# Two versions, moving independently: the chart's own version, and the +# appVersion naming the operator image it installs by default. A tag releases +# one of them -- v the operator, chart- the chart alone -- +# and the release workflow refuses a tag the chart does not declare. +# See docs/releasing.md. +CHART_VERSION = $(shell sed -n 's/^version:[[:space:]]*//p' $(CHART_DIR)/Chart.yaml | tr -d '"' | head -1) +CHART_APP_VERSION = $(shell sed -n 's/^appVersion:[[:space:]]*//p' $(CHART_DIR)/Chart.yaml | tr -d '"' | head -1) + +.PHONY: chart-version +chart-version: ## Print the chart version. + @echo "$(CHART_VERSION)" + +.PHONY: chart-app-version +chart-app-version: ## Print the operator version the chart installs (its appVersion). + @echo "$(CHART_APP_VERSION)" + +.PHONY: chart-version-check +chart-version-check: ## Fail if the chart changed without its version being bumped (against BASE_REF). + @./hack/chart-version-check.sh + +.PHONY: chart-package +chart-package: chart-verify ## Package the install chart into dist/chart, publishing nothing. + HELM=$(HELM) ./hack/chart-publish.sh + +.PHONY: chart-publish +chart-publish: chart-verify ## Publish the packaged chart into the Memgraph helm repository. Needs GH_TOKEN. + HELM=$(HELM) ./hack/chart-publish.sh --publish + +.PHONY: test-chart-publish +test-chart-publish: ## Test the cross-publish path against a local stand-in for the helm-charts repository. + HELM=$(HELM) ./hack/chart-publish-test.sh + ##@ Deployment ifndef ignore-not-found diff --git a/README.md b/README.md index 5ebec43..39a2510 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,19 @@ The operator replaces the `memgraph-high-availability` Helm chart's fire-and-for ### To install the operator -The install chart in [`charts/memgraph-operator`](charts/memgraph-operator/README.md) is the -complete install story — it ships the `MemgraphCluster` CRD, a least-privilege RBAC set, and the -controller Deployment: +The install chart is published to the Memgraph helm repository — the same one the Memgraph charts +come from. It ships the `MemgraphCluster` CRD, a least-privilege RBAC set, and the controller +Deployment: + +```sh +helm repo add memgraph https://memgraph.github.io/helm-charts +helm repo update +helm install memgraph-operator memgraph/memgraph-operator \ + --namespace memgraph-operator-system --create-namespace --wait +``` + +The chart source lives in this repository, in +[`charts/memgraph-operator`](charts/memgraph-operator/README.md), and installs from there too: ```sh helm install memgraph-operator ./charts/memgraph-operator \ @@ -126,9 +136,12 @@ make helm-lint # lint the chart and render it with defaults and with the t make test-chart # install/uninstall the chart on a throwaway Kind cluster ``` -Releases cross-publish the packaged chart into the existing +Pushing a version tag cross-publishes the packaged chart into the existing [`memgraph.github.io/helm-charts`](https://memgraph.github.io/helm-charts) index, so users install -it from the helm repository they already have configured. +it from the helm repository they already have configured. The chart version and the operator +version move independently — `v0.2.0` releases the operator, `chart-0.4.2` releases the chart +alone. See [`docs/releasing.md`](docs/releasing.md) for the procedure, the dry-run and prerelease +paths, and the credentials involved. ## Contributing diff --git a/charts/memgraph-operator/README.md b/charts/memgraph-operator/README.md index 16c2a49..1f7c0c5 100644 --- a/charts/memgraph-operator/README.md +++ b/charts/memgraph-operator/README.md @@ -13,12 +13,26 @@ they ship with. ## Install ```sh -helm install memgraph-operator ./charts/memgraph-operator \ +helm repo add memgraph https://memgraph.github.io/helm-charts +helm repo update +helm install memgraph-operator memgraph/memgraph-operator \ --namespace memgraph-operator-system --create-namespace --wait ``` The operator watches every namespace, so one release per cluster is enough. +Releases cross-publish the packaged chart into that index; the source stays here. `helm search +repo memgraph/memgraph-operator --versions` lists what is available, and the chart's `appVersion` +is the operator version an install runs by default. The two version numbers move independently — +see [`docs/releasing.md`](../../docs/releasing.md). + +Installing from a checkout works the same way: + +```sh +helm install memgraph-operator ./charts/memgraph-operator \ + --namespace memgraph-operator-system --create-namespace --wait +``` + ## Uninstall ```sh diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..151f84c --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,200 @@ +# Releasing + +A release turns a git tag into two artifacts: + +- the operator image, `docker.io/memgraph/kubernetes-operator:`, and +- the install chart, packaged and published into the + [`memgraph.github.io/helm-charts`](https://memgraph.github.io/helm-charts) index — the same + helm repository users already have configured for the Memgraph charts. + +Chart source, CRDs and RBAC stay in this repository. Only the packaged `.tgz` and its index entry +land in `memgraph/helm-charts`, as a GitHub release named `memgraph-operator-` plus a +line in `index.yaml` on that repository's `gh-pages` branch — the same layout chart-releaser +produces for the charts maintained there. + +## Two versions + +`charts/memgraph-operator/Chart.yaml` declares both, and they move independently: + +| Field | Names | Changes when | +| --- | --- | --- | +| `version` | the chart | anything under `charts/memgraph-operator` changes | +| `appVersion` | the operator image the chart installs by default | a new operator image is released | + +`appVersion` is what `deployment.yaml` uses when `image.tag` is left empty, so it is the operator +version a default install runs. + +Because the two are independent, a tag has to say which one it releases: + +| Tag | Releases | Must equal | +| --- | --- | --- | +| `v0.2.0` | the operator image, and the chart alongside it | `appVersion` | +| `chart-0.4.2` | the chart alone — no image is built | `version` | + +The workflow refuses a tag whose version the tagged tree does not declare, before anything is +built. Bump the chart on a pull request, then tag the merge commit. + +CI also refuses a pull request that changes the chart without bumping its `version`: a published +chart version is immutable, so a change that keeps its version is a change users never receive. + +### The one thing to watch + +The chart's `crds/` are generated from the Go types in the commit being released, but its +`appVersion` points at whenever the operator was last released. Nothing forces those to be the +same commit. A chart-only release cut after the API has moved on therefore ships a CRD with +fields the operator it installs has never seen. + +The release run reports this rather than refusing it — the versions are deliberately independent +— as a warning in the job summary listing what changed. When it fires, either release the +operator too (`v`, which realigns both) or cut the chart release from the commit its +`appVersion` names. + +## Releasing the operator + +1. Open a pull request bumping `appVersion` — and `version`, since the chart changed — in + `charts/memgraph-operator/Chart.yaml`: + + ```sh + $ make -s chart-version chart-app-version + ``` + +2. Merge it once CI is green. +3. Tag the merge commit and push: + + ```sh + git switch main && git pull + git tag v0.2.0 && git push origin v0.2.0 + ``` + +The [Release workflow](../.github/workflows/release.yml) then: + +- refuses to overwrite an existing image tag, then builds and pushes `linux/amd64` and + `linux/arm64` — plus `:latest`, for stable versions only; +- verifies the chart's generated CRDs and RBAC still match the Go sources, lints it, packages it, + cuts the `memgraph-operator-` release in `memgraph/helm-charts` and merges its entry + into the index; +- installs the published chart from `https://memgraph.github.io/helm-charts` on a Kind cluster + and asserts the running Deployment is the image just released; +- creates the GitHub release here, with the packaged chart attached. + +## Releasing the chart alone + +For a template fix, a new values knob, or chart documentation — no operator change: + +1. Bump only `version` in `Chart.yaml` on a pull request, and merge. +2. Tag with the `chart-` prefix: + + ```sh + git tag chart-0.4.2 && git push origin chart-0.4.2 + ``` + +No image is built. The run checks that the image named by `appVersion` exists on Docker Hub +before publishing, so the chart can never point at an operator that was never released. + +## Rehearsing a release + +Two ways, for different questions. + +**Dry run** — *does the pipeline work?* Run the workflow from the Actions tab with **publish** +off. Everything happens except publication: both architectures are built, the chart is packaged, +the index merge is computed and printed as a diff, and the chart is installed on Kind from the +local package with the locally built image. Nothing reaches Docker Hub or the index. + +**Prerelease** — *does publishing work?* Declare the prerelease version in `Chart.yaml` like any +other (`appVersion: "0.2.0-rc.1"`, or `version: 0.4.2-rc.1` for a chart-only one), then tag it: +`v0.2.0-rc.1`, `chart-0.4.2-rc.1`. The tag still has to name what the chart declares, so the +candidate is a commit like any other release. This publishes for real, against the real index, +but: + +- helm hides prerelease versions from `helm search` and `helm install` unless `--devel` is + passed, so nobody installs one by accident; +- the image is not tagged `:latest`; +- the GitHub releases are marked as prereleases. + +The public index gains an entry that ordinary use never sees. Prereleases are the only way to +exercise the credentials, the cross-repo push and GitHub Pages' propagation before a real +release depends on them. + +## Credentials + +Three repository secrets on `memgraph/kubernetes-operator`. The run checks all of them are +present before it builds anything. + +| Secret | Used for | +| --- | --- | +| `DOCKERHUB_USERNAME` | pushing the operator image | +| `DOCKERHUB_TOKEN` | pushing the operator image | +| `HELM_CHARTS_TOKEN` | creating the release in `memgraph/helm-charts` and pushing its index | + +`DOCKERHUB_USERNAME` / `DOCKERHUB_TOKEN` are the same pair the other Memgraph repositories use to +publish images, and are typically inherited from the organization. + +### `HELM_CHARTS_TOKEN` + +A **fine-grained personal access token**, scoped to nothing but the charts repository: + +- **Resource owner**: `memgraph` +- **Repository access**: *Only select repositories* → `memgraph/helm-charts` +- **Repository permissions**: **Contents: Read and write** (this covers both creating the release + and pushing `index.yaml` to `gh-pages`). Metadata read is added automatically. Nothing else. +- **Expiration**: 90 days. + +A deploy key is not enough on its own: it can push to `gh-pages`, but cannot create the GitHub +release the index entry points at. A token owned by a machine account is preferable to a personal +one — releases in `memgraph/helm-charts` are attributed to whoever owns it, and a personal token +dies with the person's access. + +Creating the token requires admin on `memgraph/helm-charts`, which is why this part of the +release setup cannot be automated from here. + +**Rotation**, before expiry or whenever someone with access leaves: + +1. Create the replacement with the scope above. +2. Update the `HELM_CHARTS_TOKEN` secret on `memgraph/kubernetes-operator`. +3. Verify with a prerelease tag — it is the only path that exercises the token end to end. +4. Delete the old token. + +An expired token fails the run at the credential check with the secret named, before anything is +published; nothing is left half-done. + +## When a release goes wrong + +Every publishing step is skipped when its result already exists, so **re-running a failed release +is safe and finishes the job** rather than duplicating it. If a run fails between pushing the +image and indexing the chart, re-run it and it will complete: + +- an image tag already pushed **from that same commit** is left alone and the build skipped — + recognised by its `org.opencontainers.image.revision` label; +- an image tag that exists but came from anywhere else stops the run, because that is someone + else's tag, not this release's; +- an existing charts release has the package re-attached; +- an index that already lists the version is left untouched. + +What cannot be undone is a published version number. Chart versions and image tags are immutable +by convention and by the trust users place in them — to fix a bad release, release the next +version. + +## Doing it by hand + +The pipeline is a thin wrapper around targets you can run locally: + +```sh +make chart-version # the chart version +make chart-app-version # the operator version it installs +make chart-version-check # chart changed => version bumped (against BASE_REF) +make chart-package # package into dist/chart, publish nothing +make test-chart-publish # exercise the whole publish path offline +GH_TOKEN=... make chart-publish # publish for real +``` + +`hack/chart-publish.sh` publishes nothing without `--publish`; with it, it prints the exact index +diff it is about to push. `make test-chart-publish` runs the whole path — package, release, +index merge, re-run — against a local stand-in for `memgraph/helm-charts`, offline, and CI runs +it on every pull request. + +## A note on the legacy image tags + +`docker.io/memgraph/kubernetes-operator` already carries tags `0.0.1` through `1.0.0`, published +by the earlier operator attempt now archived on `archive/pre-operator-mvp`. They are unrelated to +this operator. The release run refuses to overwrite any existing tag, so they cannot be clobbered +by accident — but it does mean the current `0.x` line sorts below them on Docker Hub. diff --git a/hack/chart-publish-test.sh b/hack/chart-publish-test.sh new file mode 100755 index 0000000..dfec725 --- /dev/null +++ b/hack/chart-publish-test.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# +# Test for the cross-publish path, run on every pull request. +# +# hack/chart-publish.sh writes into another repository, so the one thing that +# must not happen is discovering a mistake in it during a release. This runs it +# end to end against a local stand-in for memgraph/helm-charts -- a bare git +# repository holding a gh-pages branch with a realistic index.yaml, and a `gh` +# stub that records what it was asked to do -- and asserts on what came out: +# the index the charts repository ends up with, and the release that was cut. +# +# Offline and side-effect free: no network, nothing touched outside a temp +# directory. + +set -euo pipefail + +REPO_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +PUBLISH=${PUBLISH:-"${REPO_ROOT}/hack/chart-publish.sh"} +HELM=${HELM:-helm} + +WORK=$(mktemp -d) +trap 'rm -rf "${WORK}"' EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +# assert_indexed fails unless the published index lists exactly one entry for +# the version, pointing at the release asset that would serve it. +assert_indexed() { + local version=$1 index=$2 + local url="https://github.com/memgraph/helm-charts/releases/download/memgraph-operator-${version}/memgraph-operator-${version}.tgz" + local count + count=$(grep -cF "${url}" "${index}" || true) + [ "${count}" = "1" ] || + fail "expected exactly one index entry for ${version}, found ${count}" +} + +# published_index checks out the stand-in's gh-pages branch as the world sees it. +published_index() { + rm -rf "${WORK}/published" + git clone --quiet --branch gh-pages "${CHARTS_URL}" "${WORK}/published" + echo "${WORK}/published/index.yaml" +} + +echo "==> Building a stand-in for memgraph/helm-charts" +# file:// rather than a bare path, so the clone behaves like a remote one -- +# git ignores --depth on local clones. +CHARTS_URL="file://${WORK}/helm-charts.git" +git init --quiet --bare --initial-branch=gh-pages "${WORK}/helm-charts.git" +git init --quiet --initial-branch=gh-pages "${WORK}/seed" + +# An abridged copy of the real index: one entry for an unrelated chart, which +# the merge has to carry through untouched. +cat >"${WORK}/seed/index.yaml" <<'EOF' +apiVersion: v1 +entries: + memgraph: + - apiVersion: v2 + appVersion: 3.12.0 + created: "2026-07-22T11:28:55.491533439Z" + description: MemgraphDB Helm Chart + digest: bef59ed4e202c17d8f84f5262e819d7b96d3ddc2fc92835c949ca9c21aa25dcb + name: memgraph + type: application + urls: + - https://github.com/memgraph/helm-charts/releases/download/memgraph-1.0.5/memgraph-1.0.5.tgz + version: 1.0.5 +generated: "2026-07-22T11:28:55.490899258Z" +EOF + +git -C "${WORK}/seed" add index.yaml +git -C "${WORK}/seed" -c user.name=test -c user.email=test@example.com \ + commit --quiet -m "Seed the index" +git -C "${WORK}/seed" push --quiet "${WORK}/helm-charts.git" gh-pages + +echo "==> Installing a gh stub" +mkdir -p "${WORK}/bin" "${WORK}/releases" +cat >"${WORK}/bin/gh" <<'EOF' +#!/usr/bin/env bash +# Records every invocation, and answers `release view` from the releases it has +# been asked to create, so the script's resume path is exercised for real. +set -euo pipefail +# One invocation per line: release notes are multi-line, and an argument that +# broke the log into several lines would break every assertion on it. +{ printf '%s ' "$@" | tr '\n' ' '; printf '\n'; } >>"${GH_LOG}" +case "${1:-} ${2:-}" in + "release view") + [ -f "${GH_RELEASES}/$3" ] || exit 1 + ;; + "release create") + tag=$3 + asset=$4 + [ -f "${asset}" ] || { echo "gh stub: no such asset: ${asset}" >&2; exit 1; } + printf '%s\n' "$*" >"${GH_RELEASES}/${tag}" + ;; + "release upload") + [ -f "${GH_RELEASES}/$3" ] || { echo "gh stub: no such release: $3" >&2; exit 1; } + ;; +esac +exit 0 +EOF +chmod +x "${WORK}/bin/gh" + +export GH_LOG="${WORK}/gh.log" +export GH_RELEASES="${WORK}/releases" +: >"${GH_LOG}" + +run_publish() { + env \ + CHARTS_REPO_URL="${CHARTS_URL}" \ + OUT_DIR="${WORK}/dist" \ + GH="${WORK}/bin/gh" \ + GH_TOKEN=stub-token \ + HELM="${HELM}" \ + "${PUBLISH}" "$@" +} + +echo +echo "==> A dry run publishes nothing" +run_publish --version 9.9.9 >"${WORK}/dry-run.log" +[ -f "${WORK}/dist/memgraph-operator-9.9.9.tgz" ] || + fail "the dry run did not package the chart" +[ ! -s "${GH_LOG}" ] || + fail "the dry run called gh: $(cat "${GH_LOG}")" +if grep -q "9.9.9" "$(published_index)"; then + fail "the dry run pushed to the charts repository" +fi +grep -q "Dry run; nothing was published" "${WORK}/dry-run.log" || + fail "the dry run did not say so" + +echo +echo "==> Publishing a release cuts it and indexes it" +run_publish --version 9.9.9 --publish >/dev/null +grep -qF "release create memgraph-operator-9.9.9 " "${GH_LOG}" || + fail "no release was created: $(cat "${GH_LOG}")" +if grep -F "release create memgraph-operator-9.9.9 " "${GH_LOG}" | grep -q -- "--prerelease"; then + fail "a stable version was released as a prerelease" +fi + +index=$(published_index) +assert_indexed 9.9.9 "${index}" +grep -qF "memgraph-1.0.5.tgz" "${index}" || + fail "the merge dropped the pre-existing memgraph entry" +grep -q "^ memgraph-operator:" "${index}" || + fail "the index has no memgraph-operator entry" + +echo +echo "==> Republishing the same version changes nothing" +run_publish --version 9.9.9 --publish >/dev/null +assert_indexed 9.9.9 "$(published_index)" + +echo +echo "==> A prerelease version is marked as one" +run_publish --version 9.9.9-rc.1 --publish >/dev/null +grep -F "release create memgraph-operator-9.9.9-rc.1" "${GH_LOG}" | grep -q -- "--prerelease" || + fail "the release candidate was not marked as a prerelease" + +index=$(published_index) +assert_indexed 9.9.9-rc.1 "${index}" +assert_indexed 9.9.9 "${index}" + +echo +echo "==> Publishing without a token is refused" +if env CHARTS_REPO_URL="${WORK}/helm-charts.git" OUT_DIR="${WORK}/dist" \ + GH="${WORK}/bin/gh" GH_TOKEN="" HELM="${HELM}" \ + "${PUBLISH}" --version 9.9.8 --publish >/dev/null 2>&1; then + fail "publishing without GH_TOKEN succeeded" +fi + +echo +echo "==> Cross-publish test passed" diff --git a/hack/chart-publish.sh b/hack/chart-publish.sh new file mode 100755 index 0000000..c5480b8 --- /dev/null +++ b/hack/chart-publish.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +# +# Package the operator install chart and cross-publish it into the existing +# memgraph.github.io/helm-charts index. +# +# Chart source and CRDs stay in this repository, but users install from the +# helm repository they already have configured. That index is a plain Helm +# index.yaml served from the helm-charts repository's gh-pages branch, whose +# entries point at .tgz assets attached to GitHub releases in that same +# repository -- the layout chart-releaser produces for the charts that live +# there. Publishing from here means writing into both halves: +# +# 1. a GitHub release - in the charts repository, with the +# packaged chart attached, and +# 2. an index.yaml entry on gh-pages pointing at that asset. +# +# The release comes first: an index entry whose download URL 404s is worse than +# an asset nobody can find yet. +# +# Every step is skipped when its result already exists, so re-running after a +# half-finished release finishes the job instead of duplicating it. +# +# Nothing leaves the machine without --publish. Without it this packages the +# chart, works out the merged index, and prints what would change. +# +# Publishing needs GH_TOKEN to carry a token with contents write access to the +# charts repository, and git to be able to push there (the release workflow +# runs `gh auth setup-git` for that). See docs/releasing.md. + +set -euo pipefail + +CHART_DIR=${CHART_DIR:-charts/memgraph-operator} +CHART_NAME=${CHART_NAME:-memgraph-operator} +CHARTS_REPO=${CHARTS_REPO:-memgraph/helm-charts} +CHARTS_REPO_URL=${CHARTS_REPO_URL:-https://github.com/${CHARTS_REPO}.git} +PAGES_BRANCH=${PAGES_BRANCH:-gh-pages} +OUT_DIR=${OUT_DIR:-dist/chart} + +HELM=${HELM:-helm} +GIT=${GIT:-git} +GH=${GH:-gh} + +GIT_USER_NAME=${GIT_USER_NAME:-memgraph-operator release} +GIT_USER_EMAIL=${GIT_USER_EMAIL:-tech@memgraph.com} + +PUBLISH=false +VERSION="" + +usage() { + cat >&2 <] [--publish] + + --version chart version to publish; defaults to the version in ${CHART_DIR}/Chart.yaml + --publish actually create the release and push the index (default: dry run) +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --publish) PUBLISH=true ;; + --version) + [ $# -ge 2 ] || { usage; exit 2; } + VERSION=$2 + shift + ;; + --version=*) VERSION=${1#--version=} ;; + -h | --help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage + exit 2 + ;; + esac + shift +done + +fail() { + echo "$*" >&2 + exit 1 +} + +if [ -z "${VERSION}" ]; then + VERSION=$(sed -n 's/^version:[[:space:]]*//p' "${CHART_DIR}/Chart.yaml" | tr -d '"' | head -1) +fi +VERSION=${VERSION#v} +[ -n "${VERSION}" ] || fail "No chart version given and none found in ${CHART_DIR}/Chart.yaml." + +RELEASE_TAG="${CHART_NAME}-${VERSION}" +PACKAGE="${CHART_NAME}-${VERSION}.tgz" +# Where the index entry will point. chart-releaser builds the same URL for the +# charts already in this index, so the operator's entries look like the rest. +DOWNLOAD_URL="https://github.com/${CHARTS_REPO}/releases/download/${RELEASE_TAG}" + +# A SemVer prerelease is hidden from `helm install` unless --devel is passed, +# which is what makes a release candidate a safe rehearsal against the real +# index: everything is exercised, nobody installs it by accident. +case "${VERSION}" in + *-*) PRERELEASE=true ;; + *) PRERELEASE=false ;; +esac + +if [ "${PUBLISH}" = true ]; then + [ -n "${GH_TOKEN:-}" ] || fail "--publish needs GH_TOKEN set to a token with contents write access to ${CHARTS_REPO}." +fi + +WORK=$(mktemp -d) +trap 'rm -rf "${WORK}"' EXIT + +echo "==> Packaging ${CHART_NAME} ${VERSION}" +mkdir -p "${OUT_DIR}" +rm -f "${OUT_DIR}/${CHART_NAME}"-*.tgz +"${HELM}" package "${CHART_DIR}" --version "${VERSION}" --destination "${OUT_DIR}" +[ -f "${OUT_DIR}/${PACKAGE}" ] || fail "helm package did not produce ${OUT_DIR}/${PACKAGE}." + +# helm repo index reads a whole directory, so the package to be indexed gets a +# directory of its own -- OUT_DIR is the caller's and may hold anything. +mkdir -p "${WORK}/packages" +cp "${OUT_DIR}/${PACKAGE}" "${WORK}/packages/" + +echo "==> Fetching the ${PAGES_BRANCH} branch of ${CHARTS_REPO}" +"${GIT}" clone --quiet --depth 1 --branch "${PAGES_BRANCH}" "${CHARTS_REPO_URL}" "${WORK}/pages" +INDEX="${WORK}/pages/index.yaml" +[ -f "${INDEX}" ] || fail "${CHARTS_REPO} ${PAGES_BRANCH} has no index.yaml." + +# The download URL contains the version twice over, so this matches the exact +# version and never a prefix of it: 0.2.0 does not match 0.2.0-rc.1. +if grep -qF "/${RELEASE_TAG}/${PACKAGE}" "${INDEX}"; then + INDEX_CURRENT=true + echo " index already lists ${CHART_NAME} ${VERSION}" +else + INDEX_CURRENT=false +fi + +echo "==> Merging the index entry" +"${HELM}" repo index "${WORK}/packages" --url "${DOWNLOAD_URL}" --merge "${INDEX}" + +if [ "${PUBLISH}" != true ]; then + echo + echo "==> Dry run; nothing was published." + echo " package: ${OUT_DIR}/${PACKAGE}" + echo " would release ${RELEASE_TAG} in ${CHARTS_REPO} (prerelease: ${PRERELEASE})" + echo " would serve ${DOWNLOAD_URL}/${PACKAGE}" + echo + echo "==> Index diff" + diff -u "${INDEX}" "${WORK}/packages/index.yaml" || true + exit 0 +fi + +echo "==> Publishing ${RELEASE_TAG} to ${CHARTS_REPO}" +if "${GH}" release view "${RELEASE_TAG}" --repo "${CHARTS_REPO}" >/dev/null 2>&1; then + echo " release exists; making sure the package is attached" + "${GH}" release upload "${RELEASE_TAG}" "${OUT_DIR}/${PACKAGE}" --repo "${CHARTS_REPO}" --clobber +else + notes="Memgraph Kubernetes operator install chart ${VERSION}. + +Chart source, CRDs and RBAC live in https://github.com/memgraph/kubernetes-operator; +this release exists so the packaged chart is served from the Memgraph helm repository. + + helm repo add memgraph https://memgraph.github.io/helm-charts + helm repo update + helm install memgraph-operator memgraph/memgraph-operator --version ${VERSION}" + + prerelease_flag=() + [ "${PRERELEASE}" = true ] && prerelease_flag=(--prerelease) + + "${GH}" release create "${RELEASE_TAG}" "${OUT_DIR}/${PACKAGE}" \ + --repo "${CHARTS_REPO}" \ + --title "${RELEASE_TAG}" \ + --notes "${notes}" \ + "${prerelease_flag[@]}" +fi + +if [ "${INDEX_CURRENT}" = true ]; then + echo "==> Index already current; leaving ${PAGES_BRANCH} alone" + exit 0 +fi + +echo "==> Pushing the index to ${CHARTS_REPO} ${PAGES_BRANCH}" +cp "${WORK}/packages/index.yaml" "${INDEX}" +"${GIT}" -C "${WORK}/pages" add index.yaml +"${GIT}" -C "${WORK}/pages" \ + -c "user.name=${GIT_USER_NAME}" \ + -c "user.email=${GIT_USER_EMAIL}" \ + commit --quiet --message "Add ${CHART_NAME} ${VERSION} to the index + +Published from memgraph/kubernetes-operator." +"${GIT}" -C "${WORK}/pages" push --quiet origin "${PAGES_BRANCH}" + +echo "==> Published ${CHART_NAME} ${VERSION}" +echo " GitHub Pages needs a moment to serve the new index." diff --git a/hack/chart-version-check.sh b/hack/chart-version-check.sh new file mode 100755 index 0000000..1ba9057 --- /dev/null +++ b/hack/chart-version-check.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Chart versioning check, run on every pull request. +# +# The chart's version and its appVersion move independently: appVersion names +# the operator image the chart installs by default, version names the chart +# itself. Neither is derived from the other, so what has to hold is narrower -- +# a pull request that changes the chart has to change the chart's version too. +# Without that, the published index keeps serving the previous package under a +# version number that no longer describes its contents, and `helm upgrade` has +# nothing to act on. +# +# Compares against the base branch, so it stays quiet on pull requests that do +# not touch the chart. + +set -euo pipefail + +CHART_DIR=${CHART_DIR:-charts/memgraph-operator} +CHART_FILE="${CHART_DIR}/Chart.yaml" +BASE_REF=${BASE_REF:-origin/main} + +# Helm requires SemVer for the chart version, and the appVersion is the image +# tag the Deployment template falls back to, so both are checked the same way. +SEMVER_RE='^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$' + +fail() { + echo "$*" >&2 + exit 1 +} + +# field prints a top-level scalar from a Chart.yaml on stdin, unquoted. +# Comments never match: the field name has to start the line. +field() { + sed -n "s/^$1:[[:space:]]*//p" | tr -d '"' | head -1 +} + +[ -f "${CHART_FILE}" ] || fail "No chart at ${CHART_FILE}; set CHART_DIR." + +version=$(field version <"${CHART_FILE}") +app_version=$(field appVersion <"${CHART_FILE}") + +[[ ${version} =~ ${SEMVER_RE} ]] || + fail "${CHART_FILE}: version '${version}' is not a SemVer version (e.g. 0.2.0, 0.2.0-rc.1)." +[[ ${app_version} =~ ${SEMVER_RE} ]] || + fail "${CHART_FILE}: appVersion '${app_version}' is not a SemVer version; it is an operator image tag." + +if ! git rev-parse --verify --quiet "${BASE_REF}^{commit}" >/dev/null; then + echo "No ${BASE_REF} to compare against; checked the version fields only." + exit 0 +fi + +if git diff --quiet "${BASE_REF}" -- "${CHART_DIR}"; then + echo "Chart unchanged against ${BASE_REF} (version ${version}, appVersion ${app_version})." + exit 0 +fi + +if ! git cat-file -e "${BASE_REF}:${CHART_FILE}" 2>/dev/null; then + echo "${CHART_FILE} is new against ${BASE_REF}; nothing to compare (version ${version})." + exit 0 +fi + +base_version=$(git show "${BASE_REF}:${CHART_FILE}" | field version) + +if [ "${version}" = "${base_version}" ]; then + fail "${CHART_DIR} changed but its version is still ${version}. +A published chart version is immutable, so every change to the chart needs a new +one. Bump 'version' in ${CHART_FILE}; leave 'appVersion' alone unless this +release also ships a new operator image." +fi + +echo "Chart ${base_version} -> ${version} (appVersion ${app_version})." diff --git a/specs/operator-mvp/issues/11-release-cross-publish.md b/specs/operator-mvp/issues/11-release-cross-publish.md index ce2d96b..5394e7b 100644 --- a/specs/operator-mvp/issues/11-release-cross-publish.md +++ b/specs/operator-mvp/issues/11-release-cross-publish.md @@ -12,12 +12,24 @@ The release path from a version tag to installable artifacts: build and push the ## Acceptance criteria -- [ ] Pushing a version tag builds and publishes the operator image with that version -- [ ] The same pipeline packages the install chart and publishes it into the `memgraph.github.io/helm-charts` index -- [ ] `helm repo update && helm install` from the existing Memgraph helm repo installs the tagged operator version end-to-end -- [ ] Chart version, appVersion, and image tag agree for every release -- [ ] The cross-repo credential is a scoped fine-grained PAT or deploy key stored as a repository secret, documented for rotation -- [ ] A dry-run/prerelease path exists to validate the pipeline without polluting the public index +- [x] Pushing a version tag builds and publishes the operator image with that version +- [x] The same pipeline packages the install chart and publishes it into the `memgraph.github.io/helm-charts` index +- [x] `helm repo update && helm install` from the existing Memgraph helm repo installs the tagged operator version end-to-end +- [x] The appVersion and the operator image tag agree for every release, and the chart's own version is bumped whenever the chart changes +- [x] The cross-repo credential is a scoped fine-grained PAT or deploy key stored as a repository secret, documented for rotation +- [x] A dry-run/prerelease path exists to validate the pipeline without polluting the public index + +> **Amended during implementation.** The fourth criterion originally read "Chart version, +> appVersion, and image tag agree for every release" — chart and operator versions locked in +> lockstep. That was relaxed to the Helm-standard arrangement, where the chart's `version` and its +> `appVersion` move independently, so a chart-only fix (a template bug, a new values knob) does +> not require a redundant operator release. The invariant that remains is the load-bearing one: +> `appVersion` is the operator image tag, and the image it names must exist. +> +> Tags therefore name the artifact they release — `v` the operator, `chart-` the +> chart alone. The known cost is that a chart-only release can ship CRDs generated after the +> operator its `appVersion` installs; the pipeline reports that drift in the run summary rather +> than refusing it. See `docs/releasing.md`. ## Blocked by From 957aaf82343201dd256377c5ac0d44688997cbab Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Mon, 27 Jul 2026 15:09:33 +0200 Subject: [PATCH 15/34] docs: quickstart README and a minimal example the e2e suite applies (#31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The minutes-to-cluster story for a developer evaluating Memgraph: install the operator chart, create the license Secret, apply one manifest, watch `kubectl get mgc` converge, and connect over Bolt. The README also states the v1alpha1 contract outright — immutable counts, hands-off failover, and the deferred features (day-2 ops, external access, TLS, auth, monitoring, standalone) — and its relationship to the HA Helm chart: successor, parity then freeze, fresh-cluster migration only. The quickstart manifest is a file, examples/minimal-cluster.yaml, and the e2e suite applies it verbatim instead of an inlined copy, deriving the cluster name, topology, image and license Secret it asserts on from the file itself. The example a newcomer copies is therefore the one CI proves boots a registered, MAIN-elected cluster, and it cannot rot in place. --- README.md | 220 ++++++++++++++++++++----------- examples/minimal-cluster.yaml | 29 ++++ go.mod | 2 +- test/e2e/memgraphcluster_test.go | 121 ++++++++++------- 4 files changed, 247 insertions(+), 125 deletions(-) create mode 100644 examples/minimal-cluster.yaml diff --git a/README.md b/README.md index 39a2510..cdbbe99 100644 --- a/README.md +++ b/README.md @@ -2,25 +2,21 @@ A Kubernetes operator for running [Memgraph](https://memgraph.com) high-availability clusters. It exposes a `MemgraphCluster` custom resource (API group `memgraph.com/v1alpha1`, short name `mgc`): declare the cluster topology in a single resource and the operator provisions the workloads, bootstraps HA registration, and continuously reconciles registration state. -> **Status: early development.** This repository was reset for a fresh operator effort; the previous attempt is preserved on the `archive/pre-operator-mvp` branch. The product requirements and issue slices driving the current work live in [`specs/operator-mvp/`](specs/operator-mvp/PRD.md). +> **Status: early development (v1alpha1).** The API and its guarantees can still change between releases. Read [what v1alpha1 does and does not do](#what-v1alpha1-does-and-does-not-do) before running it anywhere that matters. The previous attempt at this operator is preserved on the `archive/pre-operator-mvp` branch; the requirements and issue slices driving the current work live in [`specs/operator-mvp/`](specs/operator-mvp/PRD.md). ## Description The operator replaces the `memgraph-high-availability` Helm chart's fire-and-forget registration Job with a controller that continuously drives the cluster toward its declared topology: one StatefulSet per role (coordinators, data instances), automatic bootstrap and MAIN promotion, and automatic re-registration of instances that lose their registration state. See the [PRD](specs/operator-mvp/PRD.md) for the full design. -## Getting Started +## Quickstart -### Prerequisites -- go version v1.24.6+ -- docker version 17.03+. -- kubectl version v1.11.3+. -- Access to a Kubernetes v1.11.3+ cluster. +From an empty cluster to a registered, MAIN-elected Memgraph HA cluster. -### To install the operator +**You need:** a Kubernetes cluster (v1.25 or newer — the CRD uses CEL validation), `kubectl`, `helm` v3, and a Memgraph enterprise license, which high availability requires. Three coordinators and two data instances need five schedulable pods and ten PersistentVolumeClaims of 1Gi each. -The install chart is published to the Memgraph helm repository — the same one the Memgraph charts -come from. It ships the `MemgraphCluster` CRD, a least-privilege RBAC set, and the controller -Deployment: +### 1. Install the operator + +The install chart ships the `MemgraphCluster` CRD, a least-privilege RBAC set, and the controller Deployment. One release per cluster is enough — the operator watches every namespace. ```sh helm repo add memgraph https://memgraph.github.io/helm-charts @@ -29,127 +25,198 @@ helm install memgraph-operator memgraph/memgraph-operator \ --namespace memgraph-operator-system --create-namespace --wait ``` -The chart source lives in this repository, in -[`charts/memgraph-operator`](charts/memgraph-operator/README.md), and installs from there too: +### 2. Create the license Secret + +The cluster reads its license from a Secret you own; no license material ever goes into the `MemgraphCluster` resource, which keeps it safe to commit to git. ```sh -helm install memgraph-operator ./charts/memgraph-operator \ - --namespace memgraph-operator-system --create-namespace --wait +kubectl create namespace memgraph +kubectl create secret generic memgraph-secrets \ + --namespace memgraph \ + --from-literal=MEMGRAPH_ENTERPRISE_LICENSE='' \ + --from-literal=MEMGRAPH_ORGANIZATION_NAME='' ``` -Uninstall with `helm uninstall memgraph-operator --namespace memgraph-operator-system`. Helm never -deletes CRDs it installed, so remove the CRD explicitly (`kubectl delete crd -memgraphclusters.memgraph.com`) once no cluster needs it — see the -[chart README](charts/memgraph-operator/README.md) for the values and the upgrade caveat. +### 3. Declare the cluster + +This is the whole resource — counts, image, and the Secret from the previous step. Everything else (storage, ports, probes, resources, cluster domain) takes its default: + +```yaml +apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + name: memgraph +spec: + coordinators: 3 + dataInstances: 2 + image: + repository: docker.io/memgraph/memgraph + tag: "3.12.0" + secrets: + name: memgraph-secrets + licenseKey: MEMGRAPH_ENTERPRISE_LICENSE + organizationKey: MEMGRAPH_ORGANIZATION_NAME +``` -### To deploy a development build on the cluster -**Build and push your image to the location specified by `IMG`:** +It is [`examples/minimal-cluster.yaml`](examples/minimal-cluster.yaml) in this repository, and the end-to-end suite applies that file unmodified on every pull request, so it stays a manifest that works: ```sh -make docker-build docker-push IMG=/kubernetes-operator:tag +kubectl apply -n memgraph \ + -f https://raw.githubusercontent.com/memgraph/kubernetes-operator/main/examples/minimal-cluster.yaml ``` -**NOTE:** This image ought to be published in the personal registry you specified. -And it is required to have access to pull the image from the working environment. -Make sure you have the proper permission to the registry if the above commands don’t work. +If your Secret has a different name, or stores the license under different keys, change the `secrets` block to match — that is the only edit the example needs. -**Install the CRDs into the cluster:** +### 4. Watch it converge + +The operator creates one StatefulSet and one headless Service per role, waits for the pods to become ready, then registers the coordinators and data instances with each other and promotes the initial MAIN. On a cluster that has to pull the Memgraph image, expect a few minutes. ```sh -make install +kubectl get mgc -n memgraph -w ``` -**Deploy the Manager to the cluster with the image specified by `IMG`:** +While the pods are still starting, `MAIN` is empty and both conditions are `False`; the converged cluster looks like this: -```sh -make deploy IMG=/kubernetes-operator:tag +``` +NAME COORDINATORS DATA MAIN READY CONVERGED AGE +memgraph 3 2 instance_0 True True 4m12s ``` -> **NOTE**: If you encounter RBAC errors, you may need to grant yourself cluster-admin -privileges or be logged in as admin. +- **`MAIN`** is the data instance the coordinators elected as MAIN — the one that accepts writes. It is observed, not decided by the operator, so it changes on failover. +- **`READY`** is True once a MAIN is elected, i.e. the cluster serves writes. +- **`CONVERGED`** is True once every declared coordinator and data instance is registered and reported healthy. -**Create instances of your solution** -You can apply the samples (examples) from the config/sample: +To block a script or a GitOps step on the cluster being usable: ```sh -kubectl apply -k config/samples/ +kubectl wait --namespace memgraph --for=condition=Converged \ + memgraphcluster/memgraph --timeout=10m ``` ->**NOTE**: Ensure that the samples has default values to test it out. - -### To Uninstall -**Delete the instances (CRs) from the cluster:** +If it does not converge, `kubectl describe mgc memgraph -n memgraph` gives the condition messages (which pods are not ready, whether a coordinator is unreachable), and the operator logs the rest: ```sh -kubectl delete -k config/samples/ +kubectl logs -n memgraph-operator-system deploy/memgraph-operator-controller-manager ``` -**Delete the APIs(CRDs) from the cluster:** +The resource's identities follow the pod ordinals. For a cluster named `memgraph`: + +| Pod | Registered as | Role | +| --- | --- | --- | +| `memgraph-coordinator-0`, `-1`, `-2` | `coordinator_1`, `coordinator_2`, `coordinator_3` | Raft coordinators | +| `memgraph-data-0`, `-1` | `instance_0`, `instance_1` | data instances (one MAIN, the rest replicas) | + +Ask a coordinator for the cluster's own view of itself: ```sh -make uninstall +kubectl exec -n memgraph memgraph-coordinator-0 -c memgraph -- \ + bash -c "echo 'SHOW INSTANCES;' | mgconsole" ``` -**UnDeploy the controller from the cluster:** +### 5. Connect over Bolt + +Every pod runs Bolt on port 7687, and the Memgraph image ships `mgconsole`, so the shortest path to a query is to pipe Cypher into the MAIN pod (`instance_0` above is `memgraph-data-0`): ```sh -make undeploy +kubectl exec -i -n memgraph memgraph-data-0 -c memgraph -- mgconsole <<'EOF' +CREATE (:Greeting {text: "hello from the operator"}); +MATCH (n:Greeting) RETURN n; +EOF ``` -## Project Distribution +`kubectl exec -it -n memgraph memgraph-data-0 -c memgraph -- mgconsole` opens the same client interactively. -Following the options to release and provide this solution to the users. +Writes only succeed against the MAIN — the other data instances are replicas and accept reads. Check `.status.main` to find it: -### By providing a bundle with all YAML files +```sh +kubectl get mgc memgraph -n memgraph -o jsonpath='{.status.main}' +``` -1. Build the installer for the image built and published in the registry: +Applications inside the cluster reach an instance at its stable DNS name in the role's headless Service: + +``` +memgraph-data-0.memgraph-data.memgraph.svc.cluster.local:7687 +``` + +v1alpha1 ships no external access (see below), so to point a local client such as [Memgraph Lab](https://memgraph.com/docs/data-visualization) at the cluster while evaluating, forward the port: ```sh -make build-installer IMG=/kubernetes-operator:tag +kubectl port-forward -n memgraph pod/memgraph-data-0 7687:7687 ``` -**NOTE:** The makefile target mentioned above generates an 'install.yaml' -file in the dist directory. This file contains all the resources built -with Kustomize, which are necessary to install this project without its -dependencies. +### 6. Clean up -2. Using the installer +```sh +kubectl delete mgc memgraph -n memgraph +``` -Users can just run 'kubectl apply -f ' to install -the project, i.e.: +Deleting the resource removes the StatefulSets and Services through garbage collection, but **the PersistentVolumeClaims are kept** — the default retention policy protects data against an accidental delete. Remove them (and with them the data) explicitly, or set `spec.storage.retentionPolicy: Delete` on dev clusters that should clean up after themselves: ```sh -kubectl apply -f https://raw.githubusercontent.com//kubernetes-operator//dist/install.yaml +kubectl delete pvc -n memgraph --all +kubectl delete namespace memgraph ``` -### By providing a Helm chart +Uninstall the operator with `helm uninstall memgraph-operator --namespace memgraph-operator-system`. Helm never deletes CRDs it installed, so `kubectl delete crd memgraphclusters.memgraph.com` once no cluster needs it — see the [chart README](charts/memgraph-operator/README.md) for the values and the upgrade caveat. + +## Configuration + +Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, retention) per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. + +[`config/samples/v1alpha1_memgraphcluster.yaml`](config/samples/v1alpha1_memgraphcluster.yaml) spells the full surface out with every default and the reasoning behind it. `kubectl explain mgc.spec --recursive` documents the same fields from the installed CRD. + +## What v1alpha1 does and does not do + +The MVP is deliberately "provision, bootstrap, observe". It does: + +- provision one StatefulSet and headless Service per role, with per-pod identity derived from the pod ordinal; +- bootstrap HA: add the coordinators, register the data instances, and promote the initial MAIN once; +- re-register continuously: every reconcile compares `SHOW INSTANCES` on the coordinator leader against the declared topology and issues only the missing registrations, so an instance that loses its registration state (say, after being rescheduled onto a fresh node) rejoins without human action; +- report the observed MAIN and the readiness and convergence conditions on the resource's status. + +What it does not do yet: + +- **Scaling.** `coordinators` and `dataInstances` are **immutable after creation** — admission rejects a change with a clear message. Changing the topology means creating a new cluster. Mutable counts are the first item on the post-v1 roadmap. +- **Failover.** The operator issues `SET INSTANCE TO MAIN` exactly once, at bootstrap, when no MAIN exists. After that, leadership belongs entirely to the Raft coordinators; the operator only observes and reports it, so two control systems never fight over which instance is MAIN. +- **Other day-2 operations**: orchestrated or rolling version upgrades, backup and restore, storage-mode changes. +- **Removing instances**: there is no `REMOVE COORDINATOR` or `UNREGISTER INSTANCE`, and no finalizer-based storage cleanup. The operator has no destructive code path. +- **External access** of any kind — no LoadBalancer, NodePort, ingress or gateway. Access is in-cluster (or `kubectl port-forward`) only; the approach is expected to change, so it was deliberately deferred rather than shipped and broken later. +- **TLS**, for Bolt or intra-cluster traffic. +- **Bolt authentication** — the operator connects to the coordinators unauthenticated, so clusters must not enable auth yet. +- **Monitoring** of the Memgraph cluster: no exporter, ServiceMonitor or dashboards. (The operator itself serves controller-runtime metrics; see the [chart README](charts/memgraph-operator/README.md).) +- **Standalone (non-HA) topology.** The API is shaped to grow one without a breaking change, but v1alpha1 provisions HA clusters only. +- **Affinity, tolerations, init containers, sidecars, snapshot-restore fields** and the rest of the HA chart's surface — parity roadmap, not MVP. -The install chart is maintained in this repository under -[`charts/memgraph-operator`](charts/memgraph-operator/README.md), next to the manifests it ships: -its CRDs and the manager's RBAC rules are generated from the Go types and the -`+kubebuilder:rbac` markers, so the chart can never drift from the controller version it -installs. +## Relationship to the memgraph-high-availability chart + +This operator is the successor to the [`memgraph-high-availability`](https://github.com/memgraph/helm-charts) Helm chart. The plan is to grow it to functional parity with the chart, publish a migration guide, and then freeze the chart (security fixes only) with a deprecation timeline. Until then the chart remains the supported way to run HA in production, and this operator is an alpha for evaluating the reconciliation core. The standalone `memgraph` and `memgraph-lab` charts are unaffected and continue independently. + +Where a concept carries over, the operator borrows the chart's vocabulary — the `secrets.name` / `secrets.licenseKey` / `secrets.organizationKey` block is the chart's block — so translating a values file is mechanical. The topology is where they deliberately differ: the chart's per-instance blocks and StatefulSet-per-instance model become two integers and one StatefulSet per role. + +**Migration is fresh-cluster only.** The operator will never adopt a chart-deployed cluster in place: the resources are shaped differently and the ownership handover cannot be made safe. Moving means standing up a new cluster and transferring the data (backup/restore, or a replication cutover). The step-by-step guide lands when the operator reaches parity — there is nothing to migrate to before then. + +## Development ```sh -make chart-sync # regenerate the chart's CRDs and RBAC rules after changing the API or markers -make helm-lint # lint the chart and render it with defaults and with the toggles flipped -make test-chart # install/uninstall the chart on a throwaway Kind cluster +make test-unit # unit tests (pure packages) +make test # unit + envtest +make lint # golangci-lint +make run # run the controller locally against the current kubeconfig +make test-e2e # KinD end-to-end suite; creates and deletes its own Kind cluster ``` -Pushing a version tag cross-publishes the packaged chart into the existing -[`memgraph.github.io/helm-charts`](https://memgraph.github.io/helm-charts) index, so users install -it from the helm repository they already have configured. The chart version and the operator -version move independently — `v0.2.0` releases the operator, `chart-0.4.2` releases the chart -alone. See [`docs/releasing.md`](docs/releasing.md) for the procedure, the dry-run and prerelease -paths, and the credentials involved. +Every pull request runs lint, unit, envtest, chart and end-to-end suites. The e2e job boots a licensed Memgraph cluster on a multi-node Kind cluster, with the license coming from repository secrets; set `MEMGRAPH_ENTERPRISE_LICENSE` and `MEMGRAPH_ORGANIZATION_NAME` to run it locally. -## Contributing +Run a development build against a cluster with `make docker-build docker-push IMG=/kubernetes-operator:tag` followed by `make install` (CRDs) and `make deploy IMG=/kubernetes-operator:tag`, or install the local chart: -Development is sliced into PR-gated issues under [`specs/operator-mvp/issues/`](specs/operator-mvp/issues). Every pull request runs lint, unit, and envtest suites. +```sh +helm install memgraph-operator ./charts/memgraph-operator \ + --namespace memgraph-operator-system --create-namespace --wait +``` -**NOTE:** Run `make help` for more information on all potential `make` targets +The install chart is maintained in this repository under [`charts/memgraph-operator`](charts/memgraph-operator/README.md), next to the manifests it ships: its CRDs and the manager's RBAC rules are generated from the Go types and the `+kubebuilder:rbac` markers (`make chart-sync`, verified in CI by `make chart-verify`), so the chart can never drift from the controller version it installs. Pushing a version tag cross-publishes the packaged chart into the [`memgraph.github.io/helm-charts`](https://memgraph.github.io/helm-charts) index. The chart version and the operator version move independently — `v0.2.0` releases the operator, `chart-0.4.2` releases the chart alone. See [`docs/releasing.md`](docs/releasing.md). -More information can be found via the [Kubebuilder Documentation](https://book.kubebuilder.io/introduction.html) +Development is sliced into PR-gated issues under [`specs/operator-mvp/issues/`](specs/operator-mvp/issues). Run `make help` for all targets, and see the [Kubebuilder documentation](https://book.kubebuilder.io/introduction.html) for the scaffolding conventions this project follows. ## License @@ -166,4 +233,3 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/examples/minimal-cluster.yaml b/examples/minimal-cluster.yaml new file mode 100644 index 0000000..507790a --- /dev/null +++ b/examples/minimal-cluster.yaml @@ -0,0 +1,29 @@ +# The quickstart cluster from the repository README: three coordinators, two +# data instances, a pinned Memgraph image, and a reference to the Secret +# holding the enterprise license. Everything else — storage, ports, probes, +# resources, cluster domain — takes its default; config/samples/ +# v1alpha1_memgraphcluster.yaml spells the full surface out. +# +# The e2e suite applies this file verbatim, so it is the manifest CI proves +# boots a registered, MAIN-elected cluster. +apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + name: memgraph +spec: + # Both counts are immutable: v1alpha1 provisions and bootstraps a fixed + # topology, and admission rejects any later change. The coordinator count + # must be odd so the Raft quorum cannot split. + coordinators: 3 + dataInstances: 2 + image: + repository: docker.io/memgraph/memgraph + tag: "3.12.0" + # The Secret holding the enterprise license, which HA requires. Point name at + # your own Secret; the two key names are the defaults, spelled out here + # because they are what you change when your Secret stores them under + # different keys. The CR itself carries no secret material. + secrets: + name: memgraph-secrets + licenseKey: MEMGRAPH_ENTERPRISE_LICENSE + organizationKey: MEMGRAPH_ORGANIZATION_NAME diff --git a/go.mod b/go.mod index 7ff709f..b1e6e9f 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( k8s.io/client-go v0.36.0 k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 sigs.k8s.io/controller-runtime v0.24.1 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -97,5 +98,4 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect - sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index 8f04e8c..35f0958 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -26,33 +26,32 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "sigs.k8s.io/yaml" + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" "github.com/memgraph/kubernetes-operator/test/utils" ) -// The declared topology of the e2e cluster and the identities the operator -// derives from it: coordinator ordinal N registers as coordinator_N+1, data -// ordinal N as instance_N. const ( clusterNamespace = "memgraph-e2e" - clusterName = "memgraph" - - coordinatorCount = 3 - dataInstanceCount = 2 - memgraphImageRepository = "docker.io/memgraph/memgraph" - memgraphImageTag = "3.12.0" - - // licenseSecretName and the env var names below follow the HA Helm chart's - // CI convention: repository secrets of the same names are exported into the - // job environment and materialize as one Kubernetes Secret the CR - // references. - licenseSecretName = "memgraph-secrets" + // exampleManifest is the quickstart manifest the README walks a newcomer + // through, applied here verbatim (only the namespace is supplied on the + // command line). Everything the suite needs to know about the cluster — + // its name, its topology, its image — is read out of the file below, so + // the example cannot drift from what CI proves works. + exampleManifest = "examples/minimal-cluster.yaml" + + // The env vars carrying the enterprise license into the suite follow the + // HA Helm chart's CI convention: repository secrets of these names are + // exported into the job environment and materialize as the one Kubernetes + // Secret the example references. licenseEnvVar = "MEMGRAPH_ENTERPRISE_LICENSE" organizationEnvVar = "MEMGRAPH_ORGANIZATION_NAME" @@ -61,9 +60,54 @@ const ( roleMain = "main" ) +// example is the parsed quickstart manifest and the source of truth for the +// topology the specs assert on. +var example = loadExample() + +// The declared topology of the e2e cluster, as the example declares it. +var ( + clusterName = example.Name + coordinatorCount = declaredCount("coordinators", example.Spec.Coordinators) + dataInstanceCount = declaredCount("dataInstances", example.Spec.DataInstances) + + memgraphImage = example.Spec.Image.Repository + ":" + example.Spec.Image.Tag + + licenseSecretName = example.Spec.Secrets.Name +) + +// declaredCount reads a replica count the example must state outright: the +// counts drive the assertions, and a count left to the CRD's default would +// leave the suite asserting on a topology the file never declared. +func declaredCount(field string, count *int32) int32 { + if count == nil { + panic(fmt.Sprintf("%s must declare spec.%s", exampleManifest, field)) + } + return *count +} + +// loadExample reads and decodes the quickstart manifest. Decoding is strict, +// so a field the example misspells fails the suite instead of being silently +// defaulted away by the API server. +func loadExample() *memgraphcomv1alpha1.MemgraphCluster { + projectDir, err := utils.GetProjectDir() + if err != nil { + panic(fmt.Sprintf("locating the project directory: %v", err)) + } + manifest, err := os.ReadFile(filepath.Join(projectDir, exampleManifest)) + if err != nil { + panic(fmt.Sprintf("reading %s: %v", exampleManifest, err)) + } + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + if err := yaml.UnmarshalStrict(manifest, cluster); err != nil { + panic(fmt.Sprintf("decoding %s: %v", exampleManifest, err)) + } + return cluster +} + // declaredInstances returns the instance names every coordinator and data // instance must appear under in SHOW INSTANCES once the operator has converged -// registration. +// registration: coordinator ordinal N registers as coordinator_N+1, data +// ordinal N as instance_N. func declaredInstances() []string { names := make([]string, 0, coordinatorCount+dataInstanceCount) for ordinal := range coordinatorCount { @@ -95,7 +139,6 @@ var _ = Describe("MemgraphCluster", Ordered, func() { "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", organizationEnvVar) By("preloading the Memgraph image into the Kind cluster") - memgraphImage := memgraphImageRepository + ":" + memgraphImageTag cmd := exec.Command("docker", "pull", memgraphImage) _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to pull the Memgraph image") @@ -219,7 +262,7 @@ var _ = Describe("MemgraphCluster", Ordered, func() { before, err := listPVCs(clusterNamespace) Expect(err).NotTo(HaveOccurred()) // Two claims (lib and log) per coordinator and data instance pod. - Expect(before).To(HaveLen(2 * (coordinatorCount + dataInstanceCount))) + Expect(before).To(HaveLen(int(2 * (coordinatorCount + dataInstanceCount)))) By("deleting the MemgraphCluster") cmd := exec.Command("kubectl", "delete", "memgraphcluster", clusterName, @@ -286,7 +329,8 @@ spec: tag: %s storage: retentionPolicy: Delete -`, retentionCluster, retentionNamespace, memgraphImageRepository, memgraphImageTag) +`, retentionCluster, retentionNamespace, + example.Spec.Image.Repository, example.Spec.Image.Tag) cmd := exec.Command("kubectl", "apply", "-f", "-") _, err := utils.RunWithInput(cmd, manifest) Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") @@ -443,9 +487,9 @@ func removeCoordinatorRegistration() (string, error) { return "", fmt.Errorf("no coordinator leader found to remove a coordinator: %w", errors.Join(errs...)) } -// createLicenseSecret applies the Secret the MemgraphCluster references. The -// manifest is piped over stdin so no secret material ever reaches the logged -// command line. +// createLicenseSecret applies the Secret the MemgraphCluster references, under +// the name and keys the example points at. The manifest is piped over stdin so +// no secret material ever reaches the logged command line. func createLicenseSecret(license, organization string) { secret := map[string]any{ "apiVersion": "v1", @@ -455,8 +499,8 @@ func createLicenseSecret(license, organization string) { "namespace": clusterNamespace, }, "stringData": map[string]string{ - licenseEnvVar: license, - organizationEnvVar: organization, + example.Spec.Secrets.LicenseKey: license, + example.Spec.Secrets.OrganizationKey: organization, }, } manifest, err := json.Marshal(secret) @@ -467,30 +511,13 @@ func createLicenseSecret(license, organization string) { Expect(err).NotTo(HaveOccurred(), "Failed to apply the license Secret") } -// applyMemgraphCluster applies the CR under test: the minimal spec of the PRD's -// first-contact story — image, counts, and a license secret reference. +// applyMemgraphCluster applies the CR under test: the README's quickstart +// manifest, unmodified, which is the minimal spec of the PRD's first-contact +// story — image, counts, and a license secret reference. The manifest declares +// no namespace, exactly as a newcomer applies it into their own. func applyMemgraphCluster() { - manifest := fmt.Sprintf(`apiVersion: memgraph.com/v1alpha1 -kind: MemgraphCluster -metadata: - name: %s - namespace: %s -spec: - coordinators: %d - dataInstances: %d - image: - repository: %s - tag: %s - secrets: - name: %s - licenseKey: %s - organizationKey: %s -`, clusterName, clusterNamespace, coordinatorCount, dataInstanceCount, - memgraphImageRepository, memgraphImageTag, - licenseSecretName, licenseEnvVar, organizationEnvVar) - - cmd := exec.Command("kubectl", "apply", "-f", "-") - _, err := utils.RunWithInput(cmd, manifest) + cmd := exec.Command("kubectl", "apply", "-n", clusterNamespace, "-f", exampleManifest) + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") } From e2ba8cb474cb144f453ba67033fa4e8a29f73609 Mon Sep 17 00:00:00 2001 From: as51340 Date: Mon, 27 Jul 2026 15:50:12 +0200 Subject: [PATCH 16/34] feat: Add support for optional log volume --- README.md | 4 +- api/v1alpha1/memgraphcluster_types.go | 33 +++++++- api/v1alpha1/zz_generated.deepcopy.go | 5 ++ .../crds/memgraph.com_memgraphclusters.yaml | 36 ++++++++ .../bases/memgraph.com_memgraphclusters.yaml | 36 ++++++++ config/samples/v1alpha1_memgraphcluster.yaml | 9 ++ .../controller/memgraphcluster_controller.go | 34 +++++++- .../memgraphcluster_controller_test.go | 40 +++++++++ .../memgraphcluster_validation_test.go | 9 +- internal/resources/resources.go | 25 ++++-- internal/resources/statefulset.go | 57 +++++++++---- internal/resources/statefulset_test.go | 84 ++++++++++++++++++- 12 files changed, 335 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index cdbbe99..496035c 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ kubectl wait --namespace memgraph --for=condition=Converged \ memgraphcluster/memgraph --timeout=10m ``` -If it does not converge, `kubectl describe mgc memgraph -n memgraph` gives the condition messages (which pods are not ready, whether a coordinator is unreachable), and the operator logs the rest: +If it does not converge, `kubectl describe mgc memgraph -n memgraph` gives the condition messages (which pods are not ready, whether a coordinator is unreachable, or — with reason `ApplyFailed` — what the API server refused about the workloads), and the operator logs the rest: ```sh kubectl logs -n memgraph-operator-system deploy/memgraph-operator-controller-manager @@ -161,7 +161,7 @@ Uninstall the operator with `helm uninstall memgraph-operator --namespace memgra ## Configuration -Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, retention) per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. +Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, whether the log claim is created at all, retention) per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. [`config/samples/v1alpha1_memgraphcluster.yaml`](config/samples/v1alpha1_memgraphcluster.yaml) spells the full surface out with every default and the reasoning behind it. `kubectl explain mgc.spec --recursive` documents the same fields from the installed CRD. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 632f4e5..6e3a52f 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -41,10 +41,11 @@ const ( DefaultLicenseSecretKey = "MEMGRAPH_ENTERPRISE_LICENSE" DefaultOrganizationSecretKey = "MEMGRAPH_ORGANIZATION_NAME" - DefaultLibPVCSize = "1Gi" - DefaultLogPVCSize = "1Gi" - DefaultStorageAccessMode = corev1.ReadWriteOnce - DefaultStorageRetention = RetentionPolicyRetain + DefaultLibPVCSize = "1Gi" + DefaultLogPVCSize = "1Gi" + DefaultCreateLogStorageClaim = true + DefaultStorageAccessMode = corev1.ReadWriteOnce + DefaultStorageRetention = RetentionPolicyRetain DefaultClusterDomain = "cluster.local" @@ -121,6 +122,12 @@ const ( // Condition reasons reported on MemgraphCluster status. Reasons are CamelCase // per Kubernetes API conventions and are stable enough for tooling to gate on. const ( + // ReasonApplyFailed is set when the API server rejected one of the desired + // workload objects, so the cluster does not run the declared spec. The + // condition message carries the rejection verbatim: the operator cannot act + // on it, but it names exactly what a human has to change. + ReasonApplyFailed = "ApplyFailed" + // ReasonWorkloadsNotReady is set while not every workload pod is ready, so // registration has not been attempted. ReasonWorkloadsNotReady = "WorkloadsNotReady" @@ -242,6 +249,24 @@ type RoleStorageSpec struct { // +optional LibStorageClassName *string `json:"libStorageClassName,omitempty"` + // createLogStorageClaim decides whether every pod of the role gets a log + // storage claim at all. With it disabled the operator drops the claim and + // passes an empty --log-file, which turns file logging off, so stderr and + // `kubectl logs` (plus whatever collects it) become the single log sink. Use + // it to avoid a second PersistentVolumeClaim per pod on clusters that ship + // logs off-node anyway. + // + // The remaining log* knobs below are ignored while this is false. + // + // Like the sizes and classes around it this is effectively a create-time + // choice: flipping it adds or removes a volumeClaimTemplate, which + // Kubernetes forbids on a live StatefulSet, so the operator's apply is + // rejected until the StatefulSet is recreated (delete it with + // --cascade=orphan and the operator rebuilds it around the running pods). + // +kubebuilder:default=true + // +optional + CreateLogStorageClaim *bool `json:"createLogStorageClaim,omitempty"` + // logPVCSize is the requested size of the log storage claim, which backs // Memgraph's log file. // +kubebuilder:default="1Gi" diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 64b52ed..9447c55 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -403,6 +403,11 @@ func (in *RoleStorageSpec) DeepCopyInto(out *RoleStorageSpec) { *out = new(string) **out = **in } + if in.CreateLogStorageClaim != nil { + in, out := &in.CreateLogStorageClaim, &out.CreateLogStorageClaim + *out = new(bool) + **out = **in + } if in.LogPVCSize != nil { in, out := &in.LogPVCSize, &out.LogPVCSize x := (*in).DeepCopy() diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml index a436abd..bd9a0a5 100644 --- a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -694,6 +694,24 @@ spec: description: coordinators configures the storage of every coordinator pod. properties: + createLogStorageClaim: + default: true + description: |- + createLogStorageClaim decides whether every pod of the role gets a log + storage claim at all. With it disabled the operator drops the claim and + passes an empty --log-file, which turns file logging off, so stderr and + `kubectl logs` (plus whatever collects it) become the single log sink. Use + it to avoid a second PersistentVolumeClaim per pod on clusters that ship + logs off-node anyway. + + The remaining log* knobs below are ignored while this is false. + + Like the sizes and classes around it this is effectively a create-time + choice: flipping it adds or removes a volumeClaimTemplate, which + Kubernetes forbids on a live StatefulSet, so the operator's apply is + rejected until the StatefulSet is recreated (delete it with + --cascade=orphan and the operator rebuilds it around the running pods). + type: boolean libPVCSize: anyOf: - type: integer @@ -758,6 +776,24 @@ spec: description: data configures the storage of every data instance pod. properties: + createLogStorageClaim: + default: true + description: |- + createLogStorageClaim decides whether every pod of the role gets a log + storage claim at all. With it disabled the operator drops the claim and + passes an empty --log-file, which turns file logging off, so stderr and + `kubectl logs` (plus whatever collects it) become the single log sink. Use + it to avoid a second PersistentVolumeClaim per pod on clusters that ship + logs off-node anyway. + + The remaining log* knobs below are ignored while this is false. + + Like the sizes and classes around it this is effectively a create-time + choice: flipping it adds or removes a volumeClaimTemplate, which + Kubernetes forbids on a live StatefulSet, so the operator's apply is + rejected until the StatefulSet is recreated (delete it with + --cascade=orphan and the operator rebuilds it around the running pods). + type: boolean libPVCSize: anyOf: - type: integer diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 6193b44..b0083f7 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -692,6 +692,24 @@ spec: description: coordinators configures the storage of every coordinator pod. properties: + createLogStorageClaim: + default: true + description: |- + createLogStorageClaim decides whether every pod of the role gets a log + storage claim at all. With it disabled the operator drops the claim and + passes an empty --log-file, which turns file logging off, so stderr and + `kubectl logs` (plus whatever collects it) become the single log sink. Use + it to avoid a second PersistentVolumeClaim per pod on clusters that ship + logs off-node anyway. + + The remaining log* knobs below are ignored while this is false. + + Like the sizes and classes around it this is effectively a create-time + choice: flipping it adds or removes a volumeClaimTemplate, which + Kubernetes forbids on a live StatefulSet, so the operator's apply is + rejected until the StatefulSet is recreated (delete it with + --cascade=orphan and the operator rebuilds it around the running pods). + type: boolean libPVCSize: anyOf: - type: integer @@ -756,6 +774,24 @@ spec: description: data configures the storage of every data instance pod. properties: + createLogStorageClaim: + default: true + description: |- + createLogStorageClaim decides whether every pod of the role gets a log + storage claim at all. With it disabled the operator drops the claim and + passes an empty --log-file, which turns file logging off, so stderr and + `kubectl logs` (plus whatever collects it) become the single log sink. Use + it to avoid a second PersistentVolumeClaim per pod on clusters that ship + logs off-node anyway. + + The remaining log* knobs below are ignored while this is false. + + Like the sizes and classes around it this is effectively a create-time + choice: flipping it adds or removes a volumeClaimTemplate, which + Kubernetes forbids on a live StatefulSet, so the operator's apply is + rejected until the StatefulSet is recreated (delete it with + --cascade=orphan and the operator rebuilds it around the running pods). + type: boolean libPVCSize: anyOf: - type: integer diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 0293f77..d561ec7 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -35,10 +35,18 @@ spec: # while the empty string turns dynamic provisioning off and binds a # pre-created PersistentVolume. Sizes, access modes and classes become # StatefulSet volumeClaimTemplates, which Kubernetes treats as immutable. + # Setting createLogStorageClaim to false drops the log claim for that role + # and passes an empty --log-file, leaving stderr (and so `kubectl logs`) as + # the only log sink; the other log* knobs are then ignored. Flipping it + # on a live cluster needs the role's StatefulSet recreated (patch this + # resource, then `kubectl delete statefulset - + # --cascade=orphan`) — adding or removing a claim template is not an update + # Kubernetes allows. coordinators: libPVCSize: 1Gi libStorageAccessMode: ReadWriteOnce # libStorageClassName: standard + createLogStorageClaim: true logPVCSize: 1Gi logStorageAccessMode: ReadWriteOnce # logStorageClassName: standard @@ -46,6 +54,7 @@ spec: libPVCSize: 1Gi libStorageAccessMode: ReadWriteOnce # libStorageClassName: standard + createLogStorageClaim: true logPVCSize: 1Gi logStorageAccessMode: ReadWriteOnce # logStorageClassName: standard diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 8621020..307251d 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -100,7 +100,10 @@ type MemgraphClusterReconciler struct { // mid-bootstrap is harmless. Registration reconciliation is continuous, not // one-shot: a converged cluster is re-observed on a periodic resync, so a // registration a pod loses (rescheduled, wiped storage) is re-issued without -// human action. Deletion needs no handling here — every object +// human action. An apply the API server rejects — an edit to a field +// Kubernetes treats as immutable, a quota denial — is reported on the resource +// as ApplyFailed rather than only in the log, because no amount of retrying +// will clear it. Deletion needs no handling here — every object // carries a controller owner reference, so garbage collection removes the // workloads with the CR. func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -122,7 +125,22 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, fmt.Errorf("setting owner reference on %T %s: %w", obj, obj.GetName(), err) } if err := r.apply(ctx, obj); err != nil { - return ctrl.Result{}, fmt.Errorf("applying %T %s: %w", obj, obj.GetName(), err) + applyErr := fmt.Errorf("applying %T %s: %w", obj, obj.GetName(), err) + // A rejected apply is retried forever behind the scenes, so report + // it on the resource: without this the conditions keep describing + // the cluster that is still running while the declared spec never + // lands, and the rejection is only visible in the operator's log. + // Both conditions go False — the workloads are not the declared + // ones, so neither serving nor convergence can be claimed for the + // spec the user asked for. + msg := truncateMessage(applyErr.Error()) + if statusErr := r.writeStatus(ctx, &cluster, cluster.Status.Main, + notReadyCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), + notConvergedCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), + ); statusErr != nil { + return ctrl.Result{}, errors.Join(applyErr, statusErr) + } + return ctrl.Result{}, applyErr } } @@ -235,6 +253,18 @@ func readyOrNot(main string) metav1.Condition { memgraphcomv1alpha1.ReasonMainElected, "Data instance "+main+" is MAIN") } +// maxConditionMessage bounds a condition message well under the API's own +// 32Ki limit: an apply rejection can carry a long field list, and the useful +// part — what the API server refused — comes first. +const maxConditionMessage = 1024 + +func truncateMessage(message string) string { + if len(message) <= maxConditionMessage { + return message + } + return message[:maxConditionMessage-3] + "..." +} + func trueCondition(condType, reason, message string) metav1.Condition { return metav1.Condition{Type: condType, Status: metav1.ConditionTrue, Reason: reason, Message: message} } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 4e19875..8ed9c87 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -630,6 +630,46 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonCoordinatorUnreachable)) }) + // A rejected apply is retried behind the scenes forever, so the resource + // itself has to say what the API server refused — otherwise the + // conditions keep describing the cluster that is still running while the + // declared spec never lands. Removing a role's log storage claim is the + // realistic trigger: Kubernetes forbids changing a StatefulSet's + // volumeClaimTemplates, so the flip needs the StatefulSet recreated. + It("should report the API server's rejection when applying a workload fails", func() { + fake.setInstances(convergedCluster()) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + Expect(condition(memgraphcomv1alpha1.ConditionReady).Status).To(Equal(metav1.ConditionTrue)) + Expect(condition(memgraphcomv1alpha1.ConditionConverged).Status).To(Equal(metav1.ConditionTrue)) + + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + cluster.Spec.Storage.Data.CreateLogStorageClaim = ptr.To(false) + Expect(k8sClient.Update(ctx, cluster)).To(Succeed()) + + _, err := reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{Name: resourceName, Namespace: resourceNamespace}, + }) + Expect(err).To(HaveOccurred(), "the API server refuses to drop a volume claim template") + + s := status() + Expect(s.Main).To(Equal("instance_0"), "the last observed MAIN survives an apply failure") + for _, condType := range []string{ + memgraphcomv1alpha1.ConditionReady, + memgraphcomv1alpha1.ConditionConverged, + } { + cond := condition(condType) + Expect(cond.Status).To(Equal(metav1.ConditionFalse), "condition %s", condType) + Expect(cond.Reason).To(Equal(memgraphcomv1alpha1.ReasonApplyFailed), "condition %s", condType) + Expect(cond.Message).To(ContainSubstring(resourceName+dataSuffix), + "condition %s must name the object that was refused", condType) + Expect(cond.Message).To(ContainSubstring("Forbidden"), + "condition %s must carry the API server's own words", condType) + } + }) + It("should report ready and converged once the cluster is bootstrapped", func() { reconcileCluster(resourceName) markWorkloadsReady(resourceName) diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index f466541..5bcbbbe 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -40,10 +40,11 @@ import ( // materialize it. func defaultRoleStorage() memgraphcomv1alpha1.RoleStorageSpec { return memgraphcomv1alpha1.RoleStorageSpec{ - LibPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize)), - LibStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, - LogPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize)), - LogStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, + LibPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize)), + LibStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, + CreateLogStorageClaim: ptr.To(memgraphcomv1alpha1.DefaultCreateLogStorageClaim), + LogPVCSize: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize)), + LogStorageAccessMode: memgraphcomv1alpha1.DefaultStorageAccessMode, } } diff --git a/internal/resources/resources.go b/internal/resources/resources.go index f5fdfe1..51629d2 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -142,9 +142,12 @@ type normalizedStorage struct { libSize resource.Quantity libAccessMode corev1.PersistentVolumeAccessMode libClass *string - logSize resource.Quantity - logAccessMode corev1.PersistentVolumeAccessMode - logClass *string + // createLogClaim is false when the role opted out of log storage; the log + // fields below are then unused. + createLogClaim bool + logSize resource.Quantity + logAccessMode corev1.PersistentVolumeAccessMode + logClass *string } func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { @@ -287,12 +290,16 @@ func normalizeEnv(spec []memgraphcomv1alpha1.EnvVar) []corev1.EnvVar { func normalizeStorage(spec memgraphcomv1alpha1.RoleStorageSpec) normalizedStorage { n := normalizedStorage{ - libSize: resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize), - libAccessMode: spec.LibStorageAccessMode, - libClass: spec.LibStorageClassName, - logSize: resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize), - logAccessMode: spec.LogStorageAccessMode, - logClass: spec.LogStorageClassName, + libSize: resource.MustParse(memgraphcomv1alpha1.DefaultLibPVCSize), + libAccessMode: spec.LibStorageAccessMode, + libClass: spec.LibStorageClassName, + createLogClaim: memgraphcomv1alpha1.DefaultCreateLogStorageClaim, + logSize: resource.MustParse(memgraphcomv1alpha1.DefaultLogPVCSize), + logAccessMode: spec.LogStorageAccessMode, + logClass: spec.LogStorageClassName, + } + if spec.CreateLogStorageClaim != nil { + n.createLogClaim = *spec.CreateLogStorageClaim } if spec.LibPVCSize != nil { n.libSize = *spec.LibPVCSize diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index 8dae5bc..8b78021 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -85,7 +85,7 @@ func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.State role := spec.dataRole container := memgraphContainer(spec, role) - container.Args = append(commonArgs(spec), role.extraArgs...) + container.Args = append(commonArgs(spec, role), role.extraArgs...) container.Ports = []corev1.ContainerPort{ {Name: boltPortName, ContainerPort: spec.ports.bolt}, {Name: managementPortName, ContainerPort: spec.ports.management}, @@ -109,7 +109,7 @@ func coordinatorStartScript( role normalizedRole, ) string { fqdnSuffix := podFQDNSuffix(cluster, CoordinatorName(cluster), spec) - args := append(commonArgs(spec), role.extraArgs...) + args := append(commonArgs(spec, role), role.extraArgs...) return fmt.Sprintf(`ordinal="${POD_NAME##*-}" exec %s \ --coordinator-id="$((ordinal + 1))" \ @@ -122,14 +122,26 @@ exec %s \ // chart's auto-appended and default logging arguments. A role's extra args are // appended after these, and Memgraph takes the last occurrence of a repeated // flag, so a user-supplied flag wins. -func commonArgs(spec normalizedSpec) []string { +// +// A role that opted out of log storage gets --log-file with an empty value, +// which is what turns file logging off. Leaving the flag out would not: the +// image ships /etc/memgraph/memgraph.conf with log_file set to the path below, +// Memgraph parses that file before the command line, and failing to open the +// resulting path is fatal — so an unmounted log directory on a read-only root +// filesystem would crash-loop the pod. --also-log-to-stderr keeps the logs in +// `kubectl logs` either way. +func commonArgs(spec normalizedSpec, role normalizedRole) []string { + logDestination := logFile + if !role.storage.createLogClaim { + logDestination = "" + } return []string{ fmt.Sprintf("--bolt-port=%d", spec.ports.bolt), fmt.Sprintf("--management-port=%d", spec.ports.management), "--data-directory=" + dataDirectory, "--log-level=TRACE", "--also-log-to-stderr", - "--log-file=" + logFile, + "--log-file=" + logDestination, "--log-retention-days=35", } } @@ -170,11 +182,7 @@ func memgraphContainer(spec normalizedSpec, role normalizedRole) corev1.Containe }, }, }, role.env...), - VolumeMounts: []corev1.VolumeMount{ - {Name: libVolumeName, MountPath: libMountPath}, - {Name: logVolumeName, MountPath: logMountPath}, - {Name: tmpVolumeName, MountPath: tmpMountPath}, - }, + VolumeMounts: volumeMounts(role.storage), SecurityContext: &corev1.SecurityContext{ AllowPrivilegeEscalation: ptr.To(false), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, @@ -185,6 +193,30 @@ func memgraphContainer(spec normalizedSpec, role normalizedRole) corev1.Containe } } +// volumeMounts are the role's container mounts: lib storage, the scratch +// directory the read-only root filesystem needs, and log storage unless the +// role opted out of it. +func volumeMounts(storage normalizedStorage) []corev1.VolumeMount { + mounts := []corev1.VolumeMount{{Name: libVolumeName, MountPath: libMountPath}} + if storage.createLogClaim { + mounts = append(mounts, corev1.VolumeMount{Name: logVolumeName, MountPath: logMountPath}) + } + return append(mounts, corev1.VolumeMount{Name: tmpVolumeName, MountPath: tmpMountPath}) +} + +// volumeClaimTemplates are the per-pod claims of the role: lib storage always, +// log storage unless the role opted out of it. +func volumeClaimTemplates(storage normalizedStorage) []corev1.PersistentVolumeClaim { + claims := []corev1.PersistentVolumeClaim{ + volumeClaimTemplate(libVolumeName, storage.libSize, storage.libAccessMode, storage.libClass), + } + if storage.createLogClaim { + claims = append(claims, + volumeClaimTemplate(logVolumeName, storage.logSize, storage.logAccessMode, storage.logClass)) + } + return claims +} + func statefulSet( cluster *memgraphcomv1alpha1.MemgraphCluster, component, name string, @@ -217,10 +249,7 @@ func statefulSet( WhenDeleted: retentionType(spec.retentionPolicy), WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, }, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ - volumeClaimTemplate(libVolumeName, storage.libSize, storage.libAccessMode, storage.libClass), - volumeClaimTemplate(logVolumeName, storage.logSize, storage.logAccessMode, storage.logClass), - }, + VolumeClaimTemplates: volumeClaimTemplates(storage), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels(cluster, component, role.podLabels), @@ -234,7 +263,7 @@ func statefulSet( RunAsNonRoot: ptr.To(true), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, - // Lib and log storage come from the volumeClaimTemplates + // Persistent storage comes from the volumeClaimTemplates // above; only the scratch directory the read-only root // filesystem still needs is ephemeral. Volumes: []corev1.Volume{ diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index 5a7797d..53652cc 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -18,6 +18,7 @@ package resources_test import ( "maps" + "slices" "testing" "github.com/google/go-cmp/cmp" @@ -238,6 +239,19 @@ func expectedVolumeMounts() []corev1.VolumeMount { } } +// expectedVolumeMountsWithoutLog is the mount set of a role that opted out of +// log storage: everything except the log volume. +func expectedVolumeMountsWithoutLog() []corev1.VolumeMount { + return slices.DeleteFunc(expectedVolumeMounts(), func(mount corev1.VolumeMount) bool { + return mount.Name == "log-storage" + }) +} + +// expectedCommand wraps a coordinator start script the way the builder does. +func expectedCommand(script string) []string { + return []string{"/bin/sh", "-ec", script} +} + // expectedVolumes covers only the ephemeral scratch volume: lib and log // storage are provisioned through volumeClaimTemplates. func expectedVolumes() []corev1.Volume { @@ -340,7 +354,7 @@ func TestCoordinatorStatefulSetDefaults(t *testing.T) { Name: memgraphName, Image: "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo", ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sh", "-ec", expectedCoordinatorScript}, + Command: expectedCommand(expectedCoordinatorScript), Env: append([]corev1.EnvVar{{ Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{ @@ -513,6 +527,72 @@ func TestStatefulSetStorageOverrides(t *testing.T) { } } +// TestStatefulSetWithoutLogStorageClaim asserts that a role which opted out of +// log storage gets no log claim, no log mount, and an empty --log-file. The +// empty flag is load-bearing rather than cosmetic: the image's +// /etc/memgraph/memgraph.conf sets log_file, Memgraph reads it before the +// command line, and it fails startup when that path cannot be opened — so +// dropping the flag would crash-loop the pod instead of disabling file logging. +// Logs still reach `kubectl logs` through --also-log-to-stderr. +func TestStatefulSetWithoutLogStorageClaim(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.Storage = memgraphcomv1alpha1.StorageSpec{ + Coordinators: memgraphcomv1alpha1.RoleStorageSpec{ + CreateLogStorageClaim: ptr.To(false), + }, + // The data instances keep their log claim: the knob is per role. + Data: memgraphcomv1alpha1.RoleStorageSpec{}, + } + + t.Run(coordinatorComponent, func(t *testing.T) { + sts := resources.CoordinatorStatefulSet(cluster) + + wantClaims := []corev1.PersistentVolumeClaim{ + expectedClaimTemplate("lib-storage", "1Gi", corev1.ReadWriteOnce, nil), + } + if diff := cmp.Diff(wantClaims, sts.Spec.VolumeClaimTemplates); diff != "" { + t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) + } + + container := sts.Spec.Template.Spec.Containers[0] + if diff := cmp.Diff(expectedVolumeMountsWithoutLog(), container.VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } + wantScript := `ordinal="${POD_NAME##*-}" +exec /usr/lib/memgraph/memgraph \ + --coordinator-id="$((ordinal + 1))" \ + --coordinator-hostname="${POD_NAME}.example-coordinator.memgraph-test.svc.cluster.local" \ + --coordinator-port=12000 \ + --bolt-port=7687 \ + --management-port=10000 \ + --data-directory=/var/lib/memgraph/mg_data \ + --log-level=TRACE \ + --also-log-to-stderr \ + --log-file= \ + --log-retention-days=35` + wantCommand := expectedCommand(wantScript) + if diff := cmp.Diff(wantCommand, container.Command); diff != "" { + t.Errorf("start script mismatch (-want +got):\n%s", diff) + } + }) + + t.Run(dataComponent, func(t *testing.T) { + sts := resources.DataStatefulSet(cluster) + + if diff := cmp.Diff(expectedClaimTemplates(), sts.Spec.VolumeClaimTemplates); diff != "" { + t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) + } + + container := sts.Spec.Template.Spec.Containers[0] + if diff := cmp.Diff(expectedVolumeMounts(), container.VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } + if !slices.Contains(container.Args, "--log-file=/var/log/memgraph/memgraph.log") { + t.Errorf("args = %v, want the log file the role still has storage for", container.Args) + } + }) +} + // TestStatefulSetRetentionPolicy pins the mapping from the spec's retention // policy onto the StatefulSet machinery that is the only deleter of this // cluster's storage. whenScaled stays Retain regardless: both replica counts @@ -593,7 +673,7 @@ func TestStatefulSetPortsAndClusterDomain(t *testing.T) { if diff := cmp.Diff(wantPorts, container.Ports); diff != "" { t.Errorf("container ports mismatch (-want +got):\n%s", diff) } - wantCommand := []string{"/bin/sh", "-ec", expectedTunedCoordinatorScript} + wantCommand := expectedCommand(expectedTunedCoordinatorScript) if diff := cmp.Diff(wantCommand, container.Command); diff != "" { t.Errorf("start script mismatch (-want +got):\n%s", diff) } From 93a615d77f835de107cb4296abc901532ab9d0db Mon Sep 17 00:00:00 2001 From: as51340 Date: Mon, 27 Jul 2026 16:16:43 +0200 Subject: [PATCH 17/34] feat: Add support for core dumps containers --- README.md | 2 +- api/v1alpha1/memgraphcluster_types.go | 160 ++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 89 +++++++ .../crds/memgraph.com_memgraphclusters.yaml | 240 ++++++++++++++++++ .../bases/memgraph.com_memgraphclusters.yaml | 240 ++++++++++++++++++ config/samples/v1alpha1_memgraphcluster.yaml | 53 ++++ .../memgraphcluster_controller_test.go | 1 + .../memgraphcluster_validation_test.go | 75 ++++++ internal/resources/resources.go | 46 +++- internal/resources/statefulset.go | 161 ++++++++++-- internal/resources/statefulset_test.go | 204 ++++++++++++++- 11 files changed, 1246 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 496035c..b25d6f2 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Uninstall the operator with `helm uninstall memgraph-operator --namespace memgra ## Configuration -Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, whether the log claim is created at all, retention) per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. +Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, whether the log claim is created at all, retention) per role, optional core dump collection with an uploader sidecar of your choice per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. [`config/samples/v1alpha1_memgraphcluster.yaml`](config/samples/v1alpha1_memgraphcluster.yaml) spells the full surface out with every default and the reasoning behind it. `kubectl explain mgc.spec --recursive` documents the same fields from the installed CRD. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 6e3a52f..419425b 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -41,6 +41,9 @@ const ( DefaultLicenseSecretKey = "MEMGRAPH_ENTERPRISE_LICENSE" DefaultOrganizationSecretKey = "MEMGRAPH_ORGANIZATION_NAME" + DefaultCoreDumpsSize = "10Gi" + DefaultConfigureCorePattern = true + DefaultLibPVCSize = "1Gi" DefaultLogPVCSize = "1Gi" DefaultCreateLogStorageClaim = true @@ -86,6 +89,11 @@ const ( // EnvPodName carries the pod's own name, from which a coordinator derives // its ordinal-dependent identity at startup. EnvPodName = "POD_NAME" + + // EnvCoreDumpsDir carries the core dumps mount path into the uploader + // sidecar, which therefore may not set it itself. It is the only variable + // the operator sets on a container other than Memgraph's. + EnvCoreDumpsDir = "CORE_DUMPS_DIR" ) // StorageRetentionPolicy decides what happens to the cluster's @@ -314,6 +322,152 @@ type StorageSpec struct { Data RoleStorageSpec `json:"data,omitzero"` } +// RoleCoreDumpsSpec is the part of core dump collection that genuinely differs +// between the roles: whether they collect at all, and how much room a dump +// needs. Everything else — the storage class, the kernel setup, the uploader — +// is the same decision for both and lives on CoreDumpsSpec. +type RoleCoreDumpsSpec struct { + // enabled provisions a core dumps volume for every pod of the role and + // mounts it at /var/core/memgraph. It is off by default: a crashing Memgraph + // is not the normal case, and the volume costs a third + // PersistentVolumeClaim per pod. + // +kubebuilder:default=false + // +optional + Enabled bool `json:"enabled,omitempty"` + + // size is the requested size of the role's core dumps claim. A dump is + // roughly as large as the crashing process' resident memory, which is why + // this is per role: a data instance holds the graph, a coordinator holds + // Raft state. Size it against the role's memory limit, not its data. + // +kubebuilder:default="10Gi" + // +optional + Size *resource.Quantity `json:"size,omitempty"` +} + +// CoreDumpsUploaderSpec is a sidecar that reads the core dumps volume. It is +// deliberately not a full core/v1 Container: the narrow shape keeps the +// operator's pod security posture non-negotiable (no privileged sidecar, no +// extra volume mounts, no valueFrom smuggling secret material into the CR) and +// keeps the CRD small enough to apply client-side, which one inlined Container +// per role does not. +// +// +kubebuilder:validation:XValidation:rule="!has(self.env) || self.env.all(e, e.name != 'CORE_DUMPS_DIR')",message="env must not set CORE_DUMPS_DIR: the operator passes the core dumps mount path in it" +type CoreDumpsUploaderSpec struct { + // image is the full sidecar image reference including its tag, for example + // "amazon/aws-cli:2.33.28". Unlike the Memgraph image the operator has no + // default for it, so a tag belongs here rather than in a separate field. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=383 + Image string `json:"image"` + + // pullPolicy is the image pull policy of the sidecar. + // +kubebuilder:validation:Enum=Always;IfNotPresent;Never + // +kubebuilder:default=IfNotPresent + // +optional + PullPolicy corev1.PullPolicy `json:"pullPolicy,omitempty"` + + // command overrides the image's entrypoint. + // +optional + Command []string `json:"command,omitempty"` + + // args are the arguments passed to the sidecar's entrypoint. + // +optional + Args []string `json:"args,omitempty"` + + // env passes literal, non-secret environment variables to the sidecar — + // bucket names, prefixes, regions. Credentials belong in envFromSecrets. + // +listType=map + // +listMapKey=name + // +optional + Env []EnvVar `json:"env,omitempty"` + + // envFromSecrets names Secrets in the cluster's namespace whose keys become + // environment variables of the sidecar. This is how credentials reach it: + // by reference, so no secret material ever appears in this resource. + // +listType=set + // +optional + EnvFromSecrets []string `json:"envFromSecrets,omitempty"` + + // resources sets the sidecar's compute resources. Leave it unset and the + // sidecar schedules without requests or limits. + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitzero"` +} + +// CoreDumpsSpec configures core dump collection: what the two roles decide for +// themselves below, and above that the settings that are one decision for the +// whole cluster — where the volumes come from, whether the operator configures +// the node, and what ships the dumps away. +// +// The memgraph-high-availability Helm chart spreads the same feature across +// storage..coreDumps* and a separate top-level coreDumpUploader block +// that silently does nothing unless the per-role claim is enabled too. Here the +// dependency is structural: an uploader with no role collecting dumps is +// rejected, not ignored. +// +// Dumps are for debugging a crash, not for the cluster to run: nothing in the +// operator reads them, and the claims follow the same storage.retentionPolicy +// as the rest of the cluster's volumes. +// +// The has() guards keep the rule evaluable against the block's empty object +// default, which the API server checks before nested field defaults apply. +// +// +kubebuilder:validation:XValidation:rule="!has(self.uploader) || (has(self.coordinators) && has(self.coordinators.enabled) && self.coordinators.enabled) || (has(self.data) && has(self.data.enabled) && self.data.enabled)",message="uploader requires core dumps enabled for at least one role — there would be no volume for it to read" +type CoreDumpsSpec struct { + // coordinators decides whether every coordinator pod collects dumps, and + // how much room it gets for them. + // +kubebuilder:default={} + // +optional + Coordinators RoleCoreDumpsSpec `json:"coordinators,omitzero"` + + // data decides whether every data instance pod collects dumps, and how much + // room it gets for them. + // +kubebuilder:default={} + // +optional + Data RoleCoreDumpsSpec `json:"data,omitzero"` + + // storageClassName is the StorageClass backing every core dumps claim of + // this cluster. Leave it unset to use the cluster's default StorageClass; + // set it to the empty string to disable dynamic provisioning and bind + // pre-created PersistentVolumes. + // +kubebuilder:validation:MaxLength=253 + // +optional + StorageClassName *string `json:"storageClassName,omitempty"` + + // configureCorePattern lets the operator point the kernel at + // /var/core/memgraph by running a privileged init container that writes + // /proc/sys/kernel/core_pattern. It uses the cluster's own Memgraph image, + // so no second image has to be pulled. + // + // It is cluster-wide rather than per role for two reasons: core_pattern is a + // property of the **node**, so it applies to every process that crashes + // there regardless of which role asked for it, and what really decides this + // is whether the namespace tolerates a privileged container at all. + // PodSecurity "restricted" does not — set this to false there, or wherever + // the platform manages core_pattern itself, and the operator only provisions + // and mounts the volumes, trusting the node to already point at them. + // +kubebuilder:default=true + // +optional + ConfigureCorePattern *bool `json:"configureCorePattern,omitempty"` + + // uploader is an optional sidecar that ships collected dumps off the volume + // — to object storage, a debug host, wherever. Any image and destination + // works, so no provider or credential vocabulary has to live in this API: + // the operator mounts the core dumps volume into the sidecar read-only, + // passes the directory as CORE_DUMPS_DIR, and gives it the same locked-down + // security context as the Memgraph container. See + // config/samples/v1alpha1_memgraphcluster.yaml for an S3 uploader + // equivalent to the Helm chart's. + // + // One definition serves both roles — the destination and credentials do not + // differ between them, and the pods are already distinguishable by hostname + // — and it joins the pods of every role that collects dumps. It counts + // toward pod readiness, so a sidecar that crash-loops keeps those roles from + // ever being registered. + // +optional + Uploader *CoreDumpsUploaderSpec `json:"uploader,omitempty"` +} + // PortsSpec configures the internal ports Memgraph listens on. The knob names // mirror the memgraph-high-availability Helm chart's ports block. // @@ -562,6 +716,12 @@ type MemgraphClusterSpec struct { // +optional Storage StorageSpec `json:"storage,omitzero"` + // coreDumps optionally collects crash dumps of either role onto a volume of + // its own. + // +kubebuilder:default={} + // +optional + CoreDumps CoreDumpsSpec `json:"coreDumps,omitzero"` + // clusterDomain is the Kubernetes cluster domain the advertised FQDN // addresses are built from: ...svc.. // Override it on clusters configured with a domain other than the default. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9447c55..2121868 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,74 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoreDumpsSpec) DeepCopyInto(out *CoreDumpsSpec) { + *out = *in + in.Coordinators.DeepCopyInto(&out.Coordinators) + in.Data.DeepCopyInto(&out.Data) + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } + if in.ConfigureCorePattern != nil { + in, out := &in.ConfigureCorePattern, &out.ConfigureCorePattern + *out = new(bool) + **out = **in + } + if in.Uploader != nil { + in, out := &in.Uploader, &out.Uploader + *out = new(CoreDumpsUploaderSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoreDumpsSpec. +func (in *CoreDumpsSpec) DeepCopy() *CoreDumpsSpec { + if in == nil { + return nil + } + out := new(CoreDumpsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CoreDumpsUploaderSpec) DeepCopyInto(out *CoreDumpsUploaderSpec) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]EnvVar, len(*in)) + copy(*out, *in) + } + if in.EnvFromSecrets != nil { + in, out := &in.EnvFromSecrets, &out.EnvFromSecrets + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Resources.DeepCopyInto(&out.Resources) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CoreDumpsUploaderSpec. +func (in *CoreDumpsUploaderSpec) DeepCopy() *CoreDumpsUploaderSpec { + if in == nil { + return nil + } + out := new(CoreDumpsUploaderSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EnvVar) DeepCopyInto(out *EnvVar) { *out = *in @@ -197,6 +265,7 @@ func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { out.Image = in.Image out.Secrets = in.Secrets in.Storage.DeepCopyInto(&out.Storage) + in.CoreDumps.DeepCopyInto(&out.CoreDumps) in.Ports.DeepCopyInto(&out.Ports) in.Probes.DeepCopyInto(&out.Probes) in.Resources.DeepCopyInto(&out.Resources) @@ -336,6 +405,26 @@ func (in *ResourcesSpec) DeepCopy() *ResourcesSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RoleCoreDumpsSpec) DeepCopyInto(out *RoleCoreDumpsSpec) { + *out = *in + if in.Size != nil { + in, out := &in.Size, &out.Size + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RoleCoreDumpsSpec. +func (in *RoleCoreDumpsSpec) DeepCopy() *RoleCoreDumpsSpec { + if in == nil { + return nil + } + out := new(RoleCoreDumpsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RoleLabelsSpec) DeepCopyInto(out *RoleLabelsSpec) { *out = *in diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml index bd9a0a5..de4a11e 100644 --- a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -88,6 +88,246 @@ spec: - message: 'coordinators is immutable: changing the coordinator count of an existing MemgraphCluster is not supported in v1alpha1' rule: self == oldSelf + coreDumps: + default: {} + description: |- + coreDumps optionally collects crash dumps of either role onto a volume of + its own. + properties: + configureCorePattern: + default: true + description: |- + configureCorePattern lets the operator point the kernel at + /var/core/memgraph by running a privileged init container that writes + /proc/sys/kernel/core_pattern. It uses the cluster's own Memgraph image, + so no second image has to be pulled. + + It is cluster-wide rather than per role for two reasons: core_pattern is a + property of the **node**, so it applies to every process that crashes + there regardless of which role asked for it, and what really decides this + is whether the namespace tolerates a privileged container at all. + PodSecurity "restricted" does not — set this to false there, or wherever + the platform manages core_pattern itself, and the operator only provisions + and mounts the volumes, trusting the node to already point at them. + type: boolean + coordinators: + default: {} + description: |- + coordinators decides whether every coordinator pod collects dumps, and + how much room it gets for them. + properties: + enabled: + default: false + description: |- + enabled provisions a core dumps volume for every pod of the role and + mounts it at /var/core/memgraph. It is off by default: a crashing Memgraph + is not the normal case, and the volume costs a third + PersistentVolumeClaim per pod. + type: boolean + size: + anyOf: + - type: integer + - type: string + default: 10Gi + description: |- + size is the requested size of the role's core dumps claim. A dump is + roughly as large as the crashing process' resident memory, which is why + this is per role: a data instance holds the graph, a coordinator holds + Raft state. Size it against the role's memory limit, not its data. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + data: + default: {} + description: |- + data decides whether every data instance pod collects dumps, and how much + room it gets for them. + properties: + enabled: + default: false + description: |- + enabled provisions a core dumps volume for every pod of the role and + mounts it at /var/core/memgraph. It is off by default: a crashing Memgraph + is not the normal case, and the volume costs a third + PersistentVolumeClaim per pod. + type: boolean + size: + anyOf: + - type: integer + - type: string + default: 10Gi + description: |- + size is the requested size of the role's core dumps claim. A dump is + roughly as large as the crashing process' resident memory, which is why + this is per role: a data instance holds the graph, a coordinator holds + Raft state. Size it against the role's memory limit, not its data. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + storageClassName: + description: |- + storageClassName is the StorageClass backing every core dumps claim of + this cluster. Leave it unset to use the cluster's default StorageClass; + set it to the empty string to disable dynamic provisioning and bind + pre-created PersistentVolumes. + maxLength: 253 + type: string + uploader: + description: |- + uploader is an optional sidecar that ships collected dumps off the volume + — to object storage, a debug host, wherever. Any image and destination + works, so no provider or credential vocabulary has to live in this API: + the operator mounts the core dumps volume into the sidecar read-only, + passes the directory as CORE_DUMPS_DIR, and gives it the same locked-down + security context as the Memgraph container. See + config/samples/v1alpha1_memgraphcluster.yaml for an S3 uploader + equivalent to the Helm chart's. + + One definition serves both roles — the destination and credentials do not + differ between them, and the pods are already distinguishable by hostname + — and it joins the pods of every role that collects dumps. It counts + toward pod readiness, so a sidecar that crash-loops keeps those roles from + ever being registered. + properties: + args: + description: args are the arguments passed to the sidecar's + entrypoint. + items: + type: string + type: array + command: + description: command overrides the image's entrypoint. + items: + type: string + type: array + env: + description: |- + env passes literal, non-secret environment variables to the sidecar — + bucket names, prefixes, regions. Credentials belong in envFromSecrets. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFromSecrets: + description: |- + envFromSecrets names Secrets in the cluster's namespace whose keys become + environment variables of the sidecar. This is how credentials reach it: + by reference, so no secret material ever appears in this resource. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is the full sidecar image reference including its tag, for example + "amazon/aws-cli:2.33.28". Unlike the Memgraph image the operator has no + default for it, so a tag belongs here rather than in a separate field. + maxLength: 383 + minLength: 1 + type: string + pullPolicy: + default: IfNotPresent + description: pullPolicy is the image pull policy of the sidecar. + enum: + - Always + - IfNotPresent + - Never + type: string + resources: + description: |- + resources sets the sidecar's compute resources. Leave it unset and the + sidecar schedules without requests or limits. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - image + type: object + x-kubernetes-validations: + - message: 'env must not set CORE_DUMPS_DIR: the operator passes + the core dumps mount path in it' + rule: '!has(self.env) || self.env.all(e, e.name != ''CORE_DUMPS_DIR'')' + type: object + x-kubernetes-validations: + - message: uploader requires core dumps enabled for at least one role + — there would be no volume for it to read + rule: '!has(self.uploader) || (has(self.coordinators) && has(self.coordinators.enabled) + && self.coordinators.enabled) || (has(self.data) && has(self.data.enabled) + && self.data.enabled)' dataInstances: default: 2 description: |- diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index b0083f7..4371232 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -86,6 +86,246 @@ spec: - message: 'coordinators is immutable: changing the coordinator count of an existing MemgraphCluster is not supported in v1alpha1' rule: self == oldSelf + coreDumps: + default: {} + description: |- + coreDumps optionally collects crash dumps of either role onto a volume of + its own. + properties: + configureCorePattern: + default: true + description: |- + configureCorePattern lets the operator point the kernel at + /var/core/memgraph by running a privileged init container that writes + /proc/sys/kernel/core_pattern. It uses the cluster's own Memgraph image, + so no second image has to be pulled. + + It is cluster-wide rather than per role for two reasons: core_pattern is a + property of the **node**, so it applies to every process that crashes + there regardless of which role asked for it, and what really decides this + is whether the namespace tolerates a privileged container at all. + PodSecurity "restricted" does not — set this to false there, or wherever + the platform manages core_pattern itself, and the operator only provisions + and mounts the volumes, trusting the node to already point at them. + type: boolean + coordinators: + default: {} + description: |- + coordinators decides whether every coordinator pod collects dumps, and + how much room it gets for them. + properties: + enabled: + default: false + description: |- + enabled provisions a core dumps volume for every pod of the role and + mounts it at /var/core/memgraph. It is off by default: a crashing Memgraph + is not the normal case, and the volume costs a third + PersistentVolumeClaim per pod. + type: boolean + size: + anyOf: + - type: integer + - type: string + default: 10Gi + description: |- + size is the requested size of the role's core dumps claim. A dump is + roughly as large as the crashing process' resident memory, which is why + this is per role: a data instance holds the graph, a coordinator holds + Raft state. Size it against the role's memory limit, not its data. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + data: + default: {} + description: |- + data decides whether every data instance pod collects dumps, and how much + room it gets for them. + properties: + enabled: + default: false + description: |- + enabled provisions a core dumps volume for every pod of the role and + mounts it at /var/core/memgraph. It is off by default: a crashing Memgraph + is not the normal case, and the volume costs a third + PersistentVolumeClaim per pod. + type: boolean + size: + anyOf: + - type: integer + - type: string + default: 10Gi + description: |- + size is the requested size of the role's core dumps claim. A dump is + roughly as large as the crashing process' resident memory, which is why + this is per role: a data instance holds the graph, a coordinator holds + Raft state. Size it against the role's memory limit, not its data. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + storageClassName: + description: |- + storageClassName is the StorageClass backing every core dumps claim of + this cluster. Leave it unset to use the cluster's default StorageClass; + set it to the empty string to disable dynamic provisioning and bind + pre-created PersistentVolumes. + maxLength: 253 + type: string + uploader: + description: |- + uploader is an optional sidecar that ships collected dumps off the volume + — to object storage, a debug host, wherever. Any image and destination + works, so no provider or credential vocabulary has to live in this API: + the operator mounts the core dumps volume into the sidecar read-only, + passes the directory as CORE_DUMPS_DIR, and gives it the same locked-down + security context as the Memgraph container. See + config/samples/v1alpha1_memgraphcluster.yaml for an S3 uploader + equivalent to the Helm chart's. + + One definition serves both roles — the destination and credentials do not + differ between them, and the pods are already distinguishable by hostname + — and it joins the pods of every role that collects dumps. It counts + toward pod readiness, so a sidecar that crash-loops keeps those roles from + ever being registered. + properties: + args: + description: args are the arguments passed to the sidecar's + entrypoint. + items: + type: string + type: array + command: + description: command overrides the image's entrypoint. + items: + type: string + type: array + env: + description: |- + env passes literal, non-secret environment variables to the sidecar — + bucket names, prefixes, regions. Credentials belong in envFromSecrets. + items: + description: |- + EnvVar is one non-secret environment variable set on a role's Memgraph + container. Only literal values are supported — there is deliberately no + valueFrom — so secret material stays confined to the secrets block and the CR + remains safe to commit. + properties: + name: + description: name is the environment variable's name. + maxLength: 253 + minLength: 1 + pattern: ^[A-Za-z_][A-Za-z0-9_]*$ + type: string + value: + description: value is the literal, non-secret value. + maxLength: 4096 + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFromSecrets: + description: |- + envFromSecrets names Secrets in the cluster's namespace whose keys become + environment variables of the sidecar. This is how credentials reach it: + by reference, so no secret material ever appears in this resource. + items: + type: string + type: array + x-kubernetes-list-type: set + image: + description: |- + image is the full sidecar image reference including its tag, for example + "amazon/aws-cli:2.33.28". Unlike the Memgraph image the operator has no + default for it, so a tag belongs here rather than in a separate field. + maxLength: 383 + minLength: 1 + type: string + pullPolicy: + default: IfNotPresent + description: pullPolicy is the image pull policy of the sidecar. + enum: + - Always + - IfNotPresent + - Never + type: string + resources: + description: |- + resources sets the sidecar's compute resources. Leave it unset and the + sidecar schedules without requests or limits. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + required: + - image + type: object + x-kubernetes-validations: + - message: 'env must not set CORE_DUMPS_DIR: the operator passes + the core dumps mount path in it' + rule: '!has(self.env) || self.env.all(e, e.name != ''CORE_DUMPS_DIR'')' + type: object + x-kubernetes-validations: + - message: uploader requires core dumps enabled for at least one role + — there would be no volume for it to read + rule: '!has(self.uploader) || (has(self.coordinators) && has(self.coordinators.enabled) + && self.coordinators.enabled) || (has(self.data) && has(self.data.enabled) + && self.data.enabled)' dataInstances: default: 2 description: |- diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index d561ec7..56e7ee3 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -58,6 +58,59 @@ spec: logPVCSize: 1Gi logStorageAccessMode: ReadWriteOnce # logStorageClassName: standard + # Core dumps are off by default. Enabling them for a role provisions a third + # claim per pod, mounts it at /var/core/memgraph — a fixed path, since only + # the kernel writes it and only the uploader reads it, and the operator points + # both at it — and, unless configureCorePattern is false, runs a privileged + # init container that sets the node's core pattern. Two things that catch + # people: core_pattern is a property of the node, so it applies to everything + # that crashes there, and the privileged init container is rejected in a + # namespace running PodSecurity "restricted". Set configureCorePattern: false + # there and have the platform point core_pattern at that directory itself. + coreDumps: + # Per role, because only these two answers differ between them: whether the + # role collects dumps at all, and how much room one needs — a dump is about + # as large as the crashing process' memory, and a data instance holds the + # graph while a coordinator holds Raft state. + coordinators: + enabled: false + size: 10Gi + data: + enabled: false + size: 10Gi + # The rest is one decision for the cluster, so it is stated once. + # storageClassName: standard + configureCorePattern: true + # An optional sidecar that ships the dumps somewhere durable, joining the + # pods of every role that collects them. The operator mounts the dumps into + # it read-only, passes the directory as CORE_DUMPS_DIR, mounts a writable + # /tmp, and runs it with the same locked-down security context as Memgraph + # (non-root uid 101, read-only root filesystem, no capabilities) — so pick an + # image that tolerates that. Credentials come from envFromSecrets, never from + # this resource. This is the operator's equivalent of the HA chart's + # coreDumpUploader, with the destination left entirely to you: + # + # uploader: + # image: amazon/aws-cli:2.33.28 + # envFromSecrets: + # - aws-s3-credentials # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY + # env: + # - name: HOME # aws-cli needs a writable HOME + # value: /tmp + # - name: S3_TARGET + # value: s3://memgraph-ha-cluster/core-dumps + # - name: AWS_REGION + # value: eu-west-1 + # command: [/bin/sh, -c] + # args: + # - | + # while true; do + # aws s3 sync "$CORE_DUMPS_DIR" "$S3_TARGET/$HOSTNAME" --region "$AWS_REGION" + # sleep 30 + # done + # resources: + # requests: + # memory: 64Mi # The cluster domain and the internal ports are part of every advertised # address the operator registers with the cluster, so they reach the # container ports, the Services and the registration commands together. diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 8ed9c87..7d110f4 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -49,6 +49,7 @@ const ( customImageTag = "3.13.0" customSecretName = "my-license" customStorageClassName = "fast-ssd" + uploaderImage = "amazon/aws-cli:2.33.28" ) // libClaim returns the lib storage claim template of a provisioned diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index 5bcbbbe..38fcd03 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -48,6 +48,24 @@ func defaultRoleStorage() memgraphcomv1alpha1.RoleStorageSpec { } } +// defaultRoleCoreDumps is one role's core dumps block as the CRD schema +// defaults materialize it: off, but with the size it would ask for. +func defaultRoleCoreDumps() memgraphcomv1alpha1.RoleCoreDumpsSpec { + return memgraphcomv1alpha1.RoleCoreDumpsSpec{ + Size: ptr.To(resource.MustParse(memgraphcomv1alpha1.DefaultCoreDumpsSize)), + } +} + +// defaultCoreDumps is the whole core dumps block as the CRD schema defaults +// materialize it. +func defaultCoreDumps() memgraphcomv1alpha1.CoreDumpsSpec { + return memgraphcomv1alpha1.CoreDumpsSpec{ + Coordinators: defaultRoleCoreDumps(), + Data: defaultRoleCoreDumps(), + ConfigureCorePattern: ptr.To(memgraphcomv1alpha1.DefaultConfigureCorePattern), + } +} + // defaultPorts are the internal ports as the CRD schema defaults materialize // them. func defaultPorts() memgraphcomv1alpha1.PortsSpec { @@ -133,6 +151,7 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Coordinators: defaultRoleStorage(), Data: defaultRoleStorage(), }, + CoreDumps: defaultCoreDumps(), ClusterDomain: memgraphcomv1alpha1.DefaultClusterDomain, Ports: defaultPorts(), // Probes, resources, labels and the env/args passthrough have no @@ -258,6 +277,31 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Expect(stored.Spec.Ports.CoordinatorPort).To(HaveValue(Equal(memgraphcomv1alpha1.DefaultCoordinatorPort))) }) + It("should accept core dumps with an uploader and default what it leaves out", func() { + stored := createAccepted("valid-core-dumps-uploader", memgraphcomv1alpha1.MemgraphClusterSpec{ + CoreDumps: memgraphcomv1alpha1.CoreDumpsSpec{ + Data: memgraphcomv1alpha1.RoleCoreDumpsSpec{ + Enabled: true, + Size: ptr.To(resource.MustParse("200Gi")), + }, + Uploader: &memgraphcomv1alpha1.CoreDumpsUploaderSpec{ + Image: uploaderImage, + Env: []memgraphcomv1alpha1.EnvVar{{Name: "S3_BUCKET", Value: "dumps"}}, + EnvFromSecrets: []string{"aws-s3-credentials"}, + }, + }, + }) + + dumps := stored.Spec.CoreDumps + Expect(dumps.Data.Enabled).To(BeTrue()) + Expect(dumps.Data.Size).To(HaveValue(Equal(resource.MustParse("200Gi")))) + Expect(dumps.ConfigureCorePattern).To(HaveValue(BeTrue())) + Expect(dumps.Uploader.PullPolicy).To(Equal(memgraphcomv1alpha1.DefaultImagePullPolicy)) + // Whether a role collects at all, and how much room it needs, stays + // its own decision: the coordinators asked for neither. + Expect(dumps.Coordinators).To(Equal(defaultRoleCoreDumps())) + }) + It("should accept a registry host carrying a port", func() { stored := createAccepted("valid-registry-port", memgraphcomv1alpha1.MemgraphClusterSpec{ Image: memgraphcomv1alpha1.ImageSpec{Repository: "registry.example.com:5000/memgraph"}, @@ -341,6 +385,37 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Entry("a cluster domain that is not a DNS name", "invalid-cluster-domain", memgraphcomv1alpha1.MemgraphClusterSpec{ClusterDomain: "Cluster_Local"}, "in body should match"), + // An uploader with no volume to read would poll an empty directory + // forever, so the dependency the Helm chart leaves implicit between + // its two blocks is enforced here. + Entry("an uploader with no role collecting dumps", "invalid-uploader-without-dumps", + memgraphcomv1alpha1.MemgraphClusterSpec{ + CoreDumps: memgraphcomv1alpha1.CoreDumpsSpec{ + Uploader: &memgraphcomv1alpha1.CoreDumpsUploaderSpec{Image: uploaderImage}, + }, + }, + "uploader requires core dumps enabled for at least one role"), + Entry("an uploader without an image", "invalid-uploader-no-image", + memgraphcomv1alpha1.MemgraphClusterSpec{ + CoreDumps: memgraphcomv1alpha1.CoreDumpsSpec{ + Data: memgraphcomv1alpha1.RoleCoreDumpsSpec{Enabled: true}, + Uploader: &memgraphcomv1alpha1.CoreDumpsUploaderSpec{}, + }, + }, + "should be at least 1 chars long"), + Entry("an uploader shadowing the core dumps path variable", "invalid-uploader-env", + memgraphcomv1alpha1.MemgraphClusterSpec{ + CoreDumps: memgraphcomv1alpha1.CoreDumpsSpec{ + Coordinators: memgraphcomv1alpha1.RoleCoreDumpsSpec{Enabled: true}, + Uploader: &memgraphcomv1alpha1.CoreDumpsUploaderSpec{ + Image: uploaderImage, + Env: []memgraphcomv1alpha1.EnvVar{{ + Name: memgraphcomv1alpha1.EnvCoreDumpsDir, Value: "/elsewhere", + }}, + }, + }, + }, + "env must not set CORE_DUMPS_DIR"), Entry("an env var name that is not a shell identifier", "invalid-env-name", memgraphcomv1alpha1.MemgraphClusterSpec{ ExtraEnv: memgraphcomv1alpha1.ExtraEnvSpec{ diff --git a/internal/resources/resources.go b/internal/resources/resources.go index 51629d2..e4c2d73 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -115,6 +115,7 @@ type normalizedPorts struct { // normalizedRole is everything the builders need that is configured per role. type normalizedRole struct { storage normalizedStorage + coreDumps normalizedCoreDumps startupProbe normalizedProbe readinessProbe normalizedProbe livenessProbe normalizedProbe @@ -126,6 +127,18 @@ type normalizedRole struct { extraArgs []string } +// normalizedCoreDumps is one role's core dump configuration with every optional +// field resolved to its CRD schema default. Everything hangs off enabled: with +// it false the rest is unused, and no claim, mount, init container or sidecar +// reaches the role's pods. +type normalizedCoreDumps struct { + enabled bool + size resource.Quantity + class *string + configurePattern bool + uploader *memgraphcomv1alpha1.CoreDumpsUploaderSpec +} + // normalizedProbe is one probe's timings; the probe type is always a TCP-socket // check against the role's own port. type normalizedProbe struct { @@ -164,6 +177,7 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { retentionPolicy: spec.Storage.RetentionPolicy, coordinatorRole: normalizeRole(roleSpec{ storage: spec.Storage.Coordinators, + coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Coordinators), probes: spec.Probes.Coordinators, resources: spec.Resources.Coordinators, labels: spec.Labels.Coordinators, @@ -174,6 +188,7 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { }), dataRole: normalizeRole(roleSpec{ storage: spec.Storage.Data, + coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Data), probes: spec.Probes.Data, resources: spec.Resources.Data, labels: spec.Labels.Data, @@ -217,7 +232,11 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { // CR, so normalization is written once and both roles resolve their defaults // the same way. type roleSpec struct { - storage memgraphcomv1alpha1.RoleStorageSpec + storage memgraphcomv1alpha1.RoleStorageSpec + // coreDumps arrives already normalized: unlike the other entries it is + // folded from two spec blocks (the cluster-wide settings and the role's + // own), which the caller does before handing it over. + coreDumps normalizedCoreDumps probes memgraphcomv1alpha1.RoleProbesSpec resources corev1.ResourceRequirements labels memgraphcomv1alpha1.RoleLabelsSpec @@ -232,6 +251,7 @@ type roleSpec struct { func normalizeRole(role roleSpec) normalizedRole { return normalizedRole{ storage: normalizeStorage(role.storage), + coreDumps: role.coreDumps, startupProbe: normalizeProbe(role.probes.StartupProbe, role.startupFailureThreshold), readinessProbe: normalizeProbe(role.probes.ReadinessProbe, memgraphcomv1alpha1.DefaultProbeFailureThreshold), livenessProbe: normalizeProbe(role.probes.LivenessProbe, memgraphcomv1alpha1.DefaultProbeFailureThreshold), @@ -316,6 +336,30 @@ func normalizeStorage(spec memgraphcomv1alpha1.RoleStorageSpec) normalizedStorag return n } +// normalizeCoreDumps folds the cluster-wide core dump settings together with +// the role's own into the single view the builders work from. Nothing is +// resolved eagerly for a disabled role beyond its defaults: the builders check +// enabled before reading the rest. +func normalizeCoreDumps( + shared memgraphcomv1alpha1.CoreDumpsSpec, + role memgraphcomv1alpha1.RoleCoreDumpsSpec, +) normalizedCoreDumps { + n := normalizedCoreDumps{ + enabled: role.Enabled, + size: resource.MustParse(memgraphcomv1alpha1.DefaultCoreDumpsSize), + class: shared.StorageClassName, + configurePattern: memgraphcomv1alpha1.DefaultConfigureCorePattern, + uploader: shared.Uploader, + } + if role.Size != nil { + n.size = *role.Size + } + if shared.ConfigureCorePattern != nil { + n.configurePattern = *shared.ConfigureCorePattern + } + return n +} + func imageRef(image memgraphcomv1alpha1.ImageSpec) string { repository := image.Repository if repository == "" { diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index 8b78021..b9171da 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -38,12 +38,21 @@ const ( libMountPath = "/var/lib/memgraph" logMountPath = "/var/log/memgraph" tmpMountPath = "/tmp" + // coreDumpsMountPath is not configurable: it is only ever written to by the + // kernel and read by the uploader, both of which the operator points at it, + // so a knob would only create ways for the two to disagree. + coreDumpsMountPath = "/var/core/memgraph" // Volume names double as the StatefulSet volumeClaimTemplate names, so the // provisioned claims are -, e.g. lib-storage-example-data-0. - libVolumeName = "lib-storage" - logVolumeName = "log-storage" - tmpVolumeName = "tmp" + libVolumeName = "lib-storage" + logVolumeName = "log-storage" + coreDumpsVolumeName = "core-dumps" + tmpVolumeName = "tmp" + + // Container names of the two optional containers core dumps bring along. + corePatternContainerName = "init-core-pattern" + uploaderContainerName = "core-dumps-uploader" ) // CoordinatorStatefulSet builds the single StatefulSet running all @@ -182,31 +191,120 @@ func memgraphContainer(spec normalizedSpec, role normalizedRole) corev1.Containe }, }, }, role.env...), - VolumeMounts: volumeMounts(role.storage), + VolumeMounts: volumeMounts(role), + SecurityContext: restrictedSecurityContext(), + } +} + +// restrictedSecurityContext is what every container the operator builds runs +// under: no privilege escalation, no capabilities, a read-only root filesystem +// and the default seccomp profile. The core pattern init container is the one +// exception — it cannot do its job under this. +func restrictedSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + ReadOnlyRootFilesystem: ptr.To(true), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +// corePatternInitContainer points the node's kernel at the role's core dumps +// directory. It runs the cluster's own Memgraph image — already pulled on the +// node, so core dumps need no second image to be configured or mirrored — and +// is the only container the operator builds that breaks the restricted security +// posture: /proc/sys is mounted read-only in an unprivileged container, so +// writing core_pattern needs privileged plus root. Nothing else about the pod +// is relaxed, and a namespace that forbids privileged pods can turn this off +// and have the platform manage core_pattern on the node instead. +func corePatternInitContainer(spec normalizedSpec) corev1.Container { + // %e.%p.%t.%s expand to the crashing executable, its pid, the time and the + // signal. + pattern := coreDumpsMountPath + "/core.%e.%p.%t.%s" + return corev1.Container{ + Name: corePatternContainerName, + Image: spec.image, + ImagePullPolicy: spec.pullPolicy, + // tee rather than a plain redirect so the pattern that was set is + // visible in the init container's logs. + Command: []string{"/bin/sh", "-ec", fmt.Sprintf("echo '%s' | tee /proc/sys/kernel/core_pattern", pattern)}, SecurityContext: &corev1.SecurityContext{ - AllowPrivilegeEscalation: ptr.To(false), - Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + Privileged: ptr.To(true), + // Kubernetes rejects a privileged container that also forbids + // privilege escalation, so this one cannot be false. + AllowPrivilegeEscalation: ptr.To(true), ReadOnlyRootFilesystem: ptr.To(true), - RunAsNonRoot: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, } } -// volumeMounts are the role's container mounts: lib storage, the scratch -// directory the read-only root filesystem needs, and log storage unless the -// role opted out of it. -func volumeMounts(storage normalizedStorage) []corev1.VolumeMount { +// uploaderSidecar builds the optional container that ships collected dumps off +// the volume. The operator owns the wiring a sidecar must not get wrong: the +// dumps are mounted read-only (an uploader has no business writing them), the +// mount path arrives as CORE_DUMPS_DIR so the path is stated once, and the +// pod's scratch volume is mounted at /tmp so the sidecar has somewhere to write +// without a writable root filesystem. +func uploaderSidecar(coreDumps normalizedCoreDumps) corev1.Container { + uploader := coreDumps.uploader + pullPolicy := uploader.PullPolicy + if pullPolicy == "" { + pullPolicy = memgraphcomv1alpha1.DefaultImagePullPolicy + } + env := append([]corev1.EnvVar{{ + Name: memgraphcomv1alpha1.EnvCoreDumpsDir, Value: coreDumpsMountPath, + }}, normalizeEnv(uploader.Env)...) + + envFrom := make([]corev1.EnvFromSource, 0, len(uploader.EnvFromSecrets)) + for _, secret := range uploader.EnvFromSecrets { + envFrom = append(envFrom, corev1.EnvFromSource{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: secret}, + }, + }) + } + + return corev1.Container{ + Name: uploaderContainerName, + Image: uploader.Image, + ImagePullPolicy: pullPolicy, + Command: uploader.Command, + Args: uploader.Args, + Env: env, + EnvFrom: envFrom, + Resources: uploader.Resources, + VolumeMounts: []corev1.VolumeMount{ + {Name: coreDumpsVolumeName, MountPath: coreDumpsMountPath, ReadOnly: true}, + {Name: tmpVolumeName, MountPath: tmpMountPath}, + }, + SecurityContext: restrictedSecurityContext(), + } +} + +// volumeMounts are the Memgraph container's mounts: lib storage, the scratch +// directory the read-only root filesystem needs, log storage unless the role +// opted out of it, and the core dumps directory when the role collects dumps. +func volumeMounts(role normalizedRole) []corev1.VolumeMount { mounts := []corev1.VolumeMount{{Name: libVolumeName, MountPath: libMountPath}} - if storage.createLogClaim { + if role.storage.createLogClaim { mounts = append(mounts, corev1.VolumeMount{Name: logVolumeName, MountPath: logMountPath}) } - return append(mounts, corev1.VolumeMount{Name: tmpVolumeName, MountPath: tmpMountPath}) + mounts = append(mounts, corev1.VolumeMount{Name: tmpVolumeName, MountPath: tmpMountPath}) + if role.coreDumps.enabled { + mounts = append(mounts, + corev1.VolumeMount{Name: coreDumpsVolumeName, MountPath: coreDumpsMountPath}) + } + return mounts } // volumeClaimTemplates are the per-pod claims of the role: lib storage always, -// log storage unless the role opted out of it. -func volumeClaimTemplates(storage normalizedStorage) []corev1.PersistentVolumeClaim { +// log storage unless the role opted out of it, and core dumps when enabled. All +// three follow the cluster's single retention policy. +func volumeClaimTemplates(role normalizedRole) []corev1.PersistentVolumeClaim { + storage := role.storage claims := []corev1.PersistentVolumeClaim{ volumeClaimTemplate(libVolumeName, storage.libSize, storage.libAccessMode, storage.libClass), } @@ -214,9 +312,36 @@ func volumeClaimTemplates(storage normalizedStorage) []corev1.PersistentVolumeCl claims = append(claims, volumeClaimTemplate(logVolumeName, storage.logSize, storage.logAccessMode, storage.logClass)) } + if role.coreDumps.enabled { + // Dumps are written by one node's kernel into one pod's directory, so + // the access mode is not a knob: ReadWriteOnce is the only one that + // describes it. + claims = append(claims, volumeClaimTemplate(coreDumpsVolumeName, + role.coreDumps.size, corev1.ReadWriteOnce, role.coreDumps.class)) + } return claims } +// podContainers is the Memgraph container plus the uploader sidecar when the +// role has one. Memgraph stays first, so `kubectl logs` without -c keeps +// showing the database. +func podContainers(memgraph corev1.Container, role normalizedRole) []corev1.Container { + containers := []corev1.Container{memgraph} + if role.coreDumps.enabled && role.coreDumps.uploader != nil { + containers = append(containers, uploaderSidecar(role.coreDumps)) + } + return containers +} + +// podInitContainers is empty unless the role asked the operator to configure +// the node's core pattern. +func podInitContainers(spec normalizedSpec, role normalizedRole) []corev1.Container { + if !role.coreDumps.enabled || !role.coreDumps.configurePattern { + return nil + } + return []corev1.Container{corePatternInitContainer(spec)} +} + func statefulSet( cluster *memgraphcomv1alpha1.MemgraphCluster, component, name string, @@ -225,7 +350,6 @@ func statefulSet( replicas int32, container corev1.Container, ) *appsv1.StatefulSet { - storage := role.storage return &appsv1.StatefulSet{ // TypeMeta is set explicitly because the controller server-side // applies builder output, and apply patches must carry the GVK. @@ -249,13 +373,14 @@ func statefulSet( WhenDeleted: retentionType(spec.retentionPolicy), WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, }, - VolumeClaimTemplates: volumeClaimTemplates(storage), + VolumeClaimTemplates: volumeClaimTemplates(role), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: labels(cluster, component, role.podLabels), }, Spec: corev1.PodSpec{ - Containers: []corev1.Container{container}, + InitContainers: podInitContainers(spec, role), + Containers: podContainers(container, role), SecurityContext: &corev1.PodSecurityContext{ RunAsUser: ptr.To(memgraphUserID), RunAsGroup: ptr.To(memgraphGroupID), diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index 53652cc..d3200a6 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -53,6 +53,12 @@ const ( statefulSetKind = "StatefulSet" serviceKind = "Service" + tmpVolume = "tmp" + shell = "/bin/sh" + defaultImageRef = "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo" + coreDumpsVolume = "core-dumps" + coreDumpsPath = "/var/core/memgraph" + // The operator's identity labels, which custom labels may never override. nameLabel = "app.kubernetes.io/name" instanceLabel = "app.kubernetes.io/instance" @@ -235,7 +241,7 @@ func expectedVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{ {Name: "lib-storage", MountPath: "/var/lib/memgraph"}, {Name: "log-storage", MountPath: "/var/log/memgraph"}, - {Name: "tmp", MountPath: "/tmp"}, + {Name: tmpVolume, MountPath: "/tmp"}, } } @@ -249,14 +255,14 @@ func expectedVolumeMountsWithoutLog() []corev1.VolumeMount { // expectedCommand wraps a coordinator start script the way the builder does. func expectedCommand(script string) []string { - return []string{"/bin/sh", "-ec", script} + return []string{shell, "-ec", script} } // expectedVolumes covers only the ephemeral scratch volume: lib and log // storage are provisioned through volumeClaimTemplates. func expectedVolumes() []corev1.Volume { return []corev1.Volume{ - {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: tmpVolume, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, } } @@ -352,7 +358,7 @@ func TestCoordinatorStatefulSetDefaults(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: memgraphName, - Image: "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo", + Image: defaultImageRef, ImagePullPolicy: corev1.PullIfNotPresent, Command: expectedCommand(expectedCoordinatorScript), Env: append([]corev1.EnvVar{{ @@ -406,7 +412,7 @@ func TestDataStatefulSetDefaults(t *testing.T) { Spec: corev1.PodSpec{ Containers: []corev1.Container{{ Name: memgraphName, - Image: "docker.io/memgraph/memgraph:3.12.0-relwithdebinfo", + Image: defaultImageRef, ImagePullPolicy: corev1.PullIfNotPresent, Args: []string{ "--bolt-port=7687", @@ -593,6 +599,194 @@ exec /usr/lib/memgraph/memgraph \ }) } +// TestStatefulSetCoreDumpsDisabledByDefault pins that a spec which never +// mentions core dumps carries no trace of the feature: no claim, no mount, no +// init container, no sidecar. +func TestStatefulSetCoreDumpsDisabledByDefault(t *testing.T) { + for _, sts := range []*appsv1.StatefulSet{ + resources.CoordinatorStatefulSet(minimalCluster()), + resources.DataStatefulSet(minimalCluster()), + } { + t.Run(sts.Name, func(t *testing.T) { + for _, claim := range sts.Spec.VolumeClaimTemplates { + if claim.Name == coreDumpsVolume { + t.Errorf("claim %s exists without core dumps being enabled", claim.Name) + } + } + podSpec := sts.Spec.Template.Spec + if len(podSpec.InitContainers) != 0 { + t.Errorf("init containers = %v, want none", podSpec.InitContainers) + } + if len(podSpec.Containers) != 1 { + t.Errorf("containers = %d, want only Memgraph's", len(podSpec.Containers)) + } + }) + } +} + +// TestStatefulSetCoreDumps covers the enabled path end to end for one role +// while the other stays untouched: the claim, the Memgraph container's mount, +// and the privileged init container that points the node's kernel at it. +func TestStatefulSetCoreDumps(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.CoreDumps = memgraphcomv1alpha1.CoreDumpsSpec{ + Data: memgraphcomv1alpha1.RoleCoreDumpsSpec{ + Enabled: true, + Size: ptr.To(resource.MustParse("20Gi")), + }, + StorageClassName: ptr.To("cheap-hdd"), + } + + t.Run(dataComponent, func(t *testing.T) { + sts := resources.DataStatefulSet(cluster) + + wantClaims := append(expectedClaimTemplates(), + expectedClaimTemplate(coreDumpsVolume, "20Gi", corev1.ReadWriteOnce, ptr.To("cheap-hdd"))) + if diff := cmp.Diff(wantClaims, sts.Spec.VolumeClaimTemplates); diff != "" { + t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) + } + + podSpec := sts.Spec.Template.Spec + wantMounts := append(expectedVolumeMounts(), + corev1.VolumeMount{Name: coreDumpsVolume, MountPath: coreDumpsPath}) + if diff := cmp.Diff(wantMounts, podSpec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } + + // The init container has to be privileged root to write a kernel sysctl, + // and it reuses the cluster's Memgraph image so nothing else is pulled. + wantInit := []corev1.Container{{ + Name: "init-core-pattern", + Image: defaultImageRef, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: expectedCommand( + "echo '/var/core/memgraph/core.%e.%p.%t.%s' | tee /proc/sys/kernel/core_pattern"), + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(true), + ReadOnlyRootFilesystem: ptr.To(true), + RunAsUser: ptr.To(int64(0)), + RunAsNonRoot: ptr.To(false), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + }} + if diff := cmp.Diff(wantInit, podSpec.InitContainers); diff != "" { + t.Errorf("init containers mismatch (-want +got):\n%s", diff) + } + if len(podSpec.Containers) != 1 { + t.Errorf("containers = %d, want only Memgraph's without an uploader", len(podSpec.Containers)) + } + }) + + // The knob is per role: coordinators asked for nothing and get nothing. + t.Run(coordinatorComponent, func(t *testing.T) { + sts := resources.CoordinatorStatefulSet(cluster) + + if diff := cmp.Diff(expectedClaimTemplates(), sts.Spec.VolumeClaimTemplates); diff != "" { + t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) + } + if got := sts.Spec.Template.Spec.InitContainers; len(got) != 0 { + t.Errorf("init containers = %v, want none", got) + } + }) +} + +// TestStatefulSetCoreDumpsWithoutCorePattern covers the restricted-namespace +// path: the volume is provisioned and mounted, but the operator runs no +// privileged container and trusts the node's own core pattern. +func TestStatefulSetCoreDumpsWithoutCorePattern(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.CoreDumps = memgraphcomv1alpha1.CoreDumpsSpec{ + Data: memgraphcomv1alpha1.RoleCoreDumpsSpec{Enabled: true}, + ConfigureCorePattern: ptr.To(false), + } + + sts := resources.DataStatefulSet(cluster) + podSpec := sts.Spec.Template.Spec + + if got := podSpec.InitContainers; len(got) != 0 { + t.Errorf("init containers = %v, want none when the node owns the core pattern", got) + } + wantMount := corev1.VolumeMount{Name: coreDumpsVolume, MountPath: coreDumpsPath} + if got := podSpec.Containers[0].VolumeMounts; !slices.Contains(got, wantMount) { + t.Errorf("volume mounts = %v, want the core dumps volume mounted anyway", got) + } + claims := sts.Spec.VolumeClaimTemplates + if diff := cmp.Diff( + expectedClaimTemplate(coreDumpsVolume, "10Gi", corev1.ReadWriteOnce, nil), + claims[len(claims)-1], + ); diff != "" { + t.Errorf("core dumps claim mismatch (-want +got):\n%s", diff) + } +} + +// TestStatefulSetCoreDumpsUploader pins the wiring the operator owns on behalf +// of the sidecar: a read-only view of the dumps, the path as CORE_DUMPS_DIR, a +// writable /tmp, credentials by Secret reference, and the same locked-down +// security context the Memgraph container runs under. +func TestStatefulSetCoreDumpsUploader(t *testing.T) { + cluster := minimalCluster() + // The uploader is declared once for the cluster; only the role that + // collects dumps gets it. + cluster.Spec.CoreDumps = memgraphcomv1alpha1.CoreDumpsSpec{ + Data: memgraphcomv1alpha1.RoleCoreDumpsSpec{Enabled: true}, + Uploader: &memgraphcomv1alpha1.CoreDumpsUploaderSpec{ + Image: "amazon/aws-cli:2.33.28", + Command: []string{shell, "-c"}, + Args: []string{"upload-loop"}, + Env: []memgraphcomv1alpha1.EnvVar{{Name: "S3_BUCKET", Value: "dumps"}}, + EnvFromSecrets: []string{"aws-s3-credentials"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + }, + } + + containers := resources.DataStatefulSet(cluster).Spec.Template.Spec.Containers + if len(containers) != 2 { + t.Fatalf("containers = %d, want Memgraph plus the uploader", len(containers)) + } + if containers[0].Name != memgraphName { + t.Errorf("first container = %q, want Memgraph to stay first", containers[0].Name) + } + + want := corev1.Container{ + Name: "core-dumps-uploader", + Image: "amazon/aws-cli:2.33.28", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{shell, "-c"}, + Args: []string{"upload-loop"}, + Env: []corev1.EnvVar{ + {Name: "CORE_DUMPS_DIR", Value: coreDumpsPath}, + {Name: "S3_BUCKET", Value: "dumps"}, + }, + EnvFrom: []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "aws-s3-credentials"}, + }, + }}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("64Mi")}, + }, + VolumeMounts: []corev1.VolumeMount{ + {Name: coreDumpsVolume, MountPath: coreDumpsPath, ReadOnly: true}, + {Name: tmpVolume, MountPath: "/tmp"}, + }, + SecurityContext: expectedContainerSecurityContext(), + } + if diff := cmp.Diff(want, containers[1]); diff != "" { + t.Errorf("uploader sidecar mismatch (-want +got):\n%s", diff) + } + + // Coordinators collect no dumps, so the shared uploader has nothing to read + // in their pods and must not be injected there. + coordinators := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec.Containers + if len(coordinators) != 1 { + t.Errorf("coordinator containers = %d, want only Memgraph's: the role collects no dumps", + len(coordinators)) + } +} + // TestStatefulSetRetentionPolicy pins the mapping from the spec's retention // policy onto the StatefulSet machinery that is the only deleter of this // cluster's storage. whenScaled stays Retain regardless: both replica counts From 4290dda1ac6ff53c8c7c34b502991612bc4248ae Mon Sep 17 00:00:00 2001 From: as51340 Date: Mon, 27 Jul 2026 16:35:20 +0200 Subject: [PATCH 18/34] feat: Add support for extraVolumes and extraVolumeMounts --- README.md | 2 +- api/v1alpha1/memgraphcluster_types.go | 66 +++++++ api/v1alpha1/zz_generated.deepcopy.go | 65 ++++++- .../crds/memgraph.com_memgraphclusters.yaml | 168 ++++++++++++++++++ .../bases/memgraph.com_memgraphclusters.yaml | 168 ++++++++++++++++++ config/samples/v1alpha1_memgraphcluster.yaml | 26 +++ .../memgraphcluster_validation_test.go | 66 +++++++ internal/resources/resources.go | 50 +++--- internal/resources/statefulset.go | 23 ++- internal/resources/statefulset_test.go | 64 +++++++ 10 files changed, 667 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index b25d6f2..3669022 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Uninstall the operator with `helm uninstall memgraph-operator --namespace memgra ## Configuration -Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, whether the log claim is created at all, retention) per role, optional core dump collection with an uploader sidecar of your choice per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform environment-variable and Memgraph-flag passthrough per role. +Beyond the quickstart's four fields, v1alpha1 exposes storage (PVC size, access mode, storage class, whether the log claim is created at all, retention) per role, optional core dump collection with an uploader sidecar of your choice per role, resource requests and limits per role, probe timings per role, custom labels on pods, StatefulSets and Services, the internal ports, the cluster domain used in advertised addresses, and a freeform passthrough per role for environment variables, Memgraph flags, and extra volumes and volume mounts. [`config/samples/v1alpha1_memgraphcluster.yaml`](config/samples/v1alpha1_memgraphcluster.yaml) spells the full surface out with every default and the reasoning behind it. `kubectl explain mgc.spec --recursive` documents the same fields from the installed CRD. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 419425b..c0b9ebc 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -653,6 +653,62 @@ type ExtraEnvSpec struct { Data []EnvVar `json:"data,omitempty"` } +// ExtraVolumesSpec adds pod volumes to a role beyond the ones the operator +// provisions, mirroring the memgraph-high-availability Helm chart's +// storage..extraVolumes block. Each entry is a core/v1 Volume: a Secret +// holding certificates, a ConfigMap, a CSI volume, an emptyDir, whatever the +// pod needs. extraVolumeMounts is what puts them in the Memgraph container. +// +// The entries are deliberately schemaless. A core/v1 Volume carries every +// volume source Kubernetes has, and inlining that schema twice grows this CRD +// past the size a client-side kubectl apply can carry — so the field accepts +// the same arbitrary volume YAML the Helm chart does, and the API server keeps +// it verbatim without validating its contents. What that costs: kubectl +// explain says nothing about the entries, and a malformed or misspelled volume +// source is caught when the operator applies the StatefulSet, surfacing on this +// resource as the ApplyFailed condition rather than as an admission error. Two +// mistakes that arrive that way in particular: reusing one of the volume names +// the operator owns (lib-storage, log-storage, core-dumps, tmp), and naming a +// volume source that does not exist. +type ExtraVolumesSpec struct { + // coordinators are added to every coordinator pod. + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + Coordinators []corev1.Volume `json:"coordinators,omitempty"` + + // data are added to every data instance pod. + // +kubebuilder:validation:Schemaless + // +kubebuilder:pruning:PreserveUnknownFields + // +optional + Data []corev1.Volume `json:"data,omitempty"` +} + +// ExtraVolumeMountsSpec mounts volumes into a role's Memgraph container beyond +// the ones the operator mounts, mirroring the memgraph-high-availability Helm +// chart's storage..extraVolumeMounts block. Each entry names a volume the +// pod has — usually one from extraVolumes. +// +// The paths the operator already mounts are off limits: two mounts cannot share +// a path, and mounting over Memgraph's data or log directory would hide it. +type ExtraVolumeMountsSpec struct { + // coordinators are added to every coordinator pod's Memgraph container. + // +listType=map + // +listMapKey=mountPath + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:XValidation:rule="self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', '/var/core/memgraph', '/tmp']))",message="extraVolumeMounts must not mount over a path the operator already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, /tmp)" + // +optional + Coordinators []corev1.VolumeMount `json:"coordinators,omitempty"` + + // data are added to every data instance pod's Memgraph container. + // +listType=map + // +listMapKey=mountPath + // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:XValidation:rule="self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', '/var/core/memgraph', '/tmp']))",message="extraVolumeMounts must not mount over a path the operator already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, /tmp)" + // +optional + Data []corev1.VolumeMount `json:"data,omitempty"` +} + // ExtraArgsSpec passes additional Memgraph flags to a role, so any flag is // usable without waiting for a typed field. The flags are appended after the // ones the operator derives, and Memgraph takes the last occurrence of a @@ -757,6 +813,16 @@ type MemgraphClusterSpec struct { // extraArgs passes additional Memgraph flags to both roles. // +optional ExtraArgs ExtraArgsSpec `json:"extraArgs,omitzero"` + + // extraVolumes adds pod volumes to both roles beyond the ones the operator + // provisions. + // +optional + ExtraVolumes ExtraVolumesSpec `json:"extraVolumes,omitzero"` + + // extraVolumeMounts mounts volumes into both roles' Memgraph containers + // beyond the ones the operator mounts. + // +optional + ExtraVolumeMounts ExtraVolumeMountsSpec `json:"extraVolumeMounts,omitzero"` } // MemgraphClusterStatus defines the observed state of MemgraphCluster. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2121868..c4530ba 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,8 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -158,6 +159,64 @@ func (in *ExtraEnvSpec) DeepCopy() *ExtraEnvSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtraVolumeMountsSpec) DeepCopyInto(out *ExtraVolumeMountsSpec) { + *out = *in + if in.Coordinators != nil { + in, out := &in.Coordinators, &out.Coordinators + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraVolumeMountsSpec. +func (in *ExtraVolumeMountsSpec) DeepCopy() *ExtraVolumeMountsSpec { + if in == nil { + return nil + } + out := new(ExtraVolumeMountsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExtraVolumesSpec) DeepCopyInto(out *ExtraVolumesSpec) { + *out = *in + if in.Coordinators != nil { + in, out := &in.Coordinators, &out.Coordinators + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExtraVolumesSpec. +func (in *ExtraVolumesSpec) DeepCopy() *ExtraVolumesSpec { + if in == nil { + return nil + } + out := new(ExtraVolumesSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageSpec) DeepCopyInto(out *ImageSpec) { *out = *in @@ -272,6 +331,8 @@ func (in *MemgraphClusterSpec) DeepCopyInto(out *MemgraphClusterSpec) { in.Labels.DeepCopyInto(&out.Labels) in.ExtraEnv.DeepCopyInto(&out.ExtraEnv) in.ExtraArgs.DeepCopyInto(&out.ExtraArgs) + in.ExtraVolumes.DeepCopyInto(&out.ExtraVolumes) + in.ExtraVolumeMounts.DeepCopyInto(&out.ExtraVolumeMounts) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MemgraphClusterSpec. @@ -289,7 +350,7 @@ func (in *MemgraphClusterStatus) DeepCopyInto(out *MemgraphClusterStatus) { *out = *in if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.Condition, len(*in)) + *out = make([]metav1.Condition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml index de4a11e..8737072 100644 --- a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -452,6 +452,174 @@ spec: rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) type: object + extraVolumeMounts: + description: |- + extraVolumeMounts mounts volumes into both roles' Memgraph containers + beyond the ones the operator mounts. + properties: + coordinators: + description: coordinators are added to every coordinator pod's + Memgraph container. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraVolumeMounts must not mount over a path the operator + already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, + /tmp) + rule: self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', + '/var/core/memgraph', '/tmp'])) + data: + description: data are added to every data instance pod's Memgraph + container. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraVolumeMounts must not mount over a path the operator + already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, + /tmp) + rule: self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', + '/var/core/memgraph', '/tmp'])) + type: object + extraVolumes: + description: |- + extraVolumes adds pod volumes to both roles beyond the ones the operator + provisions. + properties: + coordinators: + description: coordinators are added to every coordinator pod. + x-kubernetes-preserve-unknown-fields: true + data: + description: data are added to every data instance pod. + x-kubernetes-preserve-unknown-fields: true + type: object image: default: {} description: image selects the Memgraph container image run by all diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 4371232..72132bc 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -450,6 +450,174 @@ spec: rule: self.all(e, !(e.name in ['MEMGRAPH_ENTERPRISE_LICENSE', 'MEMGRAPH_ORGANIZATION_NAME', 'POD_NAME'])) type: object + extraVolumeMounts: + description: |- + extraVolumeMounts mounts volumes into both roles' Memgraph containers + beyond the ones the operator mounts. + properties: + coordinators: + description: coordinators are added to every coordinator pod's + Memgraph container. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraVolumeMounts must not mount over a path the operator + already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, + /tmp) + rule: self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', + '/var/core/memgraph', '/tmp'])) + data: + description: data are added to every data instance pod's Memgraph + container. + items: + description: VolumeMount describes a mounting of a Volume within + a container. + properties: + mountPath: + description: |- + Path within the container at which the volume should be mounted. Must + not contain ':'. + type: string + mountPropagation: + description: |- + mountPropagation determines how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is used. + This field is beta in 1.10. + When RecursiveReadOnly is set to IfPossible or to Enabled, MountPropagation must be None or unspecified + (which defaults to None). + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: |- + Mounted read-only if true, read-write otherwise (false or unspecified). + Defaults to false. + type: boolean + recursiveReadOnly: + description: |- + RecursiveReadOnly specifies whether read-only mounts should be handled + recursively. + + If ReadOnly is false, this field has no meaning and must be unspecified. + + If ReadOnly is true, and this field is set to Disabled, the mount is not made + recursively read-only. If this field is set to IfPossible, the mount is made + recursively read-only, if it is supported by the container runtime. If this + field is set to Enabled, the mount is made recursively read-only if it is + supported by the container runtime, otherwise the pod will not be started and + an error will be generated to indicate the reason. + + If this field is set to IfPossible or Enabled, MountPropagation must be set to + None (or be unspecified, which defaults to None). + + If this field is not specified, it is treated as an equivalent of Disabled. + type: string + subPath: + description: |- + Path within the volume from which the container's volume should be mounted. + Defaults to "" (volume's root). + type: string + subPathExpr: + description: |- + Expanded path within the volume from which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). + SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + maxItems: 64 + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + x-kubernetes-validations: + - message: extraVolumeMounts must not mount over a path the operator + already mounts (/var/lib/memgraph, /var/log/memgraph, /var/core/memgraph, + /tmp) + rule: self.all(m, !(m.mountPath in ['/var/lib/memgraph', '/var/log/memgraph', + '/var/core/memgraph', '/tmp'])) + type: object + extraVolumes: + description: |- + extraVolumes adds pod volumes to both roles beyond the ones the operator + provisions. + properties: + coordinators: + description: coordinators are added to every coordinator pod. + x-kubernetes-preserve-unknown-fields: true + data: + description: data are added to every data instance pod. + x-kubernetes-preserve-unknown-fields: true + type: object image: default: {} description: image selects the Memgraph container image run by all diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 56e7ee3..9963b89 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -208,3 +208,29 @@ spec: coordinators: [] data: - --storage-snapshot-on-exit=false + # Any pod volume the operator does not provision itself, plus where to mount + # it in the Memgraph container. Entries are plain core/v1 volumes and volume + # mounts, appended after the operator's own — the usual reason to reach for + # them is a Secret holding certificates or a ConfigMap of extra + # configuration. + # + # extraVolumes entries are stored without schema validation: a core/v1 volume + # carries every volume source Kubernetes has, and inlining that schema per + # role would grow the CRD past what a client-side `kubectl apply` can carry. + # So any volume source works, exactly as in the Helm chart, but a typo in one + # is only caught when the operator applies the StatefulSet — where it shows up + # as the ApplyFailed condition on this resource. The mounts, being small, are + # fully validated: admission rejects a path the operator already mounts. + extraVolumes: + coordinators: [] + data: [] + # - name: bolt-certs + # secret: + # secretName: bolt-tls + # defaultMode: 0400 + extraVolumeMounts: + coordinators: [] + data: [] + # - name: bolt-certs + # mountPath: /etc/memgraph/ssl + # readOnly: true diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index 38fcd03..99a26bc 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -302,6 +302,56 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Expect(dumps.Coordinators).To(Equal(defaultRoleCoreDumps())) }) + // The extraVolumes entries are schemaless, so nothing but this spec + // proves the API server keeps an arbitrary volume source intact instead + // of pruning the fields it has no schema for. + It("should preserve a schemaless extra volume through a round trip", func() { + stored := createAccepted("valid-extra-volumes", memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraVolumes: memgraphcomv1alpha1.ExtraVolumesSpec{ + Data: []corev1.Volume{{ + Name: "bolt-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "bolt-tls", + DefaultMode: ptr.To(int32(0o400)), + Items: []corev1.KeyToPath{{Key: "tls.crt", Path: "cert.pem"}}, + }, + }, + }}, + Coordinators: []corev1.Volume{{ + Name: "vault", + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "secrets-store.csi.k8s.io", + ReadOnly: ptr.To(true), + VolumeAttributes: map[string]string{"secretProviderClass": "memgraph"}, + }, + }, + }}, + }, + ExtraVolumeMounts: memgraphcomv1alpha1.ExtraVolumeMountsSpec{ + Data: []corev1.VolumeMount{{ + Name: "bolt-certs", MountPath: "/etc/memgraph/ssl", ReadOnly: true, + }}, + }, + }) + + volume := stored.Spec.ExtraVolumes.Data[0] + Expect(volume.Name).To(Equal("bolt-certs")) + Expect(volume.Secret).NotTo(BeNil(), "the secret source must survive a schemaless round trip") + Expect(volume.Secret.SecretName).To(Equal("bolt-tls")) + Expect(volume.Secret.DefaultMode).To(HaveValue(Equal(int32(0o400)))) + Expect(volume.Secret.Items).To(ConsistOf(corev1.KeyToPath{Key: "tls.crt", Path: "cert.pem"})) + + csi := stored.Spec.ExtraVolumes.Coordinators[0].CSI + Expect(csi).NotTo(BeNil()) + Expect(csi.Driver).To(Equal("secrets-store.csi.k8s.io")) + Expect(csi.VolumeAttributes).To(HaveKeyWithValue("secretProviderClass", "memgraph")) + + Expect(stored.Spec.ExtraVolumeMounts.Data[0].MountPath).To(Equal("/etc/memgraph/ssl")) + Expect(stored.Spec.ExtraVolumeMounts.Coordinators).To(BeEmpty()) + }) + It("should accept a registry host carrying a port", func() { stored := createAccepted("valid-registry-port", memgraphcomv1alpha1.MemgraphClusterSpec{ Image: memgraphcomv1alpha1.ImageSpec{Repository: "registry.example.com:5000/memgraph"}, @@ -382,6 +432,22 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Ports: memgraphcomv1alpha1.PortsSpec{ManagementPort: ptr.To(memgraphcomv1alpha1.DefaultBoltPort)}, }, "must all be different ports"), + // Two mounts cannot share a path, and mounting over the data or log + // directory would hide Memgraph's own storage behind another volume. + Entry("an extra mount over the data directory", "invalid-extra-mount-lib", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraVolumeMounts: memgraphcomv1alpha1.ExtraVolumeMountsSpec{ + Data: []corev1.VolumeMount{{Name: "shadow", MountPath: "/var/lib/memgraph"}}, + }, + }, + "extraVolumeMounts must not mount over a path the operator already mounts"), + Entry("an extra mount over the scratch directory", "invalid-extra-mount-tmp", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraVolumeMounts: memgraphcomv1alpha1.ExtraVolumeMountsSpec{ + Coordinators: []corev1.VolumeMount{{Name: "shadow", MountPath: "/tmp"}}, + }, + }, + "extraVolumeMounts must not mount over a path the operator already mounts"), Entry("a cluster domain that is not a DNS name", "invalid-cluster-domain", memgraphcomv1alpha1.MemgraphClusterSpec{ClusterDomain: "Cluster_Local"}, "in body should match"), diff --git a/internal/resources/resources.go b/internal/resources/resources.go index e4c2d73..7d949e7 100644 --- a/internal/resources/resources.go +++ b/internal/resources/resources.go @@ -125,6 +125,8 @@ type normalizedRole struct { serviceLabels map[string]string env []corev1.EnvVar extraArgs []string + extraVolumes []corev1.Volume + extraMounts []corev1.VolumeMount } // normalizedCoreDumps is one role's core dump configuration with every optional @@ -176,24 +178,28 @@ func normalize(spec memgraphcomv1alpha1.MemgraphClusterSpec) normalizedSpec { ports: normalizePorts(spec.Ports), retentionPolicy: spec.Storage.RetentionPolicy, coordinatorRole: normalizeRole(roleSpec{ - storage: spec.Storage.Coordinators, - coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Coordinators), - probes: spec.Probes.Coordinators, - resources: spec.Resources.Coordinators, - labels: spec.Labels.Coordinators, - env: spec.ExtraEnv.Coordinators, - extraArgs: spec.ExtraArgs.Coordinators, + storage: spec.Storage.Coordinators, + coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Coordinators), + probes: spec.Probes.Coordinators, + resources: spec.Resources.Coordinators, + labels: spec.Labels.Coordinators, + env: spec.ExtraEnv.Coordinators, + extraArgs: spec.ExtraArgs.Coordinators, + extraVolumes: spec.ExtraVolumes.Coordinators, + extraMounts: spec.ExtraVolumeMounts.Coordinators, startupFailureThreshold: memgraphcomv1alpha1.DefaultProbeFailureThreshold, }), dataRole: normalizeRole(roleSpec{ - storage: spec.Storage.Data, - coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Data), - probes: spec.Probes.Data, - resources: spec.Resources.Data, - labels: spec.Labels.Data, - env: spec.ExtraEnv.Data, - extraArgs: spec.ExtraArgs.Data, + storage: spec.Storage.Data, + coreDumps: normalizeCoreDumps(spec.CoreDumps, spec.CoreDumps.Data), + probes: spec.Probes.Data, + resources: spec.Resources.Data, + labels: spec.Labels.Data, + env: spec.ExtraEnv.Data, + extraArgs: spec.ExtraArgs.Data, + extraVolumes: spec.ExtraVolumes.Data, + extraMounts: spec.ExtraVolumeMounts.Data, // Data instances get the long startup budget: only they load // snapshots, and a large restore must not be killed mid-load. @@ -236,12 +242,14 @@ type roleSpec struct { // coreDumps arrives already normalized: unlike the other entries it is // folded from two spec blocks (the cluster-wide settings and the role's // own), which the caller does before handing it over. - coreDumps normalizedCoreDumps - probes memgraphcomv1alpha1.RoleProbesSpec - resources corev1.ResourceRequirements - labels memgraphcomv1alpha1.RoleLabelsSpec - env []memgraphcomv1alpha1.EnvVar - extraArgs []string + coreDumps normalizedCoreDumps + probes memgraphcomv1alpha1.RoleProbesSpec + resources corev1.ResourceRequirements + labels memgraphcomv1alpha1.RoleLabelsSpec + env []memgraphcomv1alpha1.EnvVar + extraArgs []string + extraVolumes []corev1.Volume + extraMounts []corev1.VolumeMount // startupFailureThreshold is this role's default startup probe failure // budget — the one default that differs between the roles. @@ -261,6 +269,8 @@ func normalizeRole(role roleSpec) normalizedRole { serviceLabels: role.labels.ServiceLabels, env: normalizeEnv(role.env), extraArgs: role.extraArgs, + extraVolumes: role.extraVolumes, + extraMounts: role.extraMounts, } } diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index b9171da..e61b2e3 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -286,7 +286,8 @@ func uploaderSidecar(coreDumps normalizedCoreDumps) corev1.Container { // volumeMounts are the Memgraph container's mounts: lib storage, the scratch // directory the read-only root filesystem needs, log storage unless the role -// opted out of it, and the core dumps directory when the role collects dumps. +// opted out of it, the core dumps directory when the role collects dumps, and +// last the role's own extra mounts. func volumeMounts(role normalizedRole) []corev1.VolumeMount { mounts := []corev1.VolumeMount{{Name: libVolumeName, MountPath: libMountPath}} if role.storage.createLogClaim { @@ -297,7 +298,18 @@ func volumeMounts(role normalizedRole) []corev1.VolumeMount { mounts = append(mounts, corev1.VolumeMount{Name: coreDumpsVolumeName, MountPath: coreDumpsMountPath}) } - return mounts + return append(mounts, role.extraMounts...) +} + +// podVolumes is the scratch directory the read-only root filesystem needs plus +// the role's extra volumes. Everything persistent comes from +// volumeClaimTemplates instead. +func podVolumes(role normalizedRole) []corev1.Volume { + volumes := make([]corev1.Volume, 0, 1+len(role.extraVolumes)) + volumes = append(volumes, corev1.Volume{ + Name: tmpVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + return append(volumes, role.extraVolumes...) } // volumeClaimTemplates are the per-pod claims of the role: lib storage always, @@ -388,12 +400,7 @@ func statefulSet( RunAsNonRoot: ptr.To(true), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, - // Persistent storage comes from the volumeClaimTemplates - // above; only the scratch directory the read-only root - // filesystem still needs is ephemeral. - Volumes: []corev1.Volume{ - {Name: tmpVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, - }, + Volumes: podVolumes(role), }, }, }, diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index d3200a6..fd0d162 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -787,6 +787,70 @@ func TestStatefulSetCoreDumpsUploader(t *testing.T) { } } +// TestStatefulSetExtraVolumes covers the passthrough: the role's volumes join +// the pod after the operator's scratch volume, its mounts join the Memgraph +// container after the operator's, and the other role is untouched. +func TestStatefulSetExtraVolumes(t *testing.T) { + certVolume := corev1.Volume{ + Name: "bolt-certs", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "bolt-tls"}, + }, + } + certMount := corev1.VolumeMount{Name: "bolt-certs", MountPath: "/etc/memgraph/ssl", ReadOnly: true} + + cluster := minimalCluster() + cluster.Spec.ExtraVolumes = memgraphcomv1alpha1.ExtraVolumesSpec{ + Data: []corev1.Volume{certVolume}, + } + cluster.Spec.ExtraVolumeMounts = memgraphcomv1alpha1.ExtraVolumeMountsSpec{ + Data: []corev1.VolumeMount{certMount}, + } + + t.Run(dataComponent, func(t *testing.T) { + podSpec := resources.DataStatefulSet(cluster).Spec.Template.Spec + + wantVolumes := append(expectedVolumes(), certVolume) + if diff := cmp.Diff(wantVolumes, podSpec.Volumes); diff != "" { + t.Errorf("volumes mismatch (-want +got):\n%s", diff) + } + wantMounts := append(expectedVolumeMounts(), certMount) + if diff := cmp.Diff(wantMounts, podSpec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } + }) + + t.Run(coordinatorComponent, func(t *testing.T) { + podSpec := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec + + if diff := cmp.Diff(expectedVolumes(), podSpec.Volumes); diff != "" { + t.Errorf("volumes mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedVolumeMounts(), podSpec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } + }) +} + +// A volume the role declares but never mounts is still a legitimate pod volume +// — the uploader sidecar or a future consumer may be its reader — so the +// builder passes it through rather than second-guessing it. +func TestStatefulSetExtraVolumeWithoutMount(t *testing.T) { + cluster := minimalCluster() + cluster.Spec.ExtraVolumes.Coordinators = []corev1.Volume{{ + Name: "scratch", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }} + + podSpec := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec + if len(podSpec.Volumes) != 2 { + t.Errorf("volumes = %v, want the scratch volume alongside tmp", podSpec.Volumes) + } + if diff := cmp.Diff(expectedVolumeMounts(), podSpec.Containers[0].VolumeMounts); diff != "" { + t.Errorf("volume mounts mismatch (-want +got):\n%s", diff) + } +} + // TestStatefulSetRetentionPolicy pins the mapping from the spec's retention // policy onto the StatefulSet machinery that is the only deleter of this // cluster's storage. whenScaled stays Retain regardless: both replica counts From 343e5e221aad3e746f3178dc4db43c9f681d5e93 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Tue, 28 Jul 2026 13:35:04 +0200 Subject: [PATCH 19/34] fix: require a coordinator leader before planning registration (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit observeCluster treated a coordinator reporting no leader as the fresh-cluster bootstrap case and planned against its SHOW INSTANCES view. The premise was wrong: a coordinator's initial Raft configuration holds itself alone and it starts as the leader of that one-member cluster, so a fresh coordinator always names itself. An absent leader means the coordinator lost track of one — quorum gone, or it stepped down or was removed — and it then answers from its own state machine, which can be arbitrarily stale and would reject every mutating query the operator could issue anyway. Such a view is now skipped and the next coordinator tried. When no coordinator names a leader, nothing is issued, Ready and Converged go False with the new NoCoordinatorLeader reason — kept apart from CoordinatorUnreachable, since up-but-quorumless pods and unreachable pods need different remedies — and the reconcile requeues. The leader-by-name lookup now falls back to the reported view's bolt server, so a leader outside the declared coordinator set resolves instead of dead-ending. That is also what coordinator scale-down will need, where the leader can be a coordinator on its way out. The fallback was masking an unfaithful test seam: fakeMemgraph modelled a fresh cluster as an empty SHOW INSTANCES, so no fake cluster ever had a leader and the planner's empty-bolt_server rule was never exercised in the controller tests. The fake now serves the connected coordinator's own row as leader with an empty bolt_server, and ADD COORDINATOR fills that server in rather than inventing a row. --- api/v1alpha1/memgraphcluster_types.go | 7 ++ internal/controller/fake_memgraph_test.go | 83 +++++++++++++++- .../controller/memgraphcluster_controller.go | 93 ++++++++++++------ .../memgraphcluster_controller_test.go | 97 +++++++++++++++++++ .../issues/13-coordinator-leader-required.md | 30 ++++++ 5 files changed, 278 insertions(+), 32 deletions(-) create mode 100644 specs/operator-mvp/issues/13-coordinator-leader-required.md diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index c0b9ebc..b44a68e 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -144,6 +144,13 @@ const ( // SHOW INSTANCES, so the cluster state cannot be observed. ReasonCoordinatorUnreachable = "CoordinatorUnreachable" + // ReasonNoCoordinatorLeader is set when coordinators answer SHOW INSTANCES + // but none of them reports a leader, so their views come from stale state + // machines and no management query would be accepted anyway. It is kept + // apart from CoordinatorUnreachable because the remedy differs: the pods are + // up and serving Bolt, what is missing is a Raft quorum. + ReasonNoCoordinatorLeader = "NoCoordinatorLeader" + // ReasonRegistrationInProgress is set while registration commands are being // issued to converge the cluster toward the declared topology. ReasonRegistrationInProgress = "RegistrationInProgress" diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go index 4fd7b30..ca5ede3 100644 --- a/internal/controller/fake_memgraph_test.go +++ b/internal/controller/fake_memgraph_test.go @@ -20,6 +20,8 @@ import ( "context" "fmt" "slices" + "strconv" + "strings" "sync" "github.com/memgraph/kubernetes-operator/internal/memgraph" @@ -34,7 +36,12 @@ type fakeMemgraph struct { mu sync.Mutex // instances is the cluster view every coordinator serves. - instances []memgraph.Instance + instances []memgraph.Instance + // staleViews replaces the shared view for individual coordinator addresses: + // a coordinator that lost the leader answers from its own state machine, so + // what it reports need not match the cluster at all. Commands still land on + // the shared view — a stale coordinator is never written to. + staleViews map[string][]memgraph.Instance connectAttempts int // connectErr, when set, makes every Connect fail — the operator's view of a // cluster whose coordinators do not yet answer Bolt. @@ -81,25 +88,74 @@ func (f *fakeMemgraph) setInstances(instances []memgraph.Instance) { f.instances = slices.Clone(instances) } +// setStaleView makes the coordinator at the given Bolt address answer +// SHOW INSTANCES with its own view instead of the cluster's. +func (f *fakeMemgraph) setStaleView(address string, instances []memgraph.Instance) { + f.mu.Lock() + defer f.mu.Unlock() + if f.staleViews == nil { + f.staleViews = map[string][]memgraph.Instance{} + } + f.staleViews[address] = slices.Clone(instances) +} + type fakeClient struct { cluster *fakeMemgraph address string closed bool } +// ShowInstances serves the shared cluster view, plus the connected +// coordinator's own row when the cluster does not know it yet. That row is how +// a real coordinator answers before it is added: its initial Raft configuration +// holds itself alone and it is started as the leader of that one-member +// cluster, so it names itself leader and reports an empty bolt_server until +// ADD COORDINATOR fills the address in. The row is not written into the shared +// view — a coordinator the cluster has lost track of speaks only for itself. func (c *fakeClient) ShowInstances(context.Context) ([]memgraph.Instance, error) { c.cluster.mu.Lock() defer c.cluster.mu.Unlock() if c.closed { return nil, fmt.Errorf("fake memgraph: connection to %s already closed", c.address) } - return slices.Clone(c.cluster.instances), nil + if stale, ok := c.cluster.staleViews[c.address]; ok { + return slices.Clone(stale), nil + } + self, err := c.selfName() + if err != nil { + return nil, err + } + view := slices.Clone(c.cluster.instances) + if !c.cluster.hasInstance(self) { + view = append(view, memgraph.Instance{Name: self, Health: "up", Role: memgraph.RoleLeader}) + } + return view, nil } func (c *fakeClient) AddCoordinator(_ context.Context, coordinator memgraph.CoordinatorSpec) error { return c.execute(fmt.Sprintf("ADD COORDINATOR %d", coordinator.ID), func() error { - if c.cluster.hasInstance(coordinator.Name()) { - return fmt.Errorf("fake memgraph: coordinator %s already exists", coordinator.Name()) + for i, instance := range c.cluster.instances { + if instance.Name != coordinator.Name() { + continue + } + if instance.BoltServer != "" { + return fmt.Errorf("fake memgraph: coordinator %s already exists", coordinator.Name()) + } + c.cluster.instances[i].BoltServer = coordinator.BoltServer + c.cluster.instances[i].CoordinatorServer = coordinator.CoordinatorServer + c.cluster.instances[i].ManagementServer = coordinator.ManagementServer + return nil + } + // Adding the coordinator that is serving this connection materializes + // the row it has been reporting for itself, so it keeps its leadership; + // any other coordinator joins the formed cluster as a follower. + self, err := c.selfName() + if err != nil { + return err + } + role := memgraph.RoleFollower + if coordinator.Name() == self { + role = memgraph.RoleLeader } c.cluster.instances = append(c.cluster.instances, memgraph.Instance{ Name: coordinator.Name(), @@ -107,7 +163,7 @@ func (c *fakeClient) AddCoordinator(_ context.Context, coordinator memgraph.Coor CoordinatorServer: coordinator.CoordinatorServer, ManagementServer: coordinator.ManagementServer, Health: "up", - Role: memgraph.RoleFollower, + Role: role, }) return nil }) @@ -167,6 +223,23 @@ func (c *fakeClient) execute(command string, apply func() error) error { return nil } +// selfName is the instance name of the coordinator this connection is served +// by. Addresses are the resource builders' pod FQDNs +// ("-...svc.:") and the +// coordinator on pod ordinal N runs with Raft ID N+1. +func (c *fakeClient) selfName() (string, error) { + pod, _, _ := strings.Cut(c.address, ".") + dash := strings.LastIndex(pod, "-") + if dash < 0 { + return "", fmt.Errorf("fake memgraph: %s is not a pod address", c.address) + } + ordinal, err := strconv.Atoi(pod[dash+1:]) + if err != nil { + return "", fmt.Errorf("fake memgraph: %s carries no pod ordinal: %w", c.address, err) + } + return fmt.Sprintf("coordinator_%d", ordinal+1), nil +} + // hasInstance must be called with the cluster lock held. func (f *fakeMemgraph) hasInstance(name string) bool { return slices.ContainsFunc(f.instances, func(instance memgraph.Instance) bool { diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 307251d..d722e01 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -179,11 +179,17 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( topology := resources.DeclaredTopology(cluster) leader, observed, err := r.observeCluster(ctx, topology) if err != nil { - log.Info("Deferred registration because no coordinator answered", "reason", err.Error()) - msg := "No coordinator answered SHOW INSTANCES" + log.Info("Deferred registration because no coordinator leader was usable", "reason", err.Error()) + reason, msg := memgraphcomv1alpha1.ReasonCoordinatorUnreachable, "No coordinator answered SHOW INSTANCES" + if errors.Is(err, errNoCoordinatorLeader) { + // The coordinators are up but have no leader between them, so their + // views are stale and no registration command would be accepted. + reason, msg = memgraphcomv1alpha1.ReasonNoCoordinatorLeader, + "No coordinator reported a leader, so the cluster has no Raft quorum" + } if statusErr := r.writeStatus(ctx, cluster, cluster.Status.Main, - notReadyCondition(memgraphcomv1alpha1.ReasonCoordinatorUnreachable, msg), - notConvergedCondition(memgraphcomv1alpha1.ReasonCoordinatorUnreachable, msg), + notReadyCondition(reason, msg), + notConvergedCondition(reason, msg), ); statusErr != nil { return ctrl.Result{}, statusErr } @@ -326,20 +332,34 @@ func (r *MemgraphClusterReconciler) workloadsReady( return true, nil } +// errNoCoordinatorLeader reports that coordinators answered SHOW INSTANCES but +// none of them named a leader. It is distinguished from unreachable +// coordinators because the two need different remedies, and it is reported as +// such on the resource. +var errNoCoordinatorLeader = errors.New("no coordinator reported a leader") + // observeCluster connects to the coordinator leader and returns its client // together with the SHOW INSTANCES view the planner diffs against. -// Coordinators are tried in ordinal order: one reporting itself leader is -// used directly, a follower redirects to the leader it reports, and when no -// leader exists yet (fresh cluster, Raft not formed) the first reachable -// coordinator is used — adding coordinators to it makes it the leader, -// mirroring the HA chart's bootstrap against its first coordinator. +// Coordinators are tried in ordinal order: one reporting itself leader is used +// directly, and one reporting another coordinator as leader redirects to it. +// +// A coordinator that names no leader is skipped, never used as planning input. +// Its view is not the fresh-cluster case: a coordinator starts with itself as +// the only member of its Raft configuration and as the leader of that +// one-member cluster, so a fresh coordinator always names itself. An absent +// leader means the coordinator lost track of one — quorum gone, or it stepped +// down or was removed from the Raft cluster — and it then answers from its own +// state machine, which can be arbitrarily stale. Every mutating query the +// planner could issue needs a leader anyway, so such a view describes a cluster +// state that is neither current nor writable. func (r *MemgraphClusterReconciler) observeCluster( ctx context.Context, topology planner.Topology, ) (memgraph.Client, []memgraph.Instance, error) { var errs []error + leaderless := false for _, coordinator := range topology.Coordinators { - leader, observed, err := r.showInstances(ctx, coordinator) + conn, observed, err := r.showInstances(ctx, coordinator.BoltServer) if err != nil { errs = append(errs, err) continue @@ -352,45 +372,64 @@ func (r *MemgraphClusterReconciler) observeCluster( break } } - if leaderName == "" || leaderName == coordinator.Name() { - return leader, observed, nil + if leaderName == coordinator.Name() { + return conn, observed, nil } - - // This coordinator is a follower; redirect to the leader it reports. - if err := leader.Close(ctx); err != nil { + if err := conn.Close(ctx); err != nil { errs = append(errs, err) } - candidate, found := coordinatorByName(topology, leaderName) + if leaderName == "" { + leaderless = true + errs = append(errs, fmt.Errorf("%s reported no leader", coordinator.Name())) + continue + } + + // This coordinator is a follower; redirect to the leader it reports. + address, found := leaderAddress(topology, observed, leaderName) if !found { - errs = append(errs, fmt.Errorf("%s reported leader %s, which is not declared", coordinator.Name(), leaderName)) + errs = append(errs, fmt.Errorf("%s reported leader %s without a Bolt address", + coordinator.Name(), leaderName)) continue } - leader, observed, err = r.showInstances(ctx, candidate) + conn, observed, err = r.showInstances(ctx, address) if err != nil { errs = append(errs, err) continue } - return leader, observed, nil + return conn, observed, nil + } + if leaderless { + return nil, nil, fmt.Errorf("%w: %w", errNoCoordinatorLeader, errors.Join(errs...)) } - return nil, nil, fmt.Errorf("no coordinator leader reachable: %w", errors.Join(errs...)) + return nil, nil, fmt.Errorf("no coordinator answered SHOW INSTANCES: %w", errors.Join(errs...)) } -func coordinatorByName(topology planner.Topology, name string) (memgraph.CoordinatorSpec, bool) { +// leaderAddress resolves the Bolt address of the coordinator reported as +// leader. The declared topology is preferred — it is the address the operator +// itself registered — but the reported view is a valid fallback, because the +// leader need not be one of the declared coordinators: a coordinator on its way +// out of the cluster can hold leadership while it is still being removed. +func leaderAddress(topology planner.Topology, observed []memgraph.Instance, name string) (string, bool) { for _, coordinator := range topology.Coordinators { if coordinator.Name() == name { - return coordinator, true + return coordinator.BoltServer, true + } + } + for _, instance := range observed { + if instance.Name == name && instance.BoltServer != "" { + return instance.BoltServer, true } } - return memgraph.CoordinatorSpec{}, false + return "", false } -// showInstances connects to one coordinator and fetches its cluster view, -// closing the connection again on query failure. +// showInstances connects to one coordinator's Bolt address and fetches its +// cluster view, closing the connection again on query failure. func (r *MemgraphClusterReconciler) showInstances( ctx context.Context, - coordinator memgraph.CoordinatorSpec, + address string, ) (memgraph.Client, []memgraph.Instance, error) { - c, err := r.Memgraph.Connect(ctx, coordinator.BoltServer) + c, err := r.Memgraph.Connect(ctx, address) if err != nil { return nil, nil, err } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 7d110f4..9211e56 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -474,6 +474,78 @@ var _ = Describe("MemgraphCluster Controller", func() { })) }) + // A coordinator that reports no leader answers from its own state + // machine: quorum is gone, or it stepped down or was removed from the + // Raft cluster. Its view can be arbitrarily stale and no management + // query it forwards would be accepted, so it is skipped rather than + // planned against. + It("should skip a coordinator reporting no leader and plan on the next one's view", func() { + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleLeader), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }) + // coordinator_1 lost the leader and still remembers a cluster that + // has both data instances registered. + fake.setStaleView(coordinatorAddress(0), []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }) + + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + leader := coordinatorAddress(1) + Expect(fake.executedCommands()).To(Equal([]string{ + leader + ": REGISTER INSTANCE instance_1", + }), "the leader's view is planned against, not the stale one that reports instance_1 registered") + }) + + It("should issue nothing while no coordinator reports a leader", func() { + // Quorum lost: every coordinator answers, none names a leader. + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + }) + + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + result := reconcileCluster(resourceName) + + Expect(fake.executedCommands()).To(BeEmpty(), + "a cluster with no coordinator leader cannot be observed or written to") + Expect(fake.connects()).To(Equal(3), "every declared coordinator is tried before giving up") + Expect(result.RequeueAfter).To(BeNumerically(">", 0), + "a cluster that lost its quorum is retried, not abandoned") + }) + + // The leader is whoever the coordinators elected, which need not be a + // coordinator the CR declares — a coordinator on its way out of the + // cluster can still hold leadership. + It("should register on a leader outside the declared coordinator set", func() { + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedCoordinator(4, memgraph.RoleLeader), + observedDataInstance(0, memgraph.RoleMain), + }) + + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + Expect(fake.executedCommands()).To(Equal([]string{ + coordinatorAddress(3) + ": REGISTER INSTANCE instance_1", + }), "the undeclared leader is redirected to, and left registered as it is") + }) + // convergedCluster is the fully registered view of the default // 3-coordinator, 2-data topology with instance_0 elected MAIN — the // steady state drift is introduced against below. @@ -631,6 +703,31 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonCoordinatorUnreachable)) }) + // Coordinators that answer but have no leader between them are a + // different problem from coordinators that do not answer at all — the + // pods are up and serving Bolt, what is missing is the Raft quorum — so + // they get their own reason. + It("should report a missing quorum apart from unreachable coordinators", func() { + fake.setInstances([]memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + }) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + for _, condType := range []string{ + memgraphcomv1alpha1.ConditionReady, + memgraphcomv1alpha1.ConditionConverged, + } { + cond := condition(condType) + Expect(cond.Status).To(Equal(metav1.ConditionFalse), "condition %s", condType) + Expect(cond.Reason).To(Equal(memgraphcomv1alpha1.ReasonNoCoordinatorLeader), "condition %s", condType) + } + }) + // A rejected apply is retried behind the scenes forever, so the resource // itself has to say what the API server refused — otherwise the // conditions keep describing the cluster that is still running while the diff --git a/specs/operator-mvp/issues/13-coordinator-leader-required.md b/specs/operator-mvp/issues/13-coordinator-leader-required.md new file mode 100644 index 0000000..63d7039 --- /dev/null +++ b/specs/operator-mvp/issues/13-coordinator-leader-required.md @@ -0,0 +1,30 @@ +# Require a coordinator leader before acting + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Close a stale-view hole in `observeCluster`: it currently treats a coordinator that reports no leader as the fresh-cluster bootstrap case and uses that coordinator's `SHOW INSTANCES` view as the planning input. The premise is wrong. A fresh coordinator's initial Raft config contains exactly itself (`coordinator_state_manager.cpp:133-134`) and it is started as the leader of that one-member cluster (`raft_state.cpp:377-380` — "By setting it to false, all coordinators are started as leaders"), and the role column is derived from `GetLeaderId()` on the leader *and* follower paths alike (`coordinator_instance.cpp:227-228`). So a fresh coordinator always names itself leader, and an absent leader means `GetLeaderId() == -1`: quorum lost, or a coordinator that stepped down or was removed from the Raft cluster. Such a coordinator answers from its own stale state machine (`ShowInstancesStatusAsFollower`), and every mutating query the operator could issue needs `LEADER_READY` or forwards to a leader — so acting on that view plans against a cluster state that is not current and cannot be written to anyway. + +Use a coordinator's view only when it names a leader: itself, or another coordinator the operator then redirects to. Skip coordinators that name none, and when no coordinator names one, issue nothing and requeue. Report it as its own reason, `NoCoordinatorLeader`, rather than folding it into `CoordinatorUnreachable` — reachable coordinators without quorum and unreachable pods have different remedies. Widen the leader-by-name lookup so a reported leader outside the declared coordinator set resolves instead of failing, which also removes the "reported leader X, which is not declared" dead end and is the prerequisite for `16-coordinator-scale-down.md`, where the leader may be a coordinator on its way out. + +The fallback has been masking an unfaithful test seam, so `fakeMemgraph` is reworked with it. Today a fresh cluster is modelled as an empty `SHOW INSTANCES` and `ADD COORDINATOR` appends a row with role `follower`, so even a fully bootstrapped fake cluster has no leader at all, and both bootstrap tests pass only because the fallback swallows it — meaning the planner's empty-`bolt_server` rule (`planner.go:104-109`) is never exercised in the controller tests. The fake must serve a fresh coordinator's own row with role `leader` and an empty `bolt_server`, and `ADD COORDINATOR` must fill that server in rather than invent a row. + +## Acceptance criteria + +- [ ] A coordinator view naming no leader is never used as planning input; the next coordinator is tried instead +- [ ] When no coordinator names a leader, no command is issued, `Ready` and `Converged` go False with reason `NoCoordinatorLeader`, and the reconcile requeues +- [ ] A leader named outside the declared coordinator set is resolved and used, not rejected +- [ ] `fakeMemgraph` serves a fresh coordinator's own row as `leader` with an empty `bolt_server`, and `ADD COORDINATOR` fills the bolt server in +- [ ] Bootstrap envtest cases pass against the faithful fake, exercising the empty-`bolt_server` rule end to end +- [ ] Envtest coverage: coordinators reporting no leader (stale follower view) produce zero commands +- [ ] The comment claiming a fresh cluster has no leader because Raft is not yet formed is corrected + +## Blocked by + +- `05-continuous-re-registration.md` +- `06-status-and-conditions.md` From db21b204ee38ad6076bb1915d36e69f5d3aba4b7 Mon Sep 17 00:00:00 2001 From: as51340 Date: Tue, 28 Jul 2026 13:35:33 +0200 Subject: [PATCH 20/34] feat: Add the rest of the issues --- .claude/skills/grill-me/SKILL.md | 12 +++++++ .../issues/14-topology-scale-up.md | 33 ++++++++++++++++++ .../issues/15-data-instance-scale-down.md | 34 +++++++++++++++++++ .../issues/16-coordinator-scale-down.md | 33 ++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 .claude/skills/grill-me/SKILL.md create mode 100644 specs/operator-mvp/issues/14-topology-scale-up.md create mode 100644 specs/operator-mvp/issues/15-data-instance-scale-down.md create mode 100644 specs/operator-mvp/issues/16-coordinator-scale-down.md diff --git a/.claude/skills/grill-me/SKILL.md b/.claude/skills/grill-me/SKILL.md new file mode 100644 index 0000000..abbf089 --- /dev/null +++ b/.claude/skills/grill-me/SKILL.md @@ -0,0 +1,12 @@ +--- +name: grill-me +description: Interview the user relentlessly about a plan or design until reaching shared understanding, resolving each branch of the decision tree. Use when user wants to stress-test a plan, get grilled on their design, or mentions "grill me". +--- + +Interview me relentlessly about every aspect of this plan until we reach a shared understanding. Walk down each branch of the design tree, resolving dependencies between decisions one-by-one. For each question, provide your recommended answer. + +Ask the questions one at a time, waiting for the feedback on each question before continuing. Asking multiple questions at once is bewildering. + +If a *fact* can be answered by exploring the codebase, look it up rather than asking me. The *decisions*, though, are mine - put each one to me and wait for my answer. + +Do not enact the plan until I confirm we have reached a shared understanding. diff --git a/specs/operator-mvp/issues/14-topology-scale-up.md b/specs/operator-mvp/issues/14-topology-scale-up.md new file mode 100644 index 0000000..b20b408 --- /dev/null +++ b/specs/operator-mvp/issues/14-topology-scale-up.md @@ -0,0 +1,33 @@ +# Mutable topology: scale-up + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Make both replica counts mutable and support growing a live cluster, reversing the contract `07-cel-immutability-validation.md` established. The CEL immutability rules come off `coordinators` and `dataInstances`; the floors replace them: coordinators get `Minimum=3` (a hard floor at creation as well as on update — this operator builds real HA clusters) with the existing odd rule retained, data instances keep `Minimum=1`. Nothing else constrains a change: any odd coordinator target ≥ 3, any data target ≥ 1, both roles changeable in one edit, any step size. Bootstrap already issues three `ADD COORDINATOR`s back to back, so sequential Raft config changes are proven; a transient Raft rejection is a retryable error the next reconcile clears. + +Growth itself needs almost nothing new — the planner's existing diff already emits `ADD COORDINATOR` and `REGISTER INSTANCE` for declared members it does not observe. What this issue builds is the structure the shrink slices then hang off, so the API is cut once. Resource builders take the replica count as an argument (`CoordinatorStatefulSet(cluster, replicas)`), keeping them pure while the controller decides the count: `declared` when growing or unchanged, and deliberately `current` when `declared < current`, so this issue can never shrink a StatefulSet. `planner.Topology` gains `RetiringCoordinators` and `RetiringDataInstances`, empty here and populated by 15 and 16. The promotion rule is tightened while it is being touched: prefer the lowest-ordinal declared instance observed `up`, falling back to `declared[0]`, which also closes the bootstrap hole where `SET INSTANCE TO MAIN` against a down instance only writes Raft state and leaves the cluster MAIN-less until the coordinators retry. + +Observability and storage semantics move with the API. `status.coordinators` and `status.dataInstances` publish the registered counts as observed (pure observation — status stays out of reconcile input), surfaced as `priority=1` print columns so the default table stays narrow. `Converged` widens from "registration matches the declared topology" to "registration matches *and* both StatefulSets' replicas equal the declared counts", so `kubectl wait --for=condition=Converged` means the scale is genuinely finished. `persistentVolumeClaimRetentionPolicy.whenScaled` stops being hard-wired `Retain` and follows `spec.storage.retentionPolicy`, one knob meaning one thing; the comment at `statefulset.go:381-386` claiming nothing ever scales down is deleted. + +Collateral: the retention e2e declares `coordinators: 1`, now rejected, so it moves to 3 and its PVC assertion moves from 4 to 8; the envtest immutability cases invert to acceptance and gain floor rejections; `make manifests generate chart-sync` regenerates the CRDs and the chart's copy; README's scaling limitation section is rewritten. + +## Acceptance criteria + +- [ ] Increasing either count on a live `MemgraphCluster` is accepted and the new members are registered without human action +- [ ] `coordinators` below 3 or even is rejected at creation and on update; `dataInstances` below 1 is rejected +- [ ] Resource builders remain pure, taking the replica count as an argument; unit tests cover the count the controller derives, including that a lower declared count never shrinks the applied StatefulSet +- [ ] A MAIN promotion targets the lowest-ordinal declared instance observed `up`, with `declared[0]` as fallback; planner unit tests cover a down `instance_0` +- [ ] `status.coordinators` and `status.dataInstances` report observed registered counts and appear as `priority=1` print columns +- [ ] `Converged` is False while a StatefulSet's replicas differ from the declared count, True once registration and both replica counts match +- [ ] `whenScaled` follows `spec.storage.retentionPolicy`; builder tests cover both values +- [ ] E2E: a dedicated scaling cluster in its own namespace (`createLogStorageClaim: false`, explicit small resource requests, teardown awaited) grows 3/2 to 5/3 and reaches `Converged` with every new member registered +- [ ] Retention e2e updated for the coordinator floor; chart CRDs regenerated and `make chart-verify` green + +## Blocked by + +- `13-coordinator-leader-required.md` diff --git a/specs/operator-mvp/issues/15-data-instance-scale-down.md b/specs/operator-mvp/issues/15-data-instance-scale-down.md new file mode 100644 index 0000000..2f77237 --- /dev/null +++ b/specs/operator-mvp/issues/15-data-instance-scale-down.md @@ -0,0 +1,34 @@ +# Mutable topology: data-instance scale-down + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Shrink the data-instance count, which means unregistering members Memgraph will not let go of while one of them is MAIN (`UNREGISTER INSTANCE` fails with `IS_MAIN`). A StatefulSet sheds only its highest ordinals — `spec.ordinals.start` slides the window but never removes a member from the middle — so the operator never chooses a victim: the retiring set is always the top ordinals, and the job is to make them safe to remove. Retiring ordinals are `[spec.dataInstances, liveStatefulSet.spec.replicas)`, derived from the operator's own prior apply, which bounds the range exactly and keeps the guarantee that an instance a human registered is never touched. + +The sequence runs inside one reconcile pass against one leader connection, because the planner can predict every intermediate state. `DEMOTE INSTANCE` is emitted only when a retiring instance is observed as MAIN, and it deliberately leaves the cluster MAIN-less: Memgraph triggers automatic failover only on a coordinator leadership change with zero MAINs or when the current MAIN fails its ping (`coordinator_instance.cpp:496`, `:1341`), so a manual demote hands the promotion choice back to the operator, which promotes the lowest-ordinal surviving instance observed `up` through the rule `14-topology-scale-up.md` already installed. `UNREGISTER INSTANCE` then removes each retiring member. Replicas are SYNC — `REGISTER INSTANCE` is issued without `AS ASYNC` or `AS STRICT_SYNC` — so promoting a survivor after a clean demote is not a data-loss gamble, and the MAIN-less window is the few milliseconds between two queries in the same pass. + +Unregistration happens before the pods go, so the coordinators never see a registered instance disappear. The smaller replica count is applied in exactly one place: at the end of the registration phase, once the plan is empty. That keeps the pre-apply replica rule from `14` free of any cluster knowledge — it never shrinks — and makes the shrink converge across two passes without a second observation source. The readiness gate stays strict: every pod of the held-at-current StatefulSet must be ready, retiring pods included, so a retiring pod that cannot become ready blocks its own removal and the resource says so rather than acting on a half-known cluster. That is a deliberate trade and belongs in the docs. + +Errors are not special-cased. Read-before-write means `ALREADY_REPLICA` or `NO_INSTANCE_WITH_NAME` only appear when something raced the operator; they surface, the reconcile retries, and the plan is recomputed from a fresh observation. + +## Acceptance criteria + +- [ ] Decreasing `dataInstances` unregisters the retiring instances and then shrinks the StatefulSet, in that order +- [ ] A retiring instance observed as MAIN is demoted and a surviving instance is promoted before it is unregistered; no `UNREGISTER INSTANCE` is ever issued against an observed MAIN +- [ ] Demote, promote and unregister issue within a single reconcile pass against one leader connection +- [ ] The StatefulSet is shrunk only after the plan is empty; a pass with pending commands never lowers the replica count +- [ ] Instances the cluster knows but the spec never declared and that fall outside the retiring range are left untouched +- [ ] `Converged` is False with reason `RetirementInProgress`, naming the retiring instances, until the pods are gone +- [ ] Planner unit tests: MAIN on a retiring ordinal, MAIN on a survivor, several retiring at once, shrink to a single instance, already-unregistered retiring member, mixed grow-and-shrink across roles +- [ ] Retiring PVCs follow `spec.storage.retentionPolicy` on scale-down +- [ ] E2E: on the scaling cluster, `instance_2` is deliberately made MAIN, then the count drops to 2 — MAIN moves to a survivor, `instance_2` is unregistered before its pod terminates, and the cluster reaches `Converged` +- [ ] Docs state that a retiring pod which cannot become ready blocks its own removal + +## Blocked by + +- `14-topology-scale-up.md` diff --git a/specs/operator-mvp/issues/16-coordinator-scale-down.md b/specs/operator-mvp/issues/16-coordinator-scale-down.md new file mode 100644 index 0000000..f5de02c --- /dev/null +++ b/specs/operator-mvp/issues/16-coordinator-scale-down.md @@ -0,0 +1,33 @@ +# Mutable topology: coordinator scale-down + +**Type**: AFK + +## Parent + +`specs/operator-mvp/PRD.md` + +## What to build + +Shrink the coordinator count, which means removing Raft members — and Raft refuses to remove its own leader (`REMOVE COORDINATOR` returns `RAFT_CANNOT_REMOVE_LEADER`). Since a StatefulSet sheds only its highest ordinals, the leader may well sit on one of them, so the operator moves leadership out of the retiring range first. `YIELD LEADERSHIP` is the lever (`MemgraphCypher.g4:669`): it must be issued *on* the current leader, which is the connection the controller already holds, and it cannot name a successor — `yield_leadership()` is called with no successor argument (`raft_state.cpp:557`), so NuRaft picks. That makes it the one command whose outcome the planner cannot predict, so it is always emitted **last** in a plan and is terminal: the controller stops after it and requeues to re-observe under whichever coordinator won. Everything the planner could safely order before it still issues in that same pass. Retiring ordinals are `[spec.coordinators, liveStatefulSet.spec.replicas)`, and because coordinators must stay odd and at or above 3, a shrink always retires an even number of them. + +Raft membership is given up before the pods are, so no removed member's pod outlives its vote. As in `15-data-instance-scale-down.md`, the smaller replica count is applied once the plan is empty, and the strict readiness gate still covers the retiring pods. + +A coordinator removed from Raft keeps running and keeps its state on purpose, and this is what makes re-growing safe. On committing the config that drops it, NuRaft fires `RemovedFromCluster` and sets `steps_to_down_ = 2` (`handle_commit.cxx:725-750`); after two election timeouts it persists `allow_election_timer(false)` and cancels its schedulers (`handle_timeout.cxx:208-234`). It never calls `system_exit`, so the container does not die and the readiness gate is not tripped — it simply goes dormant, still serving Bolt, not campaigning. Two comments state the intent outright: the removal of self from the persisted config is deliberately disabled "for the next launch", and the persistent election-timer flag exists "for the case re-joining this replica to the original cluster". A later `ADD COORDINATOR` is accepted unconditionally by `handle_join_cluster_req` (`handle_join_leave.cxx:138-196`), which takes the leader's term, saves the new config and resets commit indices; with `snapshot_distance_ = 5` and `reserved_log_items_ = 5` (`raft_state.cpp:325-326`) the leader's log is compacted hard enough that a rejoiner almost always receives a snapshot install, replacing its state machine wholesale. A dormant server appends nothing, so its log is always a prefix of the leader's and cannot diverge. No PVC wipe, no removal bookkeeping and no re-add guard are needed; a re-added coordinator on a retained volume is in the same position as one whose pod crashed and stayed down. Verify this once by hand during this issue rather than gating it in CI. + +The stale view a dormant coordinator would otherwise serve is already handled: `13-coordinator-leader-required.md` requires a named leader before any view is used, and a stepped-down coordinator reports none. + +## Acceptance criteria + +- [ ] Decreasing `coordinators` removes the retiring members from Raft and then shrinks the StatefulSet, in that order +- [ ] When the observed leader is retiring, the plan ends with `YIELD LEADERSHIP` and nothing after it; the following pass re-observes and removes under the new leader +- [ ] `REMOVE COORDINATOR` is never issued against the observed leader +- [ ] The StatefulSet is shrunk only after the plan is empty; a pass with pending commands never lowers the replica count +- [ ] `Converged` is False with reason `RetirementInProgress`, or `LeadershipTransferInProgress` while a yield is pending, until the pods are gone +- [ ] Planner unit tests: leader on a retiring ordinal, leader on a survivor, two coordinators retiring at once, an already-removed retiring member, a retiring coordinator alongside retiring data instances +- [ ] Retiring PVCs follow `spec.storage.retentionPolicy` on scale-down +- [ ] E2E: on the scaling cluster, leadership is forced onto `coordinator_4` via `YIELD LEADERSHIP`, then the count drops to 3 — leadership moves to a survivor, both retiring members leave the Raft cluster, and the cluster reaches `Converged` +- [ ] Manual verification recorded in this issue: shrink 5 to 3 and re-grow to 5 under `Retain`, confirming `SHOW INSTANCES` converges with the retained coordinator volumes + +## Blocked by + +- `15-data-instance-scale-down.md` From 5883e6fbcbb89481c86d4e1d426deacff07be0e6 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Tue, 28 Jul 2026 14:37:44 +0200 Subject: [PATCH 21/34] feat: mutable topology with scale-up support (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: mutable topology with scale-up support Reverses the v1alpha1 contract that both replica counts are fixed at creation: the CEL immutability rules come off `coordinators` and `dataInstances`, and schema floors take their place — coordinators must be odd and at least 3 (enforced at creation and on every update, since this operator builds real HA clusters), data instances at least 1. Nothing else constrains a change: any target counts, both roles in one edit, any step size. Growth needs no new planning: the diff that restores a lost registration already emits `ADD COORDINATOR` and `REGISTER INSTANCE` for declared members it does not observe. What this adds is the structure the shrink slices hang off, so the API is cut once: - Resource builders take the replica count as an argument (`CoordinatorStatefulSet(cluster, replicas)`), staying pure while the controller decides the count: declared when growing or unchanged, and deliberately current when declared < current, so this slice can never shrink a StatefulSet. - `planner.Topology` gains `RetiringCoordinators` and `RetiringDataInstances`, empty here. - The promotion rule now prefers the lowest-ordinal declared instance observed `up`, falling back to `declared[0]` — which also closes the bootstrap hole where promoting a down instance only wrote Raft state and left the cluster MAIN-less until the coordinators retried. Observability and storage semantics move with the API. `status.coordinators` and `status.dataInstances` publish the registered counts as observed, as `priority=1` print columns. `Converged` widens to "registration matches and both StatefulSets' replicas equal the declared counts", so `kubectl wait --for=condition=Converged` means the scale is genuinely finished; a held-back count reports `ScaleInProgress`. `persistentVolumeClaimRetentionPolicy.whenScaled` now follows `spec.storage.retentionPolicy` instead of being hard-wired `Retain`. Tests: planner cases for a down `instance_0` and for a grown topology, plus `Registered`; builder cases for the argument-driven replica count and both retention values; envtest inverts the immutability cases to acceptance and adds floor rejections, a grow-both-roles spec, and the never-shrink spec. A new e2e container grows a dedicated 3/2 cluster to 5/3 in its own namespace and waits for `Converged`; the retention e2e and the chart install test move to the coordinator floor. The chart version is bumped to 0.2.0 because its generated CRDs changed. * fix: Don't bump chart version --- CLAUDE.md | 2 +- README.md | 16 +- api/v1alpha1/memgraphcluster_types.go | 50 ++- .../crds/memgraph.com_memgraphclusters.yaml | 44 ++- charts/memgraph-operator/templates/NOTES.txt | 5 + .../bases/memgraph.com_memgraphclusters.yaml | 44 ++- config/samples/v1alpha1_memgraphcluster.yaml | 14 +- examples/minimal-cluster.yaml | 7 +- hack/chart-install-test.sh | 2 +- .../controller/memgraphcluster_controller.go | 176 ++++++++- .../memgraphcluster_controller_test.go | 177 ++++++++- .../memgraphcluster_validation_test.go | 116 +++--- internal/memgraph/client.go | 10 + internal/planner/planner.go | 76 +++- internal/planner/planner_test.go | 202 +++++++++- internal/resources/statefulset.go | 31 +- internal/resources/statefulset_test.go | 164 ++++++--- internal/resources/topology.go | 13 + internal/resources/topology_test.go | 4 +- test/e2e/memgraphcluster_test.go | 346 ++++++++++++++---- 20 files changed, 1237 insertions(+), 262 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 08cb7b1..d6e74d6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,7 +49,7 @@ CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit` The PRD defines seven modules with two pure cores and one mock seam. Keep this separation — it's what makes the logic testable without a cluster: -1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **immutable after creation, enforced by CEL** in the CRD, not a webhook. Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). +1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **mutable**, bounded by schema floors alone (coordinators ≥ 3 and odd, data instances ≥ 1) — enforced by the CRD, not a webhook. Raising a count grows the cluster; lowering one is accepted but not yet carried out (see `specs/operator-mvp/issues/15`, `16`). Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). 2. **Resource builders** — pure functions: spec in, desired Kubernetes objects out (one StatefulSet per role + headless Services; per-pod identity such as coordinator ID and advertised addresses derived from pod ordinals). No API calls, no side effects. Tested golden-style. 3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. 4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. `SET INSTANCE TO MAIN` is issued exactly once at bootstrap (when no MAIN exists); after that, failover belongs to the Raft coordinators — the operator only observes. diff --git a/README.md b/README.md index 3669022..2b9410d 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,9 @@ memgraph 3 2 instance_0 True True 4m12s - **`MAIN`** is the data instance the coordinators elected as MAIN — the one that accepts writes. It is observed, not decided by the operator, so it changes on failover. - **`READY`** is True once a MAIN is elected, i.e. the cluster serves writes. -- **`CONVERGED`** is True once every declared coordinator and data instance is registered and reported healthy. +- **`CONVERGED`** is True once every declared coordinator and data instance is registered and reported healthy, and both StatefulSets run the declared number of replicas. + +`kubectl get mgc -n memgraph -o wide` adds how many of each role's declared members the cluster actually has registered (`REGISTERED-COORDINATORS`, `REGISTERED-DATA`), which is what a scale-up is watched through. To block a script or a GitOps step on the cluster being usable: @@ -172,11 +174,19 @@ The MVP is deliberately "provision, bootstrap, observe". It does: - provision one StatefulSet and headless Service per role, with per-pod identity derived from the pod ordinal; - bootstrap HA: add the coordinators, register the data instances, and promote the initial MAIN once; - re-register continuously: every reconcile compares `SHOW INSTANCES` on the coordinator leader against the declared topology and issues only the missing registrations, so an instance that loses its registration state (say, after being rescheduled onto a fresh node) rejoins without human action; -- report the observed MAIN and the readiness and convergence conditions on the resource's status. +- **grow a live cluster**: raise `coordinators` or `dataInstances` (both in one edit if you like, in any step size) and the added pods are provisioned and registered by the same diff that restores a lost registration — no manual `ADD COORDINATOR` or `REGISTER INSTANCE`; +- report the observed MAIN, the registered member counts, and the readiness and convergence conditions on the resource's status. + +Growing is one edit, and `Converged` tells you when it is finished: + +```sh +kubectl patch mgc memgraph -n memgraph --type=merge -p '{"spec":{"coordinators":5,"dataInstances":3}}' +kubectl wait --namespace memgraph --for=condition=Converged memgraphcluster/memgraph --timeout=10m +``` What it does not do yet: -- **Scaling.** `coordinators` and `dataInstances` are **immutable after creation** — admission rejects a change with a clear message. Changing the topology means creating a new cluster. Mutable counts are the first item on the post-v1 roadmap. +- **Scaling down.** `coordinators` must stay odd and at or above three, `dataInstances` at or above one — both enforced at creation and on every update. *Lowering* a count is accepted by admission but not carried out: taking a pod away means unregistering a cluster member first, and the operator has no removal path yet, so it holds the StatefulSet at its current size and reports `Converged=False` with reason `ScaleInProgress` until the count is raised back. Scale-down with MAIN- and quorum-safety is the next item on the roadmap. - **Failover.** The operator issues `SET INSTANCE TO MAIN` exactly once, at bootstrap, when no MAIN exists. After that, leadership belongs entirely to the Raft coordinators; the operator only observes and reports it, so two control systems never fight over which instance is MAIN. - **Other day-2 operations**: orchestrated or rolling version upgrades, backup and restore, storage-mode changes. - **Removing instances**: there is no `REMOVE COORDINATOR` or `UNREGISTER INSTANCE`, and no finalizer-based storage cleanup. The operator has no destructive code path. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index b44a68e..7f00801 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -113,17 +113,21 @@ const ( // Condition types reported on MemgraphCluster status. Both use normal-True // polarity: True is the healthy state. Ready answers "is the cluster serving" -// (a MAIN is elected and reachable); Converged answers "does registration -// match the declared topology" (every coordinator and data instance is -// registered). A cluster can be Ready but not Converged — a MAIN still serves -// while a lost replica registration is being restored. +// (a MAIN is elected and reachable); Converged answers "does the cluster match +// the declared topology" (every declared coordinator and data instance is +// registered, and both StatefulSets run the declared number of replicas). A +// cluster can be Ready but not Converged — a MAIN still serves while a lost +// replica registration is being restored, or while a scale finishes. const ( // ConditionReady is True when a MAIN data instance is elected and the // coordinator leader is reachable. ConditionReady = "Ready" // ConditionConverged is True when the observed cluster matches the declared - // topology and no registration commands are pending. + // topology: no registration commands are pending and both StatefulSets' + // replica counts equal the declared counts. `kubectl wait + // --for=condition=Converged` therefore means a scale is genuinely finished, + // not merely accepted. ConditionConverged = "Converged" ) @@ -159,6 +163,11 @@ const ( // declared topology. ReasonAllInstancesRegistered = "AllInstancesRegistered" + // ReasonScaleInProgress is set when registration has converged but a + // StatefulSet still runs a different number of replicas than the spec + // declares, so the declared topology is not fully realized yet. + ReasonScaleInProgress = "ScaleInProgress" + // ReasonMainElected is set when a data instance is observed as MAIN. ReasonMainElected = "MainElected" @@ -745,19 +754,21 @@ type ExtraArgsSpec struct { // MemgraphClusterSpec defines the desired state of MemgraphCluster. type MemgraphClusterSpec struct { // coordinators is the number of Raft coordinator instances. It must be odd - // so the Raft quorum cannot split, and it is immutable: scaling is not - // supported in v1alpha1. - // +kubebuilder:validation:Minimum=1 + // so the Raft quorum cannot split, and at least three, which is the + // smallest quorum that survives losing a coordinator — this operator + // builds real HA clusters, so the floor holds at creation as well as on an + // update. Raising the count on a live cluster grows it: the operator adds + // the new coordinators to the Raft cluster as their pods become ready. + // +kubebuilder:validation:Minimum=3 // +kubebuilder:validation:XValidation:rule="self % 2 == 1",message="coordinators must be an odd number so the Raft quorum cannot split" - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="coordinators is immutable: changing the coordinator count of an existing MemgraphCluster is not supported in v1alpha1" // +kubebuilder:default=3 // +optional Coordinators *int32 `json:"coordinators,omitempty"` - // dataInstances is the number of data instances. It is immutable: scaling - // is not supported in v1alpha1. + // dataInstances is the number of data instances. Raising the count on a + // live cluster grows it: the operator registers the new instances as their + // pods become ready. // +kubebuilder:validation:Minimum=1 - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="dataInstances is immutable: changing the data instance count of an existing MemgraphCluster is not supported in v1alpha1" // +kubebuilder:default=2 // +optional DataInstances *int32 `json:"dataInstances,omitempty"` @@ -844,6 +855,19 @@ type MemgraphClusterStatus struct { // +optional Main string `json:"main,omitempty"` + // coordinators is how many of the declared coordinators the coordinator + // leader reports as registered members of the Raft cluster. It reaches + // spec.coordinators once registration has converged, so it is what a scale + // is watched through. + // +optional + Coordinators int32 `json:"coordinators,omitempty"` + + // dataInstances is how many of the declared data instances the coordinator + // leader reports as registered. It reaches spec.dataInstances once + // registration has converged. + // +optional + DataInstances int32 `json:"dataInstances,omitempty"` + // conditions represent the current state of the MemgraphCluster resource. // Each condition has a unique type and reflects the status of a specific aspect of the resource. // @@ -864,6 +888,8 @@ type MemgraphClusterStatus struct { // +kubebuilder:resource:shortName=mgc // +kubebuilder:printcolumn:name="Coordinators",type=integer,JSONPath=`.spec.coordinators` // +kubebuilder:printcolumn:name="Data",type=integer,JSONPath=`.spec.dataInstances` +// +kubebuilder:printcolumn:name="Registered-Coordinators",type=integer,JSONPath=`.status.coordinators`,priority=1 +// +kubebuilder:printcolumn:name="Registered-Data",type=integer,JSONPath=`.status.dataInstances`,priority=1 // +kubebuilder:printcolumn:name="Main",type=string,JSONPath=`.status.main` // +kubebuilder:printcolumn:name="Ready",type=string,JSONPath=`.status.conditions[?(@.type=="Ready")].status` // +kubebuilder:printcolumn:name="Converged",type=string,JSONPath=`.status.conditions[?(@.type=="Converged")].status` diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml index 8737072..eaf18db 100644 --- a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -25,6 +25,14 @@ spec: - jsonPath: .spec.dataInstances name: Data type: integer + - jsonPath: .status.coordinators + name: Registered-Coordinators + priority: 1 + type: integer + - jsonPath: .status.dataInstances + name: Registered-Data + priority: 1 + type: integer - jsonPath: .status.main name: Main type: string @@ -76,18 +84,18 @@ spec: default: 3 description: |- coordinators is the number of Raft coordinator instances. It must be odd - so the Raft quorum cannot split, and it is immutable: scaling is not - supported in v1alpha1. + so the Raft quorum cannot split, and at least three, which is the + smallest quorum that survives losing a coordinator — this operator + builds real HA clusters, so the floor holds at creation as well as on an + update. Raising the count on a live cluster grows it: the operator adds + the new coordinators to the Raft cluster as their pods become ready. format: int32 - minimum: 1 + minimum: 3 type: integer x-kubernetes-validations: - message: coordinators must be an odd number so the Raft quorum cannot split rule: self % 2 == 1 - - message: 'coordinators is immutable: changing the coordinator count - of an existing MemgraphCluster is not supported in v1alpha1' - rule: self == oldSelf coreDumps: default: {} description: |- @@ -331,15 +339,12 @@ spec: dataInstances: default: 2 description: |- - dataInstances is the number of data instances. It is immutable: scaling - is not supported in v1alpha1. + dataInstances is the number of data instances. Raising the count on a + live cluster grows it: the operator registers the new instances as their + pods become ready. format: int32 minimum: 1 type: integer - x-kubernetes-validations: - - message: 'dataInstances is immutable: changing the data instance - count of an existing MemgraphCluster is not supported in v1alpha1' - rule: self == oldSelf extraArgs: description: extraArgs passes additional Memgraph flags to both roles. properties: @@ -1349,6 +1354,21 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + coordinators: + description: |- + coordinators is how many of the declared coordinators the coordinator + leader reports as registered members of the Raft cluster. It reaches + spec.coordinators once registration has converged, so it is what a scale + is watched through. + format: int32 + type: integer + dataInstances: + description: |- + dataInstances is how many of the declared data instances the coordinator + leader reports as registered. It reaches spec.dataInstances once + registration has converged. + format: int32 + type: integer main: description: |- main is the name of the data instance currently observed as MAIN, as diff --git a/charts/memgraph-operator/templates/NOTES.txt b/charts/memgraph-operator/templates/NOTES.txt index 4cca918..824c3ea 100644 --- a/charts/memgraph-operator/templates/NOTES.txt +++ b/charts/memgraph-operator/templates/NOTES.txt @@ -22,6 +22,11 @@ referencing a Secret that holds your Memgraph enterprise license: licenseKey: MEMGRAPH_ENTERPRISE_LICENSE organizationKey: MEMGRAPH_ORGANIZATION_NAME +Save it as cluster.yaml and create it with apply, so that later edits — growing +the cluster, for instance — stay a clean apply too: + + kubectl apply -f cluster.yaml + Follow the cluster reaching a registered, MAIN-elected state with: kubectl get mgc -w diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 72132bc..21655a3 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -23,6 +23,14 @@ spec: - jsonPath: .spec.dataInstances name: Data type: integer + - jsonPath: .status.coordinators + name: Registered-Coordinators + priority: 1 + type: integer + - jsonPath: .status.dataInstances + name: Registered-Data + priority: 1 + type: integer - jsonPath: .status.main name: Main type: string @@ -74,18 +82,18 @@ spec: default: 3 description: |- coordinators is the number of Raft coordinator instances. It must be odd - so the Raft quorum cannot split, and it is immutable: scaling is not - supported in v1alpha1. + so the Raft quorum cannot split, and at least three, which is the + smallest quorum that survives losing a coordinator — this operator + builds real HA clusters, so the floor holds at creation as well as on an + update. Raising the count on a live cluster grows it: the operator adds + the new coordinators to the Raft cluster as their pods become ready. format: int32 - minimum: 1 + minimum: 3 type: integer x-kubernetes-validations: - message: coordinators must be an odd number so the Raft quorum cannot split rule: self % 2 == 1 - - message: 'coordinators is immutable: changing the coordinator count - of an existing MemgraphCluster is not supported in v1alpha1' - rule: self == oldSelf coreDumps: default: {} description: |- @@ -329,15 +337,12 @@ spec: dataInstances: default: 2 description: |- - dataInstances is the number of data instances. It is immutable: scaling - is not supported in v1alpha1. + dataInstances is the number of data instances. Raising the count on a + live cluster grows it: the operator registers the new instances as their + pods become ready. format: int32 minimum: 1 type: integer - x-kubernetes-validations: - - message: 'dataInstances is immutable: changing the data instance - count of an existing MemgraphCluster is not supported in v1alpha1' - rule: self == oldSelf extraArgs: description: extraArgs passes additional Memgraph flags to both roles. properties: @@ -1347,6 +1352,21 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + coordinators: + description: |- + coordinators is how many of the declared coordinators the coordinator + leader reports as registered members of the Raft cluster. It reaches + spec.coordinators once registration has converged, so it is what a scale + is watched through. + format: int32 + type: integer + dataInstances: + description: |- + dataInstances is how many of the declared data instances the coordinator + leader reports as registered. It reaches spec.dataInstances once + registration has converged. + format: int32 + type: integer main: description: |- main is the name of the data instance currently observed as MAIN, as diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 9963b89..154069b 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -6,9 +6,12 @@ metadata: app.kubernetes.io/managed-by: kustomize name: memgraphcluster-sample spec: - # Both counts are immutable: v1alpha1 provisions and bootstraps a fixed - # topology, and admission rejects any later change. The coordinator count - # must be odd so the Raft quorum cannot split. + # Raising either count on a live cluster grows it: the operator provisions the + # new pods and registers them with the cluster. The coordinator count must be + # odd so the Raft quorum cannot split, and at least three — a quorum of one + # cannot survive losing itself. Lowering a count is not supported yet: it is + # accepted at admission but the operator holds the StatefulSet at its current + # size and reports Converged=False with reason ScaleInProgress. coordinators: 3 dataInstances: 2 # repository carries the registry host and image path only — the version @@ -26,8 +29,9 @@ spec: organizationKey: MEMGRAPH_ORGANIZATION_NAME storage: # Retain (the default) leaves the PersistentVolumeClaims behind when this - # resource is deleted, so the data survives an accidental delete. Switch to - # Delete on dev clusters that should clean up after themselves. + # resource is deleted — or when a role is scaled down — so the data survives + # an accidental delete. Switch to Delete on dev clusters that should clean up + # after themselves. retentionPolicy: Retain # Each pod of a role gets a lib claim (Memgraph's data directory) and a log # claim. The storage class names have no default and are commented out on diff --git a/examples/minimal-cluster.yaml b/examples/minimal-cluster.yaml index 507790a..0cd45bc 100644 --- a/examples/minimal-cluster.yaml +++ b/examples/minimal-cluster.yaml @@ -11,9 +11,10 @@ kind: MemgraphCluster metadata: name: memgraph spec: - # Both counts are immutable: v1alpha1 provisions and bootstraps a fixed - # topology, and admission rejects any later change. The coordinator count - # must be odd so the Raft quorum cannot split. + # Raise either count later to grow the cluster: the operator provisions the + # new pods and registers them, no manual registration involved. The + # coordinator count must be odd so the Raft quorum cannot split, and at least + # three. Lowering a count is not supported yet. coordinators: 3 dataInstances: 2 image: diff --git a/hack/chart-install-test.sh b/hack/chart-install-test.sh index 9242657..a7d9dc8 100755 --- a/hack/chart-install-test.sh +++ b/hack/chart-install-test.sh @@ -99,7 +99,7 @@ metadata: name: chart-test namespace: ${CLUSTER_NAMESPACE} spec: - coordinators: 1 + coordinators: 3 dataInstances: 1 image: repository: docker.io/memgraph/memgraph diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index d722e01..dff5ded 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -20,11 +20,13 @@ import ( "context" "errors" "fmt" + "strings" "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -93,10 +95,14 @@ type MemgraphClusterReconciler struct { // Reconcile drives the cluster toward the declared MemgraphCluster spec in // two stages. First it server-side-applies the builders' desired objects: one // StatefulSet per role (coordinators, data instances), each backed by a -// headless Service. Then, once every pod is ready, it reconciles cluster -// registration: observe SHOW INSTANCES on the coordinator leader, diff -// against the declared topology, and issue only the missing commands. All -// interaction is read-before-write and idempotent, so an operator restart +// headless Service, at the replica counts replicaCounts derives. Then, once +// every pod is ready, it reconciles cluster registration: observe SHOW +// INSTANCES on the coordinator leader, diff against the declared topology, and +// issue only the missing commands — which is all growing a live cluster takes, +// because a raised count declares members the observed cluster does not have +// registered yet. +// +// All interaction is read-before-write and idempotent, so an operator restart // mid-bootstrap is harmless. Registration reconciliation is continuous, not // one-shot: a converged cluster is re-observed on a periodic resync, so a // registration a pod loses (rescheduled, wiped storage) is re-issued without @@ -114,11 +120,16 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, client.IgnoreNotFound(err) } + replicas, err := r.replicaCounts(ctx, &cluster) + if err != nil { + return ctrl.Result{}, err + } + desired := []client.Object{ resources.CoordinatorHeadlessService(&cluster), resources.DataHeadlessService(&cluster), - resources.CoordinatorStatefulSet(&cluster), - resources.DataStatefulSet(&cluster), + resources.CoordinatorStatefulSet(&cluster, replicas.coordinators.applied), + resources.DataStatefulSet(&cluster, replicas.data.applied), } for _, obj := range desired { if err := controllerutil.SetControllerReference(&cluster, obj, r.Scheme); err != nil { @@ -134,7 +145,7 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ // ones, so neither serving nor convergence can be claimed for the // spec the user asked for. msg := truncateMessage(applyErr.Error()) - if statusErr := r.writeStatus(ctx, &cluster, cluster.Status.Main, + if statusErr := r.writeStatus(ctx, &cluster, lastObserved(&cluster), notReadyCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), notConvergedCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), ); statusErr != nil { @@ -146,7 +157,91 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ log.Info("Applied desired workload objects for MemgraphCluster", "memgraphcluster", req.NamespacedName) - return r.reconcileRegistration(ctx, &cluster) + return r.reconcileRegistration(ctx, &cluster, replicas) +} + +// roleReplicas is one role's replica arithmetic for a reconcile pass: how many +// replicas the spec declares, and how many the operator applies to the role's +// StatefulSet. +type roleReplicas struct { + // name is the StatefulSet's name, so a condition message names the object + // the user can look at. + name string + declared int32 + applied int32 +} + +// replicaCounts is both roles' replica arithmetic. +type replicaCounts struct { + coordinators roleReplicas + data roleReplicas +} + +// scaleMessage describes the roles whose StatefulSet does not run the declared +// number of replicas, and is empty once both do — which is what widens +// Converged from "registration matches the declared topology" to "the declared +// topology is actually running". +func (c replicaCounts) scaleMessage() string { + var pending []string + for _, role := range []roleReplicas{c.coordinators, c.data} { + if role.applied != role.declared { + pending = append(pending, fmt.Sprintf("StatefulSet %s runs %d replica(s) while %d are declared", + role.name, role.applied, role.declared)) + } + } + return strings.Join(pending, "; ") +} + +// replicaCounts resolves the replica count to apply per role: the declared count +// while the cluster grows or holds its size, and deliberately the current count +// while a lowered count would shrink it. Shedding pods means removing members +// from the Memgraph cluster first — the coordinators otherwise keep expecting +// instances whose pods are gone — and the operator has no removal path yet, so +// it holds the size and reports the mismatch instead of acting on half of a +// scale-down it cannot finish. +func (r *MemgraphClusterReconciler) replicaCounts( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, +) (replicaCounts, error) { + var counts replicaCounts + for _, role := range []struct { + name string + declared int32 + resolved *roleReplicas + }{ + {resources.CoordinatorName(cluster), resources.DeclaredCoordinators(cluster), &counts.coordinators}, + {resources.DataName(cluster), resources.DeclaredDataInstances(cluster), &counts.data}, + } { + current, err := r.currentReplicas(ctx, cluster.Namespace, role.name) + if err != nil { + return replicaCounts{}, err + } + // The larger of the two, so growing applies the declared count while + // shrinking holds the current one. + *role.resolved = roleReplicas{ + name: role.name, declared: role.declared, applied: max(role.declared, current), + } + } + return counts, nil +} + +// currentReplicas is the replica count the operator's own previous apply left on +// a role's StatefulSet, or zero when the cluster has not been provisioned yet. +func (r *MemgraphClusterReconciler) currentReplicas( + ctx context.Context, + namespace, name string, +) (int32, error) { + var sts appsv1.StatefulSet + if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, &sts); err != nil { + if apierrors.IsNotFound(err) { + return 0, nil + } + return 0, fmt.Errorf("getting StatefulSet %s: %w", name, err) + } + if sts.Spec.Replicas == nil { + return 0, nil + } + return *sts.Spec.Replicas, nil } // reconcileRegistration converges cluster registration once the workloads are @@ -157,6 +252,7 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ func (r *MemgraphClusterReconciler) reconcileRegistration( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, + replicas replicaCounts, ) (ctrl.Result, error) { log := logf.FromContext(ctx) @@ -167,7 +263,7 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( if !ready { log.Info("Waited for workload pods to become ready before registration") msg := "Waiting for all workload pods to become ready" - if statusErr := r.writeStatus(ctx, cluster, cluster.Status.Main, + if statusErr := r.writeStatus(ctx, cluster, lastObserved(cluster), notReadyCondition(memgraphcomv1alpha1.ReasonWorkloadsNotReady, msg), notConvergedCondition(memgraphcomv1alpha1.ReasonWorkloadsNotReady, msg), ); statusErr != nil { @@ -187,7 +283,7 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( reason, msg = memgraphcomv1alpha1.ReasonNoCoordinatorLeader, "No coordinator reported a leader, so the cluster has no Raft quorum" } - if statusErr := r.writeStatus(ctx, cluster, cluster.Status.Main, + if statusErr := r.writeStatus(ctx, cluster, lastObserved(cluster), notReadyCondition(reason, msg), notConvergedCondition(reason, msg), ); statusErr != nil { @@ -201,16 +297,30 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( } }() - main := observedMain(observed) + latest := observe(topology, observed) commands := planner.Plan(topology, observed) if len(commands) == 0 { + // Registration matches the declared topology. It is only converged once + // the StatefulSets run the declared replica counts too, so a scale the + // operator is holding back keeps the condition False and says which + // role and by how much. + if pending := replicas.scaleMessage(); pending != "" { + log.Info("Held a StatefulSet short of the declared replica count", "reason", pending) + if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), + notConvergedCondition(memgraphcomv1alpha1.ReasonScaleInProgress, pending), + ); statusErr != nil { + return ctrl.Result{}, statusErr + } + return ctrl.Result{RequeueAfter: resyncInterval}, nil + } + // Converged, but keep re-observing: a registration a pod loses later // produces no watch event, so drift is only caught by resyncing. log.Info("Confirmed cluster registration is converged") converged := trueCondition(memgraphcomv1alpha1.ConditionConverged, memgraphcomv1alpha1.ReasonAllInstancesRegistered, fmt.Sprintf("All %d declared instances are registered", len(topology.Coordinators)+len(topology.DataInstances))) - if statusErr := r.writeStatus(ctx, cluster, main, readyOrNot(main), converged); statusErr != nil { + if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), converged); statusErr != nil { return ctrl.Result{}, statusErr } return ctrl.Result{RequeueAfter: resyncInterval}, nil @@ -221,7 +331,7 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( // bootstrap has no MAIN yet, so Ready is False until one is elected. inProgress := notConvergedCondition(memgraphcomv1alpha1.ReasonRegistrationInProgress, fmt.Sprintf("Issuing %d registration command(s) to converge the cluster", len(commands))) - if statusErr := r.writeStatus(ctx, cluster, main, readyOrNot(main), inProgress); statusErr != nil { + if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), inProgress); statusErr != nil { return ctrl.Result{}, statusErr } @@ -237,6 +347,38 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil } +// observation is everything a reconcile pass observed about the cluster that +// reaches the resource's status: which data instance is MAIN, and how many of +// each role's declared members are registered. It is observation only — no +// reconcile decision reads it back off the status. +type observation struct { + main string + registeredCoordinators int32 + registeredDataInstances int32 +} + +// observe reads the coordinator leader's cluster view into the status fields. +func observe(topology planner.Topology, observed []memgraph.Instance) observation { + coordinators, dataInstances := planner.Registered(topology, observed) + return observation{ + main: observedMain(observed), + registeredCoordinators: coordinators, + registeredDataInstances: dataInstances, + } +} + +// lastObserved is the observation already published on the resource. The paths +// that could not observe the cluster this pass republish it: an unready pod or +// an unreachable coordinator says nothing about what the last reachable leader +// reported. +func lastObserved(cluster *memgraphcomv1alpha1.MemgraphCluster) observation { + return observation{ + main: cluster.Status.Main, + registeredCoordinators: cluster.Status.Coordinators, + registeredDataInstances: cluster.Status.DataInstances, + } +} + // observedMain returns the name of the data instance reported as MAIN, or the // empty string when none is elected yet. func observedMain(observed []memgraph.Instance) string { @@ -287,18 +429,20 @@ func notConvergedCondition(reason, message string) metav1.Condition { } } -// writeStatus patches the status subresource with the observed MAIN and the +// writeStatus patches the status subresource with the pass's observation and the // given conditions. It uses the status subresource exclusively — spec is never // touched — and skips the patch when nothing changed, so a converged cluster // re-observed on every resync does not churn the resource version. func (r *MemgraphClusterReconciler) writeStatus( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, - main string, + observed observation, conditions ...metav1.Condition, ) error { base := cluster.DeepCopy() - cluster.Status.Main = main + cluster.Status.Main = observed.main + cluster.Status.Coordinators = observed.registeredCoordinators + cluster.Status.DataInstances = observed.registeredDataInstances for _, condition := range conditions { condition.ObservedGeneration = cluster.Generation apimeta.SetStatusCondition(&cluster.Status.Conditions, condition) diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 9211e56..64da5dc 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -228,8 +228,9 @@ var _ = Describe("MemgraphCluster Controller", func() { "claim %s must fall back to the cluster's default StorageClass", name) } - // The default keeps data safe from an accidental CR delete, and - // nothing ever scales down because both counts are immutable. + // The default keeps data safe from an accidental CR delete and + // from a scale-down alike: one retention knob, both halves of + // the policy. Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy).To(HaveValue(Equal( appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ WhenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, @@ -286,7 +287,7 @@ var _ = Describe("MemgraphCluster Controller", func() { cr := &memgraphcomv1alpha1.MemgraphCluster{ ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, Spec: memgraphcomv1alpha1.MemgraphClusterSpec{ - Coordinators: ptr.To(int32(1)), + Coordinators: ptr.To(int32(3)), DataInstances: ptr.To(int32(1)), Image: memgraphcomv1alpha1.ImageSpec{ Repository: "registry.example.com/memgraph", @@ -322,10 +323,11 @@ var _ = Describe("MemgraphCluster Controller", func() { It("should propagate spec values into the workload objects", func() { reconcileCluster(resourceName) - for _, suffix := range []string{coordinatorSuffix, dataSuffix} { + for suffix, replicas := range map[string]int32{coordinatorSuffix: 3, dataSuffix: 1} { sts := &appsv1.StatefulSet{} get(resourceName+suffix, sts) - Expect(sts.Spec.Replicas).To(HaveValue(Equal(int32(1)))) + Expect(sts.Spec.Replicas).To(HaveValue(Equal(replicas)), + "each role's StatefulSet runs the count its own spec field declares") container := sts.Spec.Template.Spec.Containers[0] Expect(container.Image).To(Equal("registry.example.com/memgraph:3.13.0")) @@ -338,8 +340,11 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(organizationRef.Name).To(Equal(customSecretName)) Expect(organizationRef.Key).To(Equal("organization")) - Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted).To( - Equal(appsv1.DeletePersistentVolumeClaimRetentionPolicyType)) + Expect(sts.Spec.PersistentVolumeClaimRetentionPolicy).To(HaveValue(Equal( + appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ + WhenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + WhenScaled: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + })), "the Delete policy covers the claims a scale-down orphans too") } }) @@ -623,6 +628,162 @@ var _ = Describe("MemgraphCluster Controller", func() { }) }) + Context("when scaling the topology of a live cluster", func() { + const resourceName = "mgc-scale" + + coordinatorAddress := func(ordinal int) string { + return fmt.Sprintf("%s-coordinator-%d.%s-coordinator.%s.svc.cluster.local:7687", + resourceName, ordinal, resourceName, resourceNamespace) + } + + status := func() memgraphcomv1alpha1.MemgraphClusterStatus { + GinkgoHelper() + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + return cluster.Status + } + + // setCounts edits the declared topology of the live cluster. + setCounts := func(coordinators, dataInstances int32) { + GinkgoHelper() + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + cluster.Spec.Coordinators = ptr.To(coordinators) + cluster.Spec.DataInstances = ptr.To(dataInstances) + Expect(k8sClient.Update(ctx, cluster)).To(Succeed()) + } + + replicas := func(suffix string) int32 { + GinkgoHelper() + sts := &appsv1.StatefulSet{} + get(resourceName+suffix, sts) + Expect(sts.Spec.Replicas).NotTo(BeNil()) + return *sts.Spec.Replicas + } + + // bootstrapped drives the default 3/2 cluster to converged, so the specs + // below start from a live, fully registered cluster. It returns how many + // commands that took, which is the baseline sinceBootstrap counts from. + bootstrapped := func() int { + GinkgoHelper() + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + reconcileCluster(resourceName) + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + Expect(apimeta.IsStatusConditionTrue(cluster.Status.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + return len(fake.executedCommands()) + } + + // sinceBootstrap is the commands the spec's own topology edit caused, so + // the assertions are not about the bootstrap that set the scene. + sinceBootstrap := func(baseline int) []string { + GinkgoHelper() + executed := fake.executedCommands() + Expect(len(executed)).To(BeNumerically(">=", baseline)) + return executed[baseline:] + } + + BeforeEach(func() { + resource := &memgraphcomv1alpha1.MemgraphCluster{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName, Namespace: resourceNamespace}, + } + Expect(k8sClient.Create(ctx, resource)).To(Succeed()) + }) + + AfterEach(func() { + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + Expect(k8sClient.Delete(ctx, cluster)).To(Succeed()) + deleteOwned(resourceName) + }) + + It("should grow both roles and register only the added members", func() { + baseline := bootstrapped() + + setCounts(5, 3) + // The added pods are not ready yet, so this pass only widens the + // StatefulSets. + reconcileCluster(resourceName) + Expect(replicas(coordinatorSuffix)).To(Equal(int32(5))) + Expect(replicas(dataSuffix)).To(Equal(int32(3))) + Expect(sinceBootstrap(baseline)).To(BeEmpty(), + "registration waits until every pod of the grown topology is ready") + + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": ADD COORDINATOR 4", + leader + ": ADD COORDINATOR 5", + leader + ": REGISTER INSTANCE instance_2", + }), "the members the cluster already has are left alone, and no MAIN is re-promoted") + + reconcileCluster(resourceName) + s := status() + Expect(s.Coordinators).To(Equal(int32(5))) + Expect(s.DataInstances).To(Equal(int32(3))) + Expect(s.Main).To(Equal("instance_0"), "growing the cluster does not move MAIN") + converged := apimeta.FindStatusCondition(s.Conditions, memgraphcomv1alpha1.ConditionConverged) + Expect(converged.Status).To(Equal(metav1.ConditionTrue)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonAllInstancesRegistered)) + }) + + It("should report Converged False while a StatefulSet has not reached the declared count", func() { + baseline := bootstrapped() + + // A count the operator cannot realize yet: shedding pods means + // removing cluster members first, which it does not do, so the + // StatefulSet is held and the resource says so. + setCounts(3, 1) + reconcileCluster(resourceName) + + Expect(replicas(dataSuffix)).To(Equal(int32(2)), + "a lower declared count must never shrink the applied StatefulSet") + converged := apimeta.FindStatusCondition(status().Conditions, memgraphcomv1alpha1.ConditionConverged) + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonScaleInProgress)) + Expect(converged.Message).To(ContainSubstring(resourceName + dataSuffix)) + Expect(sinceBootstrap(baseline)).To(BeEmpty(), + "the instance the lowered count drops stays registered: removal is not implemented") + + // Raising the count back matches what is running, which converges + // again without touching the cluster. + setCounts(3, 2) + reconcileCluster(resourceName) + + Expect(apimeta.IsStatusConditionTrue(status().Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + + It("should report the registered counts as observed, not as declared", func() { + bootstrapped() + Expect(status().DataInstances).To(Equal(int32(2))) + + // instance_1 loses its registration: the count reports what the + // cluster has, which is what makes it worth watching during a scale. + fake.setInstances([]memgraph.Instance{ + { + Name: "coordinator_1", BoltServer: coordinatorAddress(0), + Health: memgraph.HealthUp, Role: memgraph.RoleLeader, + }, + { + Name: "coordinator_2", BoltServer: coordinatorAddress(1), + Health: memgraph.HealthUp, Role: memgraph.RoleFollower, + }, + {Name: "instance_0", Health: memgraph.HealthUp, Role: memgraph.RoleMain}, + }) + reconcileCluster(resourceName) + + s := status() + Expect(s.Coordinators).To(Equal(int32(2)), "coordinator_3 is no longer a member") + Expect(s.DataInstances).To(Equal(int32(1))) + }) + }) + Context("when reporting status and conditions", func() { const resourceName = "mgc-status" @@ -777,6 +938,8 @@ var _ = Describe("MemgraphCluster Controller", func() { s := status() Expect(s.Main).To(Equal("instance_0")) + Expect(s.Coordinators).To(Equal(int32(3)), "every declared coordinator is registered") + Expect(s.DataInstances).To(Equal(int32(2))) ready := condition(memgraphcomv1alpha1.ConditionReady) Expect(ready.Status).To(Equal(metav1.ConditionTrue)) Expect(ready.Reason).To(Equal(memgraphcomv1alpha1.ReasonMainElected)) diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index 99a26bc..c314241 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -215,14 +215,14 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Expect(stored.Spec.Storage.Data.LogStorageClassName).To(BeNil()) }) - DescribeTable("should accept any odd coordinator count and any positive data instance count", + DescribeTable("should accept any odd coordinator count from three up and any positive data instance count", func(name string, coordinators, dataInstances int32) { createAccepted(name, memgraphcomv1alpha1.MemgraphClusterSpec{ Coordinators: ptr.To(coordinators), DataInstances: ptr.To(dataInstances), }) }, - Entry("single coordinator, single data instance", "valid-topology-min", int32(1), int32(1)), + Entry("the smallest quorum and a single data instance", "valid-topology-min", int32(3), int32(1)), Entry("a quorum and replica count beyond the former upper bounds", "valid-topology-large", int32(9), int32(16)), ) @@ -366,9 +366,14 @@ var _ = Describe("MemgraphCluster CRD validation", func() { }, Entry("zero coordinators", "invalid-coordinators-zero", memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(0))}, - "should be greater than or equal to 1"), + "should be greater than or equal to 3"), + // A single coordinator is a quorum of one: it cannot survive losing + // itself, which is the whole point of running HA. + Entry("a coordinator count below the HA floor", "invalid-coordinators-below-floor", + memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(1))}, + "should be greater than or equal to 3"), Entry("an even coordinator count", "invalid-coordinators-even", - memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(2))}, + memgraphcomv1alpha1.MemgraphClusterSpec{Coordinators: ptr.To(int32(4))}, "coordinators must be an odd number"), Entry("zero data instances", "invalid-data-zero", memgraphcomv1alpha1.MemgraphClusterSpec{DataInstances: ptr.To(int32(0))}, @@ -585,78 +590,103 @@ var _ = Describe("MemgraphCluster CRD validation", func() { return k8sClient.Update(ctx, stored) } - expectImmutable := func(name string, mutate func(*memgraphcomv1alpha1.MemgraphCluster), field string) { + // expectRejectedUpdate asserts that admission refused a topology change + // and said what is wrong with the new value. + expectRejectedUpdate := func( + name string, + mutate func(*memgraphcomv1alpha1.MemgraphCluster), + wantMessage string, + ) { GinkgoHelper() err := update(name, mutate) Expect(err).To(HaveOccurred(), "expected admission to reject the topology change") Expect(apierrors.IsInvalid(err)).To(BeTrue(), "expected an Invalid error, got %v", err) - Expect(err.Error()).To(ContainSubstring(field + " is immutable")) - Expect(err.Error()).To(ContainSubstring("not supported in v1alpha1")) + Expect(err.Error()).To(ContainSubstring(wantMessage)) } - It("should reject growing or shrinking the coordinator count", func() { - createAccepted("immutable-coordinators", memgraphcomv1alpha1.MemgraphClusterSpec{ - Coordinators: ptr.To(int32(3)), + It("should accept growing both counts in one edit", func() { + createAccepted("scale-up-both", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(3)), + DataInstances: ptr.To(int32(2)), }) - expectImmutable("immutable-coordinators", func(c *memgraphcomv1alpha1.MemgraphCluster) { + Expect(update("scale-up-both", func(c *memgraphcomv1alpha1.MemgraphCluster) { c.Spec.Coordinators = ptr.To(int32(5)) - }, "coordinators") - expectImmutable("immutable-coordinators", func(c *memgraphcomv1alpha1.MemgraphCluster) { - c.Spec.Coordinators = ptr.To(int32(1)) - }, "coordinators") + c.Spec.DataInstances = ptr.To(int32(3)) + })).To(Succeed(), "both counts are mutable, in any step size, in one edit") + + stored := &memgraphcomv1alpha1.MemgraphCluster{} + Expect(k8sClient.Get(ctx, + types.NamespacedName{Name: "scale-up-both", Namespace: resourceNamespace}, stored)).To(Succeed()) + Expect(stored.Spec.Coordinators).To(HaveValue(Equal(int32(5)))) + Expect(stored.Spec.DataInstances).To(HaveValue(Equal(int32(3)))) }) - It("should reject growing or shrinking the data instance count", func() { - createAccepted("immutable-data", memgraphcomv1alpha1.MemgraphClusterSpec{ - DataInstances: ptr.To(int32(2)), + It("should accept lowering either count", func() { + createAccepted("scale-down-both", memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(5)), + DataInstances: ptr.To(int32(3)), }) - expectImmutable("immutable-data", func(c *memgraphcomv1alpha1.MemgraphCluster) { - c.Spec.DataInstances = ptr.To(int32(3)) - }, "dataInstances") - expectImmutable("immutable-data", func(c *memgraphcomv1alpha1.MemgraphCluster) { + Expect(update("scale-down-both", func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(3)) c.Spec.DataInstances = ptr.To(int32(1)) - }, "dataInstances") + })).To(Succeed(), "admission constrains the target counts, nothing about the direction") }) - It("should reject a count change that arrives as a field removal", func() { + It("should accept a count change that arrives as a field removal", func() { // Dropping a non-default count from the manifest re-defaults it, - // which is a topology change dressed up as a deletion. - createAccepted("immutable-omitted", memgraphcomv1alpha1.MemgraphClusterSpec{ + // which is a real topology change and no longer refused. + createAccepted("scale-omitted", memgraphcomv1alpha1.MemgraphClusterSpec{ Coordinators: ptr.To(int32(5)), }) - expectImmutable("immutable-omitted", func(c *memgraphcomv1alpha1.MemgraphCluster) { + Expect(update("scale-omitted", func(c *memgraphcomv1alpha1.MemgraphCluster) { c.Spec.Coordinators = nil - }, "coordinators") + })).To(Succeed()) + + stored := &memgraphcomv1alpha1.MemgraphCluster{} + Expect(k8sClient.Get(ctx, + types.NamespacedName{Name: "scale-omitted", Namespace: resourceNamespace}, stored)).To(Succeed()) + Expect(stored.Spec.Coordinators).To(HaveValue(Equal(memgraphcomv1alpha1.DefaultCoordinatorCount))) }) It("should accept an update that leaves the counts alone", func() { - createAccepted("immutable-unchanged", memgraphcomv1alpha1.MemgraphClusterSpec{ + createAccepted("scale-unchanged", memgraphcomv1alpha1.MemgraphClusterSpec{ Coordinators: ptr.To(int32(3)), DataInstances: ptr.To(int32(2)), }) - Expect(update("immutable-unchanged", func(c *memgraphcomv1alpha1.MemgraphCluster) { + Expect(update("scale-unchanged", func(c *memgraphcomv1alpha1.MemgraphCluster) { c.Spec.Image.Tag = customImageTag c.Spec.Secrets.Name = "another-license" })).To(Succeed()) - - Expect(update("immutable-unchanged", func(c *memgraphcomv1alpha1.MemgraphCluster) { - c.Spec.Coordinators = ptr.To(int32(3)) - c.Spec.DataInstances = ptr.To(int32(2)) - })).To(Succeed(), "re-applying the same counts is not a topology change") }) - It("should accept an update that omits a count matching the default", func() { - createAccepted("immutable-omitted-default", memgraphcomv1alpha1.MemgraphClusterSpec{ - Coordinators: ptr.To(memgraphcomv1alpha1.DefaultCoordinatorCount), - }) + // The floors and the odd rule are creation-time validation that keeps + // applying on every update: a live cluster cannot be edited into a + // topology it could not have been created with. + DescribeTable("should reject a topology change that breaks a floor", + func(name string, mutate func(*memgraphcomv1alpha1.MemgraphCluster), wantMessage string) { + createAccepted(name, memgraphcomv1alpha1.MemgraphClusterSpec{ + Coordinators: ptr.To(int32(5)), + DataInstances: ptr.To(int32(2)), + }) - Expect(update("immutable-omitted-default", func(c *memgraphcomv1alpha1.MemgraphCluster) { - c.Spec.Coordinators = nil - })).To(Succeed(), "defaulting restores the same count, so the topology is unchanged") - }) + expectRejectedUpdate(name, mutate, wantMessage) + }, + Entry("coordinators below the HA floor", "scale-floor-coordinators", + func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(1)) + }, "should be greater than or equal to 3"), + Entry("an even coordinator count", "scale-floor-coordinators-even", + func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.Coordinators = ptr.To(int32(4)) + }, "coordinators must be an odd number"), + Entry("no data instances left", "scale-floor-data", + func(c *memgraphcomv1alpha1.MemgraphCluster) { + c.Spec.DataInstances = ptr.To(int32(0)) + }, "should be greater than or equal to 1"), + ) }) }) diff --git a/internal/memgraph/client.go b/internal/memgraph/client.go index f9a3fda..c7b9840 100644 --- a/internal/memgraph/client.go +++ b/internal/memgraph/client.go @@ -36,6 +36,11 @@ const ( RoleReplica = "replica" ) +// HealthUp is the SHOW INSTANCES health of an instance the coordinator leader +// currently reaches. Anything else — "down", or "unknown" from a coordinator +// that does not health-check the data plane — means it does not. +const HealthUp = "up" + // Instance is one row of SHOW INSTANCES: a coordinator or data instance the // cluster currently knows about. type Instance struct { @@ -57,6 +62,11 @@ func (i Instance) IsMain() bool { return strings.EqualFold(i.Role, RoleMain) } +// IsUp reports whether the coordinator leader currently reaches the instance. +func (i Instance) IsUp() bool { + return strings.EqualFold(i.Health, HealthUp) +} + // CoordinatorSpec declares one coordinator to add to the cluster. Servers are // "host:port" addresses the rest of the cluster reaches the coordinator at. type CoordinatorSpec struct { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 336efbb..cf4f16f 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -32,10 +32,19 @@ import ( ) // Topology is the declared cluster registration state: every coordinator and -// data instance the CR says must exist, with the addresses each advertises. +// data instance the CR says must exist, with the addresses each advertises, +// plus the members on their way out when a count was lowered. type Topology struct { Coordinators []memgraph.CoordinatorSpec DataInstances []memgraph.DataInstanceSpec + + // RetiringCoordinators and RetiringDataInstances are the members a lowered + // replica count is shedding: still registered and still running, but no + // longer declared. They are empty while a cluster grows or holds its size, + // which is every case the operator handles today — removing a member from + // the cluster is not implemented yet, so a plan issues no command for them. + RetiringCoordinators []memgraph.CoordinatorSpec + RetiringDataInstances []memgraph.DataInstanceSpec } // Command is one registration step to execute against the coordinator leader. @@ -92,19 +101,15 @@ func (c SetInstanceToMain) String() string { // promotion last. Instances the cluster knows but the topology does not // declare are left untouched — unregistration is out of scope for v1. func Plan(declared Topology, observed []memgraph.Instance) []Command { - registered := make(map[string]memgraph.Instance, len(observed)) + registered := index(observed) hasMain := false for _, instance := range observed { - registered[instance.Name] = instance hasMain = hasMain || instance.IsMain() } var commands []Command for _, coordinator := range declared.Coordinators { - // A coordinator reports itself in SHOW INSTANCES with an empty - // bolt_server until ADD COORDINATOR is issued for its ID, so presence - // alone does not prove registration. - if observed, ok := registered[coordinator.Name()]; !ok || observed.BoltServer == "" { + if !coordinatorRegistered(registered, coordinator) { commands = append(commands, AddCoordinator{Coordinator: coordinator}) } } @@ -114,7 +119,62 @@ func Plan(declared Topology, observed []memgraph.Instance) []Command { } } if !hasMain && len(declared.DataInstances) > 0 { - commands = append(commands, SetInstanceToMain{Name: declared.DataInstances[0].Name}) + commands = append(commands, SetInstanceToMain{Name: promotionTarget(declared, registered)}) } return commands } + +// Registered reports how many of the declared coordinators and data instances +// the observed cluster has registered. It is pure observation for the CR's +// status, and it shares Plan's definition of "registered" — so a role's count +// reaches its declared count exactly when Plan stops issuing registrations for +// it. +func Registered(declared Topology, observed []memgraph.Instance) (coordinators, dataInstances int32) { + registered := index(observed) + for _, coordinator := range declared.Coordinators { + if coordinatorRegistered(registered, coordinator) { + coordinators++ + } + } + for _, instance := range declared.DataInstances { + if _, ok := registered[instance.Name]; ok { + dataInstances++ + } + } + return coordinators, dataInstances +} + +// index keys the observed cluster view by instance name. +func index(observed []memgraph.Instance) map[string]memgraph.Instance { + registered := make(map[string]memgraph.Instance, len(observed)) + for _, instance := range observed { + registered[instance.Name] = instance + } + return registered +} + +// coordinatorRegistered reports whether the declared coordinator is a member of +// the Raft cluster. A coordinator reports itself in SHOW INSTANCES with an +// empty bolt_server until ADD COORDINATOR is issued for its ID, so presence +// alone does not prove registration. +func coordinatorRegistered(registered map[string]memgraph.Instance, coordinator memgraph.CoordinatorSpec) bool { + observed, ok := registered[coordinator.Name()] + return ok && observed.BoltServer != "" +} + +// promotionTarget picks the data instance to promote when the cluster has no +// MAIN: the lowest-ordinal declared instance the cluster observes as up. +// Promoting a down instance would only write the intent to Raft and leave the +// cluster MAIN-less until the coordinators retried it, so an instance that is +// known to be reachable is preferred over a lower-ordinal one that is not. +// +// The first declared instance is the fallback, which is what a fresh bootstrap +// uses: nothing is observed yet at the point its registrations are planned. +func promotionTarget(declared Topology, registered map[string]memgraph.Instance) string { + for _, instance := range declared.DataInstances { + if observed, ok := registered[instance.Name]; ok && observed.IsUp() { + return instance.Name + } + } + return declared.DataInstances[0].Name +} diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index eacd3d8..47b6769 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -37,11 +37,21 @@ const firstInstance = "instance_0" // declaredTopology is the canonical 3-coordinator, 2-data-instance fixture // the cases below diff observed cluster states against. func declaredTopology() planner.Topology { + return topologyOf(3, 2) +} + +// grownTopology is the same cluster after both counts were raised: 5 +// coordinators and 3 data instances. +func grownTopology() planner.Topology { + return topologyOf(5, 3) +} + +func topologyOf(coordinators int32, dataInstances int) planner.Topology { topology := planner.Topology{} - for id := int32(1); id <= 3; id++ { + for id := int32(1); id <= coordinators; id++ { topology.Coordinators = append(topology.Coordinators, coordinatorSpec(id)) } - for i := range 2 { + for i := range dataInstances { topology.DataInstances = append(topology.DataInstances, dataInstanceSpec(i)) } return topology @@ -87,17 +97,26 @@ func observedDataInstance(i int, role string) memgraph.Instance { Name: spec.Name, BoltServer: spec.BoltServer, ManagementServer: spec.ManagementServer, - Health: "up", + Health: memgraph.HealthUp, Role: role, } } -func TestPlan(t *testing.T) { - declared := declaredTopology() +// downDataInstance is a registered data instance the coordinator leader cannot +// reach — the state a promotion must route around. +func downDataInstance(i int) memgraph.Instance { + instance := observedDataInstance(i, memgraph.RoleReplica) + instance.Health = "down" + return instance +} +func TestPlan(t *testing.T) { cases := []struct { name string observed []memgraph.Instance + // declared overrides the canonical fixture for the cases about a + // topology whose counts changed. + declared *planner.Topology want []planner.Command }{ { @@ -206,6 +225,70 @@ func TestPlan(t *testing.T) { planner.RegisterInstance{Instance: dataInstanceSpec(0)}, }, }, + // Promoting a down instance only writes the intent to Raft: the cluster + // stays MAIN-less until the coordinators retry it. A registered instance + // the leader can reach is the better target even at a higher ordinal. + { + name: "a down instance_0 is skipped in favor of the lowest reachable instance", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + downDataInstance(0), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: []planner.Command{ + planner.SetInstanceToMain{Name: "instance_1"}, + }, + }, + { + name: "a reachable instance_0 is promoted ahead of its higher-ordinal peers", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: []planner.Command{ + planner.SetInstanceToMain{Name: firstInstance}, + }, + }, + // With every declared instance down there is no reachable target, so the + // first one is promoted anyway: the coordinators act on the intent once + // the instance comes back, which beats never promoting at all. + { + name: "the first instance is promoted when none is reachable", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + downDataInstance(0), + downDataInstance(1), + }, + want: []planner.Command{ + planner.SetInstanceToMain{Name: firstInstance}, + }, + }, + // A grown topology declares members the cluster has never heard of: the + // diff that restores a lost registration is the same one that registers a + // new pod, so growth needs no separate plan. + { + name: "a grown topology registers only the added members", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + declared: ptr.To(grownTopology()), + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(4)}, + planner.AddCoordinator{Coordinator: coordinatorSpec(5)}, + planner.RegisterInstance{Instance: dataInstanceSpec(2)}, + }, + }, { name: "instances the topology does not declare are left untouched", observed: []memgraph.Instance{ @@ -223,6 +306,11 @@ func TestPlan(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + declared := declaredTopology() + if tc.declared != nil { + declared = *tc.declared + } + got := planner.Plan(declared, tc.observed) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("Plan() mismatch (-want +got):\n%s", diff) @@ -231,6 +319,110 @@ func TestPlan(t *testing.T) { } } +// TestRegistered covers what the CR's status publishes: how many of each role's +// declared members the cluster has registered. It shares Plan's definition of +// registered, so the counts reach the declared ones exactly when Plan falls +// silent. +func TestRegistered(t *testing.T) { + selfReporting := observedCoordinator(2, memgraph.RoleLeader) + // The coordinator the client is connected to lists itself with an empty + // bolt_server until ADD COORDINATOR is issued for its ID. + selfReporting.BoltServer = "" + + cases := []struct { + name string + declared planner.Topology + observed []memgraph.Instance + wantCoordinators int32 + wantDataInstances int32 + }{ + { + name: "a fresh cluster has nothing registered", + declared: declaredTopology(), + }, + { + name: "a converged cluster reports the declared counts", + declared: declaredTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + wantCoordinators: 3, + wantDataInstances: 2, + }, + { + name: "a coordinator that is present but not added does not count", + declared: declaredTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleFollower), + selfReporting, + observedDataInstance(0, memgraph.RoleMain), + }, + wantCoordinators: 1, + wantDataInstances: 1, + }, + { + name: "a grown topology reports the members registered so far", + declared: grownTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedCoordinator(4, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + wantCoordinators: 4, + wantDataInstances: 2, + }, + // Counting only declared members keeps the status a report on the topology + // the user asked for, not on whatever else the cluster happens to know. + { + name: "members the topology does not declare are not counted", + declared: declaredTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedCoordinator(4, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + wantCoordinators: 3, + wantDataInstances: 2, + }, + // Health is not registration: an instance the leader cannot reach is still + // a member of the cluster. + { + name: "a down instance still counts as registered", + declared: declaredTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + downDataInstance(0), + observedDataInstance(1, memgraph.RoleMain), + }, + wantCoordinators: 3, + wantDataInstances: 2, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + coordinators, dataInstances := planner.Registered(tc.declared, tc.observed) + if coordinators != tc.wantCoordinators || dataInstances != tc.wantDataInstances { + t.Errorf("Registered() = (%d, %d), want (%d, %d)", + coordinators, dataInstances, tc.wantCoordinators, tc.wantDataInstances) + } + }) + } +} + // TestPlanUsesConfiguredPortsAndClusterDomain plans a fresh bootstrap over a // topology derived from a CR with non-default ports and cluster domain: the // registration commands must carry exactly those addresses, because they are diff --git a/internal/resources/statefulset.go b/internal/resources/statefulset.go index e61b2e3..571b146 100644 --- a/internal/resources/statefulset.go +++ b/internal/resources/statefulset.go @@ -59,7 +59,17 @@ const ( // coordinator instances. Per-pod identity (coordinator ID, advertised FQDN) // is derived from the pod ordinal at startup, so the pod template stays // uniform across replicas. -func CoordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { +// +// The replica count is an argument rather than read off the spec because the +// count to apply is a decision about the live cluster, not about the spec: it +// is the declared count while the cluster grows, and the current count while a +// lowered one is still being retired. Keeping that decision in the controller +// keeps this builder pure. DeclaredCoordinators is the count for a cluster that +// is not shrinking. +func CoordinatorStatefulSet( + cluster *memgraphcomv1alpha1.MemgraphCluster, + replicas int32, +) *appsv1.StatefulSet { spec := normalize(cluster.Spec) role := spec.coordinatorRole @@ -85,11 +95,13 @@ func CoordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv container.ReadinessProbe = tcpProbe(spec.ports.coordinator, role.readinessProbe) container.LivenessProbe = tcpProbe(spec.ports.coordinator, role.livenessProbe) - return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec, role, spec.coordinators, container) + return statefulSet(cluster, coordinatorComponent, CoordinatorName(cluster), spec, role, replicas, container) } -// DataStatefulSet builds the single StatefulSet running all data instances. -func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { +// DataStatefulSet builds the single StatefulSet running all data instances. The +// replica count is an argument for the reason CoordinatorStatefulSet documents; +// DeclaredDataInstances is the count for a cluster that is not shrinking. +func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster, replicas int32) *appsv1.StatefulSet { spec := normalize(cluster.Spec) role := spec.dataRole @@ -104,7 +116,7 @@ func DataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.State container.ReadinessProbe = tcpProbe(spec.ports.bolt, role.readinessProbe) container.LivenessProbe = tcpProbe(spec.ports.bolt, role.livenessProbe) - return statefulSet(cluster, dataComponent, DataName(cluster), spec, role, spec.dataInstances, container) + return statefulSet(cluster, dataComponent, DataName(cluster), spec, role, replicas, container) } // coordinatorStartScript derives the coordinator's identity from its pod @@ -378,12 +390,13 @@ func statefulSet( Selector: &metav1.LabelSelector{MatchLabels: selectorLabels(cluster, component)}, // The StatefulSet controller is the only thing that ever deletes // this cluster's storage; the operator owns no finalizer and runs - // no cleanup of its own. whenScaled is always Retain because both - // replica counts are immutable in v1alpha1 — nothing scales down, - // so no claim is ever orphaned by scaling. + // no cleanup of its own. Both halves of the policy follow the one + // retention knob: whether a claim is orphaned by deleting the + // cluster or by scaling a role down, the user asked the same + // question — keep this cluster's data, or do not. PersistentVolumeClaimRetentionPolicy: &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ WhenDeleted: retentionType(spec.retentionPolicy), - WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenScaled: retentionType(spec.retentionPolicy), }, VolumeClaimTemplates: volumeClaimTemplates(role), Template: corev1.PodTemplateSpec{ diff --git a/internal/resources/statefulset_test.go b/internal/resources/statefulset_test.go index fd0d162..88c69b3 100644 --- a/internal/resources/statefulset_test.go +++ b/internal/resources/statefulset_test.go @@ -82,6 +82,19 @@ func minimalCluster() *memgraphcomv1alpha1.MemgraphCluster { } } +// coordinatorStatefulSet and dataStatefulSet build a role's StatefulSet at the +// replica count the spec declares — the count the controller derives for a +// cluster that is growing or holding its size. The count a shrinking cluster is +// held at is the controller's decision, so the cases that cover it pass it to +// the builder directly. +func coordinatorStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { + return resources.CoordinatorStatefulSet(cluster, resources.DeclaredCoordinators(cluster)) +} + +func dataStatefulSet(cluster *memgraphcomv1alpha1.MemgraphCluster) *appsv1.StatefulSet { + return resources.DataStatefulSet(cluster, resources.DeclaredDataInstances(cluster)) +} + func specifiedCluster() *memgraphcomv1alpha1.MemgraphCluster { return &memgraphcomv1alpha1.MemgraphCluster{ ObjectMeta: metav1.ObjectMeta{Name: clusterName, Namespace: testNamespace}, @@ -290,12 +303,15 @@ func expectedClaimTemplates() []corev1.PersistentVolumeClaim { } } +// expectedRetentionPolicy is the claim retention policy both roles get: the one +// retention knob decides both halves, so a claim orphaned by deleting the +// cluster and one orphaned by scaling a role down are treated alike. func expectedRetentionPolicy( - whenDeleted appsv1.PersistentVolumeClaimRetentionPolicyType, + policy appsv1.PersistentVolumeClaimRetentionPolicyType, ) *appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy { return &appsv1.StatefulSetPersistentVolumeClaimRetentionPolicy{ - WhenDeleted: whenDeleted, - WhenScaled: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + WhenDeleted: policy, + WhenScaled: policy, } } @@ -385,7 +401,7 @@ func TestCoordinatorStatefulSetDefaults(t *testing.T) { }, } - got := resources.CoordinatorStatefulSet(minimalCluster()) + got := coordinatorStatefulSet(minimalCluster()) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("CoordinatorStatefulSet() mismatch (-want +got):\n%s", diff) } @@ -442,12 +458,61 @@ func TestDataStatefulSetDefaults(t *testing.T) { }, } - got := resources.DataStatefulSet(minimalCluster()) + got := dataStatefulSet(minimalCluster()) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("DataStatefulSet() mismatch (-want +got):\n%s", diff) } } +// TestStatefulSetReplicasFollowTheArgument pins the replica count to the +// builder's argument rather than to the spec. That separation is what lets the +// controller hold a role at its current size while a lowered count is being +// retired, without the builders having to know anything about the live cluster. +func TestStatefulSetReplicasFollowTheArgument(t *testing.T) { + // The spec declares fewer replicas than the cluster currently runs, which is + // the count the controller passes so a shrink never sheds pods on its own. + cluster := minimalCluster() + cluster.Spec.Coordinators = ptr.To(int32(3)) + cluster.Spec.DataInstances = ptr.To(int32(2)) + + tests := []struct { + name string + sts *appsv1.StatefulSet + want int32 + }{ + {name: coordinatorComponent, sts: resources.CoordinatorStatefulSet(cluster, 5), want: 5}, + {name: dataComponent, sts: resources.DataStatefulSet(cluster, 3), want: 3}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := *tc.sts.Spec.Replicas; got != tc.want { + t.Errorf("replicas = %d, want the given %d, not the declared count", got, tc.want) + } + }) + } +} + +// The declared counts are what the controller passes for a cluster that is not +// shrinking, so they resolve the same schema defaults the builders do. +func TestDeclaredCounts(t *testing.T) { + if got := resources.DeclaredCoordinators(minimalCluster()); got != memgraphcomv1alpha1.DefaultCoordinatorCount { + t.Errorf("DeclaredCoordinators() = %d, want the schema default %d", + got, memgraphcomv1alpha1.DefaultCoordinatorCount) + } + if got := resources.DeclaredDataInstances(minimalCluster()); got != memgraphcomv1alpha1.DefaultDataInstanceCount { + t.Errorf("DeclaredDataInstances() = %d, want the schema default %d", + got, memgraphcomv1alpha1.DefaultDataInstanceCount) + } + + cluster := specifiedCluster() + if got := resources.DeclaredCoordinators(cluster); got != 5 { + t.Errorf("DeclaredCoordinators() = %d, want 5", got) + } + if got := resources.DeclaredDataInstances(cluster); got != 3 { + t.Errorf("DeclaredDataInstances() = %d, want 3", got) + } +} + func TestStatefulSetSpecOverrides(t *testing.T) { cluster := specifiedCluster() @@ -456,8 +521,8 @@ func TestStatefulSetSpecOverrides(t *testing.T) { sts *appsv1.StatefulSet replicas int32 }{ - {name: coordinatorComponent, sts: resources.CoordinatorStatefulSet(cluster), replicas: 5}, - {name: dataComponent, sts: resources.DataStatefulSet(cluster), replicas: 3}, + {name: coordinatorComponent, sts: coordinatorStatefulSet(cluster), replicas: 5}, + {name: dataComponent, sts: dataStatefulSet(cluster), replicas: 3}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -506,7 +571,7 @@ func TestStatefulSetStorageOverrides(t *testing.T) { }{ { name: coordinatorComponent, - sts: resources.CoordinatorStatefulSet(cluster), + sts: coordinatorStatefulSet(cluster), want: []corev1.PersistentVolumeClaim{ expectedClaimTemplate("lib-storage", "4Gi", corev1.ReadWriteOncePod, ptr.To("fast-ssd")), // An empty storage class is passed through verbatim: it means @@ -516,7 +581,7 @@ func TestStatefulSetStorageOverrides(t *testing.T) { }, { name: dataComponent, - sts: resources.DataStatefulSet(cluster), + sts: dataStatefulSet(cluster), want: []corev1.PersistentVolumeClaim{ expectedClaimTemplate("lib-storage", "100Gi", corev1.ReadWriteOnce, ptr.To("gp3")), // Untouched by the spec, so it keeps every schema default. @@ -551,7 +616,7 @@ func TestStatefulSetWithoutLogStorageClaim(t *testing.T) { } t.Run(coordinatorComponent, func(t *testing.T) { - sts := resources.CoordinatorStatefulSet(cluster) + sts := coordinatorStatefulSet(cluster) wantClaims := []corev1.PersistentVolumeClaim{ expectedClaimTemplate("lib-storage", "1Gi", corev1.ReadWriteOnce, nil), @@ -583,7 +648,7 @@ exec /usr/lib/memgraph/memgraph \ }) t.Run(dataComponent, func(t *testing.T) { - sts := resources.DataStatefulSet(cluster) + sts := dataStatefulSet(cluster) if diff := cmp.Diff(expectedClaimTemplates(), sts.Spec.VolumeClaimTemplates); diff != "" { t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) @@ -604,8 +669,8 @@ exec /usr/lib/memgraph/memgraph \ // init container, no sidecar. func TestStatefulSetCoreDumpsDisabledByDefault(t *testing.T) { for _, sts := range []*appsv1.StatefulSet{ - resources.CoordinatorStatefulSet(minimalCluster()), - resources.DataStatefulSet(minimalCluster()), + coordinatorStatefulSet(minimalCluster()), + dataStatefulSet(minimalCluster()), } { t.Run(sts.Name, func(t *testing.T) { for _, claim := range sts.Spec.VolumeClaimTemplates { @@ -638,7 +703,7 @@ func TestStatefulSetCoreDumps(t *testing.T) { } t.Run(dataComponent, func(t *testing.T) { - sts := resources.DataStatefulSet(cluster) + sts := dataStatefulSet(cluster) wantClaims := append(expectedClaimTemplates(), expectedClaimTemplate(coreDumpsVolume, "20Gi", corev1.ReadWriteOnce, ptr.To("cheap-hdd"))) @@ -680,7 +745,7 @@ func TestStatefulSetCoreDumps(t *testing.T) { // The knob is per role: coordinators asked for nothing and get nothing. t.Run(coordinatorComponent, func(t *testing.T) { - sts := resources.CoordinatorStatefulSet(cluster) + sts := coordinatorStatefulSet(cluster) if diff := cmp.Diff(expectedClaimTemplates(), sts.Spec.VolumeClaimTemplates); diff != "" { t.Errorf("volume claim templates mismatch (-want +got):\n%s", diff) @@ -701,7 +766,7 @@ func TestStatefulSetCoreDumpsWithoutCorePattern(t *testing.T) { ConfigureCorePattern: ptr.To(false), } - sts := resources.DataStatefulSet(cluster) + sts := dataStatefulSet(cluster) podSpec := sts.Spec.Template.Spec if got := podSpec.InitContainers; len(got) != 0 { @@ -742,7 +807,7 @@ func TestStatefulSetCoreDumpsUploader(t *testing.T) { }, } - containers := resources.DataStatefulSet(cluster).Spec.Template.Spec.Containers + containers := dataStatefulSet(cluster).Spec.Template.Spec.Containers if len(containers) != 2 { t.Fatalf("containers = %d, want Memgraph plus the uploader", len(containers)) } @@ -780,7 +845,7 @@ func TestStatefulSetCoreDumpsUploader(t *testing.T) { // Coordinators collect no dumps, so the shared uploader has nothing to read // in their pods and must not be injected there. - coordinators := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec.Containers + coordinators := coordinatorStatefulSet(cluster).Spec.Template.Spec.Containers if len(coordinators) != 1 { t.Errorf("coordinator containers = %d, want only Memgraph's: the role collects no dumps", len(coordinators)) @@ -808,7 +873,7 @@ func TestStatefulSetExtraVolumes(t *testing.T) { } t.Run(dataComponent, func(t *testing.T) { - podSpec := resources.DataStatefulSet(cluster).Spec.Template.Spec + podSpec := dataStatefulSet(cluster).Spec.Template.Spec wantVolumes := append(expectedVolumes(), certVolume) if diff := cmp.Diff(wantVolumes, podSpec.Volumes); diff != "" { @@ -821,7 +886,7 @@ func TestStatefulSetExtraVolumes(t *testing.T) { }) t.Run(coordinatorComponent, func(t *testing.T) { - podSpec := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec + podSpec := coordinatorStatefulSet(cluster).Spec.Template.Spec if diff := cmp.Diff(expectedVolumes(), podSpec.Volumes); diff != "" { t.Errorf("volumes mismatch (-want +got):\n%s", diff) @@ -842,7 +907,7 @@ func TestStatefulSetExtraVolumeWithoutMount(t *testing.T) { VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, }} - podSpec := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec + podSpec := coordinatorStatefulSet(cluster).Spec.Template.Spec if len(podSpec.Volumes) != 2 { t.Errorf("volumes = %v, want the scratch volume alongside tmp", podSpec.Volumes) } @@ -853,28 +918,29 @@ func TestStatefulSetExtraVolumeWithoutMount(t *testing.T) { // TestStatefulSetRetentionPolicy pins the mapping from the spec's retention // policy onto the StatefulSet machinery that is the only deleter of this -// cluster's storage. whenScaled stays Retain regardless: both replica counts -// are immutable, so nothing ever scales down. +// cluster's storage. Both whenDeleted and whenScaled follow it: the claim of a +// pod a scale-down removes is the same data as the claim of a pod a cluster +// deletion removes, so one knob answers for both. func TestStatefulSetRetentionPolicy(t *testing.T) { tests := []struct { - name string - policy memgraphcomv1alpha1.StorageRetentionPolicy - whenDeleted appsv1.PersistentVolumeClaimRetentionPolicyType + name string + policy memgraphcomv1alpha1.StorageRetentionPolicy + expected appsv1.PersistentVolumeClaimRetentionPolicyType }{ { - name: "unset defaults to retain", - policy: "", - whenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + name: "unset defaults to retain", + policy: "", + expected: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, }, { - name: "retain", - policy: memgraphcomv1alpha1.RetentionPolicyRetain, - whenDeleted: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, + name: "retain", + policy: memgraphcomv1alpha1.RetentionPolicyRetain, + expected: appsv1.RetainPersistentVolumeClaimRetentionPolicyType, }, { - name: "delete", - policy: memgraphcomv1alpha1.RetentionPolicyDelete, - whenDeleted: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, + name: "delete", + policy: memgraphcomv1alpha1.RetentionPolicyDelete, + expected: appsv1.DeletePersistentVolumeClaimRetentionPolicyType, }, } for _, tc := range tests { @@ -882,10 +948,10 @@ func TestStatefulSetRetentionPolicy(t *testing.T) { cluster := minimalCluster() cluster.Spec.Storage.RetentionPolicy = tc.policy - want := expectedRetentionPolicy(tc.whenDeleted) + want := expectedRetentionPolicy(tc.expected) for _, sts := range []*appsv1.StatefulSet{ - resources.CoordinatorStatefulSet(cluster), - resources.DataStatefulSet(cluster), + coordinatorStatefulSet(cluster), + dataStatefulSet(cluster), } { got := sts.Spec.PersistentVolumeClaimRetentionPolicy if diff := cmp.Diff(want, got); diff != "" { @@ -921,7 +987,7 @@ func TestStatefulSetPortsAndClusterDomain(t *testing.T) { cluster := tunedCluster() t.Run(coordinatorComponent, func(t *testing.T) { - container := resources.CoordinatorStatefulSet(cluster).Spec.Template.Spec.Containers[0] + container := coordinatorStatefulSet(cluster).Spec.Template.Spec.Containers[0] wantPorts := []corev1.ContainerPort{ {Name: boltPortName, ContainerPort: customBoltPort}, @@ -948,7 +1014,7 @@ func TestStatefulSetPortsAndClusterDomain(t *testing.T) { }) t.Run(dataComponent, func(t *testing.T) { - container := resources.DataStatefulSet(cluster).Spec.Template.Spec.Containers[0] + container := dataStatefulSet(cluster).Spec.Template.Spec.Containers[0] wantPorts := []corev1.ContainerPort{ {Name: boltPortName, ContainerPort: customBoltPort}, @@ -997,7 +1063,7 @@ func TestStatefulSetProbeOverrides(t *testing.T) { }{ { name: coordinatorComponent, - sts: resources.CoordinatorStatefulSet(cluster), + sts: coordinatorStatefulSet(cluster), // Only the failure threshold was raised, so the timings default. startup: tunedTCPProbe(customCoordinatorPort, 30, 10, 5), // Timings tightened, failure threshold left at its default. @@ -1006,7 +1072,7 @@ func TestStatefulSetProbeOverrides(t *testing.T) { }, { name: dataComponent, - sts: resources.DataStatefulSet(cluster), + sts: dataStatefulSet(cluster), startup: tunedTCPProbe(customBoltPort, 4320, 15, 10), readiness: tunedTCPProbe(customBoltPort, 20, 10, 5), liveness: tunedTCPProbe(customBoltPort, 6, 10, 5), @@ -1038,7 +1104,7 @@ func TestStatefulSetResourceOverrides(t *testing.T) { }{ { name: coordinatorComponent, - sts: resources.CoordinatorStatefulSet(cluster), + sts: coordinatorStatefulSet(cluster), want: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}, @@ -1046,7 +1112,7 @@ func TestStatefulSetResourceOverrides(t *testing.T) { }, { name: dataComponent, - sts: resources.DataStatefulSet(cluster), + sts: dataStatefulSet(cluster), want: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4Gi")}, }, @@ -1077,14 +1143,14 @@ func TestStatefulSetLabelOverrides(t *testing.T) { }{ { name: coordinatorComponent, - sts: resources.CoordinatorStatefulSet(cluster), + sts: coordinatorStatefulSet(cluster), component: coordinatorComponent, stsLabels: map[string]string{tierLabel: "control"}, podLabels: map[string]string{teamLabel: platformTeam}, }, { name: dataComponent, - sts: resources.DataStatefulSet(cluster), + sts: dataStatefulSet(cluster), component: dataComponent, stsLabels: map[string]string{tierLabel: "storage"}, podLabels: map[string]string{teamLabel: dataComponent}, @@ -1126,7 +1192,7 @@ func TestStatefulSetCustomLabelsCannotOverrideIdentity(t *testing.T) { } want := expectedLabelsWith(coordinatorComponent, map[string]string{teamLabel: platformTeam}) - sts := resources.CoordinatorStatefulSet(cluster) + sts := coordinatorStatefulSet(cluster) for name, got := range map[string]map[string]string{ statefulSetKind: sts.Labels, "pod": sts.Spec.Template.Labels, @@ -1151,7 +1217,7 @@ func TestStatefulSetExtraEnv(t *testing.T) { }{ { name: coordinatorComponent, - sts: resources.CoordinatorStatefulSet(cluster), + sts: coordinatorStatefulSet(cluster), want: append( append([]corev1.EnvVar{{ Name: "POD_NAME", @@ -1164,7 +1230,7 @@ func TestStatefulSetExtraEnv(t *testing.T) { }, { name: dataComponent, - sts: resources.DataStatefulSet(cluster), + sts: dataStatefulSet(cluster), want: append( licenseEnv("memgraph-secrets", "MEMGRAPH_ENTERPRISE_LICENSE", "MEMGRAPH_ORGANIZATION_NAME"), corev1.EnvVar{Name: "DATA_LABEL_ONE", Value: "one"}, diff --git a/internal/resources/topology.go b/internal/resources/topology.go index 8c30497..95fba70 100644 --- a/internal/resources/topology.go +++ b/internal/resources/topology.go @@ -24,6 +24,19 @@ import ( "github.com/memgraph/kubernetes-operator/internal/planner" ) +// DeclaredCoordinators is the number of coordinators the spec declares, with +// the CRD's schema default resolved so a spec that never passed admission reads +// the same as one that did. +func DeclaredCoordinators(cluster *memgraphcomv1alpha1.MemgraphCluster) int32 { + return normalize(cluster.Spec).coordinators +} + +// DeclaredDataInstances is the number of data instances the spec declares, with +// the CRD's schema default resolved. +func DeclaredDataInstances(cluster *memgraphcomv1alpha1.MemgraphCluster) int32 { + return normalize(cluster.Spec).dataInstances +} + // DeclaredTopology derives the registration topology the planner drives the // cluster toward. Identity follows the pod ordinal exactly as the workload // pods advertise it: coordinator ordinal N is Raft coordinator N+1 (Memgraph diff --git a/internal/resources/topology_test.go b/internal/resources/topology_test.go index deb10d7..397148d 100644 --- a/internal/resources/topology_test.go +++ b/internal/resources/topology_test.go @@ -97,7 +97,7 @@ func TestDeclaredTopologyFollowsReplicaCounts(t *testing.T) { func TestDeclaredTopologyMatchesCoordinatorStartScript(t *testing.T) { cluster := minimalCluster() topology := resources.DeclaredTopology(cluster) - sts := resources.CoordinatorStatefulSet(cluster) + sts := coordinatorStatefulSet(cluster) script := strings.Join(sts.Spec.Template.Spec.Containers[0].Command, "\n") // The script derives '.' from POD_NAME; every declared @@ -175,7 +175,7 @@ func TestDeclaredTopologyPortsAndClusterDomain(t *testing.T) { func TestDeclaredTopologyMatchesTunedCoordinatorStartScript(t *testing.T) { cluster := tunedCluster() topology := resources.DeclaredTopology(cluster) - sts := resources.CoordinatorStatefulSet(cluster) + sts := coordinatorStatefulSet(cluster) script := strings.Join(sts.Spec.Template.Spec.Containers[0].Command, "\n") suffix := fmt.Sprintf("%s.%s.svc.k8s.example.com", coordinatorName, testNamespace) diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index 35f0958..41bdbb4 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -73,8 +73,37 @@ var ( memgraphImage = example.Spec.Image.Repository + ":" + example.Spec.Image.Tag licenseSecretName = example.Spec.Secrets.Name + + // quickstartCluster is the cluster the example manifest boots, which most of + // the specs below observe. + quickstartCluster = clusterUnderTest{ + namespace: clusterNamespace, + name: clusterName, + coordinators: coordinatorCount, + dataInstances: dataInstanceCount, + } ) +// clusterUnderTest is one MemgraphCluster a spec observes, together with the +// topology it declares. The suite runs several differently-shaped clusters — +// the quickstart one, the retention one, the one that is scaled — so every +// helper below takes its cluster rather than reaching for the quickstart +// globals. +type clusterUnderTest struct { + namespace string + name string + coordinators int32 + dataInstances int32 +} + +// grownTo returns the same cluster with a different declared topology, which is +// what the assertions switch to after a scale. +func (c clusterUnderTest) grownTo(coordinators, dataInstances int32) clusterUnderTest { + c.coordinators = coordinators + c.dataInstances = dataInstances + return c +} + // declaredCount reads a replica count the example must state outright: the // counts drive the assertions, and a count left to the CRD's default would // leave the suite asserting on a topology the file never declared. @@ -108,12 +137,12 @@ func loadExample() *memgraphcomv1alpha1.MemgraphCluster { // instance must appear under in SHOW INSTANCES once the operator has converged // registration: coordinator ordinal N registers as coordinator_N+1, data // ordinal N as instance_N. -func declaredInstances() []string { - names := make([]string, 0, coordinatorCount+dataInstanceCount) - for ordinal := range coordinatorCount { +func (c clusterUnderTest) declaredInstances() []string { + names := make([]string, 0, c.coordinators+c.dataInstances) + for ordinal := range c.coordinators { names = append(names, fmt.Sprintf("coordinator_%d", ordinal+1)) } - for ordinal := range dataInstanceCount { + for ordinal := range c.dataInstances { names = append(names, fmt.Sprintf("instance_%d", ordinal)) } return names @@ -131,33 +160,13 @@ func declaredInstances() []string { // same pattern — no pipeline changes. var _ = Describe("MemgraphCluster", Ordered, func() { BeforeAll(func() { - license := os.Getenv(licenseEnvVar) - organization := os.Getenv(organizationEnvVar) - Expect(license).NotTo(BeEmpty(), - "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", licenseEnvVar) - Expect(organization).NotTo(BeEmpty(), - "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", organizationEnvVar) - - By("preloading the Memgraph image into the Kind cluster") - cmd := exec.Command("docker", "pull", memgraphImage) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to pull the Memgraph image") - Expect(utils.LoadImageToKindClusterWithName(memgraphImage)).To(Succeed(), - "Failed to load the Memgraph image into Kind") - - By("creating the cluster namespace") - cmd = exec.Command("kubectl", "create", "ns", clusterNamespace) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + license, organization := licenseFromEnv() - By("labeling the namespace to enforce the restricted security policy") - cmd = exec.Command("kubectl", "label", "--overwrite", "ns", clusterNamespace, - "pod-security.kubernetes.io/enforce=restricted") - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") + preloadMemgraphImage() + createClusterNamespace(clusterNamespace) By("creating the enterprise license Secret") - createLicenseSecret(license, organization) + createLicenseSecret(clusterNamespace, license, organization) By("applying the MemgraphCluster") applyMemgraphCluster() @@ -170,30 +179,12 @@ var _ = Describe("MemgraphCluster", Ordered, func() { _, _ = utils.Run(cmd) }) - // On failure, dump everything needed to debug a broken bootstrap from CI - // logs alone. AfterEach(func() { - if !CurrentSpecReport().Failed() { - return - } - for _, args := range [][]string{ - {"get", "pods", "-n", clusterNamespace, "-o", "wide"}, - {"get", "memgraphclusters", "-n", clusterNamespace, "-o", "yaml"}, - {"get", "events", "-n", clusterNamespace, "--sort-by=.lastTimestamp"}, - {"logs", "deploy/" + controllerDeploymentName, "-n", namespace}, - } { - cmd := exec.Command("kubectl", args...) - output, err := utils.Run(cmd) - if err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Failed to collect diagnostics %v: %s\n", args, err) - continue - } - _, _ = fmt.Fprintf(GinkgoWriter, "Diagnostics kubectl %v:\n%s\n", args, output) - } + dumpDiagnosticsOnFailure(clusterNamespace) }) It("bootstraps every declared instance registered with exactly one MAIN", func() { - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) }) // The operator's reason to exist over the chart's one-shot Job: a data @@ -204,13 +195,13 @@ var _ = Describe("MemgraphCluster", Ordered, func() { const wiped = "instance_1" By("confirming the cluster is converged before wiping a registration") - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) By("unregistering a data instance on the coordinator leader") Expect(wipeInstanceRegistration(wiped)).To(Succeed()) By("confirming the instance really left the cluster view") - view, err := leaderView() + view, err := quickstartCluster.leaderView() Expect(err).NotTo(HaveOccurred()) names := make([]string, 0, len(view)) for _, instance := range view { @@ -220,7 +211,7 @@ var _ = Describe("MemgraphCluster", Ordered, func() { "the wipe must actually remove the registration for the test to be meaningful") By("waiting for the operator to converge the cluster back to fully registered") - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) }) // The coordinator analogue of the data-instance re-registration: a @@ -231,14 +222,14 @@ var _ = Describe("MemgraphCluster", Ordered, func() { // converged cluster. It("re-adds a coordinator that was removed from the cluster", func() { By("confirming the cluster is converged before removing a coordinator") - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) By("removing a follower coordinator on the coordinator leader") removed, err := removeCoordinatorRegistration() Expect(err).NotTo(HaveOccurred()) By("confirming the coordinator really left the cluster view") - view, err := leaderView() + view, err := quickstartCluster.leaderView() Expect(err).NotTo(HaveOccurred()) names := make([]string, 0, len(view)) for _, instance := range view { @@ -248,7 +239,7 @@ var _ = Describe("MemgraphCluster", Ordered, func() { "the removal must actually drop the coordinator for the test to be meaningful") By("waiting for the operator to converge the cluster back to fully registered") - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) }) // Storage survives the cluster under the default retention policy: an @@ -256,7 +247,7 @@ var _ = Describe("MemgraphCluster", Ordered, func() { // it. This deletes the CR, so it runs last in this Ordered container. It("leaves the PVCs behind when the default-retention CR is deleted", func() { By("confirming the cluster is converged before deleting it") - Eventually(verifyClusterRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + Eventually(quickstartCluster.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) By("recording the provisioned PVCs") before, err := listPVCs(clusterNamespace) @@ -322,7 +313,7 @@ metadata: name: %s namespace: %s spec: - coordinators: 1 + coordinators: 3 dataInstances: 1 image: repository: %s @@ -336,12 +327,12 @@ spec: Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") By("waiting for the claims to be provisioned") - // One lib and one log claim for the single coordinator and the single - // data instance. + // One lib and one log claim per pod: three coordinators and one data + // instance, the smallest topology admission accepts. Eventually(func(g Gomega) { claims, err := listPVCs(retentionNamespace) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(claims).To(HaveLen(4)) + g.Expect(claims).To(HaveLen(8)) }, 5*time.Minute, 5*time.Second).Should(Succeed()) By("deleting the MemgraphCluster") @@ -359,6 +350,110 @@ spec: }) }) +// Growing a live cluster: the end-to-end proof that raising either count +// registers the members it adds without human action. It gets its own container +// and namespace because it needs a cluster it may reshape, and its teardown is +// awaited — eight Memgraph pods are a large share of a Kind cluster's capacity, +// which the scenarios that may run after it need back. +var _ = Describe("MemgraphCluster topology scale-up", Ordered, func() { + const scalingNamespace = "memgraph-e2e-scaling" + const scalingClusterName = "scaling" + + // The cluster starts at the default topology and grows to five coordinators + // and three data instances. + initial := clusterUnderTest{ + namespace: scalingNamespace, name: scalingClusterName, coordinators: 3, dataInstances: 2, + } + grown := initial.grownTo(5, 3) + + BeforeAll(func() { + license, organization := licenseFromEnv() + + preloadMemgraphImage() + createClusterNamespace(scalingNamespace) + + By("creating the enterprise license Secret") + createLicenseSecret(scalingNamespace, license, organization) + + By("applying the MemgraphCluster to scale") + // No log claim and explicit small requests: this cluster runs up to eight + // pods on the same Kind nodes as the other scenarios', so it asks for as + // little as it can while still being a real HA cluster. + manifest := fmt.Sprintf(`apiVersion: memgraph.com/v1alpha1 +kind: MemgraphCluster +metadata: + name: %s + namespace: %s +spec: + coordinators: %d + dataInstances: %d + image: + repository: %s + tag: %s + secrets: + name: %s + storage: + coordinators: + createLogStorageClaim: false + data: + createLogStorageClaim: false + resources: + coordinators: + requests: + cpu: 50m + memory: 200Mi + data: + requests: + cpu: 50m + memory: 300Mi +`, scalingClusterName, scalingNamespace, initial.coordinators, initial.dataInstances, + example.Spec.Image.Repository, example.Spec.Image.Tag, licenseSecretName) + cmd := exec.Command("kubectl", "apply", "-f", "-") + _, err := utils.RunWithInput(cmd, manifest) + Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") + }) + + AfterAll(func() { + By("removing the cluster namespace and waiting for its pods to go") + cmd := exec.Command("kubectl", "delete", "ns", scalingNamespace, + "--ignore-not-found", "--wait=true", "--timeout=5m") + _, _ = utils.Run(cmd) + }) + + AfterEach(func() { + dumpDiagnosticsOnFailure(scalingNamespace) + }) + + It("bootstraps the initial topology", func() { + Eventually(initial.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + initial.awaitConverged(2 * time.Minute) + }) + + // Both counts are raised in one edit, in different step sizes, which is the + // whole contract: nothing constrains a change beyond the target counts + // themselves. + It("grows both roles in one edit and registers every added member", func() { + By("raising both counts on the live cluster") + cmd := exec.Command("kubectl", "patch", "memgraphcluster", scalingClusterName, + "-n", scalingNamespace, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"coordinators":%d,"dataInstances":%d}}`, + grown.coordinators, grown.dataInstances)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "the operator must accept a raised topology count") + + By("waiting for every member of the grown topology to be registered") + Eventually(grown.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + + By("confirming the resource reports the scale as finished") + grown.awaitConverged(5 * time.Minute) + Expect(grown.replicas("coordinator")).To(Equal("5")) + Expect(grown.replicas("data")).To(Equal("3")) + coordinators, dataInstances := grown.registeredCounts() + Expect(coordinators).To(Equal("5")) + Expect(dataInstances).To(Equal("3")) + }) +}) + // listPVCs returns the names of the PersistentVolumeClaims in a namespace, // excluding any already marked for deletion — a claim with a deletion // timestamp is gone as far as the retention contract is concerned, even while @@ -387,11 +482,11 @@ func listPVCs(namespace string) ([]string, error) { return names, nil } -// verifyClusterRegistered asserts the coordinator leader reports every declared +// verifyRegistered asserts the coordinator leader reports every declared // instance registered and healthy with exactly one MAIN — the converged steady -// state both the bootstrap and re-registration specs check for. -func verifyClusterRegistered(g Gomega) { - view, err := leaderView() +// state the bootstrap, re-registration and scaling specs all check for. +func (c clusterUnderTest) verifyRegistered(g Gomega) { + view, err := c.leaderView() g.Expect(err).NotTo(HaveOccurred()) names := make([]string, 0, len(view)) @@ -404,10 +499,45 @@ func verifyClusterRegistered(g Gomega) { mains = append(mains, instance.name) } } - g.Expect(names).To(ConsistOf(declaredInstances())) + g.Expect(names).To(ConsistOf(c.declaredInstances())) g.Expect(mains).To(HaveLen(1), "expected exactly one MAIN, got %v", mains) } +// awaitConverged waits for the operator to report the declared topology as +// realized: every declared instance registered and both StatefulSets at the +// declared replica count. It is what a user gates a scale on. +func (c clusterUnderTest) awaitConverged(timeout time.Duration) { + GinkgoHelper() + cmd := exec.Command("kubectl", "wait", "--for=condition=Converged", + "memgraphcluster/"+c.name, "-n", c.namespace, "--timeout="+timeout.String()) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "the MemgraphCluster never reported Converged") +} + +// registeredCounts reads the registered coordinator and data-instance counts the +// operator publishes on the resource's status. +func (c clusterUnderTest) registeredCounts() (string, string) { + GinkgoHelper() + read := func(field string) string { + cmd := exec.Command("kubectl", "get", "memgraphcluster", c.name, "-n", c.namespace, + "-o", fmt.Sprintf("jsonpath={.status.%s}", field)) + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to read status.%s", field) + return strings.TrimSpace(output) + } + return read("coordinators"), read("dataInstances") +} + +// replicas reads the replica count the operator applied to a role's StatefulSet. +func (c clusterUnderTest) replicas(component string) string { + GinkgoHelper() + cmd := exec.Command("kubectl", "get", "statefulset", c.name+"-"+component, "-n", c.namespace, + "-o", "jsonpath={.spec.replicas}") + output, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to read the %s StatefulSet's replicas", component) + return strings.TrimSpace(output) +} + // wipeInstanceRegistration unregisters the named data instance on the // coordinator leader, simulating registration state a pod loses when it is // rescheduled onto a fresh node. UNREGISTER INSTANCE must run on the leader — @@ -417,7 +547,7 @@ func wipeInstanceRegistration(name string) error { var errs []error for ordinal := range coordinatorCount { pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) - view, err := showInstances(pod) + view, err := quickstartCluster.showInstances(pod) if err != nil { errs = append(errs, err) continue @@ -454,7 +584,7 @@ func removeCoordinatorRegistration() (string, error) { var errs []error for ordinal := range coordinatorCount { pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) - view, err := showInstances(pod) + view, err := quickstartCluster.showInstances(pod) if err != nil { errs = append(errs, err) continue @@ -487,16 +617,84 @@ func removeCoordinatorRegistration() (string, error) { return "", fmt.Errorf("no coordinator leader found to remove a coordinator: %w", errors.Join(errs...)) } +// dumpDiagnosticsOnFailure dumps everything needed to debug a broken cluster +// from the CI logs alone: the pods, the resource itself, the namespace's events +// and the operator's log. +func dumpDiagnosticsOnFailure(namespace string) { + if !CurrentSpecReport().Failed() { + return + } + for _, args := range [][]string{ + {"get", "pods", "-n", namespace, "-o", "wide"}, + {"get", "memgraphclusters", "-n", namespace, "-o", "yaml"}, + {"get", "events", "-n", namespace, "--sort-by=.lastTimestamp"}, + {"logs", "deploy/" + controllerDeploymentName, "-n", namespace}, + } { + cmd := exec.Command("kubectl", args...) + output, err := utils.Run(cmd) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Failed to collect diagnostics %v: %s\n", args, err) + continue + } + _, _ = fmt.Fprintf(GinkgoWriter, "Diagnostics kubectl %v:\n%s\n", args, output) + } +} + +// licenseFromEnv reads the enterprise license every HA cluster in this suite +// needs, following the HA Helm chart's CI convention of repository secrets +// exported into the job environment. +func licenseFromEnv() (string, string) { + GinkgoHelper() + license := os.Getenv(licenseEnvVar) + organization := os.Getenv(organizationEnvVar) + Expect(license).NotTo(BeEmpty(), + "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", licenseEnvVar) + Expect(organization).NotTo(BeEmpty(), + "%s must be set: the e2e suite boots a licensed Memgraph HA cluster", organizationEnvVar) + return license, organization +} + +// preloadMemgraphImage puts the Memgraph image on the Kind nodes, so a cluster's +// pods do not each wait on a registry pull. It is idempotent, so every scenario +// container that boots Memgraph can call it without depending on another's +// setup. +func preloadMemgraphImage() { + GinkgoHelper() + By("preloading the Memgraph image into the Kind cluster") + cmd := exec.Command("docker", "pull", memgraphImage) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to pull the Memgraph image") + Expect(utils.LoadImageToKindClusterWithName(memgraphImage)).To(Succeed(), + "Failed to load the Memgraph image into Kind") +} + +// createClusterNamespace creates a namespace for a MemgraphCluster and enforces +// the restricted Pod Security Standard in it, so every scenario proves the +// operator's workloads run under the policy a security review demands. +func createClusterNamespace(namespace string) { + GinkgoHelper() + By("creating the cluster namespace " + namespace) + cmd := exec.Command("kubectl", "create", "ns", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create namespace") + + By("labeling the namespace to enforce the restricted security policy") + cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace, + "pod-security.kubernetes.io/enforce=restricted") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy") +} + // createLicenseSecret applies the Secret the MemgraphCluster references, under // the name and keys the example points at. The manifest is piped over stdin so // no secret material ever reaches the logged command line. -func createLicenseSecret(license, organization string) { +func createLicenseSecret(namespace, license, organization string) { secret := map[string]any{ "apiVersion": "v1", "kind": "Secret", "metadata": map[string]any{ "name": licenseSecretName, - "namespace": clusterNamespace, + "namespace": namespace, }, "stringData": map[string]string{ example.Spec.Secrets.LicenseKey: license, @@ -532,11 +730,11 @@ type instanceRow struct { // reports a MAIN. Only the coordinator leader health-checks data instances and // reports their roles (followers show them as unknown), so a view containing a // MAIN is the leader's authoritative view. -func leaderView() ([]instanceRow, error) { +func (c clusterUnderTest) leaderView() ([]instanceRow, error) { var errs []error - for ordinal := range coordinatorCount { - pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) - view, err := showInstances(pod) + for ordinal := range c.coordinators { + pod := fmt.Sprintf("%s-coordinator-%d", c.name, ordinal) + view, err := c.showInstances(pod) if err != nil { errs = append(errs, err) continue @@ -554,8 +752,8 @@ func leaderView() ([]instanceRow, error) { // showInstances runs SHOW INSTANCES through mgconsole inside the given // coordinator pod (the Memgraph image ships the client) and parses the CSV // output. -func showInstances(pod string) ([]instanceRow, error) { - cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", +func (c clusterUnderTest) showInstances(pod string) ([]instanceRow, error) { + cmd := exec.Command("kubectl", "exec", pod, "-n", c.namespace, "-c", "memgraph", "--", "bash", "-c", "echo 'SHOW INSTANCES;' | mgconsole --output-format=csv") output, err := utils.Run(cmd) if err != nil { From cff3f627992bb9cfe9ea75c2671930b11108346c Mon Sep 17 00:00:00 2001 From: as51340 Date: Tue, 28 Jul 2026 15:20:39 +0200 Subject: [PATCH 22/34] testing: Fix e2e test --- test/e2e/memgraphcluster_test.go | 42 +++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index 41bdbb4..f07e9eb 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -27,6 +27,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strings" "time" @@ -326,11 +327,16 @@ spec: _, err := utils.RunWithInput(cmd, manifest) Expect(err).NotTo(HaveOccurred(), "Failed to apply the MemgraphCluster") - By("waiting for the claims to be provisioned") + By("waiting for the claims to be provisioned and adopted by their StatefulSets") // One lib and one log claim per pod: three coordinators and one data - // instance, the smallest topology admission accepts. + // instance, the smallest topology admission accepts. Adoption is what + // the spec has to wait for, not mere existence: the Delete policy + // reaches a claim as the StatefulSet owner reference the controller + // attaches a sync *after* it creates the claim, and a claim whose set + // is deleted before it is adopted is stranded for good, not merely + // collected late. Eventually(func(g Gomega) { - claims, err := listPVCs(retentionNamespace) + claims, err := listAdoptedPVCs(retentionNamespace) g.Expect(err).NotTo(HaveOccurred()) g.Expect(claims).To(HaveLen(8)) }, 5*time.Minute, 5*time.Second).Should(Succeed()) @@ -454,6 +460,36 @@ spec: }) }) +// listAdoptedPVCs returns the names of the PersistentVolumeClaims a +// StatefulSet has taken ownership of. Only the owner reference makes a claim +// follow its StatefulSet into deletion, so this is the precondition a spec +// asserting the Delete retention policy must wait for before it deletes +// anything. It gates a spec rather than asserting one — the retention +// assertion itself stays on the claims a user would see. +func listAdoptedPVCs(namespace string) ([]string, error) { + cmd := exec.Command("kubectl", "get", "pvc", "-n", namespace, "-o", + `jsonpath={range .items[*]}{.metadata.name}{"\t"}{.metadata.ownerReferences[*].kind}{"\n"}{end}`) + output, err := utils.Run(cmd) + if err != nil { + return nil, err + } + + // An unadopted claim yields a line of "\t": the separator is always + // emitted, so the split is total, and the kinds field is simply empty. + names := []string{} + for _, line := range utils.GetNonEmptyLines(output) { + name, ownerKinds, found := strings.Cut(line, "\t") + if !found { + return nil, fmt.Errorf("unexpected kubectl get pvc output line: %q", line) + } + if !slices.Contains(strings.Fields(ownerKinds), "StatefulSet") { + continue + } + names = append(names, strings.TrimSpace(name)) + } + return names, nil +} + // listPVCs returns the names of the PersistentVolumeClaims in a namespace, // excluding any already marked for deletion — a claim with a deletion // timestamp is gone as far as the retention contract is concerned, even while From d48bd429e292246135bd7f85bd9be5c7904ec6d5 Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Tue, 28 Jul 2026 15:46:20 +0200 Subject: [PATCH 23/34] feat: data-instance scale-down with MAIN-safe retirement (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: data-instance scale-down with MAIN-safe retirement Carries out a lowered `dataInstances` count, which `14-topology-scale-up.md` accepted at admission but deliberately held back. A StatefulSet sheds only its highest ordinals, so the operator never picks a victim: the retiring set is `[spec.dataInstances, liveStatefulSet.spec.replicas)`, derived from the operator's own prior apply, which bounds the range exactly and keeps an instance a human registered out of it. The planner drives the whole removal in one pass against one leader connection, because it can predict every intermediate state: - `DEMOTE INSTANCE` is emitted only when a retiring instance is observed as MAIN — Memgraph refuses to unregister the MAIN — and it deliberately leaves the cluster MAIN-less, since failover only triggers on a coordinator leadership change or a failed ping. - The promotion that follows is the rule 14 installed (lowest-ordinal declared instance observed `up`), so the operator hands MAIN to a survivor itself. Only declared instances are candidates, so a member on its way out is never the target. Replicas are SYNC, so promoting after a clean demote is not a data-loss gamble, and the MAIN-less window is the time between two queries. - `UNREGISTER INSTANCE` then removes each retiring member that is still registered. Unregistration happens before the pods go: the smaller replica count is applied in exactly one place, at the end of the registration phase once the plan is empty. That keeps 14's pre-apply replica rule free of any cluster knowledge — it still never shrinks — and converges the shrink across two passes with no second observation source. The readiness gate stays strict, retiring pods included, so a retiring pod that cannot become ready blocks its own removal and the resource reports `WorkloadsNotReady` instead of the operator acting on a half-known cluster; that trade is documented. `Converged` is False with reason `RetirementInProgress`, naming the retiring instances, for as long as the StatefulSet still runs them. Errors are not special-cased: read-before-write means `ALREADY_REPLICA` or `NO_INSTANCE_WITH_NAME` only appear when something raced the operator, so they surface and the plan is recomputed from a fresh observation. Tests: planner cases for MAIN on a retiring ordinal, MAIN on a survivor, a down survivor, several retiring at once, shrink to a single instance, an already-unregistered retiring member, mixed grow-and-shrink across roles, and an undeclared instance outside the range; `RetiringDataInstances` bounds in both directions; envtest specs for the retirement order, the MAIN move, the unready retiring pod, and — rewritten for the role that still holds — the lowered coordinator count. The scaling e2e container parks MAIN on `instance_2`, drops the count to 2, and asserts the instance leaves the cluster before its pod is shed, that MAIN moved to a survivor, that the cluster converges, and that the retired claim is kept by the default retention policy. The fake cluster gains demote and unregister, rejecting a non-MAIN demote and an unregistration aimed at a MAIN, so any plan that is not read-before-write fails the suite loudly. No CRD or RBAC change, so the chart is untouched. * testing: Make more reliable e2e test --- CLAUDE.md | 4 +- README.md | 16 +- api/v1alpha1/memgraphcluster_types.go | 7 + config/samples/v1alpha1_memgraphcluster.yaml | 13 +- examples/minimal-cluster.yaml | 3 +- internal/controller/fake_memgraph_test.go | 38 ++++ .../controller/memgraphcluster_controller.go | 135 +++++++++--- .../memgraphcluster_controller_test.go | 161 ++++++++++++-- internal/memgraph/bolt.go | 10 + internal/memgraph/client.go | 19 +- internal/memgraph/queries.go | 8 + internal/memgraph/queries_test.go | 14 ++ internal/planner/planner.go | 100 +++++++-- internal/planner/planner_test.go | 165 ++++++++++++++- internal/resources/topology.go | 50 ++++- internal/resources/topology_test.go | 86 +++++++- test/e2e/memgraphcluster_test.go | 197 +++++++++++++++--- 17 files changed, 924 insertions(+), 102 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d6e74d6..ae7d638 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -52,7 +52,7 @@ The PRD defines seven modules with two pure cores and one mock seam. Keep this s 1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **mutable**, bounded by schema floors alone (coordinators ≥ 3 and odd, data instances ≥ 1) — enforced by the CRD, not a webhook. Raising a count grows the cluster; lowering one is accepted but not yet carried out (see `specs/operator-mvp/issues/15`, `16`). Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). 2. **Resource builders** — pure functions: spec in, desired Kubernetes objects out (one StatefulSet per role + headless Services; per-pod identity such as coordinator ID and advertised addresses derived from pod ordinals). No API calls, no side effects. Tested golden-style. 3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. -4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. `SET INSTANCE TO MAIN` is issued exactly once at bootstrap (when no MAIN exists); after that, failover belongs to the Raft coordinators — the operator only observes. +4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. A MAIN is promoted only when the cluster has none — at bootstrap, and after the planner itself demoted a data instance that is retiring; a MAIN that is staying is never overridden, because failover belongs to the Raft coordinators. A lowered `dataInstances` count is the one removal the planner drives: `DEMOTE INSTANCE` the retiring MAIN, promote a survivor, `UNREGISTER INSTANCE` the retiring members, and only then does the controller shed their pods (see `specs/operator-mvp/issues/15`). 5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. 6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. The chart's `version` and its `appVersion` (the operator image tag) move **independently**: tag `v` releases the operator, `chart-` releases the chart alone — see `docs/releasing.md`. 7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. @@ -63,6 +63,6 @@ Test philosophy (from the PRD): assert external behavior, never internal call or - Spec knob names mirror the HA Helm chart's vocabulary where the concept carries over (e.g. the `secrets.name` / `secrets.licenseKey` / `secrets.organizationKey` block) — check the chart before inventing a name. - No secret material in spec or status; secrets are consumed by reference only. -- No destructive code paths in v1: no finalizer-based storage cleanup, no instance unregistration; PVC retention maps to the StatefulSet PVC retention policy (default `Retain`). +- Storage is never deleted by the operator: no finalizer-based cleanup; PVC retention (deletion *and* scale-down) maps to the StatefulSet PVC retention policy (default `Retain`). The only cluster members the operator removes are the data instances a lowered `dataInstances` count retires; coordinators are never removed (`REMOVE COORDINATOR` arrives with `specs/operator-mvp/issues/16`). - Workload pods: non-root uid 101 / gid 103, seccomp RuntimeDefault, all capabilities dropped. - Log messages follow Kubernetes style: capital first letter, no trailing period, past tense, object type named (see AGENTS.md for examples). diff --git a/README.md b/README.md index 2b9410d..bdfb2fb 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ memgraph 3 2 instance_0 True True 4m12s - **`READY`** is True once a MAIN is elected, i.e. the cluster serves writes. - **`CONVERGED`** is True once every declared coordinator and data instance is registered and reported healthy, and both StatefulSets run the declared number of replicas. -`kubectl get mgc -n memgraph -o wide` adds how many of each role's declared members the cluster actually has registered (`REGISTERED-COORDINATORS`, `REGISTERED-DATA`), which is what a scale-up is watched through. +`kubectl get mgc -n memgraph -o wide` adds how many of each role's declared members the cluster actually has registered (`REGISTERED-COORDINATORS`, `REGISTERED-DATA`), which is what a scale is watched through. To block a script or a GitOps step on the cluster being usable: @@ -175,21 +175,27 @@ The MVP is deliberately "provision, bootstrap, observe". It does: - bootstrap HA: add the coordinators, register the data instances, and promote the initial MAIN once; - re-register continuously: every reconcile compares `SHOW INSTANCES` on the coordinator leader against the declared topology and issues only the missing registrations, so an instance that loses its registration state (say, after being rescheduled onto a fresh node) rejoins without human action; - **grow a live cluster**: raise `coordinators` or `dataInstances` (both in one edit if you like, in any step size) and the added pods are provisioned and registered by the same diff that restores a lost registration — no manual `ADD COORDINATOR` or `REGISTER INSTANCE`; +- **shrink the data instances**: lower `dataInstances` and the instances above the new count are retired — MAIN moved off them if one of them holds it, then `UNREGISTER INSTANCE`, and only then are their pods shed, so the coordinators never expect an instance whose pod is gone; - report the observed MAIN, the registered member counts, and the readiness and convergence conditions on the resource's status. -Growing is one edit, and `Converged` tells you when it is finished: +Scaling is one edit, and `Converged` tells you when it is finished: ```sh kubectl patch mgc memgraph -n memgraph --type=merge -p '{"spec":{"coordinators":5,"dataInstances":3}}' kubectl wait --namespace memgraph --for=condition=Converged memgraphcluster/memgraph --timeout=10m ``` +A scale-down reports `Converged=False` with reason `RetirementInProgress`, naming the instances on their way out, until their pods are gone. Two things to know about it: + +- **A retiring pod that cannot become ready blocks its own removal.** The operator only touches the cluster when every pod of both StatefulSets is ready, and until the shrink is applied the retiring pods still belong to the data StatefulSet. So an instance that is stuck (crash-looping, unschedulable, wedged in a snapshot restore) keeps its own retirement waiting, and the resource reports `WorkloadsNotReady` rather than the operator writing to a cluster whose state it only half knows. Fix the pod, or delete it if it is genuinely unrecoverable, and the retirement continues. +- **The claims of a retired instance follow `spec.storage.retentionPolicy`**, the same knob that decides what happens to storage when the cluster is deleted — `Retain` (the default) keeps them, so a shrink made by accident loses no data, and re-raising the count reattaches them. + What it does not do yet: -- **Scaling down.** `coordinators` must stay odd and at or above three, `dataInstances` at or above one — both enforced at creation and on every update. *Lowering* a count is accepted by admission but not carried out: taking a pod away means unregistering a cluster member first, and the operator has no removal path yet, so it holds the StatefulSet at its current size and reports `Converged=False` with reason `ScaleInProgress` until the count is raised back. Scale-down with MAIN- and quorum-safety is the next item on the roadmap. -- **Failover.** The operator issues `SET INSTANCE TO MAIN` exactly once, at bootstrap, when no MAIN exists. After that, leadership belongs entirely to the Raft coordinators; the operator only observes and reports it, so two control systems never fight over which instance is MAIN. +- **Scaling the coordinators down.** `coordinators` must stay odd and at or above three, `dataInstances` at or above one — all enforced at creation and on every update. Lowering `coordinators` is accepted by admission but not carried out: dropping a coordinator means removing a Raft member, which the operator does not do yet, so it holds the StatefulSet at its current size and reports `Converged=False` with reason `ScaleInProgress` until the count is raised back. +- **Failover.** The operator promotes a MAIN only when the cluster has none: once at bootstrap, and once more when it demotes an instance that is retiring. It never overrides a MAIN that is staying — leadership belongs to the Raft coordinators, so two control systems never fight over which instance is MAIN. - **Other day-2 operations**: orchestrated or rolling version upgrades, backup and restore, storage-mode changes. -- **Removing instances**: there is no `REMOVE COORDINATOR` or `UNREGISTER INSTANCE`, and no finalizer-based storage cleanup. The operator has no destructive code path. +- **Removing coordinators**: there is no `REMOVE COORDINATOR`, and no finalizer-based storage cleanup — deleting storage is left entirely to the StatefulSet's own retention policy. - **External access** of any kind — no LoadBalancer, NodePort, ingress or gateway. Access is in-cluster (or `kubectl port-forward`) only; the approach is expected to change, so it was deliberately deferred rather than shipped and broken later. - **TLS**, for Bolt or intra-cluster traffic. - **Bolt authentication** — the operator connects to the coordinators unauthenticated, so clusters must not enable auth yet. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 7f00801..7146d7d 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -168,6 +168,13 @@ const ( // declares, so the declared topology is not fully realized yet. ReasonScaleInProgress = "ScaleInProgress" + // ReasonRetirementInProgress is set while a lowered dataInstances count is + // being carried out: the instances beyond the declared count are still + // members of the cluster, or their pods are still being shed. The message + // names them, so a scale-down that stalls says which instance it is waiting + // on. + ReasonRetirementInProgress = "RetirementInProgress" + // ReasonMainElected is set when a data instance is observed as MAIN. ReasonMainElected = "MainElected" diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 154069b..24bf81c 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -9,9 +9,16 @@ spec: # Raising either count on a live cluster grows it: the operator provisions the # new pods and registers them with the cluster. The coordinator count must be # odd so the Raft quorum cannot split, and at least three — a quorum of one - # cannot survive losing itself. Lowering a count is not supported yet: it is - # accepted at admission but the operator holds the StatefulSet at its current - # size and reports Converged=False with reason ScaleInProgress. + # cannot survive losing itself. + # + # Lowering dataInstances shrinks the cluster: the instances above the new count + # are retired (MAIN moved off them, then UNREGISTER INSTANCE) before their pods + # are shed, reported as Converged=False with reason RetirementInProgress. Note + # that the operator only touches the cluster while every pod is ready, so a + # retiring pod that cannot become ready blocks its own removal. Lowering + # coordinators is not supported yet: it is accepted at admission but the + # operator holds the StatefulSet at its current size and reports + # Converged=False with reason ScaleInProgress. coordinators: 3 dataInstances: 2 # repository carries the registry host and image path only — the version diff --git a/examples/minimal-cluster.yaml b/examples/minimal-cluster.yaml index 0cd45bc..3098272 100644 --- a/examples/minimal-cluster.yaml +++ b/examples/minimal-cluster.yaml @@ -14,7 +14,8 @@ spec: # Raise either count later to grow the cluster: the operator provisions the # new pods and registers them, no manual registration involved. The # coordinator count must be odd so the Raft quorum cannot split, and at least - # three. Lowering a count is not supported yet. + # three. Lowering dataInstances retires the instances above the new count — + # unregistered before their pods go; lowering coordinators is not supported yet. coordinators: 3 dataInstances: 2 image: diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go index ca5ede3..e3bee3f 100644 --- a/internal/controller/fake_memgraph_test.go +++ b/internal/controller/fake_memgraph_test.go @@ -202,6 +202,44 @@ func (c *fakeClient) SetInstanceToMain(_ context.Context, name string) error { }) } +// DemoteInstance turns the named MAIN back into a replica, and — like the real +// thing — refuses an instance that is not MAIN, so the operator's read-before-write +// is what has to keep this call meaningful. +func (c *fakeClient) DemoteInstance(_ context.Context, name string) error { + return c.execute("DEMOTE INSTANCE "+name, func() error { + for i, instance := range c.cluster.instances { + if instance.Name != name { + continue + } + if !instance.IsMain() { + return fmt.Errorf("fake memgraph: instance %s is not MAIN", name) + } + c.cluster.instances[i].Role = memgraph.RoleReplica + return nil + } + return fmt.Errorf("fake memgraph: instance %s is not registered", name) + }) +} + +// UnregisterInstance removes the named data instance from the cluster view. It +// rejects an unregistered name and, as Memgraph does, the MAIN — so a plan that +// aims an unregistration at a MAIN fails the suite loudly. +func (c *fakeClient) UnregisterInstance(_ context.Context, name string) error { + return c.execute("UNREGISTER INSTANCE "+name, func() error { + for i, instance := range c.cluster.instances { + if instance.Name != name { + continue + } + if instance.IsMain() { + return fmt.Errorf("fake memgraph: instance %s is MAIN", name) + } + c.cluster.instances = slices.Delete(c.cluster.instances, i, i+1) + return nil + } + return fmt.Errorf("fake memgraph: instance %s is not registered", name) + }) +} + func (c *fakeClient) Close(context.Context) error { c.cluster.mu.Lock() defer c.cluster.mu.Unlock() diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index dff5ded..3ea5e6d 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -100,7 +100,9 @@ type MemgraphClusterReconciler struct { // INSTANCES on the coordinator leader, diff against the declared topology, and // issue only the missing commands — which is all growing a live cluster takes, // because a raised count declares members the observed cluster does not have -// registered yet. +// registered yet. A lowered dataInstances count runs the same loop in reverse: +// the members beyond the declared count are demoted if one of them holds MAIN, +// unregistered, and only then are their pods shed. // // All interaction is read-before-write and idempotent, so an operator restart // mid-bootstrap is harmless. Registration reconciliation is continuous, not @@ -125,39 +127,51 @@ func (r *MemgraphClusterReconciler) Reconcile(ctx context.Context, req ctrl.Requ return ctrl.Result{}, err } - desired := []client.Object{ + if err := r.applyDesired(ctx, &cluster, resources.CoordinatorHeadlessService(&cluster), resources.DataHeadlessService(&cluster), resources.CoordinatorStatefulSet(&cluster, replicas.coordinators.applied), resources.DataStatefulSet(&cluster, replicas.data.applied), + ); err != nil { + return ctrl.Result{}, err } + + log.Info("Applied desired workload objects for MemgraphCluster", "memgraphcluster", req.NamespacedName) + + return r.reconcileRegistration(ctx, &cluster, replicas) +} + +// applyDesired server-side-applies the desired workload objects, each owned by +// the cluster so garbage collection removes it with the CR. +// +// A rejected apply is retried forever behind the scenes, so it is reported on +// the resource before the error is returned: without that the conditions keep +// describing the cluster that is still running while the declared spec never +// lands, and the rejection is only visible in the operator's log. Both +// conditions go False — the workloads are not the declared ones, so neither +// serving nor convergence can be claimed for the spec the user asked for. +func (r *MemgraphClusterReconciler) applyDesired( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, + desired ...client.Object, +) error { for _, obj := range desired { - if err := controllerutil.SetControllerReference(&cluster, obj, r.Scheme); err != nil { - return ctrl.Result{}, fmt.Errorf("setting owner reference on %T %s: %w", obj, obj.GetName(), err) + if err := controllerutil.SetControllerReference(cluster, obj, r.Scheme); err != nil { + return fmt.Errorf("setting owner reference on %T %s: %w", obj, obj.GetName(), err) } if err := r.apply(ctx, obj); err != nil { applyErr := fmt.Errorf("applying %T %s: %w", obj, obj.GetName(), err) - // A rejected apply is retried forever behind the scenes, so report - // it on the resource: without this the conditions keep describing - // the cluster that is still running while the declared spec never - // lands, and the rejection is only visible in the operator's log. - // Both conditions go False — the workloads are not the declared - // ones, so neither serving nor convergence can be claimed for the - // spec the user asked for. msg := truncateMessage(applyErr.Error()) - if statusErr := r.writeStatus(ctx, &cluster, lastObserved(&cluster), + if statusErr := r.writeStatus(ctx, cluster, lastObserved(cluster), notReadyCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), notConvergedCondition(memgraphcomv1alpha1.ReasonApplyFailed, msg), ); statusErr != nil { - return ctrl.Result{}, errors.Join(applyErr, statusErr) + return errors.Join(applyErr, statusErr) } - return ctrl.Result{}, applyErr + return applyErr } } - - log.Info("Applied desired workload objects for MemgraphCluster", "memgraphcluster", req.NamespacedName) - - return r.reconcileRegistration(ctx, &cluster, replicas) + return nil } // roleReplicas is one role's replica arithmetic for a reconcile pass: how many @@ -192,13 +206,35 @@ func (c replicaCounts) scaleMessage() string { return strings.Join(pending, "; ") } +// retirementMessage names the data instances a lowered count is shedding, and is +// empty when none are. It is non-empty for exactly as long as the retirement is +// unfinished: the retiring set is derived from the replica count the operator's +// own StatefulSet still runs, so it empties only once the shrink that removes +// those pods has been applied. +func retirementMessage(topology planner.Topology) string { + if len(topology.RetiringDataInstances) == 0 { + return "" + } + names := make([]string, 0, len(topology.RetiringDataInstances)) + for _, instance := range topology.RetiringDataInstances { + names = append(names, instance.Name) + } + return "Retiring data instance(s) " + strings.Join(names, ", ") + + " before their pods are shed" +} + // replicaCounts resolves the replica count to apply per role: the declared count // while the cluster grows or holds its size, and deliberately the current count // while a lowered count would shrink it. Shedding pods means removing members // from the Memgraph cluster first — the coordinators otherwise keep expecting -// instances whose pods are gone — and the operator has no removal path yet, so -// it holds the size and reports the mismatch instead of acting on half of a -// scale-down it cannot finish. +// instances whose pods are gone — so this rule never shrinks anything, which +// keeps it free of any knowledge about the cluster's state. +// +// Lowering the data-instance count is carried out at the end of the registration +// phase instead, once the retiring members have actually left the cluster (see +// reconcileRegistration). Lowering the coordinator count is not carried out at +// all yet: the size is held and the mismatch reported, rather than acting on half +// of a scale-down the operator cannot finish. func (r *MemgraphClusterReconciler) replicaCounts( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, @@ -246,9 +282,21 @@ func (r *MemgraphClusterReconciler) currentReplicas( // reconcileRegistration converges cluster registration once the workloads are // ready: find the coordinator leader, plan against its SHOW INSTANCES view, -// and execute the missing commands. Unreachable coordinators are retried on a -// delay rather than surfaced as errors — Bolt endpoints lagging pod readiness -// is a normal startup phase, not a failure. +// and execute the planned commands against that one connection. Unreachable +// coordinators are retried on a delay rather than surfaced as errors — Bolt +// endpoints lagging pod readiness is a normal startup phase, not a failure. +// +// The readiness gate is deliberately strict about the pods a lowered count is +// retiring too: they belong to the StatefulSet the operator is still holding at +// its current size, so a retiring pod that cannot become ready blocks its own +// removal, and the resource reports WorkloadsNotReady rather than the operator +// acting on a half-known cluster. +// +// This is also where a data-instance scale-down finishes. Once the plan comes +// back empty — meaning the retiring instances have left the cluster — the data +// StatefulSet is applied at the declared count, shedding their pods. That is the +// one place the operator ever lowers a replica count, so the coordinators never +// see a registered instance's pod disappear. func (r *MemgraphClusterReconciler) reconcileRegistration( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, @@ -273,6 +321,14 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( } topology := resources.DeclaredTopology(cluster) + // The members a lowered count is shedding are the ordinals the operator's own + // previous apply still runs beyond the declared count, so the range is bounded + // by what the operator itself created. + topology.RetiringDataInstances = resources.RetiringDataInstances(cluster, replicas.data.applied) + // Whether a retirement is in flight is decided once per pass, from the + // topology alone: it is what both the condition and the shrink below key off. + retiring := retirementMessage(topology) + leader, observed, err := r.observeCluster(ctx, topology) if err != nil { log.Info("Deferred registration because no coordinator leader was usable", "reason", err.Error()) @@ -300,6 +356,25 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( latest := observe(topology, observed) commands := planner.Plan(topology, observed) if len(commands) == 0 { + // No retiring instance is a member of the cluster any more — the plan + // would carry an UNREGISTER INSTANCE otherwise — so their pods can go. + if retiring != "" { + if err := r.applyDesired(ctx, cluster, + resources.DataStatefulSet(cluster, replicas.data.declared)); err != nil { + return ctrl.Result{}, err + } + log.Info("Shrank the data StatefulSet to the declared replica count", + "statefulset", replicas.data.name, "replicas", replicas.data.declared) + if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), + notConvergedCondition(memgraphcomv1alpha1.ReasonRetirementInProgress, retiring), + ); statusErr != nil { + return ctrl.Result{}, statusErr + } + // The shrink was applied, not yet observed back: the next pass sees the + // lowered count, finds nothing retiring, and reports convergence. + return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil + } + // Registration matches the declared topology. It is only converged once // the StatefulSets run the declared replica counts too, so a scale the // operator is holding back keeps the condition False and says which @@ -328,9 +403,15 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( // Report the in-progress state before mutating the cluster: a MAIN already // serving stays Ready while a lost registration is restored; a fresh - // bootstrap has no MAIN yet, so Ready is False until one is elected. - inProgress := notConvergedCondition(memgraphcomv1alpha1.ReasonRegistrationInProgress, - fmt.Sprintf("Issuing %d registration command(s) to converge the cluster", len(commands))) + // bootstrap has no MAIN yet, so Ready is False until one is elected. A + // retirement in flight is named as such — it is the more specific operation, + // and the one whose pending members a user wants to see. + reason := memgraphcomv1alpha1.ReasonRegistrationInProgress + message := fmt.Sprintf("Issuing %d registration command(s) to converge the cluster", len(commands)) + if retiring != "" { + reason, message = memgraphcomv1alpha1.ReasonRetirementInProgress, retiring + } + inProgress := notConvergedCondition(reason, message) if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), inProgress); statusErr != nil { return ctrl.Result{}, statusErr } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index 64da5dc..cfeb6ca 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -636,6 +636,33 @@ var _ = Describe("MemgraphCluster Controller", func() { resourceName, ordinal, resourceName, resourceNamespace) } + // convergedWithMainOn is the fully registered default 3/2 topology with the + // data instance on the given ordinal elected MAIN, so a spec can put MAIN + // where it needs it before lowering a count. + convergedWithMainOn := func(mainOrdinal int) []memgraph.Instance { + instances := make([]memgraph.Instance, 0, 5) + for id := 1; id <= 3; id++ { + role := memgraph.RoleFollower + if id == 1 { + role = memgraph.RoleLeader + } + instances = append(instances, memgraph.Instance{ + Name: fmt.Sprintf("coordinator_%d", id), BoltServer: coordinatorAddress(id - 1), + Health: memgraph.HealthUp, Role: role, + }) + } + for ordinal := range 2 { + role := memgraph.RoleReplica + if ordinal == mainOrdinal { + role = memgraph.RoleMain + } + instances = append(instances, memgraph.Instance{ + Name: fmt.Sprintf("instance_%d", ordinal), Health: memgraph.HealthUp, Role: role, + }) + } + return instances + } + status := func() memgraphcomv1alpha1.MemgraphClusterStatus { GinkgoHelper() cluster := &memgraphcomv1alpha1.MemgraphCluster{} @@ -643,6 +670,11 @@ var _ = Describe("MemgraphCluster Controller", func() { return cluster.Status } + convergedCondition := func() *metav1.Condition { + GinkgoHelper() + return apimeta.FindStatusCondition(status().Conditions, memgraphcomv1alpha1.ConditionConverged) + } + // setCounts edits the declared topology of the live cluster. setCounts := func(coordinators, dataInstances int32) { GinkgoHelper() @@ -732,33 +764,138 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonAllInstancesRegistered)) }) - It("should report Converged False while a StatefulSet has not reached the declared count", func() { - baseline := bootstrapped() + // Lowering the coordinator count is the scale the operator still cannot + // realize: removing a Raft member is not implemented, so the StatefulSet is + // held at its current size and the resource says so rather than the + // operator acting on half of a scale-down it cannot finish. + It("should report Converged False while a lowered coordinator count is held back", func() { + bootstrapped() - // A count the operator cannot realize yet: shedding pods means - // removing cluster members first, which it does not do, so the - // StatefulSet is held and the resource says so. - setCounts(3, 1) + By("growing the coordinators so there is a member to drop") + setCounts(5, 2) + reconcileCluster(resourceName) + markWorkloadsReady(resourceName) + reconcileCluster(resourceName) reconcileCluster(resourceName) + Expect(apimeta.IsStatusConditionTrue(status().Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + grown := len(fake.executedCommands()) - Expect(replicas(dataSuffix)).To(Equal(int32(2)), + setCounts(3, 2) + reconcileCluster(resourceName) + + Expect(replicas(coordinatorSuffix)).To(Equal(int32(5)), "a lower declared count must never shrink the applied StatefulSet") - converged := apimeta.FindStatusCondition(status().Conditions, memgraphcomv1alpha1.ConditionConverged) + converged := convergedCondition() Expect(converged.Status).To(Equal(metav1.ConditionFalse)) Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonScaleInProgress)) - Expect(converged.Message).To(ContainSubstring(resourceName + dataSuffix)) - Expect(sinceBootstrap(baseline)).To(BeEmpty(), - "the instance the lowered count drops stays registered: removal is not implemented") + Expect(converged.Message).To(ContainSubstring(resourceName + coordinatorSuffix)) + Expect(sinceBootstrap(grown)).To(BeEmpty(), + "the coordinators the lowered count drops stay registered: removal is not implemented") // Raising the count back matches what is running, which converges // again without touching the cluster. - setCounts(3, 2) + setCounts(5, 2) reconcileCluster(resourceName) Expect(apimeta.IsStatusConditionTrue(status().Conditions, memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) }) + // The whole point of the shrink: the member beyond the declared count leaves + // the cluster before its pod does, so the coordinators never expect an + // instance whose pod is gone. + It("should unregister a retiring data instance and only then shed its pod", func() { + baseline := bootstrapped() + Expect(status().Main).To(Equal("instance_0")) + + setCounts(3, 1) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": UNREGISTER INSTANCE instance_1", + }), "the MAIN survives the shrink, so nothing but the removal is issued") + Expect(replicas(dataSuffix)).To(Equal(int32(2)), + "a pass with pending commands must never lower the replica count") + converged := convergedCondition() + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonRetirementInProgress)) + Expect(converged.Message).To(ContainSubstring("instance_1"), + "the condition must name the instance being retired") + + By("shedding the pod once the instance has left the cluster") + reconcileCluster(resourceName) + Expect(replicas(dataSuffix)).To(Equal(int32(1))) + Expect(sinceBootstrap(baseline)).To(HaveLen(1), "the removal is not re-issued") + Expect(convergedCondition().Reason).To(Equal(memgraphcomv1alpha1.ReasonRetirementInProgress)) + + By("reporting the shrink as finished once the StatefulSet runs the declared count") + reconcileCluster(resourceName) + s := status() + Expect(s.DataInstances).To(Equal(int32(1))) + Expect(s.Main).To(Equal("instance_0"), "the surviving MAIN was never moved") + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + + // Memgraph refuses to unregister the MAIN, so a retiring instance holding it + // is demoted and a survivor promoted in its place — within the same pass, on + // the same leader connection, so the cluster is MAIN-less for the time + // between two queries. + It("should move MAIN off a retiring data instance before unregistering it", func() { + fake.setInstances(convergedWithMainOn(1)) + baseline := bootstrapped() + Expect(status().Main).To(Equal("instance_1")) + + setCounts(3, 1) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": DEMOTE INSTANCE instance_1", + leader + ": SET INSTANCE instance_0 TO MAIN", + leader + ": UNREGISTER INSTANCE instance_1", + }), "demote, promote and unregister issue in one pass against one leader") + Expect(replicas(dataSuffix)).To(Equal(int32(2)), + "a pass with pending commands must never lower the replica count") + + reconcileCluster(resourceName) + Expect(replicas(dataSuffix)).To(Equal(int32(1))) + reconcileCluster(resourceName) + + s := status() + Expect(s.Main).To(Equal("instance_0"), "MAIN moved to the surviving instance") + Expect(s.DataInstances).To(Equal(int32(1))) + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + + // The readiness gate covers the pods on their way out too: they belong to + // the StatefulSet the operator is still holding at its current size. A + // retiring pod that cannot become ready therefore blocks its own removal, + // which is a deliberate trade — the alternative is acting on a cluster whose + // state is only half known. + It("should not retire anything while a pod of the held StatefulSet is unready", func() { + baseline := bootstrapped() + + setCounts(3, 1) + sts := &appsv1.StatefulSet{} + get(resourceName+dataSuffix, sts) + sts.Status.ReadyReplicas = *sts.Spec.Replicas - 1 + sts.Status.AvailableReplicas = sts.Status.ReadyReplicas + Expect(k8sClient.Status().Update(ctx, sts)).To(Succeed()) + + reconcileCluster(resourceName) + + Expect(sinceBootstrap(baseline)).To(BeEmpty(), + "a half-known cluster is not written to, retirement included") + Expect(replicas(dataSuffix)).To(Equal(int32(2))) + converged := convergedCondition() + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonWorkloadsNotReady)) + }) + It("should report the registered counts as observed, not as declared", func() { bootstrapped() Expect(status().DataInstances).To(Equal(int32(2))) diff --git a/internal/memgraph/bolt.go b/internal/memgraph/bolt.go index cbec9ef..6abbc13 100644 --- a/internal/memgraph/bolt.go +++ b/internal/memgraph/bolt.go @@ -75,6 +75,16 @@ func (c *boltClient) SetInstanceToMain(ctx context.Context, name string) error { return err } +func (c *boltClient) DemoteInstance(ctx context.Context, name string) error { + _, err := c.run(ctx, demoteInstanceQuery(name)) + return err +} + +func (c *boltClient) UnregisterInstance(ctx context.Context, name string) error { + _, err := c.run(ctx, unregisterInstanceQuery(name)) + return err +} + func (c *boltClient) Close(ctx context.Context) error { return c.driver.Close(ctx) } diff --git a/internal/memgraph/client.go b/internal/memgraph/client.go index c7b9840..891c3fd 100644 --- a/internal/memgraph/client.go +++ b/internal/memgraph/client.go @@ -16,9 +16,10 @@ limitations under the License. // Package memgraph provides the narrow client surface the operator uses to // drive a Memgraph high-availability cluster over Bolt: show instances, add -// coordinator, register instance, set main. All higher layers depend on the -// Client and Connector interfaces, never on the Bolt driver — this package is -// the mock seam for testing and the only place the driver is referenced. +// coordinator, register instance, set main, and — for a data instance a lowered +// replica count is retiring — demote and unregister. All higher layers depend on +// the Client and Connector interfaces, never on the Bolt driver — this package +// is the mock seam for testing and the only place the driver is referenced. package memgraph import ( @@ -98,6 +99,18 @@ type Client interface { AddCoordinator(ctx context.Context, coordinator CoordinatorSpec) error RegisterInstance(ctx context.Context, instance DataInstanceSpec) error SetInstanceToMain(ctx context.Context, name string) error + + // DemoteInstance turns the named MAIN back into a replica, which is what + // makes a MAIN on its way out of the cluster unregisterable: Memgraph + // refuses to unregister the MAIN. It deliberately leaves the cluster + // MAIN-less — the coordinators fail over only on a leadership change or a + // failed ping, so the caller promotes a survivor itself. + DemoteInstance(ctx context.Context, name string) error + + // UnregisterInstance removes the named data instance from the cluster, so + // the coordinators stop expecting it before its pod goes away. + UnregisterInstance(ctx context.Context, name string) error + Close(ctx context.Context) error } diff --git a/internal/memgraph/queries.go b/internal/memgraph/queries.go index f19b9a9..70ce04c 100644 --- a/internal/memgraph/queries.go +++ b/internal/memgraph/queries.go @@ -48,3 +48,11 @@ func registerInstanceQuery(instance DataInstanceSpec) string { func setInstanceToMainQuery(name string) string { return fmt.Sprintf("SET INSTANCE %s TO MAIN", name) } + +func demoteInstanceQuery(name string) string { + return fmt.Sprintf("DEMOTE INSTANCE %s", name) +} + +func unregisterInstanceQuery(name string) string { + return fmt.Sprintf("UNREGISTER INSTANCE %s", name) +} diff --git a/internal/memgraph/queries_test.go b/internal/memgraph/queries_test.go index 572d64e..a3a7a99 100644 --- a/internal/memgraph/queries_test.go +++ b/internal/memgraph/queries_test.go @@ -64,6 +64,20 @@ func TestSetInstanceToMainQuery(t *testing.T) { } } +func TestDemoteInstanceQuery(t *testing.T) { + got := demoteInstanceQuery(testInstanceName) + if want := "DEMOTE INSTANCE instance_1"; got != want { + t.Errorf("demoteInstanceQuery() = %q, want %q", got, want) + } +} + +func TestUnregisterInstanceQuery(t *testing.T) { + got := unregisterInstanceQuery(testInstanceName) + if want := "UNREGISTER INSTANCE instance_1"; got != want { + t.Errorf("unregisterInstanceQuery() = %q, want %q", got, want) + } +} + func TestInstanceFromRecord(t *testing.T) { record := &db.Record{ Keys: []string{ diff --git a/internal/planner/planner.go b/internal/planner/planner.go index cf4f16f..e5f48b4 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -18,10 +18,15 @@ limitations under the License. // observed Memgraph HA cluster toward its declared topology. The logic is a // pure diff — declared topology plus observed SHOW INSTANCES output in, // commands out (empty when converged) — so reconciliation stays idempotent -// and read-before-write: only missing registrations are re-issued, and the -// initial MAIN promotion happens exactly once, when no MAIN exists. After -// bootstrap, failover belongs to the Raft coordinators; the planner never -// overrides an existing MAIN. +// and read-before-write: only missing registrations are re-issued, and a MAIN is +// promoted only when the cluster has none. After bootstrap, failover belongs to +// the Raft coordinators; the planner never overrides a MAIN that is staying. +// +// Members a lowered replica count is retiring are the one thing the planner +// removes, and it can order the whole removal in a single pass because it +// predicts every intermediate state: a retiring MAIN is demoted, a survivor is +// promoted in its place, and the retiring members are then unregistered — so no +// UNREGISTER INSTANCE is ever aimed at a MAIN, which Memgraph would refuse. package planner import ( @@ -40,9 +45,12 @@ type Topology struct { // RetiringCoordinators and RetiringDataInstances are the members a lowered // replica count is shedding: still registered and still running, but no - // longer declared. They are empty while a cluster grows or holds its size, - // which is every case the operator handles today — removing a member from - // the cluster is not implemented yet, so a plan issues no command for them. + // longer declared. They are empty while a cluster grows or holds its size. + // + // A retiring data instance is demoted if it holds MAIN and then + // unregistered, so the cluster stops expecting it before its pod goes. + // RetiringCoordinators is still always empty: removing a Raft member is not + // implemented yet, so a plan issues no command for one. RetiringCoordinators []memgraph.CoordinatorSpec RetiringDataInstances []memgraph.DataInstanceSpec } @@ -81,7 +89,8 @@ func (c RegisterInstance) String() string { return "REGISTER INSTANCE " + c.Instance.Name } -// SetInstanceToMain promotes the named data instance to MAIN at bootstrap. +// SetInstanceToMain promotes the named data instance to MAIN: at bootstrap, +// when the cluster has no MAIN yet, and after a retiring MAIN was demoted. type SetInstanceToMain struct { Name string } @@ -95,16 +104,59 @@ func (c SetInstanceToMain) String() string { return fmt.Sprintf("SET INSTANCE %s TO MAIN", c.Name) } +// DemoteInstance turns a retiring MAIN back into a replica, which is what makes +// it unregisterable. It is only ever aimed at an instance on its way out of the +// cluster: demoting one that is staying would be the operator overriding a +// failover decision that belongs to the coordinators. +type DemoteInstance struct { + Name string +} + +// Run implements Command. +func (c DemoteInstance) Run(ctx context.Context, client memgraph.Client) error { + return client.DemoteInstance(ctx, c.Name) +} + +func (c DemoteInstance) String() string { + return "DEMOTE INSTANCE " + c.Name +} + +// UnregisterInstance removes a retiring data instance from the cluster, so the +// coordinators stop expecting it before its pod is shed. +type UnregisterInstance struct { + Name string +} + +// Run implements Command. +func (c UnregisterInstance) Run(ctx context.Context, client memgraph.Client) error { + return client.UnregisterInstance(ctx, c.Name) +} + +func (c UnregisterInstance) String() string { + return "UNREGISTER INSTANCE " + c.Name +} + // Plan diffs the declared topology against the observed instances and returns // the commands still needed, in execution order: coordinators before data -// instances (registration requires a formed Raft cluster), the initial MAIN -// promotion last. Instances the cluster knows but the topology does not -// declare are left untouched — unregistration is out of scope for v1. +// instances (registration requires a formed Raft cluster), then the retirement +// of the members a lowered count sheds — demote a retiring MAIN, promote a +// survivor in its place, unregister every retiring member. The promotion sits +// between the two so that no UNREGISTER INSTANCE is ever aimed at an observed +// MAIN, and so the cluster is MAIN-less only for the few milliseconds between +// two queries of the same pass. +// +// Instances the cluster knows but the topology neither declares nor retires are +// left untouched: the retiring set is bounded by the operator's own prior apply, +// so an instance a human registered is never removed. func Plan(declared Topology, observed []memgraph.Instance) []Command { registered := index(observed) + retiring := retiringNames(declared) + + // A MAIN on its way out does not count as one: it is demoted below, and the + // cluster needs a survivor promoted in its place. hasMain := false for _, instance := range observed { - hasMain = hasMain || instance.IsMain() + hasMain = hasMain || (instance.IsMain() && !retiring[instance.Name]) } var commands []Command @@ -118,9 +170,19 @@ func Plan(declared Topology, observed []memgraph.Instance) []Command { commands = append(commands, RegisterInstance{Instance: instance}) } } + for _, instance := range declared.RetiringDataInstances { + if observed, ok := registered[instance.Name]; ok && observed.IsMain() { + commands = append(commands, DemoteInstance{Name: instance.Name}) + } + } if !hasMain && len(declared.DataInstances) > 0 { commands = append(commands, SetInstanceToMain{Name: promotionTarget(declared, registered)}) } + for _, instance := range declared.RetiringDataInstances { + if _, ok := registered[instance.Name]; ok { + commands = append(commands, UnregisterInstance{Name: instance.Name}) + } + } return commands } @@ -144,6 +206,16 @@ func Registered(declared Topology, observed []memgraph.Instance) (coordinators, return coordinators, dataInstances } +// retiringNames is the set of data instances the topology is shedding, keyed by +// the name they are registered under. +func retiringNames(declared Topology) map[string]bool { + retiring := make(map[string]bool, len(declared.RetiringDataInstances)) + for _, instance := range declared.RetiringDataInstances { + retiring[instance.Name] = true + } + return retiring +} + // index keys the observed cluster view by instance name. func index(observed []memgraph.Instance) map[string]memgraph.Instance { registered := make(map[string]memgraph.Instance, len(observed)) @@ -168,6 +240,10 @@ func coordinatorRegistered(registered map[string]memgraph.Instance, coordinator // cluster MAIN-less until the coordinators retried it, so an instance that is // known to be reachable is preferred over a lower-ordinal one that is not. // +// Only declared instances are candidates, which is what makes the rule serve a +// retirement too: promoting a survivor is the same choice as promoting at +// bootstrap, and a member on its way out can never be the target. +// // The first declared instance is the fallback, which is what a fresh bootstrap // uses: nothing is observed yet at the point its registrations are planned. func promotionTarget(declared Topology, registered map[string]memgraph.Instance) string { diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 47b6769..26c8e0a 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -30,9 +30,14 @@ import ( "github.com/memgraph/kubernetes-operator/internal/resources" ) -// firstInstance is the data instance the planner promotes at bootstrap: the -// first one the topology declares. -const firstInstance = "instance_0" +// The data instances the cases below name. firstInstance is the one a bootstrap +// promotes — the first the topology declares — and the one a shrink keeps; the +// higher ordinals are what a lowered count retires. +const ( + firstInstance = "instance_0" + secondInstance = "instance_1" + thirdInstance = "instance_2" +) // declaredTopology is the canonical 3-coordinator, 2-data-instance fixture // the cases below diff observed cluster states against. @@ -46,6 +51,25 @@ func grownTopology() planner.Topology { return topologyOf(5, 3) } +// shrunkTopology is a cluster whose dataInstances count was lowered to the given +// number while its StatefulSet still runs `running` data pods: the ordinals in +// between are retiring. +func shrunkTopology(declared, running int) planner.Topology { + topology := topologyOf(3, declared) + for i := declared; i < running; i++ { + topology.RetiringDataInstances = append(topology.RetiringDataInstances, dataInstanceSpec(i)) + } + return topology +} + +// mixedTopology is one edit moving both counts in opposite directions: the +// coordinators grow from 3 to 5 while the data instances shrink from 3 to 2. +func mixedTopology() planner.Topology { + topology := topologyOf(5, 2) + topology.RetiringDataInstances = append(topology.RetiringDataInstances, dataInstanceSpec(2)) + return topology +} + func topologyOf(coordinators int32, dataInstances int) planner.Topology { topology := planner.Topology{} for id := int32(1); id <= coordinators; id++ { @@ -238,7 +262,7 @@ func TestPlan(t *testing.T) { observedDataInstance(1, memgraph.RoleReplica), }, want: []planner.Command{ - planner.SetInstanceToMain{Name: "instance_1"}, + planner.SetInstanceToMain{Name: secondInstance}, }, }, { @@ -302,6 +326,139 @@ func TestPlan(t *testing.T) { }, want: nil, }, + // A lowered count with MAIN on a survivor needs nothing but the removal: + // the cluster keeps serving from the instance it is already serving from. + { + name: "a retiring instance is unregistered with the surviving MAIN untouched", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(2, 3)), + want: []planner.Command{ + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // MAIN on the ordinal that is going away: Memgraph refuses to unregister + // the MAIN, so it is demoted, a survivor is promoted in its place, and only + // then is it removed — all in this one plan. + { + name: "a retiring MAIN is demoted, a survivor promoted, and only then unregistered", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // The promotion follows the same rule as at bootstrap, so a survivor the + // leader cannot reach is not the one that gets MAIN. + { + name: "a retiring MAIN hands MAIN to the lowest reachable survivor", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + downDataInstance(0), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: secondInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + { + name: "several retiring instances are removed down to a single survivor", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleMain), + observedDataInstance(2, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(1, 3)), + want: []planner.Command{ + planner.DemoteInstance{Name: secondInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: secondInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // Read-before-write: a retiring member the cluster no longer knows about + // gets no command, so a reconcile that crashed between the unregistration + // and the shrink re-plans to just the rest of the work. + { + name: "an already-unregistered retiring instance is not unregistered again", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(2, 4)), + want: []planner.Command{ + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // Both counts change in one edit, in opposite directions: the coordinators + // grow while the data instances shrink, and each role's work is planned + // independently of the other's. + { + name: "a mixed grow-and-shrink adds coordinators and retires a data instance", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(mixedTopology()), + want: []planner.Command{ + planner.AddCoordinator{Coordinator: coordinatorSpec(4)}, + planner.AddCoordinator{Coordinator: coordinatorSpec(5)}, + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // The retiring range is bounded by the operator's own prior apply, which is + // what keeps an instance a human registered out of it — even one at a + // higher ordinal than everything the operator ever ran. + { + name: "an undeclared instance outside the retiring range is left registered", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + observedDataInstance(3, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(2, 3)), + want: []planner.Command{ + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, } for _, tc := range cases { diff --git a/internal/resources/topology.go b/internal/resources/topology.go index 95fba70..05f74f5 100644 --- a/internal/resources/topology.go +++ b/internal/resources/topology.go @@ -60,17 +60,53 @@ func DeclaredTopology(cluster *memgraphcomv1alpha1.MemgraphCluster) planner.Topo }) } for ordinal := range spec.dataInstances { - fqdn := podFQDN(cluster, DataName(cluster), spec, ordinal) - topology.DataInstances = append(topology.DataInstances, memgraph.DataInstanceSpec{ - Name: fmt.Sprintf("instance_%d", ordinal), - BoltServer: hostPort(fqdn, spec.ports.bolt), - ManagementServer: hostPort(fqdn, spec.ports.management), - ReplicationServer: hostPort(fqdn, spec.ports.replication), - }) + topology.DataInstances = append(topology.DataInstances, dataInstance(cluster, spec, ordinal)) } return topology } +// RetiringDataInstances is the data instances a lowered dataInstances count is +// shedding: pod ordinals [declared, applied), where applied is the replica count +// the operator's own previous apply left on the data StatefulSet. It is empty +// while a cluster grows or holds its size. +// +// The operator never picks which member retires. A StatefulSet sheds its highest +// ordinals and nothing else, so the range is fully determined by the two counts — +// which is also what keeps an instance the operator did not create out of it. +func RetiringDataInstances( + cluster *memgraphcomv1alpha1.MemgraphCluster, + applied int32, +) []memgraph.DataInstanceSpec { + spec := normalize(cluster.Spec) + if applied <= spec.dataInstances { + return nil + } + + retiring := make([]memgraph.DataInstanceSpec, 0, applied-spec.dataInstances) + for ordinal := spec.dataInstances; ordinal < applied; ordinal++ { + retiring = append(retiring, dataInstance(cluster, spec, ordinal)) + } + return retiring +} + +// dataInstance describes the data instance running on the given pod ordinal, as +// the pod itself advertises it. Retiring instances are described the same way as +// declared ones: they are registered under the addresses the operator registered +// them with, whether or not the spec still declares them. +func dataInstance( + cluster *memgraphcomv1alpha1.MemgraphCluster, + spec normalizedSpec, + ordinal int32, +) memgraph.DataInstanceSpec { + fqdn := podFQDN(cluster, DataName(cluster), spec, ordinal) + return memgraph.DataInstanceSpec{ + Name: fmt.Sprintf("instance_%d", ordinal), + BoltServer: hostPort(fqdn, spec.ports.bolt), + ManagementServer: hostPort(fqdn, spec.ports.management), + ReplicationServer: hostPort(fqdn, spec.ports.replication), + } +} + // podFQDNSuffix returns the DNS suffix a pod name is appended to for pods of // the given headless Service: "..svc.", where the // domain is the configured cluster domain. diff --git a/internal/resources/topology_test.go b/internal/resources/topology_test.go index 397148d..05abf2a 100644 --- a/internal/resources/topology_test.go +++ b/internal/resources/topology_test.go @@ -22,12 +22,19 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "k8s.io/utils/ptr" + memgraphcomv1alpha1 "github.com/memgraph/kubernetes-operator/api/v1alpha1" "github.com/memgraph/kubernetes-operator/internal/memgraph" "github.com/memgraph/kubernetes-operator/internal/planner" "github.com/memgraph/kubernetes-operator/internal/resources" ) +// secondDataInstance is the data instance on pod ordinal 1: the one the default +// topology's second replica registers as, and the first one a lowered count +// retires. +const secondDataInstance = "instance_1" + func TestDeclaredTopologyDefaults(t *testing.T) { got := resources.DeclaredTopology(minimalCluster()) @@ -67,7 +74,7 @@ func TestDeclaredTopologyDefaults(t *testing.T) { ReplicationServer: dataFQDN(0) + ":20000", }, { - Name: "instance_1", + Name: secondDataInstance, BoltServer: dataFQDN(1) + ":7687", ManagementServer: dataFQDN(1) + ":10000", ReplicationServer: dataFQDN(1) + ":20000", @@ -91,6 +98,81 @@ func TestDeclaredTopologyFollowsReplicaCounts(t *testing.T) { } } +// TestRetiringDataInstances covers the range a lowered dataInstances count +// sheds: the pod ordinals the applied StatefulSet still runs beyond the declared +// count, and nothing else. The bounds are what keep an instance the operator did +// not create out of the range, so they are pinned in both directions. +func TestRetiringDataInstances(t *testing.T) { + cases := []struct { + name string + cluster *memgraphcomv1alpha1.MemgraphCluster + applied int32 + want []string + }{ + { + name: "a cluster holding its size retires nothing", + cluster: minimalCluster(), + applied: 2, + }, + { + name: "a growing cluster retires nothing", + cluster: minimalCluster(), + applied: 1, + }, + { + name: "the highest ordinal retires when the count drops by one", + cluster: dataInstancesCluster(2), + applied: 3, + want: []string{"instance_2"}, + }, + { + name: "every ordinal above the declared count retires at once", + cluster: dataInstancesCluster(1), + applied: 4, + want: []string{secondDataInstance, "instance_2", "instance_3"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var names []string + for _, instance := range resources.RetiringDataInstances(tc.cluster, tc.applied) { + names = append(names, instance.Name) + } + if diff := cmp.Diff(tc.want, names); diff != "" { + t.Errorf("RetiringDataInstances() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// dataInstancesCluster is the minimal cluster with a lowered dataInstances count, +// the spec side of a scale-down. +func dataInstancesCluster(dataInstances int32) *memgraphcomv1alpha1.MemgraphCluster { + cluster := minimalCluster() + cluster.Spec.DataInstances = ptr.To(dataInstances) + return cluster +} + +// A retiring instance is registered under the addresses the operator registered +// it with, so it must be described exactly as the declared instance on the same +// ordinal was — otherwise the plan would aim its removal at a name the cluster +// does not know. +func TestRetiringDataInstanceMatchesItsDeclaredForm(t *testing.T) { + // The tuned cluster (non-default ports and cluster domain) declares two + // instances. Lowering the count to one leaves instance_1 retiring, which must + // equal the instance_1 the same spec declared before the edit, verbatim. + declared := resources.DeclaredTopology(tunedCluster()).DataInstances + + shrunk := tunedCluster() + shrunk.Spec.DataInstances = ptr.To(int32(1)) + got := resources.RetiringDataInstances(shrunk, int32(len(declared))) + + if diff := cmp.Diff(declared[1:], got); diff != "" { + t.Errorf("RetiringDataInstances() mismatch (-want +got):\n%s", diff) + } +} + // The registration topology must advertise exactly the identity the // coordinator pods derive for themselves at startup, otherwise the Raft // cluster and the registrations disagree about who is who. @@ -156,7 +238,7 @@ func TestDeclaredTopologyPortsAndClusterDomain(t *testing.T) { ReplicationServer: dataFQDN(0) + ":20001", }, { - Name: "instance_1", + Name: secondDataInstance, BoltServer: dataFQDN(1) + ":7777", ManagementServer: dataFQDN(1) + ":10001", ReplicationServer: dataFQDN(1) + ":20001", diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index f07e9eb..cecb7e7 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -97,9 +97,9 @@ type clusterUnderTest struct { dataInstances int32 } -// grownTo returns the same cluster with a different declared topology, which is -// what the assertions switch to after a scale. -func (c clusterUnderTest) grownTo(coordinators, dataInstances int32) clusterUnderTest { +// withTopology returns the same cluster with a different declared topology, +// which is what the assertions switch to after a scale in either direction. +func (c clusterUnderTest) withTopology(coordinators, dataInstances int32) clusterUnderTest { c.coordinators = coordinators c.dataInstances = dataInstances return c @@ -356,12 +356,14 @@ spec: }) }) -// Growing a live cluster: the end-to-end proof that raising either count -// registers the members it adds without human action. It gets its own container -// and namespace because it needs a cluster it may reshape, and its teardown is -// awaited — eight Memgraph pods are a large share of a Kind cluster's capacity, -// which the scenarios that may run after it need back. -var _ = Describe("MemgraphCluster topology scale-up", Ordered, func() { +// Scaling a live cluster in both directions: the end-to-end proof that raising a +// count registers the members it adds, and that lowering the data-instance count +// retires the members it drops safely — MAIN moved off them, unregistered before +// their pods go. It gets its own container and namespace because it needs a +// cluster it may reshape, and its teardown is awaited — eight Memgraph pods are a +// large share of a Kind cluster's capacity, which the scenarios that may run +// after it need back. +var _ = Describe("MemgraphCluster topology scaling", Ordered, func() { const scalingNamespace = "memgraph-e2e-scaling" const scalingClusterName = "scaling" @@ -370,7 +372,7 @@ var _ = Describe("MemgraphCluster topology scale-up", Ordered, func() { initial := clusterUnderTest{ namespace: scalingNamespace, name: scalingClusterName, coordinators: 3, dataInstances: 2, } - grown := initial.grownTo(5, 3) + grown := initial.withTopology(5, 3) BeforeAll(func() { license, organization := licenseFromEnv() @@ -458,6 +460,82 @@ spec: Expect(coordinators).To(Equal("5")) Expect(dataInstances).To(Equal("3")) }) + + // The shrink, against the 5/3 cluster the growth spec left behind (Ordered), + // with MAIN deliberately parked on the ordinal that has to go — the case the + // whole safety argument is about: Memgraph refuses to unregister a MAIN, so the + // operator has to move it first, and it has to unregister before the pod goes + // or the coordinators are left expecting an instance that is not there. + It("retires a data instance holding MAIN and only then sheds its pod", func() { + shrunk := grown.withTopology(5, 2) + const retiring = "instance_2" + + By("parking MAIN on the data instance the shrink retires") + // Retried as a whole: the operator promotes a survivor itself if it + // observes the cluster MAIN-less between the two statements. + Eventually(func(g Gomega) { + g.Expect(grown.makeMain(retiring)).To(Succeed()) + view, err := grown.leaderView() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(mainOf(view)).To(Equal(retiring)) + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("lowering the data-instance count on the live cluster") + cmd := exec.Command("kubectl", "patch", "memgraphcluster", scalingClusterName, + "-n", scalingNamespace, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"dataInstances":%d}}`, shrunk.dataInstances)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "the operator must accept a lowered topology count") + + By("waiting for the retiring instance to leave the cluster while its pod is still there") + Eventually(func(g Gomega) { + // The replica count is read before the registration view, and that + // order carries the whole assertion. A view read afterwards that still + // lists the retiring instance proves it was registered at a moment the + // StatefulSet had already been shrunk — the reverse order the operator + // must never produce, because the coordinators would be left expecting + // an instance whose pod is gone. Reading the view first would prove + // nothing: the operator's own UNREGISTER can land between the two + // reads, so a pre-unregistration view paired with a post-shrink count + // is the correct sequence misread as a violation. + replicas, err := shrunk.replicas("data") + g.Expect(err).NotTo(HaveOccurred()) + + view, err := shrunk.leaderView() + g.Expect(err).NotTo(HaveOccurred()) + names := instanceNames(view) + + if replicas != "3" && slices.Contains(names, retiring) { + // Not something to retry: the ordering this catches is broken for + // good by the time it is observable. + StopTrying(fmt.Sprintf( + "the data StatefulSet was scaled to %s replicas while %s was still registered", + replicas, retiring)).Now() + } + g.Expect(names).NotTo(ContainElement(retiring)) + }, 10*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for its pod to be shed") + Eventually(func(g Gomega) { + g.Expect(shrunk.replicas("data")).To(Equal("2")) + g.Expect(shrunk.podExists("data", 2)).To(BeFalse()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + By("confirming the shrunk cluster is registered, converged, and led by a survivor") + Eventually(shrunk.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + shrunk.awaitConverged(5 * time.Minute) + _, dataInstances := shrunk.registeredCounts() + Expect(dataInstances).To(Equal("2")) + view, err := shrunk.leaderView() + Expect(err).NotTo(HaveOccurred()) + Expect(mainOf(view)).To(BeElementOf("instance_0", "instance_1")) + + By("confirming the retired instance's claims are kept by the default retention policy") + claims, err := listPVCs(scalingNamespace) + Expect(err).NotTo(HaveOccurred()) + Expect(claims).To(ContainElement(fmt.Sprintf("lib-storage-%s-data-2", scalingClusterName)), + "whenScaled follows spec.storage.retentionPolicy, which defaults to Retain") + }) }) // listAdoptedPVCs returns the names of the PersistentVolumeClaims a @@ -565,13 +643,29 @@ func (c clusterUnderTest) registeredCounts() (string, string) { } // replicas reads the replica count the operator applied to a role's StatefulSet. -func (c clusterUnderTest) replicas(component string) string { - GinkgoHelper() +// The error is returned rather than asserted so the value can be read inside a +// polled assertion, where a transient kubectl failure has to retry. +func (c clusterUnderTest) replicas(component string) (string, error) { cmd := exec.Command("kubectl", "get", "statefulset", c.name+"-"+component, "-n", c.namespace, "-o", "jsonpath={.spec.replicas}") output, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to read the %s StatefulSet's replicas", component) - return strings.TrimSpace(output) + if err != nil { + return "", fmt.Errorf("reading the %s StatefulSet's replicas: %w", component, err) + } + return strings.TrimSpace(output), nil +} + +// podExists reports whether the pod with the given ordinal of a role's +// StatefulSet is still there at all — the state a shed pod leaves behind once the +// StatefulSet controller is done with it. +func (c clusterUnderTest) podExists(component string, ordinal int32) (bool, error) { + cmd := exec.Command("kubectl", "get", "pod", fmt.Sprintf("%s-%s-%d", c.name, component, ordinal), + "-n", c.namespace, "--ignore-not-found", "-o", "name") + output, err := utils.Run(cmd) + if err != nil { + return false, err + } + return strings.TrimSpace(output) != "", nil } // wipeInstanceRegistration unregisters the named data instance on the @@ -656,15 +750,17 @@ func removeCoordinatorRegistration() (string, error) { // dumpDiagnosticsOnFailure dumps everything needed to debug a broken cluster // from the CI logs alone: the pods, the resource itself, the namespace's events // and the operator's log. -func dumpDiagnosticsOnFailure(namespace string) { +// The operator's log comes from the namespace the operator is installed in, not +// the cluster's — a parameter named namespace would shadow that constant. +func dumpDiagnosticsOnFailure(clusterNamespace string) { if !CurrentSpecReport().Failed() { return } for _, args := range [][]string{ - {"get", "pods", "-n", namespace, "-o", "wide"}, - {"get", "memgraphclusters", "-n", namespace, "-o", "yaml"}, - {"get", "events", "-n", namespace, "--sort-by=.lastTimestamp"}, - {"logs", "deploy/" + controllerDeploymentName, "-n", namespace}, + {"get", "pods", "-n", clusterNamespace, "-o", "wide"}, + {"get", "memgraphclusters", "-n", clusterNamespace, "-o", "yaml"}, + {"get", "events", "-n", clusterNamespace, "--sort-by=.lastTimestamp"}, + {"logs", "deploy/" + controllerDeploymentName, "-n", namespace, "--tail=200"}, } { cmd := exec.Command("kubectl", args...) output, err := utils.Run(cmd) @@ -767,6 +863,15 @@ type instanceRow struct { // reports their roles (followers show them as unknown), so a view containing a // MAIN is the leader's authoritative view. func (c clusterUnderTest) leaderView() ([]instanceRow, error) { + _, view, err := c.leaderPod() + return view, err +} + +// leaderPod locates the coordinator leader — the coordinator whose view reports a +// MAIN — and returns its pod name together with that view. Management queries a +// test issues by hand have to run there: only the leader holds the authoritative +// cluster state and accepts a mutation of it. +func (c clusterUnderTest) leaderPod() (string, []instanceRow, error) { var errs []error for ordinal := range c.coordinators { pod := fmt.Sprintf("%s-coordinator-%d", c.name, ordinal) @@ -775,14 +880,58 @@ func (c clusterUnderTest) leaderView() ([]instanceRow, error) { errs = append(errs, err) continue } - for _, instance := range view { - if instance.role == roleMain { - return view, nil - } + if mainOf(view) != "" { + return pod, view, nil } errs = append(errs, fmt.Errorf("%s reports no MAIN among %d instances", pod, len(view))) } - return nil, errors.Join(errs...) + return "", nil, errors.Join(errs...) +} + +// makeMain moves MAIN onto the named data instance by hand, which is how a spec +// arranges for the instance a scale-down retires to be the one holding MAIN. +// +// The demotion and the promotion go out as one mgconsole invocation because the +// operator promotes a survivor itself the moment it observes a MAIN-less cluster: +// the window between the two statements is the whole point of keeping them +// together. It is a no-op when the instance already is MAIN, so a caller can +// simply retry it. +func (c clusterUnderTest) makeMain(name string) error { + pod, view, err := c.leaderPod() + if err != nil { + return err + } + main := mainOf(view) + if main == name { + return nil + } + query := fmt.Sprintf("DEMOTE INSTANCE %s; SET INSTANCE %s TO MAIN;", main, name) + cmd := exec.Command("kubectl", "exec", pod, "-n", c.namespace, "-c", "memgraph", "--", + "bash", "-c", fmt.Sprintf("echo '%s' | mgconsole", query)) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("moving MAIN from %s to %s on %s: %w", main, name, pod, err) + } + return nil +} + +// mainOf returns the name of the data instance a view reports as MAIN, or the +// empty string when it reports none. +func mainOf(view []instanceRow) string { + for _, instance := range view { + if instance.role == roleMain { + return instance.name + } + } + return "" +} + +// instanceNames are the instance names a SHOW INSTANCES view lists. +func instanceNames(view []instanceRow) []string { + names := make([]string, 0, len(view)) + for _, instance := range view { + names = append(names, instance.name) + } + return names } // showInstances runs SHOW INSTANCES through mgconsole inside the given From c6e02dd0f05e7da4d598b34ffa15e405651a017b Mon Sep 17 00:00:00 2001 From: Andi Skrgat Date: Wed, 29 Jul 2026 10:10:04 +0200 Subject: [PATCH 24/34] feat: coordinator scale-down with leadership-safe Raft removal (#35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: coordinator scale-down with leadership-safe Raft removal Carries out a lowered `coordinators` count, the last scale `14-topology-scale-up.md` accepted at admission but held back. Removing a coordinator means removing a Raft member, and Raft refuses to remove its own leader (`RAFT_CANNOT_REMOVE_LEADER`) — while a StatefulSet sheds only its highest ordinals, so the leader may well sit on one of them. Retiring ordinals are `[spec.coordinators, liveStatefulSet.spec.replicas)`, and because the count must stay odd a shrink always retires an even number of members, so the surviving Raft cluster keeps an odd membership throughout. `YIELD LEADERSHIP` is the lever, and it is the one command whose outcome the planner cannot predict: it must be issued on the current leader — the connection the controller already holds — and it names no successor, because `yield_leadership()` is called without one and NuRaft's election decides. So it is always a plan's **last** command and terminal: the controller stops after it and requeues to re-observe under whichever coordinator won, asking again if the election happened to pick another retiring member. Everything the planner can still order safely goes out ahead of it in that same pass — the retiring coordinators that are not the leader, and the whole data-instance retirement. `REMOVE COORDINATOR` is therefore never aimed at the observed leader. Raft membership is given up before the pods are, so no removed member's vote outlives its pod: the shrink is applied in the one place a replica count is ever lowered, at the end of the registration phase once the plan is empty. The readiness gate stays strict, retiring pods included. `Converged` is False with reason `RetirementInProgress` — now naming the retiring members of both roles — or `LeadershipTransferInProgress` while a yield is pending. A coordinator removed from Raft keeps running and keeps its state on purpose: NuRaft fires `RemovedFromCluster`, stops it campaigning after two election timeouts, and never calls `system_exit`, so the container does not die and the readiness gate is not tripped. It appends nothing, so its log stays a prefix of the leader's and cannot diverge, and a later `ADD COORDINATOR` is accepted unconditionally — a re-added coordinator on a retained volume is in the same position as one whose pod crashed and stayed down. No PVC wipe, no removal bookkeeping, no re-add guard. Its stale view is already handled by `13-coordinator-leader-required.md`. `ScaleInProgress` goes away with this: `applied` is `max(declared, current)`, so a mismatch between the two is now exactly a retirement in flight, and the reason became unreachable rather than merely unused. Tests: planner cases for the leader on a retiring ordinal, on a survivor and outside the retiring set, two coordinators retiring at once, an already-removed retiring member, a retiring leader with nothing else to order, and a retiring coordinator alongside retiring data instances; `RetiringCoordinators` bounds in both directions plus its declared-form equality; envtest specs for the removal order, the yield and the pass that removes under the new leader, the raised-back count, and both roles retiring in one edit. The fake cluster gains `REMOVE COORDINATOR` and `YIELD LEADERSHIP`, refusing a removal aimed at its own leader, so a plan that skipped the yield fails the suite loudly. The scaling e2e container forces leadership onto `coordinator_4`, drops the count to 3, and asserts both members leave the Raft cluster before their pods are shed, that leadership lands on a survivor, that the cluster converges, and that the retired claims are kept by the default retention policy. No CRD or RBAC change, so the chart is untouched. * fix: Finding out who is the leader --- CLAUDE.md | 8 +- README.md | 12 +- api/v1alpha1/memgraphcluster_types.go | 22 +- config/samples/v1alpha1_memgraphcluster.yaml | 17 +- examples/minimal-cluster.yaml | 5 +- internal/controller/fake_memgraph_test.go | 68 +++++ .../controller/memgraphcluster_controller.go | 157 ++++++---- .../memgraphcluster_controller_test.go | 141 ++++++++- internal/memgraph/bolt.go | 10 + internal/memgraph/client.go | 21 +- internal/memgraph/queries.go | 9 + internal/memgraph/queries_test.go | 16 ++ internal/planner/planner.go | 107 ++++++- internal/planner/planner_test.go | 151 ++++++++++ internal/resources/topology.go | 49 +++- internal/resources/topology_test.go | 79 +++++ test/e2e/memgraphcluster_test.go | 270 +++++++++++++----- test/utils/names.go | 47 +++ 18 files changed, 989 insertions(+), 200 deletions(-) create mode 100644 test/utils/names.go diff --git a/CLAUDE.md b/CLAUDE.md index ae7d638..e47cad8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,10 +49,10 @@ CI (`.github/workflows/`) runs `make lint-config`, `make lint`, `make test-unit` The PRD defines seven modules with two pure cores and one mock seam. Keep this separation — it's what makes the logic testable without a cluster: -1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **mutable**, bounded by schema floors alone (coordinators ≥ 3 and odd, data instances ≥ 1) — enforced by the CRD, not a webhook. Raising a count grows the cluster; lowering one is accepted but not yet carried out (see `specs/operator-mvp/issues/15`, `16`). Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). +1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **mutable in both directions**, bounded by schema floors alone (coordinators ≥ 3 and odd, data instances ≥ 1) — enforced by the CRD, not a webhook. Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). 2. **Resource builders** — pure functions: spec in, desired Kubernetes objects out (one StatefulSet per role + headless Services; per-pod identity such as coordinator ID and advertised addresses derived from pod ordinals). No API calls, no side effects. Tested golden-style. -3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. -4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. A MAIN is promoted only when the cluster has none — at bootstrap, and after the planner itself demoted a data instance that is retiring; a MAIN that is staying is never overridden, because failover belongs to the Raft coordinators. A lowered `dataInstances` count is the one removal the planner drives: `DEMOTE INSTANCE` the retiring MAIN, promote a survivor, `UNREGISTER INSTANCE` the retiring members, and only then does the controller shed their pods (see `specs/operator-mvp/issues/15`). +3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main, demote/unregister instance, remove coordinator, yield leadership) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. +4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. A MAIN is promoted only when the cluster has none — at bootstrap, and after the planner itself demoted a data instance that is retiring; a MAIN that is staying is never overridden, because failover belongs to the Raft coordinators. A lowered count is the one removal the planner drives: `DEMOTE INSTANCE` the retiring MAIN, promote a survivor, `UNREGISTER INSTANCE` the retiring data instances, `REMOVE COORDINATOR` the retiring coordinators, and only then does the controller shed their pods (see `specs/operator-mvp/issues/15`, `16`). One command breaks the pure-diff mould: `YIELD LEADERSHIP`, needed because Raft refuses to remove its own leader. It names no successor, so it is always a plan's **last** command and terminal — the controller requeues and re-observes under whichever coordinator won the election. 5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. 6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. The chart's `version` and its `appVersion` (the operator image tag) move **independently**: tag `v` releases the operator, `chart-` releases the chart alone — see `docs/releasing.md`. 7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. @@ -63,6 +63,6 @@ Test philosophy (from the PRD): assert external behavior, never internal call or - Spec knob names mirror the HA Helm chart's vocabulary where the concept carries over (e.g. the `secrets.name` / `secrets.licenseKey` / `secrets.organizationKey` block) — check the chart before inventing a name. - No secret material in spec or status; secrets are consumed by reference only. -- Storage is never deleted by the operator: no finalizer-based cleanup; PVC retention (deletion *and* scale-down) maps to the StatefulSet PVC retention policy (default `Retain`). The only cluster members the operator removes are the data instances a lowered `dataInstances` count retires; coordinators are never removed (`REMOVE COORDINATOR` arrives with `specs/operator-mvp/issues/16`). +- Storage is never deleted by the operator: no finalizer-based cleanup; PVC retention (deletion *and* scale-down) maps to the StatefulSet PVC retention policy (default `Retain`). The only cluster members the operator removes are the ones a lowered replica count retires; a coordinator removed from Raft keeps running and keeps its state on purpose, which is what makes re-growing onto a retained volume safe. - Workload pods: non-root uid 101 / gid 103, seccomp RuntimeDefault, all capabilities dropped. - Log messages follow Kubernetes style: capital first letter, no trailing period, past tense, object type named (see AGENTS.md for examples). diff --git a/README.md b/README.md index bdfb2fb..d9d791f 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ The MVP is deliberately "provision, bootstrap, observe". It does: - bootstrap HA: add the coordinators, register the data instances, and promote the initial MAIN once; - re-register continuously: every reconcile compares `SHOW INSTANCES` on the coordinator leader against the declared topology and issues only the missing registrations, so an instance that loses its registration state (say, after being rescheduled onto a fresh node) rejoins without human action; - **grow a live cluster**: raise `coordinators` or `dataInstances` (both in one edit if you like, in any step size) and the added pods are provisioned and registered by the same diff that restores a lost registration — no manual `ADD COORDINATOR` or `REGISTER INSTANCE`; -- **shrink the data instances**: lower `dataInstances` and the instances above the new count are retired — MAIN moved off them if one of them holds it, then `UNREGISTER INSTANCE`, and only then are their pods shed, so the coordinators never expect an instance whose pod is gone; +- **shrink a live cluster**: lower `dataInstances` or `coordinators` and the members above the new count are retired before their pods are shed — a data instance has MAIN moved off it if it holds it and is then `UNREGISTER INSTANCE`d, a coordinator is `REMOVE COORDINATOR`ed out of the Raft cluster — so the coordinators never expect an instance whose pod is gone, and no removed member's pod outlives its vote; - report the observed MAIN, the registered member counts, and the readiness and convergence conditions on the resource's status. Scaling is one edit, and `Converged` tells you when it is finished: @@ -185,17 +185,17 @@ kubectl patch mgc memgraph -n memgraph --type=merge -p '{"spec":{"coordinators": kubectl wait --namespace memgraph --for=condition=Converged memgraphcluster/memgraph --timeout=10m ``` -A scale-down reports `Converged=False` with reason `RetirementInProgress`, naming the instances on their way out, until their pods are gone. Two things to know about it: +Both counts have a floor the schema enforces at creation and on every update: `coordinators` must stay odd and at or above three, `dataInstances` at or above one. A scale-down reports `Converged=False` with reason `RetirementInProgress`, naming the members on their way out, until their pods are gone. Three things to know about it: -- **A retiring pod that cannot become ready blocks its own removal.** The operator only touches the cluster when every pod of both StatefulSets is ready, and until the shrink is applied the retiring pods still belong to the data StatefulSet. So an instance that is stuck (crash-looping, unschedulable, wedged in a snapshot restore) keeps its own retirement waiting, and the resource reports `WorkloadsNotReady` rather than the operator writing to a cluster whose state it only half knows. Fix the pod, or delete it if it is genuinely unrecoverable, and the retirement continues. -- **The claims of a retired instance follow `spec.storage.retentionPolicy`**, the same knob that decides what happens to storage when the cluster is deleted — `Retain` (the default) keeps them, so a shrink made by accident loses no data, and re-raising the count reattaches them. +- **A retiring pod that cannot become ready blocks its own removal.** The operator only touches the cluster when every pod of both StatefulSets is ready, and until the shrink is applied the retiring pods still belong to their StatefulSet. So a member that is stuck (crash-looping, unschedulable, wedged in a snapshot restore) keeps its own retirement waiting, and the resource reports `WorkloadsNotReady` rather than the operator writing to a cluster whose state it only half knows. Fix the pod, or delete it if it is genuinely unrecoverable, and the retirement continues. +- **The claims of a retired member follow `spec.storage.retentionPolicy`**, the same knob that decides what happens to storage when the cluster is deleted — `Retain` (the default) keeps them, so a shrink made by accident loses no data, and re-raising the count reattaches them. A coordinator removed from Raft keeps running and keeps its state on purpose, which is what makes re-growing onto a retained volume safe: it is in the same position as one whose pod crashed and stayed down, and a later `ADD COORDINATOR` brings it back in. +- **A coordinator shrink may have to wait for a Raft election.** Raft refuses to remove its own leader, and a StatefulSet sheds only its highest ordinals, so a leader sitting in the retiring range is asked to `YIELD LEADERSHIP` first — which cannot name a successor. The resource reports `Converged=False` with reason `LeadershipTransferInProgress` while that is pending, and the operator asks again if the election happens to pick another retiring coordinator. What it does not do yet: -- **Scaling the coordinators down.** `coordinators` must stay odd and at or above three, `dataInstances` at or above one — all enforced at creation and on every update. Lowering `coordinators` is accepted by admission but not carried out: dropping a coordinator means removing a Raft member, which the operator does not do yet, so it holds the StatefulSet at its current size and reports `Converged=False` with reason `ScaleInProgress` until the count is raised back. - **Failover.** The operator promotes a MAIN only when the cluster has none: once at bootstrap, and once more when it demotes an instance that is retiring. It never overrides a MAIN that is staying — leadership belongs to the Raft coordinators, so two control systems never fight over which instance is MAIN. - **Other day-2 operations**: orchestrated or rolling version upgrades, backup and restore, storage-mode changes. -- **Removing coordinators**: there is no `REMOVE COORDINATOR`, and no finalizer-based storage cleanup — deleting storage is left entirely to the StatefulSet's own retention policy. +- **Deleting storage**: the operator owns no finalizer and runs no cleanup of its own — deleting a volume is left entirely to the StatefulSet's own retention policy. - **External access** of any kind — no LoadBalancer, NodePort, ingress or gateway. Access is in-cluster (or `kubectl port-forward`) only; the approach is expected to change, so it was deliberately deferred rather than shipped and broken later. - **TLS**, for Bolt or intra-cluster traffic. - **Bolt authentication** — the operator connects to the coordinators unauthenticated, so clusters must not enable auth yet. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 7146d7d..d78027b 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -163,18 +163,20 @@ const ( // declared topology. ReasonAllInstancesRegistered = "AllInstancesRegistered" - // ReasonScaleInProgress is set when registration has converged but a - // StatefulSet still runs a different number of replicas than the spec - // declares, so the declared topology is not fully realized yet. - ReasonScaleInProgress = "ScaleInProgress" - - // ReasonRetirementInProgress is set while a lowered dataInstances count is - // being carried out: the instances beyond the declared count are still - // members of the cluster, or their pods are still being shed. The message - // names them, so a scale-down that stalls says which instance it is waiting - // on. + // ReasonRetirementInProgress is set while a lowered count of either role is + // being carried out: the members beyond the declared count are still part of + // the cluster, or their pods are still being shed. The message names them, so + // a scale-down that stalls says which member it is waiting on. ReasonRetirementInProgress = "RetirementInProgress" + // ReasonLeadershipTransferInProgress is set while a lowered coordinators + // count is waiting on Raft leadership to move: Raft refuses to remove its own + // leader, so a retiring coordinator holding leadership is asked to yield it + // first. YIELD LEADERSHIP cannot name a successor, so the operator re-observes + // the cluster under whichever coordinator won the election and may have to ask + // again — which is exactly what this reason means when it persists. + ReasonLeadershipTransferInProgress = "LeadershipTransferInProgress" + // ReasonMainElected is set when a data instance is observed as MAIN. ReasonMainElected = "MainElected" diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 24bf81c..8bba63b 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -11,14 +11,15 @@ spec: # odd so the Raft quorum cannot split, and at least three — a quorum of one # cannot survive losing itself. # - # Lowering dataInstances shrinks the cluster: the instances above the new count - # are retired (MAIN moved off them, then UNREGISTER INSTANCE) before their pods - # are shed, reported as Converged=False with reason RetirementInProgress. Note - # that the operator only touches the cluster while every pod is ready, so a - # retiring pod that cannot become ready blocks its own removal. Lowering - # coordinators is not supported yet: it is accepted at admission but the - # operator holds the StatefulSet at its current size and reports - # Converged=False with reason ScaleInProgress. + # Lowering either count shrinks the cluster: the members above the new count are + # retired before their pods are shed — a data instance has MAIN moved off it and + # is then UNREGISTER INSTANCEd, a coordinator is REMOVE COORDINATORed out of the + # Raft cluster — reported as Converged=False with reason RetirementInProgress. + # Two notes. The operator only touches the cluster while every pod is ready, so + # a retiring pod that cannot become ready blocks its own removal. And Raft + # refuses to remove its own leader, so a leader in the retiring range is asked + # to YIELD LEADERSHIP first, reported as LeadershipTransferInProgress while that + # is pending. coordinators: 3 dataInstances: 2 # repository carries the registry host and image path only — the version diff --git a/examples/minimal-cluster.yaml b/examples/minimal-cluster.yaml index 3098272..9c13fe0 100644 --- a/examples/minimal-cluster.yaml +++ b/examples/minimal-cluster.yaml @@ -14,8 +14,9 @@ spec: # Raise either count later to grow the cluster: the operator provisions the # new pods and registers them, no manual registration involved. The # coordinator count must be odd so the Raft quorum cannot split, and at least - # three. Lowering dataInstances retires the instances above the new count — - # unregistered before their pods go; lowering coordinators is not supported yet. + # three. Lower either count and the members above it are retired first — a data + # instance unregistered, a coordinator removed from the Raft cluster — before + # their pods go. coordinators: 3 dataInstances: 2 image: diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go index e3bee3f..69734fb 100644 --- a/internal/controller/fake_memgraph_test.go +++ b/internal/controller/fake_memgraph_test.go @@ -88,6 +88,23 @@ func (f *fakeMemgraph) setInstances(instances []memgraph.Instance) { f.instances = slices.Clone(instances) } +// setLeader moves Raft leadership onto the named coordinator, which is how a spec +// parks it where a scale-down cannot remove it: on an ordinal the shrink retires. +func (f *fakeMemgraph) setLeader(name string) { + f.mu.Lock() + defer f.mu.Unlock() + for i, instance := range f.instances { + if !strings.HasPrefix(instance.Name, "coordinator_") { + continue + } + role := memgraph.RoleFollower + if instance.Name == name { + role = memgraph.RoleLeader + } + f.instances[i].Role = role + } +} + // setStaleView makes the coordinator at the given Bolt address answer // SHOW INSTANCES with its own view instead of the cluster's. func (f *fakeMemgraph) setStaleView(address string, instances []memgraph.Instance) { @@ -240,6 +257,57 @@ func (c *fakeClient) UnregisterInstance(_ context.Context, name string) error { }) } +// RemoveCoordinator drops the coordinator with the given Raft ID from the cluster +// view and — as Raft does — refuses the current leader, so a plan that aims a +// removal at the leader fails the suite loudly instead of quietly working. +func (c *fakeClient) RemoveCoordinator(_ context.Context, id int32) error { + name := fmt.Sprintf("coordinator_%d", id) + return c.execute(fmt.Sprintf("REMOVE COORDINATOR %d", id), func() error { + for i, instance := range c.cluster.instances { + if instance.Name != name { + continue + } + if instance.IsLeader() { + return fmt.Errorf("fake memgraph: %s is the leader", name) + } + c.cluster.instances = slices.Delete(c.cluster.instances, i, i+1) + return nil + } + return fmt.Errorf("fake memgraph: coordinator %s is not a member", name) + }) +} + +// YieldLeadership moves leadership off the coordinator serving this connection to +// the lowest-numbered remaining member, standing in for the election NuRaft runs. +// A test cannot rely on which coordinator wins — that is the point of the command +// — only on leadership having moved, which is what the operator has to converge +// around. +func (c *fakeClient) YieldLeadership(context.Context) error { + self, err := c.selfName() + if err != nil { + return err + } + return c.execute("YIELD LEADERSHIP", func() error { + successor := -1 + for i, instance := range c.cluster.instances { + if strings.HasPrefix(instance.Name, "coordinator_") && instance.Name != self { + successor = i + break + } + } + if successor < 0 { + return fmt.Errorf("fake memgraph: %s is the only coordinator, so leadership cannot be yielded", self) + } + for i, instance := range c.cluster.instances { + if instance.Name == self { + c.cluster.instances[i].Role = memgraph.RoleFollower + } + } + c.cluster.instances[successor].Role = memgraph.RoleLeader + return nil + }) +} + func (c *fakeClient) Close(context.Context) error { c.cluster.mu.Lock() defer c.cluster.mu.Unlock() diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 3ea5e6d..79d90f8 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -191,50 +191,68 @@ type replicaCounts struct { data roleReplicas } -// scaleMessage describes the roles whose StatefulSet does not run the declared -// number of replicas, and is empty once both do — which is what widens -// Converged from "registration matches the declared topology" to "the declared -// topology is actually running". -func (c replicaCounts) scaleMessage() string { - var pending []string - for _, role := range []roleReplicas{c.coordinators, c.data} { - if role.applied != role.declared { - pending = append(pending, fmt.Sprintf("StatefulSet %s runs %d replica(s) while %d are declared", - role.name, role.applied, role.declared)) - } +// retirementMessage names the members a lowered count is shedding, and is empty +// when none are. It is non-empty for exactly as long as the retirement is +// unfinished: the retiring sets are derived from the replica counts the +// operator's own StatefulSets still run, so they empty only once the shrink that +// removes those pods has been applied. +func retirementMessage(topology planner.Topology) string { + var retiring []string + if names := coordinatorNames(topology.RetiringCoordinators); len(names) > 0 { + retiring = append(retiring, "coordinator(s) "+strings.Join(names, ", ")) + } + if names := instanceNames(topology.RetiringDataInstances); len(names) > 0 { + retiring = append(retiring, "data instance(s) "+strings.Join(names, ", ")) + } + if len(retiring) == 0 { + return "" } - return strings.Join(pending, "; ") + return "Retiring " + strings.Join(retiring, " and ") + " before their pods are shed" } -// retirementMessage names the data instances a lowered count is shedding, and is -// empty when none are. It is non-empty for exactly as long as the retirement is -// unfinished: the retiring set is derived from the replica count the operator's -// own StatefulSet still runs, so it empties only once the shrink that removes -// those pods has been applied. -func retirementMessage(topology planner.Topology) string { - if len(topology.RetiringDataInstances) == 0 { - return "" +func coordinatorNames(coordinators []memgraph.CoordinatorSpec) []string { + names := make([]string, 0, len(coordinators)) + for _, coordinator := range coordinators { + names = append(names, coordinator.Name()) } - names := make([]string, 0, len(topology.RetiringDataInstances)) - for _, instance := range topology.RetiringDataInstances { + return names +} + +func instanceNames(instances []memgraph.DataInstanceSpec) []string { + names := make([]string, 0, len(instances)) + for _, instance := range instances { names = append(names, instance.Name) } - return "Retiring data instance(s) " + strings.Join(names, ", ") + - " before their pods are shed" + return names +} + +// yieldedLeader is the retiring coordinator a plan ends by moving Raft leadership +// off, or the empty string when the plan does not do that. A yield is always the +// plan's last command, because nothing after it could be planned: the election +// picks the successor, so the pass stops there and the next one observes the +// cluster under whoever won. +func yieldedLeader(commands []planner.Command) string { + if len(commands) == 0 { + return "" + } + yield, ok := commands[len(commands)-1].(planner.YieldLeadership) + if !ok { + return "" + } + return yield.Leader } // replicaCounts resolves the replica count to apply per role: the declared count // while the cluster grows or holds its size, and deliberately the current count // while a lowered count would shrink it. Shedding pods means removing members // from the Memgraph cluster first — the coordinators otherwise keep expecting -// instances whose pods are gone — so this rule never shrinks anything, which -// keeps it free of any knowledge about the cluster's state. +// instances whose pods are gone, and a removed coordinator's vote must be given +// up before its pod is — so this rule never shrinks anything, which keeps it free +// of any knowledge about the cluster's state. // -// Lowering the data-instance count is carried out at the end of the registration +// A lowered count of either role is carried out at the end of the registration // phase instead, once the retiring members have actually left the cluster (see -// reconcileRegistration). Lowering the coordinator count is not carried out at -// all yet: the size is held and the mismatch reported, rather than acting on half -// of a scale-down the operator cannot finish. +// reconcileRegistration). func (r *MemgraphClusterReconciler) replicaCounts( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, @@ -292,11 +310,12 @@ func (r *MemgraphClusterReconciler) currentReplicas( // removal, and the resource reports WorkloadsNotReady rather than the operator // acting on a half-known cluster. // -// This is also where a data-instance scale-down finishes. Once the plan comes -// back empty — meaning the retiring instances have left the cluster — the data -// StatefulSet is applied at the declared count, shedding their pods. That is the -// one place the operator ever lowers a replica count, so the coordinators never -// see a registered instance's pod disappear. +// This is also where a scale-down finishes. Once the plan comes back empty — +// meaning the retiring instances have been unregistered and the retiring +// coordinators have left the Raft cluster — the shrinking role's StatefulSet is +// applied at the declared count, shedding their pods. That is the one place the +// operator ever lowers a replica count, so the coordinators never see a registered +// instance's pod disappear, and no removed member's pod outlives its vote. func (r *MemgraphClusterReconciler) reconcileRegistration( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, @@ -324,6 +343,7 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( // The members a lowered count is shedding are the ordinals the operator's own // previous apply still runs beyond the declared count, so the range is bounded // by what the operator itself created. + topology.RetiringCoordinators = resources.RetiringCoordinators(cluster, replicas.coordinators.applied) topology.RetiringDataInstances = resources.RetiringDataInstances(cluster, replicas.data.applied) // Whether a retirement is in flight is decided once per pass, from the // topology alone: it is what both the condition and the shrink below key off. @@ -356,15 +376,13 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( latest := observe(topology, observed) commands := planner.Plan(topology, observed) if len(commands) == 0 { - // No retiring instance is a member of the cluster any more — the plan - // would carry an UNREGISTER INSTANCE otherwise — so their pods can go. + // No retiring member belongs to the cluster any more — the plan would + // carry an UNREGISTER INSTANCE or a REMOVE COORDINATOR otherwise — so + // their pods can go. if retiring != "" { - if err := r.applyDesired(ctx, cluster, - resources.DataStatefulSet(cluster, replicas.data.declared)); err != nil { + if err := r.shedRetiredPods(ctx, cluster, topology, replicas); err != nil { return ctrl.Result{}, err } - log.Info("Shrank the data StatefulSet to the declared replica count", - "statefulset", replicas.data.name, "replicas", replicas.data.declared) if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), notConvergedCondition(memgraphcomv1alpha1.ReasonRetirementInProgress, retiring), ); statusErr != nil { @@ -375,20 +393,6 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil } - // Registration matches the declared topology. It is only converged once - // the StatefulSets run the declared replica counts too, so a scale the - // operator is holding back keeps the condition False and says which - // role and by how much. - if pending := replicas.scaleMessage(); pending != "" { - log.Info("Held a StatefulSet short of the declared replica count", "reason", pending) - if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), - notConvergedCondition(memgraphcomv1alpha1.ReasonScaleInProgress, pending), - ); statusErr != nil { - return ctrl.Result{}, statusErr - } - return ctrl.Result{RequeueAfter: resyncInterval}, nil - } - // Converged, but keep re-observing: a registration a pod loses later // produces no watch event, so drift is only caught by resyncing. log.Info("Confirmed cluster registration is converged") @@ -405,12 +409,21 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( // serving stays Ready while a lost registration is restored; a fresh // bootstrap has no MAIN yet, so Ready is False until one is elected. A // retirement in flight is named as such — it is the more specific operation, - // and the one whose pending members a user wants to see. + // and the one whose pending members a user wants to see. A pending leadership + // yield is more specific still: it is the one step whose outcome nobody can + // predict, so a scale-down circling it says so rather than looking stuck on + // the removal it cannot reach yet. reason := memgraphcomv1alpha1.ReasonRegistrationInProgress message := fmt.Sprintf("Issuing %d registration command(s) to converge the cluster", len(commands)) if retiring != "" { reason, message = memgraphcomv1alpha1.ReasonRetirementInProgress, retiring } + if yielded := yieldedLeader(commands); yielded != "" { + reason = memgraphcomv1alpha1.ReasonLeadershipTransferInProgress + message = fmt.Sprintf( + "Retiring coordinator %s holds Raft leadership, which cannot be removed: yielding it to another member", + yielded) + } inProgress := notConvergedCondition(reason, message) if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), inProgress); statusErr != nil { return ctrl.Result{}, statusErr @@ -428,6 +441,38 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( return ctrl.Result{RequeueAfter: requeueAfterRegistration}, nil } +// shedRetiredPods applies the shrinking roles' StatefulSets at their declared +// replica counts — the one place the operator ever lowers a replica count. It is +// reached only after the plan came back empty, so every pod it sheds belongs to a +// member that has already left the Memgraph cluster: an unregistered data +// instance, or a coordinator whose Raft vote is gone. +func (r *MemgraphClusterReconciler) shedRetiredPods( + ctx context.Context, + cluster *memgraphcomv1alpha1.MemgraphCluster, + topology planner.Topology, + replicas replicaCounts, +) error { + log := logf.FromContext(ctx) + for _, role := range []struct { + retiring int + replicas roleReplicas + build func(*memgraphcomv1alpha1.MemgraphCluster, int32) *appsv1.StatefulSet + }{ + {len(topology.RetiringCoordinators), replicas.coordinators, resources.CoordinatorStatefulSet}, + {len(topology.RetiringDataInstances), replicas.data, resources.DataStatefulSet}, + } { + if role.retiring == 0 { + continue + } + if err := r.applyDesired(ctx, cluster, role.build(cluster, role.replicas.declared)); err != nil { + return err + } + log.Info("Shrank a StatefulSet to the declared replica count", + "statefulset", role.replicas.name, "replicas", role.replicas.declared) + } + return nil +} + // observation is everything a reconcile pass observed about the cluster that // reaches the resource's status: which data instance is MAIN, and how many of // each role's declared members are registered. It is observation only — no diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index cfeb6ca..e77a45d 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -764,14 +764,13 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonAllInstancesRegistered)) }) - // Lowering the coordinator count is the scale the operator still cannot - // realize: removing a Raft member is not implemented, so the StatefulSet is - // held at its current size and the resource says so rather than the - // operator acting on half of a scale-down it cannot finish. - It("should report Converged False while a lowered coordinator count is held back", func() { + // grownToFive drives the cluster to a converged five-coordinator topology, + // which is the only shape a coordinator shrink can start from: the count must + // stay odd and at or above three, so five is the smallest cluster with members + // to drop. It returns the command count the shrink assertions start from. + grownToFive := func() int { + GinkgoHelper() bootstrapped() - - By("growing the coordinators so there is a member to drop") setCounts(5, 2) reconcileCluster(resourceName) markWorkloadsReady(resourceName) @@ -779,29 +778,141 @@ var _ = Describe("MemgraphCluster Controller", func() { reconcileCluster(resourceName) Expect(apimeta.IsStatusConditionTrue(status().Conditions, memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) - grown := len(fake.executedCommands()) + return len(fake.executedCommands()) + } + + // Raft membership is given up before the pods are, so no removed member's pod + // outlives its vote. With the leader on a survivor that is the whole shrink: + // removing a follower needs no leadership dance. + It("should remove retiring coordinators from Raft and only then shed their pods", func() { + baseline := grownToFive() setCounts(3, 2) reconcileCluster(resourceName) + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": REMOVE COORDINATOR 4", + leader + ": REMOVE COORDINATOR 5", + }), "both retiring members leave the Raft cluster in one pass under a surviving leader") Expect(replicas(coordinatorSuffix)).To(Equal(int32(5)), - "a lower declared count must never shrink the applied StatefulSet") + "a pass with pending commands must never lower the replica count") converged := convergedCondition() Expect(converged.Status).To(Equal(metav1.ConditionFalse)) - Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonScaleInProgress)) - Expect(converged.Message).To(ContainSubstring(resourceName + coordinatorSuffix)) - Expect(sinceBootstrap(grown)).To(BeEmpty(), - "the coordinators the lowered count drops stay registered: removal is not implemented") + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonRetirementInProgress)) + Expect(converged.Message).To(ContainSubstring("coordinator_4"), + "the condition must name the coordinators being retired") - // Raising the count back matches what is running, which converges - // again without touching the cluster. + By("shedding the pods once the members have left the Raft cluster") + reconcileCluster(resourceName) + Expect(replicas(coordinatorSuffix)).To(Equal(int32(3))) + Expect(sinceBootstrap(baseline)).To(HaveLen(2), "the removals are not re-issued") + Expect(convergedCondition().Reason).To(Equal(memgraphcomv1alpha1.ReasonRetirementInProgress)) + + By("reporting the shrink as finished once the StatefulSet runs the declared count") + reconcileCluster(resourceName) + s := status() + Expect(s.Coordinators).To(Equal(int32(3))) + Expect(s.Main).To(Equal("instance_0"), "shrinking the coordinators does not move MAIN") + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + + // A StatefulSet sheds its highest ordinals, so the Raft leader may well sit on + // one of them — and Raft refuses to remove its own leader. The plan then ends + // with YIELD LEADERSHIP and nothing after it, because the election picks the + // successor: the pass stops there and the next one removes under whoever won. + // The fake refuses a removal aimed at its leader, so a plan that skipped the + // yield would fail this spec rather than quietly working. + It("should yield leadership off a retiring coordinator before removing it", func() { + baseline := grownToFive() + + By("parking Raft leadership on the coordinator the shrink retires") + fake.setLeader("coordinator_4") + + setCounts(3, 2) + reconcileCluster(resourceName) + + retiringLeader := coordinatorAddress(3) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + retiringLeader + ": REMOVE COORDINATOR 5", + retiringLeader + ": YIELD LEADERSHIP", + }), "the yield comes last, after the removal the planner could still order safely") + converged := convergedCondition() + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonLeadershipTransferInProgress)) + Expect(converged.Message).To(ContainSubstring("coordinator_4"), + "the condition must name the coordinator being moved off leadership") + Expect(replicas(coordinatorSuffix)).To(Equal(int32(5)), + "a pass with a pending yield must never lower the replica count") + + By("removing the former leader on the next pass, under whichever coordinator won") + reconcileCluster(resourceName) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + retiringLeader + ": REMOVE COORDINATOR 5", + retiringLeader + ": YIELD LEADERSHIP", + coordinatorAddress(0) + ": REMOVE COORDINATOR 4", + })) + Expect(convergedCondition().Reason).To(Equal(memgraphcomv1alpha1.ReasonRetirementInProgress)) + + By("shedding the pods and converging once the Raft cluster is down to three") + reconcileCluster(resourceName) + Expect(replicas(coordinatorSuffix)).To(Equal(int32(3))) + reconcileCluster(resourceName) + s := status() + Expect(s.Coordinators).To(Equal(int32(3))) + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + + // Raising the count back to what is already running is a no-op scale: the + // members are still registered, so nothing is planned and nothing is applied. + It("should converge again when a lowered coordinator count is raised back", func() { + baseline := grownToFive() + + setCounts(3, 2) setCounts(5, 2) reconcileCluster(resourceName) + Expect(sinceBootstrap(baseline)).To(BeEmpty(), + "a shrink that was undone before it was acted on touches the cluster not at all") + Expect(replicas(coordinatorSuffix)).To(Equal(int32(5))) Expect(apimeta.IsStatusConditionTrue(status().Conditions, memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) }) + // Both counts lowered in one edit: each role's retirement is planned + // independently, and both StatefulSets shrink once the plan is empty. + It("should retire members of both roles in one edit", func() { + baseline := grownToFive() + + setCounts(3, 1) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": UNREGISTER INSTANCE instance_1", + leader + ": REMOVE COORDINATOR 4", + leader + ": REMOVE COORDINATOR 5", + }), "the surviving MAIN is left alone, and each role's removals are planned on their own") + Expect(replicas(coordinatorSuffix)).To(Equal(int32(5))) + Expect(replicas(dataSuffix)).To(Equal(int32(2))) + Expect(convergedCondition().Message).To(SatisfyAll( + ContainSubstring("coordinator_4"), ContainSubstring("instance_1")), + "the condition must name the retiring members of both roles") + + reconcileCluster(resourceName) + Expect(replicas(coordinatorSuffix)).To(Equal(int32(3))) + Expect(replicas(dataSuffix)).To(Equal(int32(1))) + + reconcileCluster(resourceName) + s := status() + Expect(s.Coordinators).To(Equal(int32(3))) + Expect(s.DataInstances).To(Equal(int32(1))) + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + // The whole point of the shrink: the member beyond the declared count leaves // the cluster before its pod does, so the coordinators never expect an // instance whose pod is gone. diff --git a/internal/memgraph/bolt.go b/internal/memgraph/bolt.go index 6abbc13..b587bda 100644 --- a/internal/memgraph/bolt.go +++ b/internal/memgraph/bolt.go @@ -85,6 +85,16 @@ func (c *boltClient) UnregisterInstance(ctx context.Context, name string) error return err } +func (c *boltClient) RemoveCoordinator(ctx context.Context, id int32) error { + _, err := c.run(ctx, removeCoordinatorQuery(id)) + return err +} + +func (c *boltClient) YieldLeadership(ctx context.Context) error { + _, err := c.run(ctx, yieldLeadershipQuery) + return err +} + func (c *boltClient) Close(ctx context.Context) error { return c.driver.Close(ctx) } diff --git a/internal/memgraph/client.go b/internal/memgraph/client.go index 891c3fd..aa6f6bd 100644 --- a/internal/memgraph/client.go +++ b/internal/memgraph/client.go @@ -16,8 +16,9 @@ limitations under the License. // Package memgraph provides the narrow client surface the operator uses to // drive a Memgraph high-availability cluster over Bolt: show instances, add -// coordinator, register instance, set main, and — for a data instance a lowered -// replica count is retiring — demote and unregister. All higher layers depend on +// coordinator, register instance, set main, and — for the members a lowered +// replica count is retiring — demote and unregister a data instance, yield +// coordinator leadership and remove a coordinator. All higher layers depend on // the Client and Connector interfaces, never on the Bolt driver — this package // is the mock seam for testing and the only place the driver is referenced. package memgraph @@ -111,6 +112,22 @@ type Client interface { // the coordinators stop expecting it before its pod goes away. UnregisterInstance(ctx context.Context, name string) error + // RemoveCoordinator drops the coordinator with the given Raft ID from the + // Raft cluster, so its vote is gone before its pod is. Raft refuses to + // remove its own leader, so the caller must never aim this at the leader — + // YieldLeadership moves leadership away first. + // + // The removed coordinator keeps running and keeps its state: NuRaft only + // stops it from campaigning, which is what makes a later ADD COORDINATOR on + // the retained volume safe. + RemoveCoordinator(ctx context.Context, id int32) error + + // YieldLeadership makes the coordinator this client is connected to give up + // Raft leadership. It has to be issued on the leader itself and cannot name + // a successor — NuRaft's election picks one — so its outcome is not + // predictable and the caller must re-observe the cluster afterwards. + YieldLeadership(ctx context.Context) error + Close(ctx context.Context) error } diff --git a/internal/memgraph/queries.go b/internal/memgraph/queries.go index 70ce04c..9b01e2c 100644 --- a/internal/memgraph/queries.go +++ b/internal/memgraph/queries.go @@ -56,3 +56,12 @@ func demoteInstanceQuery(name string) string { func unregisterInstanceQuery(name string) string { return fmt.Sprintf("UNREGISTER INSTANCE %s", name) } + +func removeCoordinatorQuery(id int32) string { + return fmt.Sprintf("REMOVE COORDINATOR %d", id) +} + +// yieldLeadershipQuery takes no argument on purpose: Memgraph's grammar has no +// successor to name, so the coordinator it runs on hands leadership to whichever +// member NuRaft's election picks. +const yieldLeadershipQuery = "YIELD LEADERSHIP" diff --git a/internal/memgraph/queries_test.go b/internal/memgraph/queries_test.go index a3a7a99..ecb944e 100644 --- a/internal/memgraph/queries_test.go +++ b/internal/memgraph/queries_test.go @@ -78,6 +78,22 @@ func TestUnregisterInstanceQuery(t *testing.T) { } } +func TestRemoveCoordinatorQuery(t *testing.T) { + got := removeCoordinatorQuery(4) + if want := "REMOVE COORDINATOR 4"; got != want { + t.Errorf("removeCoordinatorQuery() = %q, want %q", got, want) + } +} + +// YIELD LEADERSHIP names no successor: the coordinator it runs on is the subject, +// and NuRaft picks who takes over. A query that grew an argument would mean the +// planner could suddenly predict the outcome, so the shape is pinned. +func TestYieldLeadershipQuery(t *testing.T) { + if want := "YIELD LEADERSHIP"; yieldLeadershipQuery != want { + t.Errorf("yieldLeadershipQuery = %q, want %q", yieldLeadershipQuery, want) + } +} + func TestInstanceFromRecord(t *testing.T) { record := &db.Record{ Keys: []string{ diff --git a/internal/planner/planner.go b/internal/planner/planner.go index e5f48b4..2be75c5 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -23,10 +23,17 @@ limitations under the License. // the Raft coordinators; the planner never overrides a MAIN that is staying. // // Members a lowered replica count is retiring are the one thing the planner -// removes, and it can order the whole removal in a single pass because it -// predicts every intermediate state: a retiring MAIN is demoted, a survivor is -// promoted in its place, and the retiring members are then unregistered — so no -// UNREGISTER INSTANCE is ever aimed at a MAIN, which Memgraph would refuse. +// removes, and it orders each removal so that Memgraph never has to refuse it: a +// retiring MAIN is demoted and a survivor promoted in its place before any +// UNREGISTER INSTANCE, and a retiring coordinator is only ever removed from Raft +// while it is not the leader. +// +// One command breaks the pure-diff mould: YIELD LEADERSHIP, which moves +// coordinator leadership off a retiring coordinator so it can be removed at all. +// It cannot name a successor, so its outcome is the one thing the planner cannot +// predict — which is why it is always the last command of a plan. Everything the +// planner can still order safely goes out ahead of it in the same pass, and the +// caller re-observes the cluster under whichever coordinator won the election. package planner import ( @@ -48,9 +55,10 @@ type Topology struct { // longer declared. They are empty while a cluster grows or holds its size. // // A retiring data instance is demoted if it holds MAIN and then - // unregistered, so the cluster stops expecting it before its pod goes. - // RetiringCoordinators is still always empty: removing a Raft member is not - // implemented yet, so a plan issues no command for one. + // unregistered, so the cluster stops expecting it before its pod goes. A + // retiring coordinator is removed from the Raft cluster, which Raft only + // allows for a member that is not the leader — so leadership is yielded away + // from a retiring leader first. RetiringCoordinators []memgraph.CoordinatorSpec RetiringDataInstances []memgraph.DataInstanceSpec } @@ -136,14 +144,63 @@ func (c UnregisterInstance) String() string { return "UNREGISTER INSTANCE " + c.Name } +// RemoveCoordinator drops a retiring coordinator from the Raft cluster, so its +// vote is gone before its pod is. It is never aimed at the observed leader: Raft +// refuses to remove its own leader, which is what YieldLeadership is for. +// +// The removed coordinator keeps running and keeps its state — NuRaft only stops +// it campaigning — so nothing here has to be undone before a raised count adds +// it back on its retained volume. +type RemoveCoordinator struct { + Coordinator memgraph.CoordinatorSpec +} + +// Run implements Command. +func (c RemoveCoordinator) Run(ctx context.Context, client memgraph.Client) error { + return client.RemoveCoordinator(ctx, c.Coordinator.ID) +} + +func (c RemoveCoordinator) String() string { + return fmt.Sprintf("REMOVE COORDINATOR %d", c.Coordinator.ID) +} + +// YieldLeadership hands Raft leadership away from the retiring coordinator that +// currently holds it, which is the only way it can then be removed. It runs on +// the leader — the connection the caller already holds — and cannot name a +// successor, so the plan it ends says nothing about who takes over: the caller +// re-observes the cluster and plans again under the new leader. +type YieldLeadership struct { + // Leader is the retiring coordinator giving leadership up. It is carried for + // the sake of whoever is watching the scale-down; the query itself has no + // argument, and no successor can be named. + Leader string +} + +// Run implements Command. +func (c YieldLeadership) Run(ctx context.Context, client memgraph.Client) error { + return client.YieldLeadership(ctx) +} + +func (c YieldLeadership) String() string { + return "YIELD LEADERSHIP" +} + // Plan diffs the declared topology against the observed instances and returns // the commands still needed, in execution order: coordinators before data // instances (registration requires a formed Raft cluster), then the retirement // of the members a lowered count sheds — demote a retiring MAIN, promote a -// survivor in its place, unregister every retiring member. The promotion sits -// between the two so that no UNREGISTER INSTANCE is ever aimed at an observed -// MAIN, and so the cluster is MAIN-less only for the few milliseconds between -// two queries of the same pass. +// survivor in its place, unregister every retiring data instance, remove every +// retiring coordinator from Raft. The promotion sits between the demotion and the +// unregistrations so that no UNREGISTER INSTANCE is ever aimed at an observed +// MAIN, and so the cluster is MAIN-less only for the few milliseconds between two +// queries of the same pass. +// +// A retiring coordinator that holds Raft leadership cannot be removed at all, so +// the plan ends with YIELD LEADERSHIP instead and stops there — the retiring +// coordinators that are not the leader still go out ahead of it in that same +// pass. Nothing follows a yield, because nothing after it could be planned: the +// election picks the next leader, and the caller has to observe the cluster again +// to learn who won. // // Instances the cluster knows but the topology neither declares nor retires are // left untouched: the retiring set is bounded by the operator's own prior apply, @@ -183,9 +240,37 @@ func Plan(declared Topology, observed []memgraph.Instance) []Command { commands = append(commands, UnregisterInstance{Name: instance.Name}) } } + + leader := leaderName(observed) + yieldFrom := "" + for _, coordinator := range declared.RetiringCoordinators { + if coordinator.Name() == leader { + // Raft refuses to remove its own leader, so this one waits for the + // yield below to move leadership to another member. + yieldFrom = leader + continue + } + if coordinatorRegistered(registered, coordinator) { + commands = append(commands, RemoveCoordinator{Coordinator: coordinator}) + } + } + if yieldFrom != "" { + commands = append(commands, YieldLeadership{Leader: yieldFrom}) + } return commands } +// leaderName is the coordinator the observed view reports as Raft leader, or the +// empty string when it reports none. +func leaderName(observed []memgraph.Instance) string { + for _, instance := range observed { + if instance.IsLeader() { + return instance.Name + } + } + return "" +} + // Registered reports how many of the declared coordinators and data instances // the observed cluster has registered. It is pure observation for the CR's // status, and it shares Plan's definition of "registered" — so a role's count diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 26c8e0a..bab10e1 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -39,6 +39,11 @@ const ( thirdInstance = "instance_2" ) +// fourthCoordinator is the lowest-numbered coordinator a shrink from five to +// three retires, and the one the cases below park Raft leadership on: a +// StatefulSet sheds its highest ordinals, so the leader may well sit on one. +const fourthCoordinator = "coordinator_4" + // declaredTopology is the canonical 3-coordinator, 2-data-instance fixture // the cases below diff observed cluster states against. func declaredTopology() planner.Topology { @@ -70,6 +75,26 @@ func mixedTopology() planner.Topology { return topology } +// shrunkCoordinators is a cluster whose coordinators count was lowered from five +// to three — the smallest coordinator shrink the schema floors allow, and an even +// number of members either way — while its StatefulSet still runs all five pods: +// coordinator_4 and coordinator_5 are Raft members on their way out. +func shrunkCoordinators() planner.Topology { + topology := topologyOf(3, 2) + for id := int32(4); id <= 5; id++ { + topology.RetiringCoordinators = append(topology.RetiringCoordinators, coordinatorSpec(id)) + } + return topology +} + +// retiringBothRoles is one edit lowering both counts: the coordinators shrink from +// 5 to 3 while the data instances shrink from 3 to 2. +func retiringBothRoles() planner.Topology { + topology := shrunkCoordinators() + topology.RetiringDataInstances = append(topology.RetiringDataInstances, dataInstanceSpec(2)) + return topology +} + func topologyOf(coordinators int32, dataInstances int) planner.Topology { topology := planner.Topology{} for id := int32(1); id <= coordinators; id++ { @@ -115,6 +140,22 @@ func observedCoordinator(id int32, role string) memgraph.Instance { } } +// observedCoordinators is the Raft membership a leader reports: one row per given +// coordinator ID, the one named by leaderID reporting itself leader. Which +// coordinator holds leadership is what decides whether a shrink can remove +// members at all, so every retirement case below states it outright. +func observedCoordinators(leaderID int32, ids ...int32) []memgraph.Instance { + view := make([]memgraph.Instance, 0, len(ids)) + for _, id := range ids { + role := memgraph.RoleFollower + if id == leaderID { + role = memgraph.RoleLeader + } + view = append(view, observedCoordinator(id, role)) + } + return view +} + func observedDataInstance(i int, role string) memgraph.Instance { spec := dataInstanceSpec(i) return memgraph.Instance{ @@ -440,6 +481,116 @@ func TestPlan(t *testing.T) { planner.UnregisterInstance{Name: thirdInstance}, }, }, + // Coordinators the count drops are removed from Raft outright when the + // leader is a survivor: their votes are gone before their pods are, and + // removing a follower needs no leadership dance. + { + name: "retiring coordinators are removed from Raft under a surviving leader", + observed: append(observedCoordinators(1, 1, 2, 3, 4, 5), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + declared: ptr.To(shrunkCoordinators()), + want: []planner.Command{ + planner.RemoveCoordinator{Coordinator: coordinatorSpec(4)}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + }, + }, + // Raft refuses to remove its own leader, so a leader on a retiring ordinal + // is asked to yield — last in the plan, with nothing after it, because the + // election picks the successor and only a fresh observation can say who won. + // The other retiring member still goes in this same pass: removing a + // follower is safe and predictable. + { + name: "a retiring leader yields last, after every removal it can still order", + observed: append(observedCoordinators(4, 1, 2, 3, 4, 5), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + declared: ptr.To(shrunkCoordinators()), + want: []planner.Command{ + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + planner.YieldLeadership{Leader: fourthCoordinator}, + }, + }, + // Read-before-write: a coordinator already gone from the Raft membership gets + // no removal, so a pass that crashed between two removals re-plans to just + // the rest of the work. + { + name: "an already-removed retiring coordinator is not removed again", + observed: append(observedCoordinators(1, 1, 2, 3, 4), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + declared: ptr.To(shrunkCoordinators()), + want: []planner.Command{ + planner.RemoveCoordinator{Coordinator: coordinatorSpec(4)}, + }, + }, + // Nothing is left to order ahead of the yield: the plan is the yield alone, + // and the removal of the leader itself waits for the next pass. + { + name: "a retiring leader with nothing else to remove plans only the yield", + observed: append(observedCoordinators(4, 1, 2, 3, 4), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + declared: ptr.To(shrunkCoordinators()), + want: []planner.Command{ + planner.YieldLeadership{Leader: fourthCoordinator}, + }, + }, + // Leadership on a coordinator the topology neither declares nor retires — one + // a human added — is left where it is: it is not in the way of any removal. + { + name: "a leader outside the retiring set is not asked to yield", + observed: append(observedCoordinators(6, 1, 2, 3, 4, 5, 6), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + declared: ptr.To(shrunkCoordinators()), + want: []planner.Command{ + planner.RemoveCoordinator{Coordinator: coordinatorSpec(4)}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + }, + }, + // Both roles shrinking in one edit: the data instances are retired first + // (MAIN moved off the one going away), then the Raft members are removed. + { + name: "both roles retire in one pass under a surviving leader", + observed: append(observedCoordinators(1, 1, 2, 3, 4, 5), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + ), + declared: ptr.To(retiringBothRoles()), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(4)}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + }, + }, + // The same edit with leadership in the way: the data-instance retirement is + // fully ordered and issues in this pass regardless — only the removal of the + // leader itself has to wait behind the yield. + { + name: "a retiring leader does not hold up the data-instance retirement", + observed: append(observedCoordinators(4, 1, 2, 3, 4, 5), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + ), + declared: ptr.To(retiringBothRoles()), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + planner.YieldLeadership{Leader: fourthCoordinator}, + }, + }, // The retiring range is bounded by the operator's own prior apply, which is // what keeps an instance a human registered out of it — even one at a // higher ordinal than everything the operator ever ran. diff --git a/internal/resources/topology.go b/internal/resources/topology.go index 05f74f5..7105de4 100644 --- a/internal/resources/topology.go +++ b/internal/resources/topology.go @@ -51,13 +51,7 @@ func DeclaredTopology(cluster *memgraphcomv1alpha1.MemgraphCluster) planner.Topo DataInstances: make([]memgraph.DataInstanceSpec, 0, spec.dataInstances), } for ordinal := range spec.coordinators { - fqdn := podFQDN(cluster, CoordinatorName(cluster), spec, ordinal) - topology.Coordinators = append(topology.Coordinators, memgraph.CoordinatorSpec{ - ID: ordinal + 1, - BoltServer: hostPort(fqdn, spec.ports.bolt), - CoordinatorServer: hostPort(fqdn, spec.ports.coordinator), - ManagementServer: hostPort(fqdn, spec.ports.management), - }) + topology.Coordinators = append(topology.Coordinators, coordinator(cluster, spec, ordinal)) } for ordinal := range spec.dataInstances { topology.DataInstances = append(topology.DataInstances, dataInstance(cluster, spec, ordinal)) @@ -65,6 +59,29 @@ func DeclaredTopology(cluster *memgraphcomv1alpha1.MemgraphCluster) planner.Topo return topology } +// RetiringCoordinators is the coordinators a lowered coordinators count is +// shedding: pod ordinals [declared, applied), where applied is the replica count +// the operator's own previous apply left on the coordinator StatefulSet. It is +// empty while a cluster grows or holds its size. +// +// Because the count must stay odd, a shrink always retires an even number of +// coordinators, so the surviving Raft cluster keeps an odd membership throughout. +func RetiringCoordinators( + cluster *memgraphcomv1alpha1.MemgraphCluster, + applied int32, +) []memgraph.CoordinatorSpec { + spec := normalize(cluster.Spec) + if applied <= spec.coordinators { + return nil + } + + retiring := make([]memgraph.CoordinatorSpec, 0, applied-spec.coordinators) + for ordinal := spec.coordinators; ordinal < applied; ordinal++ { + retiring = append(retiring, coordinator(cluster, spec, ordinal)) + } + return retiring +} + // RetiringDataInstances is the data instances a lowered dataInstances count is // shedding: pod ordinals [declared, applied), where applied is the replica count // the operator's own previous apply left on the data StatefulSet. It is empty @@ -89,6 +106,24 @@ func RetiringDataInstances( return retiring } +// coordinator describes the coordinator running on the given pod ordinal, as the +// pod itself advertises it. Retiring coordinators are described the same way as +// declared ones: they are members of the Raft cluster under the ID and addresses +// the operator added them with, whether or not the spec still declares them. +func coordinator( + cluster *memgraphcomv1alpha1.MemgraphCluster, + spec normalizedSpec, + ordinal int32, +) memgraph.CoordinatorSpec { + fqdn := podFQDN(cluster, CoordinatorName(cluster), spec, ordinal) + return memgraph.CoordinatorSpec{ + ID: ordinal + 1, + BoltServer: hostPort(fqdn, spec.ports.bolt), + CoordinatorServer: hostPort(fqdn, spec.ports.coordinator), + ManagementServer: hostPort(fqdn, spec.ports.management), + } +} + // dataInstance describes the data instance running on the given pod ordinal, as // the pod itself advertises it. Retiring instances are described the same way as // declared ones: they are registered under the addresses the operator registered diff --git a/internal/resources/topology_test.go b/internal/resources/topology_test.go index 05abf2a..32e26e7 100644 --- a/internal/resources/topology_test.go +++ b/internal/resources/topology_test.go @@ -146,6 +146,85 @@ func TestRetiringDataInstances(t *testing.T) { } } +// TestRetiringCoordinators covers the range a lowered coordinators count sheds. +// The bounds matter more here than for data instances: every member in the range +// loses a Raft vote, so a range that reached past what the operator applied would +// try to remove a coordinator a human added. +func TestRetiringCoordinators(t *testing.T) { + cases := []struct { + name string + cluster *memgraphcomv1alpha1.MemgraphCluster + applied int32 + want []string + }{ + { + name: "a cluster holding its size retires nothing", + cluster: minimalCluster(), + applied: 3, + }, + { + name: "a growing cluster retires nothing", + cluster: coordinatorsCluster(5), + applied: 3, + }, + // The count must stay odd, so a shrink always retires an even number of + // coordinators and the surviving Raft membership stays odd throughout. + { + name: "both ordinals above the declared count retire at once", + cluster: coordinatorsCluster(3), + applied: 5, + want: []string{"coordinator_4", "coordinator_5"}, + }, + { + name: "a larger shrink retires every ordinal above the declared count", + cluster: coordinatorsCluster(3), + applied: 7, + want: []string{"coordinator_4", "coordinator_5", "coordinator_6", "coordinator_7"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var names []string + for _, coordinator := range resources.RetiringCoordinators(tc.cluster, tc.applied) { + names = append(names, coordinator.Name()) + } + if diff := cmp.Diff(tc.want, names); diff != "" { + t.Errorf("RetiringCoordinators() mismatch (-want +got):\n%s", diff) + } + }) + } +} + +// A retiring coordinator is a Raft member under the ID and addresses the operator +// added it with, so it must be described exactly as the declared coordinator on +// the same ordinal was — otherwise the plan would aim REMOVE COORDINATOR at the +// wrong ID. +func TestRetiringCoordinatorMatchesItsDeclaredForm(t *testing.T) { + // The tuned cluster (non-default ports and cluster domain) declares three + // coordinators. Lowering the count is not possible below three, so the declared + // form is taken from a five-coordinator variant of the same spec. + grown := tunedCluster() + grown.Spec.Coordinators = ptr.To(int32(5)) + declared := resources.DeclaredTopology(grown).Coordinators + + shrunk := tunedCluster() + shrunk.Spec.Coordinators = ptr.To(int32(3)) + got := resources.RetiringCoordinators(shrunk, int32(len(declared))) + + if diff := cmp.Diff(declared[3:], got); diff != "" { + t.Errorf("RetiringCoordinators() mismatch (-want +got):\n%s", diff) + } +} + +// coordinatorsCluster is the minimal cluster with a different coordinators count, +// the spec side of a coordinator scale. +func coordinatorsCluster(coordinators int32) *memgraphcomv1alpha1.MemgraphCluster { + cluster := minimalCluster() + cluster.Spec.Coordinators = ptr.To(coordinators) + return cluster +} + // dataInstancesCluster is the minimal cluster with a lowered dataInstances count, // the spec side of a scale-down. func dataInstancesCluster(dataInstances int32) *memgraphcomv1alpha1.MemgraphCluster { diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index cecb7e7..5e7ebcf 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -59,6 +59,11 @@ const ( // roleMain is the MAIN data-instance role reported in the SHOW INSTANCES role // column. roleMain = "main" + + // roleLeader is the Raft leader coordinator role reported in the same column. + // Which coordinator holds it decides whether a shrink can remove a member at + // all: Raft refuses to remove its own leader. + roleLeader = "leader" ) // example is the parsed quickstart manifest and the source of truth for the @@ -141,10 +146,10 @@ func loadExample() *memgraphcomv1alpha1.MemgraphCluster { func (c clusterUnderTest) declaredInstances() []string { names := make([]string, 0, c.coordinators+c.dataInstances) for ordinal := range c.coordinators { - names = append(names, fmt.Sprintf("coordinator_%d", ordinal+1)) + names = append(names, utils.CoordinatorName(ordinal)) } for ordinal := range c.dataInstances { - names = append(names, fmt.Sprintf("instance_%d", ordinal)) + names = append(names, utils.DataInstanceName(ordinal)) } return names } @@ -536,6 +541,94 @@ spec: Expect(claims).To(ContainElement(fmt.Sprintf("lib-storage-%s-data-2", scalingClusterName)), "whenScaled follows spec.storage.retentionPolicy, which defaults to Retain") }) + + // The coordinator shrink, against the 5-coordinator cluster the specs above left + // behind (Ordered), with Raft leadership deliberately parked on a coordinator + // that has to go — the case the whole safety argument is about: Raft returns + // RAFT_CANNOT_REMOVE_LEADER for its own leader, and a StatefulSet sheds only its + // highest ordinals, so the operator has to move leadership out of the retiring + // range before it can remove anything there. + It("retires the coordinator holding Raft leadership and only then sheds its pod", func() { + // The topology the data shrink left running, and the one this spec drops to. + // Three is the floor, so a shrink from five retires an even number of members + // and the surviving Raft cluster keeps an odd membership throughout. + running := grown.withTopology(5, 2) + shrunk := grown.withTopology(3, 2) + retiring := []string{"coordinator_4", "coordinator_5"} + + By("forcing Raft leadership onto a coordinator the shrink retires") + // Retried as a whole: YIELD LEADERSHIP names no successor, so each attempt + // hands leadership to whichever member NuRaft nominates. Either retiring + // member satisfies the precondition — what the spec is about is leadership + // sitting inside the retiring range, not on one particular ordinal, and + // insisting on one would make the wait depend on which peer NuRaft happens + // to nominate. + Eventually(func(g Gomega) { + g.Expect(running.makeLeader(retiring...)).To(Succeed()) + view, err := running.leaderView() + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(coordinatorLeaderOf(view)).To(BeElementOf(retiring)) + }, 10*time.Minute, 10*time.Second).Should(Succeed()) + + By("lowering the coordinator count on the live cluster") + cmd := exec.Command("kubectl", "patch", "memgraphcluster", scalingClusterName, + "-n", scalingNamespace, "--type=merge", "-p", + fmt.Sprintf(`{"spec":{"coordinators":%d}}`, shrunk.coordinators)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "the operator must accept a lowered coordinator count") + + By("waiting for both retiring members to leave the Raft cluster while their pods are still there") + Eventually(func(g Gomega) { + // The replica count is read before the membership view for the reason the + // data shrink above spells out: a view read afterwards that still lists a + // retiring member proves it held a vote at a moment its pod had already + // been scaled away, which is the order the operator must never produce. + replicas, err := shrunk.replicas("coordinator") + g.Expect(err).NotTo(HaveOccurred()) + + // Read through the five-coordinator view: while leadership is still moving + // it may sit on a retiring ordinal, which the shrunk cluster does not scan. + view, err := running.leaderView() + g.Expect(err).NotTo(HaveOccurred()) + names := instanceNames(view) + + for _, name := range retiring { + if replicas != "5" && slices.Contains(names, name) { + // Not something to retry: the ordering this catches is broken for + // good by the time it is observable. + StopTrying(fmt.Sprintf( + "the coordinator StatefulSet was scaled to %s replicas while %s was still a Raft member", + replicas, name)).Now() + } + g.Expect(names).NotTo(ContainElement(name)) + } + }, 10*time.Minute, 5*time.Second).Should(Succeed()) + + By("waiting for their pods to be shed") + Eventually(func(g Gomega) { + g.Expect(shrunk.replicas("coordinator")).To(Equal("3")) + g.Expect(shrunk.podExists("coordinator", 3)).To(BeFalse()) + g.Expect(shrunk.podExists("coordinator", 4)).To(BeFalse()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + + By("confirming the shrunk cluster is registered, converged, and led by a survivor") + Eventually(shrunk.verifyRegistered, 10*time.Minute, 10*time.Second).Should(Succeed()) + shrunk.awaitConverged(5 * time.Minute) + coordinators, _ := shrunk.registeredCounts() + Expect(coordinators).To(Equal("3")) + view, err := shrunk.leaderView() + Expect(err).NotTo(HaveOccurred()) + Expect(coordinatorLeaderOf(view)).To(BeElementOf("coordinator_1", "coordinator_2", "coordinator_3")) + + By("confirming the retired coordinators' claims are kept by the default retention policy") + claims, err := listPVCs(scalingNamespace) + Expect(err).NotTo(HaveOccurred()) + for _, ordinal := range []int{3, 4} { + Expect(claims).To(ContainElement( + fmt.Sprintf("lib-storage-%s-coordinator-%d", scalingClusterName, ordinal)), + "whenScaled follows spec.storage.retentionPolicy, which defaults to Retain") + } + }) }) // listAdoptedPVCs returns the names of the PersistentVolumeClaims a @@ -671,80 +764,44 @@ func (c clusterUnderTest) podExists(component string, ordinal int32) (bool, erro // wipeInstanceRegistration unregisters the named data instance on the // coordinator leader, simulating registration state a pod loses when it is // rescheduled onto a fresh node. UNREGISTER INSTANCE must run on the leader — -// only it holds the authoritative cluster view — so the leader is located the -// same way leaderView does: the coordinator that reports a MAIN. +// only it holds the authoritative cluster view — which leaderPod locates. func wipeInstanceRegistration(name string) error { - var errs []error - for ordinal := range coordinatorCount { - pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) - view, err := quickstartCluster.showInstances(pod) - if err != nil { - errs = append(errs, err) - continue - } - isLeader := false - for _, instance := range view { - if instance.role == roleMain { - isLeader = true - break - } - } - if !isLeader { - continue - } - cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", - "bash", "-c", fmt.Sprintf("echo 'UNREGISTER INSTANCE %s;' | mgconsole", name)) - if _, err := utils.Run(cmd); err != nil { - return fmt.Errorf("unregistering %s on %s: %w", name, pod, err) - } - return nil + pod, _, err := quickstartCluster.leaderPod() + if err != nil { + return fmt.Errorf("no coordinator leader found to unregister %s: %w", name, err) + } + cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", + "bash", "-c", fmt.Sprintf("echo 'UNREGISTER INSTANCE %s;' | mgconsole", name)) + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("unregistering %s on %s: %w", name, pod, err) } - return fmt.Errorf("no coordinator leader found to unregister %s: %w", name, errors.Join(errs...)) + return nil } // removeCoordinatorRegistration removes a follower coordinator from the Raft // cluster on the coordinator leader, simulating a coordinator that fell out of // the cluster view (e.g. rescheduled onto a fresh node). REMOVE COORDINATOR -// mutates Raft membership, so it must run on the leader — located the same way -// leaderView does: the coordinator that reports a MAIN. A follower is chosen -// (never the leader itself) so the leader keeps the authoritative view it needs -// to accept the removal and observe the operator's re-ADD. It returns the -// instance name of the coordinator it removed. +// mutates Raft membership, so it must run on the leader — which leaderPod +// locates. A follower is chosen (never the leader itself): Raft refuses to remove +// its own leader, and the leader keeps the authoritative view it needs to accept +// the removal and observe the operator's re-ADD. It returns the instance name of +// the coordinator it removed. func removeCoordinatorRegistration() (string, error) { - var errs []error - for ordinal := range coordinatorCount { - pod := fmt.Sprintf("%s-coordinator-%d", clusterName, ordinal) - view, err := quickstartCluster.showInstances(pod) - if err != nil { - errs = append(errs, err) - continue - } - isLeader := false - for _, instance := range view { - if instance.role == roleMain { - isLeader = true - break - } - } - if !isLeader { - continue - } - // The leader hosts coordinator_ordinal+1; remove a different - // coordinator so the leader keeps quorum and its authoritative view. - leaderID := ordinal + 1 - removeID := 1 - if leaderID == 1 { - removeID = 2 - } - name := fmt.Sprintf("coordinator_%d", removeID) - cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", - "bash", "-c", fmt.Sprintf("echo 'REMOVE COORDINATOR %d;' | mgconsole", removeID)) - if _, err := utils.Run(cmd); err != nil { - return "", fmt.Errorf("removing coordinator %d on %s: %w", removeID, pod, err) - } - return name, nil + pod, view, err := quickstartCluster.leaderPod() + if err != nil { + return "", fmt.Errorf("no coordinator leader found to remove a coordinator: %w", err) + } + ordinal := int32(0) + if coordinatorLeaderOf(view) == utils.CoordinatorName(ordinal) { + ordinal = 1 } - return "", fmt.Errorf("no coordinator leader found to remove a coordinator: %w", errors.Join(errs...)) + name := utils.CoordinatorName(ordinal) + cmd := exec.Command("kubectl", "exec", pod, "-n", clusterNamespace, "-c", "memgraph", "--", + "bash", "-c", fmt.Sprintf("echo 'REMOVE COORDINATOR %d;' | mgconsole", ordinal+1)) + if _, err := utils.Run(cmd); err != nil { + return "", fmt.Errorf("removing %s on %s: %w", name, pod, err) + } + return name, nil } // dumpDiagnosticsOnFailure dumps everything needed to debug a broken cluster @@ -858,36 +915,57 @@ type instanceRow struct { role string } -// leaderView returns the SHOW INSTANCES view of the first coordinator that -// reports a MAIN. Only the coordinator leader health-checks data instances and -// reports their roles (followers show them as unknown), so a view containing a -// MAIN is the leader's authoritative view. +// leaderView returns the coordinator leader's SHOW INSTANCES view, the +// authoritative one: only the leader health-checks the data instances it reports +// on. func (c clusterUnderTest) leaderView() ([]instanceRow, error) { _, view, err := c.leaderPod() return view, err } -// leaderPod locates the coordinator leader — the coordinator whose view reports a -// MAIN — and returns its pod name together with that view. Management queries a -// test issues by hand have to run there: only the leader holds the authoritative -// cluster state and accepts a mutation of it. +// leaderPod locates the coordinator leader and returns its pod name together with +// its view. Management queries a test issues by hand have to run there: only the +// leader holds the authoritative cluster state and accepts a mutation of it — a +// follower rejects YIELD LEADERSHIP, SET INSTANCE TO MAIN and REMOVE COORDINATOR +// outright. +// +// The leader is read out of the role column, never inferred from a view reporting +// a MAIN: a coordinator forwards SHOW INSTANCES to the leader and answers with the +// leader's view, so every coordinator reports the MAIN and only the role column +// says who holds Raft leadership. That same forwarding is why one read is enough — +// the view a follower returns is already the authoritative one, and only the pod to +// send mutations to has to be looked up from it. func (c clusterUnderTest) leaderPod() (string, []instanceRow, error) { var errs []error for ordinal := range c.coordinators { - pod := fmt.Sprintf("%s-coordinator-%d", c.name, ordinal) + pod := c.coordinatorPod(ordinal) view, err := c.showInstances(pod) if err != nil { errs = append(errs, err) continue } - if mainOf(view) != "" { - return pod, view, nil + leader := coordinatorLeaderOf(view) + if leader == "" { + errs = append(errs, fmt.Errorf("%s reports no coordinator leader among %d instances", + pod, len(view))) + continue } - errs = append(errs, fmt.Errorf("%s reports no MAIN among %d instances", pod, len(view))) + leaderOrdinal, err := utils.CoordinatorOrdinal(leader) + if err != nil { + errs = append(errs, fmt.Errorf("%s named %s as leader: %w", pod, leader, err)) + continue + } + return c.coordinatorPod(leaderOrdinal), view, nil } return "", nil, errors.Join(errs...) } +// coordinatorPod is the pod the coordinator with the given StatefulSet ordinal +// runs in. +func (c clusterUnderTest) coordinatorPod(ordinal int32) string { + return fmt.Sprintf("%s-coordinator-%d", c.name, ordinal) +} + // makeMain moves MAIN onto the named data instance by hand, which is how a spec // arranges for the instance a scale-down retires to be the one holding MAIN. // @@ -914,6 +992,40 @@ func (c clusterUnderTest) makeMain(name string) error { return nil } +// makeLeader nudges Raft leadership toward one of the named coordinators by +// yielding it on whichever coordinator currently holds it, which is how a spec +// parks leadership where a scale-down cannot remove it. +// +// One call is an attempt, not a guarantee: YIELD LEADERSHIP takes no successor, so +// NuRaft decides who takes over. Callers retry until one of the targets wins. It is +// a no-op when a target already holds leadership. +func (c clusterUnderTest) makeLeader(names ...string) error { + pod, view, err := c.leaderPod() + if err != nil { + return err + } + if slices.Contains(names, coordinatorLeaderOf(view)) { + return nil + } + cmd := exec.Command("kubectl", "exec", pod, "-n", c.namespace, "-c", "memgraph", "--", + "bash", "-c", "echo 'YIELD LEADERSHIP;' | mgconsole") + if _, err := utils.Run(cmd); err != nil { + return fmt.Errorf("yielding leadership on %s: %w", pod, err) + } + return nil +} + +// coordinatorLeaderOf returns the coordinator a view reports as Raft leader, or the +// empty string when it reports none. +func coordinatorLeaderOf(view []instanceRow) string { + for _, instance := range view { + if instance.role == roleLeader { + return instance.name + } + } + return "" +} + // mainOf returns the name of the data instance a view reports as MAIN, or the // empty string when it reports none. func mainOf(view []instanceRow) string { diff --git a/test/utils/names.go b/test/utils/names.go new file mode 100644 index 0000000..c0bd73b --- /dev/null +++ b/test/utils/names.go @@ -0,0 +1,47 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "fmt" + +// The names a cluster member appears under in SHOW INSTANCES, derived from its +// StatefulSet pod ordinal. They are spelled out here rather than taken from the +// operator's own packages on purpose: a test that asked production code what it +// named an instance could never catch it renaming one. + +// CoordinatorName is the SHOW INSTANCES name of the coordinator in the pod with +// the given ordinal: ordinal N registers as coordinator_N+1, because Raft +// coordinator IDs start at one. +func CoordinatorName(ordinal int32) string { + return fmt.Sprintf("coordinator_%d", ordinal+1) +} + +// CoordinatorOrdinal is the inverse of CoordinatorName: the ordinal of the pod +// running the coordinator a view names. +func CoordinatorOrdinal(name string) (int32, error) { + var id int32 + if _, err := fmt.Sscanf(name, "coordinator_%d", &id); err != nil { + return 0, fmt.Errorf("parsing coordinator name %q: %w", name, err) + } + return id - 1, nil +} + +// DataInstanceName is the SHOW INSTANCES name of the data instance in the pod +// with the given ordinal: ordinal N registers as instance_N. +func DataInstanceName(ordinal int32) string { + return fmt.Sprintf("instance_%d", ordinal) +} From f8da2b8ecc50631e5aeebda211a5facdb4736dda Mon Sep 17 00:00:00 2001 From: as51340 Date: Wed, 29 Jul 2026 11:44:52 +0200 Subject: [PATCH 25/34] fix: Avoid two connections on observeCluster --- .../controller/memgraphcluster_controller.go | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 79d90f8..25dfbcb 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -586,6 +586,10 @@ func (r *MemgraphClusterReconciler) writeStatus( // ready. Registration waits for the full topology: coordinators cannot form a // Raft cluster and data instances cannot be registered until every advertised // address resolves to a running pod. +// +// A StatefulSet the apply just created is not ready, not an error: the read goes +// through the informer cache, which lags the apply, so an absent StatefulSet is +// the same waiting state as one whose pods have not come up yet. func (r *MemgraphClusterReconciler) workloadsReady( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, @@ -593,6 +597,9 @@ func (r *MemgraphClusterReconciler) workloadsReady( for _, name := range []string{resources.CoordinatorName(cluster), resources.DataName(cluster)} { var sts appsv1.StatefulSet if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: cluster.Namespace}, &sts); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } return false, fmt.Errorf("getting StatefulSet %s: %w", name, err) } if sts.Spec.Replicas == nil || sts.Status.ReadyReplicas < *sts.Spec.Replicas { @@ -611,7 +618,11 @@ var errNoCoordinatorLeader = errors.New("no coordinator reported a leader") // observeCluster connects to the coordinator leader and returns its client // together with the SHOW INSTANCES view the planner diffs against. // Coordinators are tried in ordinal order: one reporting itself leader is used -// directly, and one reporting another coordinator as leader redirects to it. +// directly, and one reporting another coordinator as leader hands over the +// address to connect to. The view is read once either way — a follower forwards +// SHOW INSTANCES to the leader, so what it answers with is already the leader's +// view, and only the planner's mutating commands need the leader connection +// itself. // // A coordinator that names no leader is skipped, never used as planning input. // Its view is not the fresh-cluster case: a coordinator starts with itself as @@ -654,19 +665,21 @@ func (r *MemgraphClusterReconciler) observeCluster( continue } - // This coordinator is a follower; redirect to the leader it reports. + // This coordinator is a follower, so it answered with the leader's + // forwarded view: keep that view and open the connection the mutating + // commands need on the leader itself. address, found := leaderAddress(topology, observed, leaderName) if !found { errs = append(errs, fmt.Errorf("%s reported leader %s without a Bolt address", coordinator.Name(), leaderName)) continue } - conn, observed, err = r.showInstances(ctx, address) + leader, err := r.Memgraph.Connect(ctx, address) if err != nil { errs = append(errs, err) continue } - return conn, observed, nil + return leader, observed, nil } if leaderless { return nil, nil, fmt.Errorf("%w: %w", errNoCoordinatorLeader, errors.Join(errs...)) From c779f7213f7d1ac3985f8ffa8bf1a688a2e2a347 Mon Sep 17 00:00:00 2001 From: as51340 Date: Wed, 29 Jul 2026 13:03:35 +0200 Subject: [PATCH 26/34] feat: Take into account replication lag before demotion --- CLAUDE.md | 4 +- api/v1alpha1/memgraphcluster_types.go | 9 + internal/controller/fake_memgraph_test.go | 56 ++- .../controller/memgraphcluster_controller.go | 31 +- .../memgraphcluster_controller_test.go | 55 +++ internal/memgraph/bolt.go | 87 +++++ internal/memgraph/client.go | 63 +++- internal/memgraph/queries.go | 5 + internal/memgraph/queries_test.go | 170 +++++++++ internal/planner/planner.go | 140 ++++++- internal/planner/planner_test.go | 342 +++++++++++++++++- 11 files changed, 931 insertions(+), 31 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e47cad8..e337bfc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -51,8 +51,8 @@ The PRD defines seven modules with two pure cores and one mock seam. Keep this s 1. **API types** (`api/v1alpha1/`) — spec/status with kubebuilder + CEL markers. Replica counts (coordinators, data instances) are **mutable in both directions**, bounded by schema floors alone (coordinators ≥ 3 and odd, data instances ≥ 1) — enforced by the CRD, not a webhook. Defaults are declared as CRD schema defaults *and* mirrored as Go constants in `memgraphcluster_types.go` so resource builders behave correctly on specs that never passed admission (unit tests). 2. **Resource builders** — pure functions: spec in, desired Kubernetes objects out (one StatefulSet per role + headless Services; per-pod identity such as coordinator ID and advertised addresses derived from pod ordinals). No API calls, no side effects. Tested golden-style. -3. **Memgraph HA client** — a narrow Go interface (show instances, register instance, add coordinator, set main, demote/unregister instance, remove coordinator, yield leadership) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. -4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. A MAIN is promoted only when the cluster has none — at bootstrap, and after the planner itself demoted a data instance that is retiring; a MAIN that is staying is never overridden, because failover belongs to the Raft coordinators. A lowered count is the one removal the planner drives: `DEMOTE INSTANCE` the retiring MAIN, promote a survivor, `UNREGISTER INSTANCE` the retiring data instances, `REMOVE COORDINATOR` the retiring coordinators, and only then does the controller shed their pods (see `specs/operator-mvp/issues/15`, `16`). One command breaks the pure-diff mould: `YIELD LEADERSHIP`, needed because Raft refuses to remove its own leader. It names no successor, so it is always a plan's **last** command and terminal — the controller requeues and re-observes under whichever coordinator won the election. +3. **Memgraph HA client** — a narrow Go interface (show instances, show replication lag, register instance, add coordinator, set main, demote/unregister instance, remove coordinator, yield leadership) over the Bolt driver. Everything above depends on the interface, never the driver — this is the mock seam. +4. **Registration planner** — pure diff: declared topology + observed `SHOW INSTANCES` in, ordered registration commands out (empty when converged). Reconciliation semantics live here: read-before-write, idempotent, re-issue only missing registrations. A MAIN is promoted only when the cluster has none — at bootstrap, and after the planner itself demoted a data instance that is retiring; a MAIN that is staying is never overridden, because failover belongs to the Raft coordinators. A lowered count is the one removal the planner drives: `DEMOTE INSTANCE` the retiring MAIN, promote a survivor, `UNREGISTER INSTANCE` the retiring data instances, `REMOVE COORDINATOR` the retiring coordinators, and only then does the controller shed their pods (see `specs/operator-mvp/issues/15`, `16`). Moving MAIN off a retiring instance is the one step that can lose data, so it has a precondition nothing else does: a survivor that is both reachable and reported by `SHOW REPLICATION LAG` as holding every transaction the MAIN committed, in every database. Without one, the demotion, the promotion and that instance's unregistration are all left out of the plan and the retiring MAIN keeps serving — a scale-down that pauses, not one that drops writes. Because that state plans *nothing*, an empty plan is not proof a retirement finished: `planner.Retired` is what gates shedding the pods. One command breaks the pure-diff mould: `YIELD LEADERSHIP`, needed because Raft refuses to remove its own leader. It names no successor, so it is always a plan's **last** command and terminal — the controller requeues and re-observes under whichever coordinator won the election. 5. **Controller** (`internal/controller/`) — fetch CR, server-side-apply builder output, gate on pod readiness, run planner against the HA client, write status/conditions. 6. **Operator install chart** (`charts/memgraph-operator/`) — lives in this repo, cross-published to `memgraph.github.io/helm-charts` at release. Its `crds/` and `rbac/manager-rules.yaml` are **generated** (`make chart-sync`, verified by `make chart-verify`): the manager's ClusterRole comes from the `+kubebuilder:rbac` markers, so tightening or widening the controller's permissions means editing the markers, never the chart. The e2e suite installs the operator through this chart, so every scenario runs under the RBAC users get. The chart's `version` and its `appVersion` (the operator image tag) move **independently**: tag `v` releases the operator, `chart-` releases the chart alone — see `docs/releasing.md`. 7. **E2E harness** (`test/e2e/`, build tag `e2e`) — multi-node KinD with real Memgraph images. diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index d78027b..74797dc 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -177,6 +177,15 @@ const ( // again — which is exactly what this reason means when it persists. ReasonLeadershipTransferInProgress = "LeadershipTransferInProgress" + // ReasonNoCaughtUpSurvivor is set while a lowered dataInstances count is + // waiting to move MAIN off the instance it retires: no surviving instance is + // both reachable and holding every transaction the MAIN has committed, so + // demoting it now would drop those writes. The retiring MAIN keeps serving + // until one catches up, which is a scale-down that pauses rather than one that + // loses data. It persisting means replication is not progressing — the + // survivors are down, or too far behind to catch up. + ReasonNoCaughtUpSurvivor = "NoCaughtUpSurvivor" + // ReasonMainElected is set when a data instance is observed as MAIN. ReasonMainElected = "MainElected" diff --git a/internal/controller/fake_memgraph_test.go b/internal/controller/fake_memgraph_test.go index 69734fb..61a2f11 100644 --- a/internal/controller/fake_memgraph_test.go +++ b/internal/controller/fake_memgraph_test.go @@ -41,7 +41,12 @@ type fakeMemgraph struct { // a coordinator that lost the leader answers from its own state machine, so // what it reports need not match the cluster at all. Commands still land on // the shared view — a stale coordinator is never written to. - staleViews map[string][]memgraph.Instance + staleViews map[string][]memgraph.Instance + // behind is how many transactions a data instance trails the MAIN by, for the + // instances a test puts behind. A registered instance absent from the map is + // caught up, which is what a healthy cluster looks like — falling behind is the + // exceptional state a test opts into. + behind map[string]int64 connectAttempts int // connectErr, when set, makes every Connect fail — the operator's view of a // cluster whose coordinators do not yet answer Bolt. @@ -105,6 +110,17 @@ func (f *fakeMemgraph) setLeader(name string) { } } +// setBehind puts the named data instance the given number of transactions behind +// the MAIN, which is how a spec keeps it from being promoted. +func (f *fakeMemgraph) setBehind(name string, txns int64) { + f.mu.Lock() + defer f.mu.Unlock() + if f.behind == nil { + f.behind = map[string]int64{} + } + f.behind[name] = txns +} + // setStaleView makes the coordinator at the given Bolt address answer // SHOW INSTANCES with its own view instead of the cluster's. func (f *fakeMemgraph) setStaleView(address string, instances []memgraph.Instance) { @@ -149,6 +165,44 @@ func (c *fakeClient) ShowInstances(context.Context) ([]memgraph.Instance, error) return view, nil } +// ShowReplicationLag answers as the real query does: the counts come from the +// MAIN, so a cluster with no MAIN reports nothing at all rather than failing, and +// the MAIN reports itself at zero behind. Every other registered data instance is +// caught up unless a test put it behind. +func (c *fakeClient) ShowReplicationLag(context.Context) ([]memgraph.ReplicationLag, error) { + c.cluster.mu.Lock() + defer c.cluster.mu.Unlock() + if c.closed { + return nil, fmt.Errorf("fake memgraph: connection to %s already closed", c.address) + } + if !slices.ContainsFunc(c.cluster.instances, func(instance memgraph.Instance) bool { return instance.IsMain() }) { + return nil, nil + } + + var lag []memgraph.ReplicationLag + for _, instance := range c.cluster.instances { + if strings.HasPrefix(instance.Name, "coordinator_") { + continue + } + // Lag is measured against the MAIN, so the MAIN is zero behind itself + // whatever a test set for it — what it set describes the instance as a + // replica, which is what it becomes once demoted. + behind := c.cluster.behind[instance.Name] + if instance.IsMain() { + behind = 0 + } + lag = append(lag, memgraph.ReplicationLag{ + Instance: instance.Name, + Databases: []memgraph.DatabaseLag{{ + Database: "memgraph", + CommittedTxns: 100 - behind, + TxnsBehindMain: behind, + }}, + }) + } + return lag, nil +} + func (c *fakeClient) AddCoordinator(_ context.Context, coordinator memgraph.CoordinatorSpec) error { return c.execute(fmt.Sprintf("ADD COORDINATOR %d", coordinator.ID), func() error { for i, instance := range c.cluster.instances { diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 25dfbcb..920b7dd 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -373,13 +373,36 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( } }() + // How far behind the MAIN each data instance is, which is what decides whether + // a retiring MAIN can hand over. The read is not allowed to fail the pass: it + // is consulted for that one decision, every other one has to keep working + // without it, and an empty view already means "no survivor is known to be + // caught up" — the same conclusion, reached by the planner. + lag, err := leader.ShowReplicationLag(ctx) + if err != nil { + log.Info("Could not read replication lag, so no MAIN handover will be planned", "reason", err.Error()) + lag = nil + } + latest := observe(topology, observed) - commands := planner.Plan(topology, observed) + commands := planner.Plan(topology, observed, lag) if len(commands) == 0 { - // No retiring member belongs to the cluster any more — the plan would - // carry an UNREGISTER INSTANCE or a REMOVE COORDINATOR otherwise — so - // their pods can go. if retiring != "" { + // An empty plan is not on its own proof that the retirement finished: a + // handover waiting for a caught-up survivor plans nothing either, because + // no command would make a lagging replica ready. Shedding pods on that + // would delete a registered MAIN, so the membership is what gates it. + if !planner.Retired(topology, observed) { + log.Info("Deferred retirement because no surviving data instance is caught up with MAIN") + msg := "Waiting for a surviving data instance that is reachable and caught up with MAIN " + + "before moving MAIN off the instance being retired" + if statusErr := r.writeStatus(ctx, cluster, latest, readyOrNot(latest.main), + notConvergedCondition(memgraphcomv1alpha1.ReasonNoCaughtUpSurvivor, msg), + ); statusErr != nil { + return ctrl.Result{}, statusErr + } + return ctrl.Result{RequeueAfter: requeueWhilePending}, nil + } if err := r.shedRetiredPods(ctx, cluster, topology, replicas); err != nil { return ctrl.Result{}, err } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index e77a45d..eb9e134 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -982,6 +982,61 @@ var _ = Describe("MemgraphCluster Controller", func() { memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) }) + // Moving MAIN is the one step of a retirement that can lose data, so it waits + // for a survivor that actually holds the writes. Everything else about the + // scale-down waits with it: the demotion would leave the cluster serving from + // an instance missing transactions, and the unregistration cannot precede the + // demotion at all. + It("should not move MAIN off a retiring instance while every survivor is behind", func() { + fake.setInstances(convergedWithMainOn(1)) + fake.setBehind("instance_0", 12) + baseline := bootstrapped() + Expect(status().Main).To(Equal("instance_1")) + + setCounts(3, 1) + reconcileCluster(resourceName) + + Expect(sinceBootstrap(baseline)).To(BeEmpty(), + "no demotion, no promotion, and no unregistration of a MAIN that cannot be demoted") + Expect(replicas(dataSuffix)).To(Equal(int32(2)), + "the retiring pod must outlive its registration, so the count is held") + converged := convergedCondition() + Expect(converged.Status).To(Equal(metav1.ConditionFalse)) + Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonNoCaughtUpSurvivor)) + // The retiring MAIN is still MAIN, and still serving: a paused scale-down + // costs availability nothing, which is what makes waiting the better trade. + Expect(apimeta.IsStatusConditionTrue(status().Conditions, + memgraphcomv1alpha1.ConditionReady)).To(BeTrue()) + Expect(status().Main).To(Equal("instance_1")) + + By("holding there for as long as the survivor stays behind") + reconcileCluster(resourceName) + Expect(sinceBootstrap(baseline)).To(BeEmpty()) + Expect(replicas(dataSuffix)).To(Equal(int32(2))) + Expect(convergedCondition().Reason).To(Equal(memgraphcomv1alpha1.ReasonNoCaughtUpSurvivor)) + + By("handing MAIN over once the survivor has caught up") + fake.setBehind("instance_0", 0) + reconcileCluster(resourceName) + + leader := coordinatorAddress(0) + Expect(sinceBootstrap(baseline)).To(Equal([]string{ + leader + ": DEMOTE INSTANCE instance_1", + leader + ": SET INSTANCE instance_0 TO MAIN", + leader + ": UNREGISTER INSTANCE instance_1", + }), "the retirement resumes from where it stalled, in one pass") + + reconcileCluster(resourceName) + Expect(replicas(dataSuffix)).To(Equal(int32(1))) + reconcileCluster(resourceName) + + s := status() + Expect(s.Main).To(Equal("instance_0")) + Expect(s.DataInstances).To(Equal(int32(1))) + Expect(apimeta.IsStatusConditionTrue(s.Conditions, + memgraphcomv1alpha1.ConditionConverged)).To(BeTrue()) + }) + // The readiness gate covers the pods on their way out too: they belong to // the StatefulSet the operator is still holding at its current size. A // retiring pod that cannot become ready therefore blocks its own removal, diff --git a/internal/memgraph/bolt.go b/internal/memgraph/bolt.go index b587bda..3d168dc 100644 --- a/internal/memgraph/bolt.go +++ b/internal/memgraph/bolt.go @@ -19,6 +19,8 @@ package memgraph import ( "context" "fmt" + "maps" + "slices" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j/db" @@ -60,6 +62,22 @@ func (c *boltClient) ShowInstances(ctx context.Context) ([]Instance, error) { return instances, nil } +func (c *boltClient) ShowReplicationLag(ctx context.Context) ([]ReplicationLag, error) { + records, err := c.run(ctx, showReplicationLagQuery) + if err != nil { + return nil, err + } + lag := make([]ReplicationLag, 0, len(records)) + for _, record := range records { + instance, err := replicationLagFromRecord(record) + if err != nil { + return nil, err + } + lag = append(lag, instance) + } + return lag, nil +} + func (c *boltClient) AddCoordinator(ctx context.Context, coordinator CoordinatorSpec) error { _, err := c.run(ctx, addCoordinatorQuery(coordinator)) return err @@ -130,6 +148,53 @@ func instanceFromRecord(record *db.Record) Instance { } } +// replicationLagFromRecord maps one SHOW REPLICATION LAG row to a +// ReplicationLag: an instance_name and a data_info map keyed by database name, +// each entry carrying that database's counters. +// +// Unlike instanceFromRecord this refuses a row it cannot read rather than +// filling in zero values. The leniency there is safe because a missing column +// leaves an empty string, which reads as neither up nor MAIN; here a missing +// counter would read as zero transactions behind, which is precisely the answer +// that makes an instance promotable. This view must never invent that. +func replicationLagFromRecord(record *db.Record) (ReplicationLag, error) { + name := stringColumn(record, "instance_name") + if name == "" { + return ReplicationLag{}, fmt.Errorf("%s row carries no instance_name", showReplicationLagQuery) + } + databases, ok := mapColumn(record, "data_info") + if !ok { + return ReplicationLag{}, fmt.Errorf("%s row for %s carries no data_info map", showReplicationLagQuery, name) + } + + lag := ReplicationLag{Instance: name} + // Databases come out ordered by name so the view a caller compares is stable + // across reconciles; the driver hands back an unordered map. + for _, database := range slices.Sorted(maps.Keys(databases)) { + counters, ok := asMap(databases[database]) + if !ok { + return ReplicationLag{}, fmt.Errorf("%s row for %s carries no counters for database %s", + showReplicationLagQuery, name, database) + } + committed, ok := intEntry(counters, "num_committed_txns") + if !ok { + return ReplicationLag{}, fmt.Errorf("%s row for %s is missing num_committed_txns for database %s", + showReplicationLagQuery, name, database) + } + behind, ok := intEntry(counters, "num_txns_behind_main") + if !ok { + return ReplicationLag{}, fmt.Errorf("%s row for %s is missing num_txns_behind_main for database %s", + showReplicationLagQuery, name, database) + } + lag.Databases = append(lag.Databases, DatabaseLag{ + Database: database, + CommittedTxns: committed, + TxnsBehindMain: behind, + }) + } + return lag, nil +} + func stringColumn(record *db.Record, key string) string { value, ok := record.Get(key) if !ok { @@ -141,3 +206,25 @@ func stringColumn(record *db.Record, key string) string { } return s } + +func mapColumn(record *db.Record, key string) (map[string]any, bool) { + value, ok := record.Get(key) + if !ok { + return nil, false + } + return asMap(value) +} + +func asMap(value any) (map[string]any, bool) { + m, ok := value.(map[string]any) + return m, ok +} + +func intEntry(m map[string]any, key string) (int64, bool) { + value, ok := m[key] + if !ok { + return 0, false + } + i, ok := value.(int64) + return i, ok +} diff --git a/internal/memgraph/client.go b/internal/memgraph/client.go index aa6f6bd..9200d97 100644 --- a/internal/memgraph/client.go +++ b/internal/memgraph/client.go @@ -15,12 +15,13 @@ limitations under the License. */ // Package memgraph provides the narrow client surface the operator uses to -// drive a Memgraph high-availability cluster over Bolt: show instances, add -// coordinator, register instance, set main, and — for the members a lowered -// replica count is retiring — demote and unregister a data instance, yield -// coordinator leadership and remove a coordinator. All higher layers depend on -// the Client and Connector interfaces, never on the Bolt driver — this package -// is the mock seam for testing and the only place the driver is referenced. +// drive a Memgraph high-availability cluster over Bolt: show instances, show +// replication lag, add coordinator, register instance, set main, and — for the +// members a lowered replica count is retiring — demote and unregister a data +// instance, yield coordinator leadership and remove a coordinator. All higher +// layers depend on the Client and Connector interfaces, never on the Bolt driver +// — this package is the mock seam for testing and the only place the driver is +// referenced. package memgraph import ( @@ -93,10 +94,60 @@ type DataInstanceSpec struct { ReplicationServer string } +// DatabaseLag is one database's replication progress on one data instance: how +// many transactions it has committed, and how many that leaves it behind the +// MAIN. The count behind can be negative for a moment after a failover — a SYNC +// replica can hold transactions the new MAIN never saw — so "not behind" is the +// condition worth testing, never "exactly equal". +type DatabaseLag struct { + Database string + CommittedTxns int64 + TxnsBehindMain int64 +} + +// ReplicationLag is one row of SHOW REPLICATION LAG: one data instance's +// replication progress across every database it holds. The MAIN reports itself +// too, at zero behind, because the lag of every other instance is measured +// against it. +type ReplicationLag struct { + Instance string + Databases []DatabaseLag +} + +// IsCaughtUp reports whether the instance holds every transaction the MAIN has +// committed, in every one of its databases — which is what makes it promotable +// without losing writes. +// +// An instance with no databases reported is not caught up. That is the answer +// for anything the view does not cover: an instance the MAIN does not list, or a +// whole view that came back empty because there is no MAIN to measure against. +// Unknown has to read as "not safe to promote", because the alternative is +// promoting on an assumption and discarding whatever the survivor never received. +func (l ReplicationLag) IsCaughtUp() bool { + if len(l.Databases) == 0 { + return false + } + for _, database := range l.Databases { + if database.TxnsBehindMain > 0 { + return false + } + } + return true +} + // Client is the narrow surface of a single coordinator's Bolt endpoint. Every // method issues exactly one HA management query. type Client interface { ShowInstances(ctx context.Context) ([]Instance, error) + + // ShowReplicationLag reports how far behind the MAIN every data instance the + // cluster knows is, counted in committed transactions. Only a coordinator + // answers it, and the answer is relayed from the MAIN itself — so a cluster + // with no MAIN, or one whose MAIN the coordinator leader cannot reach, reports + // no rows rather than failing. An empty view therefore means "cannot tell", + // which is why nothing is promoted on the strength of it. + ShowReplicationLag(ctx context.Context) ([]ReplicationLag, error) + AddCoordinator(ctx context.Context, coordinator CoordinatorSpec) error RegisterInstance(ctx context.Context, instance DataInstanceSpec) error SetInstanceToMain(ctx context.Context, name string) error diff --git a/internal/memgraph/queries.go b/internal/memgraph/queries.go index 9b01e2c..c5841e0 100644 --- a/internal/memgraph/queries.go +++ b/internal/memgraph/queries.go @@ -25,6 +25,11 @@ import "fmt" const showInstancesQuery = "SHOW INSTANCES" +// showReplicationLagQuery is answered only by a coordinator, which relays the +// counts from the MAIN. Like YIELD LEADERSHIP it takes no argument: the answer +// covers every instance the cluster knows at once. +const showReplicationLagQuery = "SHOW REPLICATION LAG" + func addCoordinatorQuery(coordinator CoordinatorSpec) string { return fmt.Sprintf( `ADD COORDINATOR %d WITH CONFIG {"bolt_server": %q, "coordinator_server": %q, "management_server": %q}`, diff --git a/internal/memgraph/queries_test.go b/internal/memgraph/queries_test.go index ecb944e..e14b6d2 100644 --- a/internal/memgraph/queries_test.go +++ b/internal/memgraph/queries_test.go @@ -25,6 +25,19 @@ import ( const testInstanceName = "instance_1" +// The SHOW REPLICATION LAG column and map keys, plus the two databases the cases +// below report on. They are spelled out here rather than shared with the parser +// on purpose: pinning the names the parser reads off the wire is what these tests +// are for, and a constant shared with it would let a rename pass unnoticed. +const ( + instanceNameColumn = "instance_name" + dataInfoColumn = "data_info" + committedTxnsKey = "num_committed_txns" + behindMainKey = "num_txns_behind_main" + defaultDatabase = "memgraph" + otherDatabase = "analytics" +) + func TestAddCoordinatorQuery(t *testing.T) { got := addCoordinatorQuery(CoordinatorSpec{ ID: 2, @@ -94,6 +107,163 @@ func TestYieldLeadershipQuery(t *testing.T) { } } +// SHOW REPLICATION LAG takes no argument: one call covers every instance the +// cluster knows, so a query that grew one would mean the caller had to ask per +// instance and could no longer compare them from a single view. +func TestShowReplicationLagQuery(t *testing.T) { + if want := "SHOW REPLICATION LAG"; showReplicationLagQuery != want { + t.Errorf("showReplicationLagQuery = %q, want %q", showReplicationLagQuery, want) + } +} + +func TestReplicationLagFromRecord(t *testing.T) { + record := &db.Record{ + Keys: []string{instanceNameColumn, dataInfoColumn}, + Values: []any{testInstanceName, map[string]any{ + defaultDatabase: map[string]any{ + committedTxnsKey: int64(42), + behindMainKey: int64(0), + }, + // Sorted by database name, so this one comes out first. + otherDatabase: map[string]any{ + committedTxnsKey: int64(40), + behindMainKey: int64(2), + }, + }}, + } + want := ReplicationLag{ + Instance: testInstanceName, + Databases: []DatabaseLag{ + {Database: otherDatabase, CommittedTxns: 40, TxnsBehindMain: 2}, + {Database: defaultDatabase, CommittedTxns: 42, TxnsBehindMain: 0}, + }, + } + + got, err := replicationLagFromRecord(record) + if err != nil { + t.Fatalf("replicationLagFromRecord() error = %v", err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("replicationLagFromRecord() mismatch (-want +got):\n%s", diff) + } +} + +// A row the parsing cannot read has to fail rather than default. Every field it +// could default is zero, and zero transactions behind is exactly the answer that +// makes an instance promotable — so a malformed row must never be quietly read as +// a caught-up one. +func TestReplicationLagFromRecordRejectsUnreadableRows(t *testing.T) { + cases := []struct { + name string + record *db.Record + }{ + { + name: "no instance_name", + record: &db.Record{Keys: []string{dataInfoColumn}, Values: []any{map[string]any{}}}, + }, + { + name: "no data_info", + record: &db.Record{Keys: []string{instanceNameColumn}, Values: []any{testInstanceName}}, + }, + { + name: "data_info is not a map of maps", + record: &db.Record{ + Keys: []string{instanceNameColumn, dataInfoColumn}, + Values: []any{testInstanceName, map[string]any{defaultDatabase: int64(3)}}, + }, + }, + { + name: "no num_txns_behind_main", + record: &db.Record{ + Keys: []string{instanceNameColumn, dataInfoColumn}, + Values: []any{testInstanceName, map[string]any{ + defaultDatabase: map[string]any{committedTxnsKey: int64(42)}, + }}, + }, + }, + { + name: "no num_committed_txns", + record: &db.Record{ + Keys: []string{instanceNameColumn, dataInfoColumn}, + Values: []any{testInstanceName, map[string]any{ + defaultDatabase: map[string]any{behindMainKey: int64(0)}, + }}, + }, + }, + { + name: "counters are not integers", + record: &db.Record{ + Keys: []string{instanceNameColumn, dataInfoColumn}, + Values: []any{testInstanceName, map[string]any{ + defaultDatabase: map[string]any{committedTxnsKey: "42", behindMainKey: "0"}, + }}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := replicationLagFromRecord(tc.record); err == nil { + t.Error("replicationLagFromRecord() succeeded on an unreadable row, want error") + } + }) + } +} + +func TestReplicationLagIsCaughtUp(t *testing.T) { + cases := []struct { + name string + lag ReplicationLag + want bool + }{ + { + name: "every database at the MAIN's offset", + lag: ReplicationLag{Instance: testInstanceName, Databases: []DatabaseLag{ + {Database: otherDatabase, TxnsBehindMain: 0}, + {Database: defaultDatabase, TxnsBehindMain: 0}, + }}, + want: true, + }, + { + name: "behind in one database of several", + lag: ReplicationLag{Instance: testInstanceName, Databases: []DatabaseLag{ + {Database: otherDatabase, TxnsBehindMain: 0}, + {Database: defaultDatabase, TxnsBehindMain: 1}, + }}, + want: false, + }, + // SYNC replication can leave a replica holding transactions a newly promoted + // MAIN never saw. Ahead is not behind, so it does not disqualify. + { + name: "ahead of the MAIN", + lag: ReplicationLag{Instance: testInstanceName, Databases: []DatabaseLag{ + {Database: defaultDatabase, TxnsBehindMain: -2}, + }}, + want: true, + }, + // Nothing reported is not the same as nothing behind: an instance the MAIN + // does not list is one whose progress is unknown. + { + name: "no databases reported", + lag: ReplicationLag{Instance: testInstanceName}, + want: false, + }, + { + name: "the zero value, as a lookup miss reads back", + lag: ReplicationLag{}, + want: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.lag.IsCaughtUp(); got != tc.want { + t.Errorf("IsCaughtUp() = %t, want %t", got, tc.want) + } + }) + } +} + func TestInstanceFromRecord(t *testing.T) { record := &db.Record{ Keys: []string{ diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 2be75c5..e667d54 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -28,6 +28,13 @@ limitations under the License. // UNREGISTER INSTANCE, and a retiring coordinator is only ever removed from Raft // while it is not the leader. // +// Moving MAIN off a retiring instance is the one thing here that can lose data, +// so it has a precondition the rest do not: a survivor that is both reachable and +// fully caught up with the MAIN, per SHOW REPLICATION LAG. Without one the +// retirement is planned as nothing at all and the retiring MAIN keeps serving — +// waiting is always better than promoting onto an instance that never received +// the writes. +// // One command breaks the pure-diff mould: YIELD LEADERSHIP, which moves // coordinator leadership off a retiring coordinator so it can be removed at all. // It cannot name a successor, so its outcome is the one thing the planner cannot @@ -195,6 +202,15 @@ func (c YieldLeadership) String() string { // MAIN, and so the cluster is MAIN-less only for the few milliseconds between two // queries of the same pass. // +// The handover off a retiring MAIN is conditional on a survivor being able to +// take it: reachable, and holding every transaction the MAIN has committed. When +// none is, the demotion, the promotion and that instance's unregistration are all +// left out of the plan — the retiring MAIN stays MAIN and stays registered, and a +// later pass tries again once a survivor has caught up. The rest of the +// retirement still goes out: retiring instances that are not MAIN are +// unregistered, and retiring coordinators are removed, because neither depends on +// where MAIN sits. +// // A retiring coordinator that holds Raft leadership cannot be removed at all, so // the plan ends with YIELD LEADERSHIP instead and stops there — the retiring // coordinators that are not the leader still go out ahead of it in that same @@ -205,16 +221,31 @@ func (c YieldLeadership) String() string { // Instances the cluster knows but the topology neither declares nor retires are // left untouched: the retiring set is bounded by the operator's own prior apply, // so an instance a human registered is never removed. -func Plan(declared Topology, observed []memgraph.Instance) []Command { +func Plan(declared Topology, observed []memgraph.Instance, lag []memgraph.ReplicationLag) []Command { registered := index(observed) retiring := retiringNames(declared) - // A MAIN on its way out does not count as one: it is demoted below, and the - // cluster needs a survivor promoted in its place. - hasMain := false + // Which instance holds MAIN, and whether it is one on its way out. A retiring + // MAIN is the cluster's MAIN for as long as it stays: it stops counting as one + // only once this pass commits to demoting it, which is what the handover below + // decides. + retiringMain, hasMain := "", false for _, instance := range observed { - hasMain = hasMain || (instance.IsMain() && !retiring[instance.Name]) + switch { + case !instance.IsMain(): + case retiring[instance.Name]: + retiringMain = instance.Name + default: + hasMain = true + } } + // The survivor a retiring MAIN can hand over to, and the empty string when + // none qualifies — which is what defers the whole retirement to a later pass. + successor := "" + if retiringMain != "" { + successor = handoverTarget(declared, registered, indexLag(lag)) + } + handover := retiringMain != "" && successor != "" var commands []Command for _, coordinator := range declared.Coordinators { @@ -227,18 +258,33 @@ func Plan(declared Topology, observed []memgraph.Instance) []Command { commands = append(commands, RegisterInstance{Instance: instance}) } } - for _, instance := range declared.RetiringDataInstances { - if observed, ok := registered[instance.Name]; ok && observed.IsMain() { - commands = append(commands, DemoteInstance{Name: instance.Name}) - } + if handover { + commands = append(commands, DemoteInstance{Name: retiringMain}) } - if !hasMain && len(declared.DataInstances) > 0 { + switch { + case handover && !hasMain: + // The demotion above left the cluster MAIN-less on purpose; the survivor + // picked for the handover takes over in the next command. + commands = append(commands, SetInstanceToMain{Name: successor}) + case retiringMain == "" && !hasMain && len(declared.DataInstances) > 0: + // No MAIN and none retiring: a fresh bootstrap, a MAIN whose promotion never + // landed, or a pass that died between a retiring MAIN's demotion and the + // promotion meant to follow it. Lag is measured against a MAIN, so with none + // there is nothing to measure and this promotion goes by reachability alone. + // That is also why the handover is gated before the demotion rather than + // after: it is the last moment at which the choice is still free. commands = append(commands, SetInstanceToMain{Name: promotionTarget(declared, registered)}) } for _, instance := range declared.RetiringDataInstances { - if _, ok := registered[instance.Name]; ok { - commands = append(commands, UnregisterInstance{Name: instance.Name}) + if _, ok := registered[instance.Name]; !ok { + continue } + if instance.Name == retiringMain && !handover { + // Memgraph refuses to unregister the MAIN, and the demotion that would + // make this one unregisterable is waiting for a survivor to take over. + continue + } + commands = append(commands, UnregisterInstance{Name: instance.Name}) } leader := leaderName(observed) @@ -271,6 +317,31 @@ func leaderName(observed []memgraph.Instance) string { return "" } +// Retired reports whether every member a lowered replica count is shedding has +// left the cluster: no retiring data instance is still registered, and no +// retiring coordinator is still a Raft member. It is the precondition for +// shedding their pods. +// +// Plan coming back empty does not establish that on its own. A retirement whose +// handover is waiting for a caught-up survivor also plans nothing — there is no +// command that would make a lagging replica ready — so a caller that read +// emptiness as "done" would delete the pod of a registered MAIN. This is the +// question it has to ask instead. +func Retired(declared Topology, observed []memgraph.Instance) bool { + registered := index(observed) + for _, instance := range declared.RetiringDataInstances { + if _, ok := registered[instance.Name]; ok { + return false + } + } + for _, coordinator := range declared.RetiringCoordinators { + if coordinatorRegistered(registered, coordinator) { + return false + } + } + return true +} + // Registered reports how many of the declared coordinators and data instances // the observed cluster has registered. It is pure observation for the CR's // status, and it shares Plan's definition of "registered" — so a role's count @@ -310,6 +381,17 @@ func index(observed []memgraph.Instance) map[string]memgraph.Instance { return registered } +// indexLag keys the observed replication lag by instance name. A name the view +// does not cover reads back as the zero ReplicationLag, which reports itself as +// not caught up — the safe answer for an instance nothing is known about. +func indexLag(lag []memgraph.ReplicationLag) map[string]memgraph.ReplicationLag { + byInstance := make(map[string]memgraph.ReplicationLag, len(lag)) + for _, instance := range lag { + byInstance[instance.Instance] = instance + } + return byInstance +} + // coordinatorRegistered reports whether the declared coordinator is a member of // the Raft cluster. A coordinator reports itself in SHOW INSTANCES with an // empty bolt_server until ADD COORDINATOR is issued for its ID, so presence @@ -319,6 +401,40 @@ func coordinatorRegistered(registered map[string]memgraph.Instance, coordinator return ok && observed.BoltServer != "" } +// handoverTarget picks the survivor a retiring MAIN hands MAIN over to: the +// lowest-ordinal declared instance the coordinator leader observes as up and that +// SHOW REPLICATION LAG reports as holding every transaction the MAIN has +// committed, in every database. It returns the empty string when no survivor +// qualifies. +// +// Both conditions are needed and neither implies the other. Health says the +// leader can reach the instance, which a caught-up replica can still fail — +// registration outlives reachability, and the lag view is relayed from the MAIN's +// own cached progress for each replica, so an instance that went down a moment +// ago still appears there at the offset it last reached. Lag says the instance +// holds the writes, which a reachable one need not. +// +// There is deliberately no fallback, which is what separates this from +// promotionTarget. A cluster with no MAIN is worse off than one whose promotion +// has to be retried, so that one guesses rather than stall. A retiring MAIN is +// still serving: waiting costs nothing but the scale-down's completion, while +// promoting a lagging survivor discards every transaction it never received. +func handoverTarget( + declared Topology, + registered map[string]memgraph.Instance, + lag map[string]memgraph.ReplicationLag, +) string { + for _, instance := range declared.DataInstances { + if observed, ok := registered[instance.Name]; !ok || !observed.IsUp() { + continue + } + if lag[instance.Name].IsCaughtUp() { + return instance.Name + } + } + return "" +} + // promotionTarget picks the data instance to promote when the cluster has no // MAIN: the lowest-ordinal declared instance the cluster observes as up. // Promoting a down instance would only write the intent to Raft and leave the diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index bab10e1..af49ada 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -175,6 +175,47 @@ func downDataInstance(i int) memgraph.Instance { return instance } +// caughtUp is the SHOW REPLICATION LAG view of the given data instances with every +// one of them holding all of the MAIN's transactions: the state that lets a +// retiring MAIN hand over. The MAIN reports itself in the view too, at zero +// behind, which is what every other row is measured against. +func caughtUp(indices ...int) []memgraph.ReplicationLag { + return behindBy(0, indices...) +} + +// behindBy is the same view with every named instance trailing the MAIN by the +// given number of transactions. A negative count is a replica briefly ahead of a +// new MAIN, which SYNC replication can produce and which is not "behind". +func behindBy(txns int64, indices ...int) []memgraph.ReplicationLag { + lag := make([]memgraph.ReplicationLag, 0, len(indices)) + for _, i := range indices { + lag = append(lag, memgraph.ReplicationLag{ + Instance: dataInstanceSpec(i).Name, + Databases: []memgraph.DatabaseLag{{ + Database: "memgraph", + CommittedTxns: 100 - txns, + TxnsBehindMain: txns, + }}, + }) + } + return lag +} + +// multiDatabaseLag is one instance's view across two databases, which is how an +// enterprise cluster reports: an instance is only promotable when it is caught up +// in every one of them. +func multiDatabaseLag(i int, behind ...int64) []memgraph.ReplicationLag { + lag := memgraph.ReplicationLag{Instance: dataInstanceSpec(i).Name} + for db, txns := range behind { + lag.Databases = append(lag.Databases, memgraph.DatabaseLag{ + Database: fmt.Sprintf("db_%d", db), + CommittedTxns: 100 - txns, + TxnsBehindMain: txns, + }) + } + return []memgraph.ReplicationLag{lag} +} + func TestPlan(t *testing.T) { cases := []struct { name string @@ -182,7 +223,12 @@ func TestPlan(t *testing.T) { // declared overrides the canonical fixture for the cases about a // topology whose counts changed. declared *planner.Topology - want []planner.Command + // lag is the SHOW REPLICATION LAG view. It is nil for every case with no + // retiring MAIN to move, because that is what the real query answers with + // when there is no MAIN to measure against — and the only decision that + // consults it is the handover off a retiring MAIN. + lag []memgraph.ReplicationLag + want []planner.Command }{ { name: "fresh cluster bootstraps everything and promotes one MAIN", @@ -398,14 +444,16 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleMain), }, declared: ptr.To(shrunkTopology(2, 3)), + lag: caughtUp(0, 1, 2), want: []planner.Command{ planner.DemoteInstance{Name: thirdInstance}, planner.SetInstanceToMain{Name: firstInstance}, planner.UnregisterInstance{Name: thirdInstance}, }, }, - // The promotion follows the same rule as at bootstrap, so a survivor the - // leader cannot reach is not the one that gets MAIN. + // A survivor the leader cannot reach does not get MAIN even when the lag view + // still reports it caught up: that view is the MAIN's cached record of how far + // each replica got, so it outlives the replica's reachability. { name: "a retiring MAIN hands MAIN to the lowest reachable survivor", observed: []memgraph.Instance{ @@ -417,6 +465,142 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleMain), }, declared: ptr.To(shrunkTopology(2, 3)), + lag: caughtUp(0, 1, 2), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: secondInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // The handover skips a survivor that is behind, exactly as it skips one that + // is down: promoting it would drop the transactions it never received. + { + name: "a retiring MAIN hands MAIN to the lowest caught-up survivor", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(behindBy(7, 0), caughtUp(1, 2)...), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: secondInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // The whole retirement waits: no demotion, no promotion, and no + // unregistration of the MAIN that cannot be demoted. The cluster keeps + // serving from the instance on its way out until a survivor catches up, + // which is a scale-down that pauses rather than one that loses writes. + { + name: "a retiring MAIN is left alone while every survivor is behind", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(behindBy(7, 0, 1), caughtUp(2)...), + want: nil, + }, + // Down and behind are separate reasons to refuse, and either one alone is + // enough: here the one caught-up survivor is unreachable and the one + // reachable survivor is behind. + { + name: "a retiring MAIN is left alone when no survivor is both up and caught up", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + downDataInstance(0), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(caughtUp(0, 2), behindBy(3, 1)...), + want: nil, + }, + // No lag view at all — the MAIN unreachable from the coordinator leader, or + // the read having failed — is not evidence that anything is caught up, so the + // handover waits on it exactly as it waits on a lagging survivor. + { + name: "a retiring MAIN is left alone when the lag view is empty", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: nil, + want: nil, + }, + // A survivor the lag view does not mention is not caught up either: an + // instance the MAIN does not list is one it is not replicating to. + { + name: "a survivor missing from the lag view does not get MAIN", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(caughtUp(1), caughtUp(2)...), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: secondInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // A replica ahead of the MAIN is not behind it. SYNC replication can leave one + // holding transactions a newly promoted MAIN never saw, which is a negative + // count and no reason to refuse the handover. + { + name: "a survivor ahead of the MAIN is still promotable", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(behindBy(-2, 0), caughtUp(1, 2)...), + want: []planner.Command{ + planner.DemoteInstance{Name: thirdInstance}, + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // Every database has to be caught up, not just one: a survivor holding all of + // one database's writes and none of another's would lose the second on + // promotion. + { + name: "a survivor behind in one of several databases does not get MAIN", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: append(multiDatabaseLag(0, 0, 4), + append(multiDatabaseLag(1, 0, 0), multiDatabaseLag(2, 0, 0)...)...), want: []planner.Command{ planner.DemoteInstance{Name: thirdInstance}, planner.SetInstanceToMain{Name: secondInstance}, @@ -434,6 +618,7 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleReplica), }, declared: ptr.To(shrunkTopology(1, 3)), + lag: caughtUp(0, 1, 2), want: []planner.Command{ planner.DemoteInstance{Name: secondInstance}, planner.SetInstanceToMain{Name: firstInstance}, @@ -441,6 +626,49 @@ func TestPlan(t *testing.T) { planner.UnregisterInstance{Name: thirdInstance}, }, }, + // A blocked handover holds up only the MAIN's own retirement. The other + // retiring instance is not MAIN, so unregistering it needs nothing from the + // survivors and goes out in this pass. + { + name: "a blocked handover still unregisters the retiring instances that are not MAIN", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleMain), + observedDataInstance(2, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(1, 3)), + lag: append(behindBy(9, 0), caughtUp(1, 2)...), + want: []planner.Command{ + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, + // A pass that died between the demotion and the promotion meant to follow it + // leaves the cluster with no MAIN at all, which is the one state the caught-up + // rule cannot be applied in: lag is measured against a MAIN, and there is + // none. The plan falls back to the MAIN-less rule and promotes by + // reachability, because a cluster serving nothing is worse off than one whose + // promotion has to be retried — and gating the handover ahead of the demotion + // is what keeps this window as narrow as a single pair of queries. + { + name: "an already-demoted retiring MAIN leaves a promotion by reachability", + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedCoordinator(2, memgraph.RoleFollower), + observedCoordinator(3, memgraph.RoleFollower), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + declared: ptr.To(shrunkTopology(2, 3)), + lag: nil, + want: []planner.Command{ + planner.SetInstanceToMain{Name: firstInstance}, + planner.UnregisterInstance{Name: thirdInstance}, + }, + }, // Read-before-write: a retiring member the cluster no longer knows about // gets no command, so a reconcile that crashed between the unregistration // and the shrink re-plans to just the rest of the work. @@ -473,6 +701,7 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleMain), }, declared: ptr.To(mixedTopology()), + lag: caughtUp(0, 1, 2), want: []planner.Command{ planner.AddCoordinator{Coordinator: coordinatorSpec(4)}, planner.AddCoordinator{Coordinator: coordinatorSpec(5)}, @@ -564,6 +793,7 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleMain), ), declared: ptr.To(retiringBothRoles()), + lag: caughtUp(0, 1, 2), want: []planner.Command{ planner.DemoteInstance{Name: thirdInstance}, planner.SetInstanceToMain{Name: firstInstance}, @@ -572,6 +802,22 @@ func TestPlan(t *testing.T) { planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, }, }, + // The coordinator side of the same edit is independent of where MAIN sits, so + // a handover the survivors cannot take does not hold up the Raft removals. + { + name: "a blocked handover does not hold up the coordinator removals", + observed: append(observedCoordinators(1, 1, 2, 3, 4, 5), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + ), + declared: ptr.To(retiringBothRoles()), + lag: append(behindBy(5, 0, 1), caughtUp(2)...), + want: []planner.Command{ + planner.RemoveCoordinator{Coordinator: coordinatorSpec(4)}, + planner.RemoveCoordinator{Coordinator: coordinatorSpec(5)}, + }, + }, // The same edit with leadership in the way: the data-instance retirement is // fully ordered and issues in this pass regardless — only the removal of the // leader itself has to wait behind the yield. @@ -583,6 +829,7 @@ func TestPlan(t *testing.T) { observedDataInstance(2, memgraph.RoleMain), ), declared: ptr.To(retiringBothRoles()), + lag: caughtUp(0, 1, 2), want: []planner.Command{ planner.DemoteInstance{Name: thirdInstance}, planner.SetInstanceToMain{Name: firstInstance}, @@ -619,7 +866,7 @@ func TestPlan(t *testing.T) { declared = *tc.declared } - got := planner.Plan(declared, tc.observed) + got := planner.Plan(declared, tc.observed, tc.lag) if diff := cmp.Diff(tc.want, got); diff != "" { t.Errorf("Plan() mismatch (-want +got):\n%s", diff) } @@ -627,6 +874,89 @@ func TestPlan(t *testing.T) { } } +// TestRetired covers the question the controller has to ask before it sheds a +// retiring member's pod. Plan falling silent is not the same question — a +// handover waiting for a caught-up survivor is also silent — so the two disagree +// on purpose in the blocked case below. +func TestRetired(t *testing.T) { + cases := []struct { + name string + declared planner.Topology + observed []memgraph.Instance + want bool + }{ + { + name: "nothing retiring is retired", + declared: declaredTopology(), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedDataInstance(0, memgraph.RoleMain), + }, + want: true, + }, + { + name: "a retiring instance the cluster has forgotten is retired", + declared: shrunkTopology(2, 3), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + }, + want: true, + }, + { + name: "a retiring instance still registered is not retired", + declared: shrunkTopology(2, 3), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleReplica), + }, + want: false, + }, + // The state a blocked handover leaves behind: Plan has nothing to issue, and + // the pod still belongs to a registered MAIN. + { + name: "a retiring MAIN whose handover is blocked is not retired", + declared: shrunkTopology(2, 3), + observed: []memgraph.Instance{ + observedCoordinator(1, memgraph.RoleLeader), + observedDataInstance(0, memgraph.RoleReplica), + observedDataInstance(1, memgraph.RoleReplica), + observedDataInstance(2, memgraph.RoleMain), + }, + want: false, + }, + { + name: "a retiring coordinator still in Raft is not retired", + declared: shrunkCoordinators(), + observed: append(observedCoordinators(1, 1, 2, 3, 4), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + want: false, + }, + { + name: "retiring coordinators all out of Raft are retired", + declared: shrunkCoordinators(), + observed: append(observedCoordinators(1, 1, 2, 3), + observedDataInstance(0, memgraph.RoleMain), + observedDataInstance(1, memgraph.RoleReplica), + ), + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := planner.Retired(tc.declared, tc.observed); got != tc.want { + t.Errorf("Retired() = %t, want %t", got, tc.want) + } + }) + } +} + // TestRegistered covers what the CR's status publishes: how many of each role's // declared members the cluster has registered. It shares Plan's definition of // registered, so the counts reach the declared ones exactly when Plan falls @@ -769,7 +1099,7 @@ func TestPlanUsesConfiguredPortsAndClusterDomain(t *testing.T) { planner.SetInstanceToMain{Name: firstInstance}, } - got := planner.Plan(resources.DeclaredTopology(cluster), nil) + got := planner.Plan(resources.DeclaredTopology(cluster), nil, nil) if diff := cmp.Diff(want, got); diff != "" { t.Errorf("Plan() mismatch (-want +got):\n%s", diff) } @@ -810,7 +1140,7 @@ func TestPlanConvergedOnConfiguredPorts(t *testing.T) { }, } - if got := planner.Plan(declared, observed); got != nil { + if got := planner.Plan(declared, observed, nil); got != nil { t.Errorf("Plan() = %v, want no commands", got) } } From 19f6b48baa771931f6d90d770c7b72ade352fd67 Mon Sep 17 00:00:00 2001 From: as51340 Date: Wed, 29 Jul 2026 13:12:44 +0200 Subject: [PATCH 27/34] fix: Data instances comparison --- .../controller/memgraphcluster_controller.go | 28 ++++++++++----- .../memgraphcluster_controller_test.go | 34 +++++++++++++++++++ 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/internal/controller/memgraphcluster_controller.go b/internal/controller/memgraphcluster_controller.go index 920b7dd..80f1b7c 100644 --- a/internal/controller/memgraphcluster_controller.go +++ b/internal/controller/memgraphcluster_controller.go @@ -323,7 +323,7 @@ func (r *MemgraphClusterReconciler) reconcileRegistration( ) (ctrl.Result, error) { log := logf.FromContext(ctx) - ready, err := r.workloadsReady(ctx, cluster) + ready, err := r.workloadsReady(ctx, cluster, replicas) if err != nil { return ctrl.Result{}, err } @@ -610,22 +610,34 @@ func (r *MemgraphClusterReconciler) writeStatus( // Raft cluster and data instances cannot be registered until every advertised // address resolves to a running pod. // -// A StatefulSet the apply just created is not ready, not an error: the read goes -// through the informer cache, which lags the apply, so an absent StatefulSet is -// the same waiting state as one whose pods have not come up yet. +// The count each role must reach is the one this pass applied, never the +// spec.replicas read back off the StatefulSet. Reads go through the informer +// cache, which lags the apply, so the object read back a few lines after a +// scale-up is still the pre-apply snapshot — and in that snapshot the old +// spec.replicas and the old status.readyReplicas agree, because the cluster +// genuinely was converged at the old size. Comparing those two stale numbers +// against each other reports a grown topology as ready and lets registration run +// against a pod Kubernetes has not been asked to create yet. Comparing a stale +// readyReplicas against the count this pass intends cannot fail that way: a lagging +// status only ever reads as not-yet-ready. +// +// A StatefulSet the apply just created is not ready, not an error: the same lag +// makes an absent StatefulSet the same waiting state as one whose pods have not +// come up yet. func (r *MemgraphClusterReconciler) workloadsReady( ctx context.Context, cluster *memgraphcomv1alpha1.MemgraphCluster, + replicas replicaCounts, ) (bool, error) { - for _, name := range []string{resources.CoordinatorName(cluster), resources.DataName(cluster)} { + for _, role := range []roleReplicas{replicas.coordinators, replicas.data} { var sts appsv1.StatefulSet - if err := r.Get(ctx, types.NamespacedName{Name: name, Namespace: cluster.Namespace}, &sts); err != nil { + if err := r.Get(ctx, types.NamespacedName{Name: role.name, Namespace: cluster.Namespace}, &sts); err != nil { if apierrors.IsNotFound(err) { return false, nil } - return false, fmt.Errorf("getting StatefulSet %s: %w", name, err) + return false, fmt.Errorf("getting StatefulSet %s: %w", role.name, err) } - if sts.Spec.Replicas == nil || sts.Status.ReadyReplicas < *sts.Spec.Replicas { + if sts.Status.ReadyReplicas < role.applied { return false, nil } } diff --git a/internal/controller/memgraphcluster_controller_test.go b/internal/controller/memgraphcluster_controller_test.go index eb9e134..ee115e1 100644 --- a/internal/controller/memgraphcluster_controller_test.go +++ b/internal/controller/memgraphcluster_controller_test.go @@ -1062,6 +1062,40 @@ var _ = Describe("MemgraphCluster Controller", func() { Expect(converged.Reason).To(Equal(memgraphcomv1alpha1.ReasonWorkloadsNotReady)) }) + // The gate has to key off the count the pass applied, not the spec.replicas it + // can read back. Production reads through the informer cache, so a few lines + // after a scale-up that field is still the pre-apply value — and it agrees + // with a status.readyReplicas from the same old snapshot, because the cluster + // really was converged at the old size. Two stale numbers that agree report a + // grown topology as ready, and registration then names a pod Kubernetes has + // not been asked to create. + // + // The gate is called directly here: the envtest client is uncached, so the + // staleness itself cannot be reproduced, only the comparison it would defeat. + // A StatefulSet left at 2 ready out of 2 is exactly what that stale read looks + // like, and the pass that intends 3 must not accept it. + It("should gate readiness on the applied count, not the StatefulSet's own spec", func() { + bootstrapped() + cluster := &memgraphcomv1alpha1.MemgraphCluster{} + get(resourceName, cluster) + Expect(replicas(dataSuffix)).To(Equal(int32(2))) + + held := replicaCounts{ + coordinators: roleReplicas{name: resourceName + coordinatorSuffix, declared: 3, applied: 3}, + data: roleReplicas{name: resourceName + dataSuffix, declared: 2, applied: 2}, + } + ready, err := reconciler.workloadsReady(ctx, cluster, held) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeTrue(), "the cluster is ready at the size this pass applies") + + grown := held + grown.data.declared, grown.data.applied = 3, 3 + ready, err = reconciler.workloadsReady(ctx, cluster, grown) + Expect(err).NotTo(HaveOccurred()) + Expect(ready).To(BeFalse(), + "a pass applying 3 must not read 2-ready-of-2 as ready, whatever spec.replicas still says") + }) + It("should report the registered counts as observed, not as declared", func() { bootstrapped() Expect(status().DataInstances).To(Equal(int32(2))) From 99929e69cb5cfc4ef4c1865a78f5d4b95478249f Mon Sep 17 00:00:00 2001 From: as51340 Date: Wed, 29 Jul 2026 14:14:27 +0200 Subject: [PATCH 28/34] fix: gflags and CI flakiness --- api/v1alpha1/memgraphcluster_types.go | 4 +- .../crds/memgraph.com_memgraphclusters.yaml | 22 +++---- .../bases/memgraph.com_memgraphclusters.yaml | 22 +++---- config/samples/v1alpha1_memgraphcluster.yaml | 2 +- .../memgraphcluster_validation_test.go | 35 +++++++++- test/e2e/memgraphcluster_test.go | 66 ++++++++++++++++++- 6 files changed, 120 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/memgraphcluster_types.go b/api/v1alpha1/memgraphcluster_types.go index 74797dc..5c381da 100644 --- a/api/v1alpha1/memgraphcluster_types.go +++ b/api/v1alpha1/memgraphcluster_types.go @@ -756,7 +756,7 @@ type ExtraArgsSpec struct { // +kubebuilder:validation:MaxItems=64 // +kubebuilder:validation:items:MinLength=1 // +kubebuilder:validation:items:MaxLength=4096 - // +kubebuilder:validation:XValidation:rule="self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', '--coordinator-hostname', '--coordinator-port'].exists(f, a.startsWith(f)))",message="extraArgs must not set a port or the coordinator identity the operator derives (--bolt-port, --management-port, --coordinator-id, --coordinator-hostname, --coordinator-port); configure ports through spec.ports" + // +kubebuilder:validation:XValidation:rule="self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= ])'))",message="extraArgs must not set a port or the coordinator identity the operator derives (bolt-port, management-port, coordinator-id, coordinator-hostname, coordinator-port), in any spelling gflags accepts; configure ports through spec.ports" // +optional Coordinators []string `json:"coordinators,omitempty"` @@ -764,7 +764,7 @@ type ExtraArgsSpec struct { // +kubebuilder:validation:MaxItems=64 // +kubebuilder:validation:items:MinLength=1 // +kubebuilder:validation:items:MaxLength=4096 - // +kubebuilder:validation:XValidation:rule="self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', '--coordinator-hostname', '--coordinator-port'].exists(f, a.startsWith(f)))",message="extraArgs must not set a port or the coordinator identity the operator derives (--bolt-port, --management-port, --coordinator-id, --coordinator-hostname, --coordinator-port); configure ports through spec.ports" + // +kubebuilder:validation:XValidation:rule="self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= ])'))",message="extraArgs must not set a port or the coordinator identity the operator derives (bolt-port, management-port, coordinator-id, coordinator-hostname, coordinator-port), in any spelling gflags accepts; configure ports through spec.ports" // +optional Data []string `json:"data,omitempty"` } diff --git a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml index eaf18db..4b72a4d 100644 --- a/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml +++ b/charts/memgraph-operator/crds/memgraph.com_memgraphclusters.yaml @@ -359,12 +359,11 @@ spec: type: array x-kubernetes-validations: - message: extraArgs must not set a port or the coordinator identity - the operator derives (--bolt-port, --management-port, --coordinator-id, - --coordinator-hostname, --coordinator-port); configure ports - through spec.ports - rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', - '--coordinator-hostname', '--coordinator-port'].exists(f, - a.startsWith(f))) + the operator derives (bolt-port, management-port, coordinator-id, + coordinator-hostname, coordinator-port), in any spelling gflags + accepts; configure ports through spec.ports + rule: self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= + ])')) data: description: data are appended to every data instance pod's Memgraph flags. @@ -376,12 +375,11 @@ spec: type: array x-kubernetes-validations: - message: extraArgs must not set a port or the coordinator identity - the operator derives (--bolt-port, --management-port, --coordinator-id, - --coordinator-hostname, --coordinator-port); configure ports - through spec.ports - rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', - '--coordinator-hostname', '--coordinator-port'].exists(f, - a.startsWith(f))) + the operator derives (bolt-port, management-port, coordinator-id, + coordinator-hostname, coordinator-port), in any spelling gflags + accepts; configure ports through spec.ports + rule: self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= + ])')) type: object extraEnv: description: |- diff --git a/config/crd/bases/memgraph.com_memgraphclusters.yaml b/config/crd/bases/memgraph.com_memgraphclusters.yaml index 21655a3..13eb8a4 100644 --- a/config/crd/bases/memgraph.com_memgraphclusters.yaml +++ b/config/crd/bases/memgraph.com_memgraphclusters.yaml @@ -357,12 +357,11 @@ spec: type: array x-kubernetes-validations: - message: extraArgs must not set a port or the coordinator identity - the operator derives (--bolt-port, --management-port, --coordinator-id, - --coordinator-hostname, --coordinator-port); configure ports - through spec.ports - rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', - '--coordinator-hostname', '--coordinator-port'].exists(f, - a.startsWith(f))) + the operator derives (bolt-port, management-port, coordinator-id, + coordinator-hostname, coordinator-port), in any spelling gflags + accepts; configure ports through spec.ports + rule: self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= + ])')) data: description: data are appended to every data instance pod's Memgraph flags. @@ -374,12 +373,11 @@ spec: type: array x-kubernetes-validations: - message: extraArgs must not set a port or the coordinator identity - the operator derives (--bolt-port, --management-port, --coordinator-id, - --coordinator-hostname, --coordinator-port); configure ports - through spec.ports - rule: self.all(a, !['--bolt-port', '--management-port', '--coordinator-id', - '--coordinator-hostname', '--coordinator-port'].exists(f, - a.startsWith(f))) + the operator derives (bolt-port, management-port, coordinator-id, + coordinator-hostname, coordinator-port), in any spelling gflags + accepts; configure ports through spec.ports + rule: self.all(a, !a.replace('-', '_').matches('^_{1,2}(bolt_port|management_port|coordinator_id|coordinator_hostname|coordinator_port)($|[= + ])')) type: object extraEnv: description: |- diff --git a/config/samples/v1alpha1_memgraphcluster.yaml b/config/samples/v1alpha1_memgraphcluster.yaml index 8bba63b..59d20f1 100644 --- a/config/samples/v1alpha1_memgraphcluster.yaml +++ b/config/samples/v1alpha1_memgraphcluster.yaml @@ -21,7 +21,7 @@ spec: # to YIELD LEADERSHIP first, reported as LeadershipTransferInProgress while that # is pending. coordinators: 3 - dataInstances: 2 + dataInstances: 0 # repository carries the registry host and image path only — the version # belongs in tag, and admission rejects a digest or a tag smuggled into the # repository. diff --git a/internal/controller/memgraphcluster_validation_test.go b/internal/controller/memgraphcluster_validation_test.go index c314241..b047b00 100644 --- a/internal/controller/memgraphcluster_validation_test.go +++ b/internal/controller/memgraphcluster_validation_test.go @@ -253,7 +253,10 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Data: []memgraphcomv1alpha1.EnvVar{{Name: "DATA_LABEL_ONE", Value: "one"}}, }, ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{ - Data: []string{"--storage-snapshot-on-exit=true"}, + // The second one shares a prefix with the reserved --bolt-port + // without being it: the guard matches whole flag names, so a + // legitimate neighbour is not caught by it. + Data: []string{"--storage-snapshot-on-exit=true", "--bolt-num-workers=8"}, }, }) @@ -263,7 +266,8 @@ var _ = Describe("MemgraphCluster CRD validation", func() { Expect(stored.Spec.Probes.Data.ReadinessProbe).To(Equal(memgraphcomv1alpha1.ProbeSpec{}), "an unset probe stays unset; its defaults are resolved by the builders, not the schema") Expect(stored.Spec.ExtraEnv.Data).To(HaveLen(1)) - Expect(stored.Spec.ExtraArgs.Data).To(ConsistOf("--storage-snapshot-on-exit=true")) + Expect(stored.Spec.ExtraArgs.Data).To(ConsistOf( + "--storage-snapshot-on-exit=true", "--bolt-num-workers=8")) }) It("should default the ports a partially specified block leaves out", func() { @@ -526,6 +530,33 @@ var _ = Describe("MemgraphCluster CRD validation", func() { ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Coordinators: []string{"--coordinator-id=9"}}, }, "the coordinator identity the operator derives"), + // Memgraph's flags are gflags, which treats one dash as two and a hyphen as + // an underscore — the flags are declared bolt_port, coordinator_id and so + // on, and the operator's own --bolt-port only works because of that. Every + // spelling reaches the same flag, so the guard has to reject all of them or + // it rejects none. + Entry("a reserved flag spelled with one dash", "invalid-args-single-dash", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Data: []string{"-bolt-port=7777"}}, + }, + "configure ports through spec.ports"), + Entry("a reserved flag spelled with underscores", "invalid-args-underscores", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Data: []string{"--bolt_port=7777"}}, + }, + "configure ports through spec.ports"), + Entry("a reserved flag spelled with one dash and underscores", "invalid-args-single-underscore", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Coordinators: []string{"-coordinator_id=9"}}, + }, + "the coordinator identity the operator derives"), + // gflags takes a non-boolean flag's value as the next argument too, so the + // flag can arrive as an element of its own. + Entry("a reserved flag with its value in the next element", "invalid-args-separate-value", + memgraphcomv1alpha1.MemgraphClusterSpec{ + ExtraArgs: memgraphcomv1alpha1.ExtraArgsSpec{Data: []string{"--bolt-port", "7777"}}, + }, + "configure ports through spec.ports"), Entry("a probe timing below one", "invalid-probe-period", memgraphcomv1alpha1.MemgraphClusterSpec{ Probes: memgraphcomv1alpha1.ProbesSpec{ diff --git a/test/e2e/memgraphcluster_test.go b/test/e2e/memgraphcluster_test.go index 5e7ebcf..08e5813 100644 --- a/test/e2e/memgraphcluster_test.go +++ b/test/e2e/memgraphcluster_test.go @@ -24,6 +24,7 @@ import ( "encoding/json" "errors" "fmt" + "maps" "os" "os/exec" "path/filepath" @@ -340,10 +341,17 @@ spec: // attaches a sync *after* it creates the claim, and a claim whose set // is deleted before it is adopted is stranded for good, not merely // collected late. + // + // The claims are named rather than counted. A count is satisfied by any + // eight adopted claims, so it cannot distinguish the state this guard + // exists to wait for from any other view that happens to be eight rows + // long — and a guard that passes before adoption completes hands the rest + // of the spec the exact race it was put here to exclude. + wantClaims := expectedClaims(retentionCluster, map[string]int{"coordinator": 3, "data": 1}) Eventually(func(g Gomega) { claims, err := listAdoptedPVCs(retentionNamespace) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(claims).To(HaveLen(8)) + g.Expect(claims).To(ConsistOf(wantClaims)) }, 5*time.Minute, 5*time.Second).Should(Succeed()) By("deleting the MemgraphCluster") @@ -352,11 +360,30 @@ spec: _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to delete the MemgraphCluster") + // Claim deletion is garbage collection following the owner reference, so + // the sets have to be gone before their absence can take the claims with + // them. Waiting on that first is also what separates the two ways this can + // fail: sets that linger name the workloads, claims that linger after the + // sets are gone name the collection. + By("waiting for garbage collection to remove the workloads") + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "statefulsets", "-n", retentionNamespace, + "-o", "jsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(strings.TrimSpace(output)).To(BeEmpty()) + }, 5*time.Minute, 5*time.Second).Should(Succeed()) + By("waiting for the StatefulSet machinery to take the claims down with it") Eventually(func(g Gomega) { claims, err := listPVCs(retentionNamespace) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(claims).To(BeEmpty()) + // A claim that outlives its owner says nothing on its own about why, so + // the owner references it still carries go into the failure: whether it + // was never adopted, or adopted by a set that is somehow still around, + // is the whole diagnosis and it is not recoverable after the fact. + g.Expect(claims).To(BeEmpty(), "claims left behind, with their owners:\n%s", + describeClaimOwners(retentionNamespace)) }, 5*time.Minute, 5*time.Second).Should(Succeed()) }) }) @@ -637,6 +664,41 @@ spec: // asserting the Delete retention policy must wait for before it deletes // anything. It gates a spec rather than asserting one — the retention // assertion itself stays on the claims a user would see. +// expectedClaims is every claim name the given cluster's StatefulSets provision: +// the two volume claim templates, for each pod of each role. Claim names are +// "