Improved Reaper integration

charts/cass-operator-chart/crds/customresourcedefinition.yaml

@@ -5938,16 +5938,27 @@ spec:
             reaper:
               properties:
                 enabled:
+                  description: When set to true, cass-operator configures and enables
+                    remote JMX. Changing this after the cluster has been deployed
+                    will result in a rolling restart of the cluster.
                   type: boolean
                 image:
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
                   type: string
                 imagePullPolicy:
-                  description: PullPolicy describes a policy for if/when to pull a
-                    container image
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
+                  type: string
+                jmxSecretName:
+                  description: The name of the secret that stores the credentials
+                    used for JMX authentication. The secret should be created in the
+                    same namespace as the CassandraDatacenter. If this property is
+                    not set, the operator will generate a default secret.
                   type: string
                 resources:
-                  description: Kubernetes resource requests and limits per reaper
-                    container.
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
                   properties:
                     limits:
                       additionalProperties:

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975
 	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/api v0.17.4

go.sum

@@ -909,6 +909,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=

mage/ginkgo/lib.go

@@ -298,6 +298,13 @@ func (ns *NsWrapper) WaitForDatacenterReadyWithTimeouts(dcName string, podCountT
 	ns.WaitForDatacenterOperatorProgress(dcName, "Ready", dcReadyTimeout)
 }
 
+func (ns *NsWrapper) WaitForReaperReady(reaperName string, timeout int) {
+	step := "waiting for reaper to become ready"
+	json := "jsonpath={.status.ready}"
+	k := kubectl.Get("reaper", reaperName).FormatOutput(json)
+	ns.WaitForOutputAndLog(step, k, "true", timeout)
+}
+
 func (ns *NsWrapper) WaitForPodNotStarted(podName string) {
 	step := "verify that the pod is no longer marked as started"
 	k := kubectl.Get("pod").
@@ -403,6 +410,16 @@ func (ns *NsWrapper) WaitForOperatorReady() {
 	ns.WaitForOutputAndLog(step, k, "true", 240)
 }
 
+func (ns *NsWrapper) WaitForReaperOperatorReady() {
+	step := "waiting for reaper-operator to become ready"
+	json := "jsonpath={.items[0].status.containerStatuses[0].ready}"
+	k := kubectl.Get("pods").
+		WithLabel("control-plane=reaper-operator").
+		WithFlag("field-selector", "status.phase=Running").
+		FormatOutput(json)
+	ns.WaitForOutputAndLog(step, k, "true", 240)
+}
+
 // kubectl create secret docker-registry github-docker-registry --docker-username=USER --docker-password=PASS --docker-server docker.pkg.github.com
 func CreateDockerRegistrySecret(name string, namespace string) {
 	args := []string{"secret", "docker-registry", name}

operator/deploy/crds/cassandra.datastax.com_cassandradatacenters_crd.yaml

@@ -5950,16 +5950,27 @@ spec:
             reaper:
               properties:
                 enabled:
+                  description: When set to true, cass-operator configures and enables
+                    remote JMX. Changing this after the cluster has been deployed
+                    will result in a rolling restart of the cluster.
                   type: boolean
                 image:
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
                   type: string
                 imagePullPolicy:
-                  description: PullPolicy describes a policy for if/when to pull a
-                    container image
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
+                  type: string
+                jmxSecretName:
+                  description: The name of the secret that stores the credentials
+                    used for JMX authentication. The secret should be created in the
+                    same namespace as the CassandraDatacenter. If this property is
+                    not set, the operator will generate a default secret.
                   type: string
                 resources:
-                  description: Kubernetes resource requests and limits per reaper
-                    container.
+                  description: Deprecated - this property only applied to Reaper sidecar
+                    which is no longer used.
                   properties:
                     limits:
                       additionalProperties:

operator/pkg/apis/cassandra/v1beta1/cassandradatacenter_types.go

@@ -347,13 +347,22 @@ type ManagementApiAuthConfig struct {
 }
 
 type ReaperConfig struct {
+	// When set to true, cass-operator configures and enables remote JMX. Changing this
+	// after the cluster has been deployed will result in a rolling restart of the cluster.
 	Enabled bool `json:"enabled,omitempty"`
 
+	// The name of the secret that stores the credentials used for JMX authentication. The
+	// secret should be created in the same namespace as the CassandraDatacenter. If this
+	// property is not set, the operator will generate a default secret.
+	JmxSecretName string `json:"jmxSecretName,omitempty"`
+
+	// Deprecated - this property only applied to Reaper sidecar which is no longer used.
 	Image string `json:"image,omitempty"`
 
+	// Deprecated - this property only applied to Reaper sidecar which is no longer used.
 	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
 
-	// Kubernetes resource requests and limits per reaper container.
+	// Deprecated - this property only applied to Reaper sidecar which is no longer used.
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 }
 
@@ -418,6 +427,16 @@ func (dc *CassandraDatacenter) IsReaperEnabled() bool {
 	return false
 }
 
+func (dc *CassandraDatacenter) GetReaperJmxSecretName() string {
+	if !dc.IsReaperEnabled() {
+		return ""
+	}
+	if len(dc.Spec.Reaper.JmxSecretName) > 0 {
+		return dc.Spec.Reaper.JmxSecretName
+	}
+	return dc.Spec.ClusterName + "-reaper-jmx"
+}
+
 func (status *CassandraDatacenterStatus) GetConditionStatus(conditionType DatacenterConditionType) corev1.ConditionStatus {
 	for _, condition := range status.Conditions {
 		if condition.Type == conditionType {

operator/pkg/images/images.go

@@ -275,6 +275,10 @@ func GetSystemLoggerImage() string {
 	}
 }
 
+func GetJmxCredentialsImage() string {
+	return GetImage(BusyBox)
+}
+
 func AddDefaultRegistryImagePullSecrets(podSpec *corev1.PodSpec) bool {
 	secretName := os.Getenv(envDefaultRegistryOverridePullSecrets)
 	if secretName != "" {

operator/pkg/reconciliation/construct_podtemplatespec.go

@@ -26,6 +26,7 @@ const (
 	CassandraContainerName               = "cassandra"
 	PvcName                              = "server-data"
 	SystemLoggerContainerName            = "server-system-logger"
+	JmxCredentialsContainerName          = "jmx-credentials"
 )
 
 // calculateNodeAffinity provides a way to pin all pods within a statefulset to the same zone
@@ -262,7 +263,46 @@ func buildInitContainers(dc *api.CassandraDatacenter, rackName string, baseTempl
 	if !foundOverrides {
 		// Note that append makes a copy, so we must do this after
 		// serverCfg has been properly set up.
-		baseTemplate.Spec.InitContainers = append(baseTemplate.Spec.InitContainers, *serverCfg)
+		//baseTemplate.Spec.InitContainers = append(baseTemplate.Spec.InitContainers, *serverCfg)
+		baseTemplate.Spec.InitContainers = append([]corev1.Container{*serverCfg}, baseTemplate.Spec.InitContainers...)
+	}
+
+	if dc.IsReaperEnabled() {
+		initJmxCredentials := &corev1.Container{
+			Name: JmxCredentialsContainerName,
+			Image: images.GetJmxCredentialsImage(),
+			Args: []string{
+				"/bin/sh",
+				"-c",
+				"echo -n \"$JMX_USERNAME $JMX_PASSWORD\" > /config/jmxremote.password",
+			},
+			Env: []corev1.EnvVar{
+				{
+					Name: "JMX_USERNAME",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: dc.GetReaperJmxSecretName(),
+							},
+							Key: "username",
+						},
+					},
+				},
+				{
+					Name: "JMX_PASSWORD",
+					ValueFrom: &corev1.EnvVarSource{
+						SecretKeyRef: &corev1.SecretKeySelector{
+							LocalObjectReference: corev1.LocalObjectReference{
+								Name: dc.GetReaperJmxSecretName(),
+							},
+							Key: "password",
+						},
+					},
+				},
+			},
+			VolumeMounts: []corev1.VolumeMount{serverCfgMount},
+		}
+		baseTemplate.Spec.InitContainers = append(baseTemplate.Spec.InitContainers, *initJmxCredentials)
 	}
 
 	return nil
@@ -276,21 +316,16 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 
 	cassContainer := &corev1.Container{}
 	loggerContainer := &corev1.Container{}
-	reaperContainer := &corev1.Container{}
 
 	foundCass := false
 	foundLogger := false
-	foundReaper := false
 	for i, c := range baseTemplate.Spec.Containers {
 		if c.Name == CassandraContainerName {
 			foundCass = true
 			cassContainer = &baseTemplate.Spec.Containers[i]
 		} else if c.Name == SystemLoggerContainerName {
 			foundLogger = true
 			loggerContainer = &baseTemplate.Spec.Containers[i]
-		} else if c.Name == ReaperContainerName {
-			foundReaper = true
-			reaperContainer = &baseTemplate.Spec.Containers[i]
 		}
 	}
 
@@ -349,6 +384,10 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 			corev1.EnvVar{Name: "JVM_EXTRA_OPTS", Value: getJvmExtraOpts(dc)})
 	}
 
+	if dc.IsReaperEnabled() {
+		envDefaults = append(envDefaults, corev1.EnvVar{Name: "LOCAL_JMX", Value: "no"})
+	}
+
 	cassContainer.Env = combineEnvSlices(envDefaults, cassContainer.Env)
 
 	// Combine ports
@@ -404,12 +443,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 
 	loggerContainer.Resources = *getResourcesOrDefault(&dc.Spec.SystemLoggerResources, &DefaultsLoggerContainer)
 
-	// Reaper Container
-
-	if dc.IsReaperEnabled() {
-		buildReaperContainer(dc, reaperContainer)
-	}
-
 	// Note that append() can make copies of each element,
 	// so we call it after modifying any existing elements.
 
@@ -421,12 +454,6 @@ func buildContainers(dc *api.CassandraDatacenter, baseTemplate *corev1.PodTempla
 		baseTemplate.Spec.Containers = append(baseTemplate.Spec.Containers, *loggerContainer)
 	}
 
-	if dc.IsReaperEnabled() {
-		if !foundReaper {
-			baseTemplate.Spec.Containers = append(baseTemplate.Spec.Containers, *reaperContainer)
-		}
-	}
-
 	return nil
 }