Fix multiple starts causing errors

Author: khewonc

pkg/fleet/daemon.go

@@ -185,9 +185,13 @@ func (d *Daemon) startDatadogAgentExperiment(ctx context.Context, req remoteAPIR
 	}
 
 	if err := canStart(getExperimentPhase(dda)); err != nil {
-		// If the experiment is already running with the same ID, treat as idempotent success.
-		if dda.Status.Experiment != nil && dda.Status.Experiment.ID == req.ID {
-			logger.Info("Experiment already running with this ID, acknowledging", "phase", dda.Status.Experiment.Phase)
+		// If an experiment is already running, treat repeat start signals as idempotent.
+		// The backend retries with new task UUIDs until it sees an ack, and in-memory
+		// state (ExperimentConfigVersion) doesn't survive operator restarts, so we
+		// cannot reliably match on config version. The backend is responsible for not
+		// sending a start for a different experiment while one is already running.
+		if getExperimentPhase(dda) == v2alpha1.ExperimentPhaseRunning {
+			logger.Info("Experiment already running, acknowledging start signal as idempotent", "experimentID", dda.Status.Experiment.ID)
 			return nil
 		}
 		return fmt.Errorf("start DatadogAgent experiment: %w", err)

pkg/fleet/daemon_test.go

@@ -127,24 +127,20 @@ func TestStartDatadogAgentExperiment_DDANotFound(t *testing.T) {
 	assert.Error(t, d.startDatadogAgentExperiment(context.Background(), testStartRequest()))
 }
 
-func TestStartDatadogAgentExperiment_Running(t *testing.T) {
-	d, _ := testDaemon(testDDAObject(v2alpha1.ExperimentPhaseRunning), testInstallerConfigWithDDA())
-	assert.Error(t, d.startDatadogAgentExperiment(context.Background(), testStartRequest()))
-}
-
-func TestStartDatadogAgentExperiment_Running_SameID_Idempotent(t *testing.T) {
+func TestStartDatadogAgentExperiment_Running_Idempotent(t *testing.T) {
 	dda := testDDAObject(v2alpha1.ExperimentPhaseRunning)
-	req := testStartRequest()
-	dda.Status.Experiment.ID = req.ID // same ID as the incoming request
 	d, c := testDaemon(dda, testInstallerConfigWithDDA())
+	// Backend retries with a new task UUID; should be treated as idempotent.
+	req := testStartRequest()
+	req.ID = "retry-task-uuid"
 	require.NoError(t, d.startDatadogAgentExperiment(context.Background(), req))
 
 	// DDA should be unchanged — no re-patch, no status update.
 	got := &v2alpha1.DatadogAgent{}
 	require.NoError(t, c.Get(context.Background(), testDDANSN, got))
 	require.NotNil(t, got.Status.Experiment)
 	assert.Equal(t, v2alpha1.ExperimentPhaseRunning, got.Status.Experiment.Phase)
-	assert.Equal(t, req.ID, got.Status.Experiment.ID)
+	assert.Equal(t, "old-exp", got.Status.Experiment.ID)
 }
 
 func TestStartDatadogAgentExperiment_Stopped(t *testing.T) {