diff --git a/cmd/serve_jobs_v2.go b/cmd/serve_jobs_v2.go index 6524e68e..c2604222 100644 --- a/cmd/serve_jobs_v2.go +++ b/cmd/serve_jobs_v2.go @@ -229,14 +229,15 @@ var ( ) const ( - jobsV2SchedulerMinDelay = 100 * time.Millisecond - jobsV2SchedulerIdleDelay = time.Minute - jobsV2SchedulerErrorDelay = 200 * time.Millisecond - jobsV2WorkerMinDelay = 100 * time.Millisecond - jobsV2WorkerIdleDelay = time.Minute - jobsV2WorkerErrorDelay = 200 * time.Millisecond - jobsV2FinishRunAttempts = 3 - jobsV2FinishRunRetryDelay = 50 * time.Millisecond + jobsV2SchedulerMinDelay = 100 * time.Millisecond + jobsV2SchedulerIdleDelay = time.Minute + jobsV2SchedulerErrorDelay = 200 * time.Millisecond + jobsV2WorkerMinDelay = 100 * time.Millisecond + jobsV2WorkerIdleDelay = time.Minute + jobsV2WorkerErrorDelay = 200 * time.Millisecond + jobsV2FinishRunRetryDelay = 50 * time.Millisecond + jobsV2FinishRunRetryMaxDelay = time.Second + jobsV2FinishRunRetryTimeout = 30 * time.Second ) type jobsV2ProgramConfig struct { @@ -1417,19 +1418,31 @@ func (m *jobsV2Manager) executeRun(run jobsV2Run) { } func (m *jobsV2Manager) finishRunWithRetry(runID string, status jobsV2RunStatus, result jobsV2RunResult, runErr error, policyAttempt int) { + deadline := time.Now().Add(jobsV2FinishRunRetryTimeout) + delay := jobsV2FinishRunRetryDelay + attempts := 0 var lastErr error - for attempt := 1; attempt <= jobsV2FinishRunAttempts; attempt++ { + for { + attempts++ if err := m.finishRun(runID, status, result, runErr, policyAttempt); err != nil { lastErr = err - if attempt < jobsV2FinishRunAttempts { - time.Sleep(jobsV2FinishRunRetryDelay) - continue + remaining := time.Until(deadline) + if remaining <= 0 { + break } - } else { - return + wait := min(delay, remaining) + select { + case <-time.After(wait): + case <-m.doneChan(): + log.Printf("jobs v2: failed to finish run %q as %s after %d attempts; manager is closing and run may remain active until restart recovery: %v", runID, status, attempts, lastErr) + return + } + delay = min(delay*2, jobsV2FinishRunRetryMaxDelay) + continue } + return } - log.Printf("jobs v2: failed to finish run %q as %s after %d attempts; run may remain active: %v", runID, status, jobsV2FinishRunAttempts, lastErr) + log.Printf("jobs v2: failed to finish run %q as %s after %d attempts over %s; run may remain active until restart recovery: %v", runID, status, attempts, jobsV2FinishRunRetryTimeout, lastErr) } func (m *jobsV2Manager) finishRun(runID string, status jobsV2RunStatus, result jobsV2RunResult, runErr error, attempt int) error { diff --git a/cmd/serve_jobs_v2_test.go b/cmd/serve_jobs_v2_test.go index 6c4a295f..a5834438 100644 --- a/cmd/serve_jobs_v2_test.go +++ b/cmd/serve_jobs_v2_test.go @@ -1748,7 +1748,7 @@ func TestJobsV2ExecuteRunRetriesFinishRunAfterTransientDBLock(t *testing.T) { return jobsV2RunResult{}, err } go func() { - time.Sleep(70 * time.Millisecond) + time.Sleep(250 * time.Millisecond) _ = tx.Rollback() _ = lockDB.Close() close(lockReleased)