Retry failed precondition sandbox registrations

Author: YunchuWang

durabletask-azuremanaged/durabletask/azuremanaged/preview/sandboxes/worker.py

@@ -243,14 +243,15 @@ def _resolve_dts_sandbox_identifier() -> str:
 _RETRIABLE_REGISTRATION_STATUS_CODES = {
     grpc.StatusCode.CANCELLED,
     grpc.StatusCode.DEADLINE_EXCEEDED,
+    grpc.StatusCode.FAILED_PRECONDITION,
     grpc.StatusCode.INTERNAL,
     grpc.StatusCode.RESOURCE_EXHAUSTED,
     grpc.StatusCode.UNAVAILABLE,
     grpc.StatusCode.UNKNOWN,
 }
 
-# TODO: Selectively retry FAILED_PRECONDITION only after sandbox worker
-# registration exposes structured, machine-readable transient reasons.
+# TODO: Stop retrying permanent FAILED_PRECONDITION responses once sandbox
+# worker registration exposes structured, machine-readable failure reasons.
 
 
 def _is_retriable_registration_failure(ex: Exception) -> bool:

tests/durabletask-azuremanaged/test_sandboxes_extension.py

@@ -755,14 +755,14 @@ def RemoteHello(_ctx, value):
         assert start.start.sandbox_provider == pb.SANDBOX_PROVIDER_KIND_UNSPECIFIED
 
 
-def test_sandbox_registration_does_not_retry_failed_precondition_without_structured_reasons() -> None:
+def test_sandbox_registration_retries_failed_precondition_until_structured_reasons() -> None:
     assert sandbox_worker._is_retriable_registration_failure(
         _FakeRpcError(grpc.StatusCode.UNAVAILABLE))
-    assert not sandbox_worker._is_retriable_registration_failure(
+    assert sandbox_worker._is_retriable_registration_failure(
         _FakeRpcError(grpc.StatusCode.FAILED_PRECONDITION, "sandbox not ready"))
     assert not sandbox_worker._is_retriable_registration_failure(
         _FakeRpcError(grpc.StatusCode.INVALID_ARGUMENT))
-    assert not sandbox_worker._is_retriable_registration_failure(
+    assert sandbox_worker._is_retriable_registration_failure(
         _FakeRpcError(grpc.StatusCode.FAILED_PRECONDITION, "worker profile does not match"))