diff --git a/diffusers/src/diffusers/hooks/group_offloading.py b/diffusers/src/diffusers/hooks/group_offloading.py
index 49509cb..76c7d22 100644
--- a/diffusers/src/diffusers/hooks/group_offloading.py
+++ b/diffusers/src/diffusers/hooks/group_offloading.py
@@ -259,17 +259,16 @@ def _onload_from_disk(self):
         current_stream = self._torch_accelerator_module.current_stream() if self.record_stream else None
 
         with context:
-            # Load to CPU (if using streams) or directly to target device, pin, and async copy to device
-            device = str(self.onload_device) if self.stream is None else "cpu"
-            loaded_tensors = safetensors.torch.load_file(self.safetensors_file_path, device=device)
-
             if self.stream is not None:
+                # Load to CPU, pin memory, then async transfer to device
+                loaded_tensors = safetensors.torch.load_file(self.safetensors_file_path, device="cpu")
                 for key, tensor_obj in self.key_to_tensor.items():
                     pinned_tensor = loaded_tensors[key].pin_memory()
                     tensor_obj.data = pinned_tensor.to(self.onload_device, non_blocking=self.non_blocking)
                     if self.record_stream:
                         tensor_obj.data.record_stream(current_stream)
             else:
+                # Load directly to target device
                 onload_device = (
                     self.onload_device.type if isinstance(self.onload_device, torch.device) else self.onload_device
                 )