skip unimplemented error; update workflow

Author: Yejing-Lai

.github/workflows/cpu-inference.yml

@@ -63,4 +63,7 @@ jobs:
           unset TORCH_CUDA_ARCH_LIST # only jit compile for current arch
           if [[ -d ./torch-extensions ]]; then rm -rf ./torch-extensions; fi
           cd tests
-          TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'inference' unit/inference/test_inference_config.py
+          pytest -v -s unit/autotuning/ unit/checkpoint/ unit/comm/ unit/compression/ unit/elasticity/ unit/launcher/ unit/profiling/ unit/ops
+          TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'seq_inference' unit/
+          TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest -m 'inference_ops' unit/
+          TRANSFORMERS_CACHE=~/tmp/transformers_cache/ TORCH_EXTENSIONS_DIR=./torch-extensions pytest --forked -n 4 -m 'inference' unit/

accelerator/abstract_accelerator.py

@@ -156,6 +156,10 @@ def is_bf16_supported(self):
     def is_fp16_supported(self):
         ...
 
+    @abc.abstractmethod
+    def supported_dtypes(self):
+        ...
+
     # Misc
     @abc.abstractmethod
     def amp(self):

accelerator/cpu_accelerator.py

@@ -183,7 +183,10 @@ def is_bf16_supported(self):
         return True
 
     def is_fp16_supported(self):
-        return True
+        return False
+
+    def supported_dtypes(self):
+        return [torch.float, torch.bfloat16]
 
     # Tensor operations
 

setup.py

@@ -153,6 +153,7 @@ def op_enabled(op_name):
 for op_name, builder in ALL_OPS.items():
     op_compatible = builder.is_compatible()
     compatible_ops[op_name] = op_compatible
+    compatible_ops["deepspeed_not_implemented"] = False
 
     # If op is requested but not available, throw an error.
     if op_enabled(op_name) and not op_compatible:

tests/unit/checkpoint/test_latest_checkpoint.py

@@ -5,10 +5,15 @@
 
 import deepspeed
 
+import pytest
 from unit.common import DistributedTest
 from unit.simple_model import *
 
 from unit.checkpoint.common import checkpoint_correctness_verification
+from deepspeed.ops.op_builder import FusedAdamBuilder
+
+if not deepspeed.ops.__compatible_ops__[FusedAdamBuilder.NAME]:
+    pytest.skip("This op had not been implemented on this system.", allow_module_level=True)
 
 
 class TestLatestCheckpoint(DistributedTest):

tests/unit/inference/test_inference.py

@@ -20,6 +20,10 @@
 from deepspeed.model_implementations import DeepSpeedTransformerInference
 from torch import nn
 from deepspeed.accelerator import get_accelerator
+from deepspeed.ops.op_builder import InferenceBuilder
+
+if not deepspeed.ops.__compatible_ops__[InferenceBuilder.NAME]:
+    pytest.skip("This op had not been implemented on this system.", allow_module_level=True)
 
 rocm_version = OpBuilder.installed_rocm_version()
 if rocm_version != (0, 0):

tests/unit/inference/test_model_profiling.py

@@ -11,6 +11,10 @@
 from transformers import pipeline
 from unit.common import DistributedTest
 from deepspeed.accelerator import get_accelerator
+from deepspeed.ops.op_builder import InferenceBuilder
+
+if not deepspeed.ops.__compatible_ops__[InferenceBuilder.NAME]:
+    pytest.skip("This op had not been implemented on this system.", allow_module_level=True)
 
 
 @pytest.fixture

tests/unit/ops/accelerators/test_accelerator_backward.py

@@ -19,6 +19,8 @@
 #pytest.skip(
 #    "transformer kernels are temporarily disabled because of unexplained failures",
 #    allow_module_level=True)
+if torch.half not in get_accelerator().supported_dtypes():
+    pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True)
 
 
 def check_equal(first, second, atol=1e-2, verbose=False):

tests/unit/ops/accelerators/test_accelerator_forward.py

@@ -15,6 +15,9 @@
 from deepspeed.accelerator import get_accelerator
 from unit.common import DistributedTest
 
+if torch.half not in get_accelerator().supported_dtypes():
+    pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True)
+
 
 def check_equal(first, second, atol=1e-2, verbose=False):
     if verbose:

tests/unit/ops/adam/test_adamw.py

@@ -11,7 +11,10 @@
 from deepspeed.ops.adam import DeepSpeedCPUAdam
 from unit.common import DistributedTest
 from unit.simple_model import SimpleModel
+from deepspeed.accelerator import get_accelerator
 
+if torch.half not in get_accelerator().supported_dtypes():
+    pytest.skip(f"fp16 not supported, valid dtype: {get_accelerator().supported_dtypes()}", allow_module_level=True)
 # yapf: disable
 #'optimizer, zero_offload, torch_adam, adam_w_mode, resulting_optimizer
 adam_configs = [["AdamW", False, False, False, (FusedAdam, True)],