Migrate tests to Suite/Provider/Area traits

.azure/pipelines/build.yaml

@@ -46,8 +46,8 @@ parameters:
     default:
     - net8.0
     - net10.0
-  - name: tests_categories
-    displayName: Test categories
+  - name: suites
+    displayName: Test suites
     type: object
     default:
     - BVT
@@ -98,5 +98,5 @@ extends:
           publish_nightly: false
           publish_nuget: false
           frameworks: ${{ parameters.frameworks }}
-          tests_categories: ${{ parameters.tests_categories }}
+          suites: ${{ parameters.suites }}
           runCodeQL3000: ${{ parameters.runCodeQL3000 }}

.azure/pipelines/templates/build.yaml

@@ -40,8 +40,8 @@ parameters:
     default:
     - net8.0
     - net10.0
-  - name: tests_categories
-    displayName: Test categories
+  - name: suites
+    displayName: Test suites
     type: object
     default:
     - BVT
@@ -175,17 +175,17 @@ jobs:
 
 # Tests
 - ${{ if and(eq(parameters.skip_test, false), ne(variables.runCodeQL3000, 'true')) }}:
-  - ${{ each category in parameters.tests_categories }}:
+  - ${{ each suite in parameters.suites }}:
     - ${{ each framework in parameters.frameworks }}:
       - job:
-        displayName: ${{category}} on ${{framework}}
+        displayName: ${{suite}} on ${{framework}}
         timeoutInMinutes: 120
         dependsOn: Build
         templateContext:
           outputs:
           - output: pipelineArtifact
-            targetPath: '$(Build.ArtifactStagingDirectory)/test_outputs_${{category}}_${{framework}}_$(Build.BuildId)'
-            artifactName: 'test_outputs_${{category}}_${{framework}}_$(System.JobAttempt)'
+            targetPath: '$(Build.ArtifactStagingDirectory)/test_outputs_${{suite}}_${{framework}}_$(Build.BuildId)'
+            artifactName: 'test_outputs_${{suite}}_${{framework}}_$(System.JobAttempt)'
             condition: succeededOrFailed()
         variables:
           # Transform "net8.0" to "8.0.x" for the second DotNetCoreCLI@2
@@ -243,8 +243,8 @@ jobs:
               SYSTEM_ACCESSTOKEN: $(System.AccessToken)
           inputs:
             command: 'test'
-            testRunTitle: ${{category}} on ${{framework}}
-            arguments: '--no-build --logger "trx;LogFilePrefix=testresults-${{framework}}-${{category}}" --framework ${{framework}} --configuration "${{parameters.build_configuration}}" --filter Category=${{category}} --blame-crash-dump-type full --blame-hang-timeout 10m --blame-hang-dump-type full -- -parallel none -noshadow'
+            testRunTitle: ${{suite}} on ${{framework}}
+            arguments: '--no-build --logger "trx;LogFilePrefix=testresults-${{framework}}-${{suite}}" --framework ${{framework}} --configuration "${{parameters.build_configuration}}" --filter "Provider=None&Suite=${{suite}}" --blame-crash-dump-type full --blame-hang-timeout 10m --blame-hang-dump-type full -- -parallel none -noshadow'
             publishTestResults: false # Doesn't merge correctly, use the explicit PublishTestResults task instead
         - task: PublishTestResults@2
           displayName: Publishing test results
@@ -253,18 +253,18 @@ jobs:
             testResultsFormat: VSTest
             testResultsFiles: '**/testresults-*.trx'
             mergeTestResults: true
-            testRunTitle: ${{category}} on ${{framework}}
+            testRunTitle: ${{suite}} on ${{framework}}
         - task: CopyFiles@2
           displayName: 'Copy test logs'
           condition: succeededOrFailed()
           inputs:
             Contents: '**\*.log'
-            TargetFolder: '$(Build.ArtifactStagingDirectory)/test_outputs_${{category}}_${{framework}}_$(Build.BuildId)'
+            TargetFolder: '$(Build.ArtifactStagingDirectory)/test_outputs_${{suite}}_${{framework}}_$(Build.BuildId)'
             OverWrite: true
         - task: CopyFiles@2
           displayName: 'Copy crash dumps'
           condition: succeededOrFailed()
           inputs:
             Contents: '**\*.dmp'
-            TargetFolder: '$(Build.ArtifactStagingDirectory)/test_outputs_${{category}}_${{framework}}_$(Build.BuildId)'
+            TargetFolder: '$(Build.ArtifactStagingDirectory)/test_outputs_${{suite}}_${{framework}}_$(Build.BuildId)'
             OverWrite: true

.github/workflows/ci.yml

@@ -66,7 +66,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -102,7 +102,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Clustering)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -150,7 +150,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -194,7 +194,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -240,7 +240,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -259,7 +259,40 @@
         path: |
           **/TestResults/*
           **/logs/*
+  test-sqlite:
+    name: SQLite provider tests
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        provider: ["Sqlite"]
+        framework: ["net8.0", "net10.0"]
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        global-json-file: global.json
+    - name: Test
+      run: dotnet test
+        --framework ${{ matrix.framework }}
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
+        --blame-hang-timeout 10m
+        --blame-crash-dump-type full
+        --blame-hang-dump-type full
+        --logger "trx;LogFileName=test_results_${{ matrix.provider }}_${{ matrix.framework }}.trx"
+        --
+        -parallel none -noshadow
+    - name: Archive Test Results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test_output_${{ github.job }}_${{ matrix.provider }}_${{ matrix.framework }}
+        retention-days: 1
+        path: |
+          **/TestResults/*
+          **/logs/*
   test-azure-storage:
     name: Azure Storage provider tests
     runs-on: ubuntu-latest
     continue-on-error: true
@@ -288,7 +321,7 @@
         global-json-file: global.json
     - name: Test
       run: dotnet test
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --framework ${{ matrix.framework }}
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
@@ -308,7 +341,7 @@
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: test_output_${{ github.job }}_${{ matrix.framework }}
+        name: test_output_${{ github.job }}_${{ matrix.provider }}_${{ matrix.framework }}
         retention-days: 1
         path: |
           **/TestResults/*
@@ -357,7 +390,7 @@
         global-json-file: global.json
     - name: Test
       run: dotnet test
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional|Category=Streaming)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --framework ${{ matrix.framework }}
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
@@ -431,7 +464,7 @@
         done
     - name: Test
       run: |
-        cosmos_filter='Category=${{ matrix.provider }}&Category!=Performance&Category!=Stress'
+        cosmos_filter='Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)&Category!=Performance&Category!=Stress'
         # The Linux vNext emulator still misses a narrow set of ETag/version concurrency semantics.
         cosmos_filter+='&FullyQualifiedName!=Tester.Cosmos.Clustering.CosmosMembershipTableTests.MembershipTable_Cosmos_ReadRow_Insert_Read'
         cosmos_filter+='&FullyQualifiedName!=Tester.Cosmos.Clustering.CosmosMembershipTableTests.MembershipTable_Cosmos_UpdateRow'
@@ -495,7 +528,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -533,7 +566,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -552,56 +585,66 @@
           **/TestResults/*
           **/logs/*
   test-dynamodb:
-    name: AWS DynamoDB provider tests
+    name: AWS provider tests (${{ matrix.provider }})
     runs-on: ubuntu-latest
     continue-on-error: true
     strategy:
       matrix:
-        provider: ["DynamoDB"]
+        provider: ["DynamoDB", "SQS"]
         framework: ["net8.0", "net10.0"]
     services:
       dynamodb:
         image: amazon/dynamodb-local:latest
         ports:
         - 8000:8000
         env:
 # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]
           AWS_ACCESS_KEY_ID: root
 # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]
           AWS_SECRET_ACCESS_KEY: pass
           AWS_REGION: us-east-1
+      elasticmq:
+        image: softwaremill/elasticmq-native:latest
+        ports:
+        - 9324:9324
     steps:
     - uses: actions/checkout@v4
+    - name: Wait for ElasticMQ
+      run: |
+        echo "Waiting for ElasticMQ to be ready..."
+        timeout 60 bash -c 'until nc -z localhost 9324; do sleep 1; done'
+        echo "ElasticMQ is ready"
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
         global-json-file: global.json
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}&(Category=BVT|Category=SlowBVT|Category=Functional)"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
         --logger "trx;LogFileName=test_results_${{ matrix.provider }}_${{ matrix.framework }}.trx"
         --
         -parallel none -noshadow
       env:
         ORLEANSDYNAMODBSERVICE: "http://127.0.0.1:8000"
 # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]
         ORLEANSDYNAMODBACCESSKEY: "root"
 # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Not a secret")]
         ORLEANSDYNAMODBSECRETKEY: "pass"
+        ORLEANSSQSCONNECTIONSTRING: "Service=http://127.0.0.1:9324"
     - name: Archive Test Results
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: test_output_${{ github.job }}_${{ matrix.framework }}
+        name: test_output_${{ github.job }}_${{ matrix.provider }}_${{ matrix.framework }}
         retention-days: 1
         path: |
           **/TestResults/*
           **/logs/*
   test-nats:
     name: NATS stream provider tests
     runs-on: ubuntu-latest
     continue-on-error: true
@@ -637,7 +680,7 @@
       run: dotnet test
         test/Extensions/Orleans.Streaming.NATS.Tests/Orleans.Streaming.NATS.Tests.csproj
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.provider }}"
+        --filter "Provider=${{ matrix.provider }}&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -680,7 +723,7 @@
     - name: Test
       run: dotnet test
         --framework ${{ matrix.framework }}
-        --filter "Category=${{ matrix.suite }}&Category!=Consul"
+        --filter "Provider=None&Suite=${{ matrix.suite }}&Area!=CodeGen"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
@@ -711,13 +754,23 @@
       with:
         global-json-file: global.json
     - name: Test Code Generator
-      run: dotnet test
-        test/Orleans.CodeGenerator.Tests/Orleans.CodeGenerator.Tests.csproj
+      run: dotnet test test/Orleans.CodeGenerator.Tests/Orleans.CodeGenerator.Tests.csproj
+        --framework ${{ matrix.framework }}
+        --filter "Provider=None&Area=CodeGen&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
+        --blame-hang-timeout 10m
+        --blame-crash-dump-type full
+        --blame-hang-dump-type full
+        --logger "trx;LogFileName=test_results_codegen_${{ matrix.framework }}.trx"
+        --
+        -parallel none -noshadow
+        &&
+        dotnet test test/Orleans.DefaultCluster.Tests/Orleans.DefaultCluster.Tests.csproj
         --framework ${{ matrix.framework }}
+        --filter "Provider=None&Area=CodeGen&(Suite=BVT|Suite=SlowBVT|Suite=Functional)"
         --blame-hang-timeout 10m
         --blame-crash-dump-type full
         --blame-hang-dump-type full
-        --logger "trx;LogFileName=test_results_${{ matrix.framework }}.trx"
+        --logger "trx;LogFileName=test_results_defaultcluster_codegen_${{ matrix.framework }}.trx"
         --
         -parallel none -noshadow
     - name: Archive Test Results

test/Benchmarks/Serialization/Comparison/ClassDeserializeBenchmark.cs

@@ -19,6 +19,9 @@ namespace Benchmarks.Serialization.Comparison;
 /// Compares Orleans deserialization performance against other popular serializers for class types.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 [BenchmarkCategory("Serialization")]
 //[DisassemblyDiagnoser(recursiveDepth: 2, printSource: true)]

test/Benchmarks/Serialization/Comparison/ClassSerializeBenchmark.cs

@@ -21,6 +21,9 @@ namespace Benchmarks.Serialization.Comparison;
 /// Compares Orleans serialization performance against other popular serializers for class types.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 [BenchmarkCategory("Serialization")]
 [PayloadSizeColumn]

test/Benchmarks/Serialization/Comparison/StructDeserializeBenchmark.cs

@@ -18,6 +18,9 @@ namespace Benchmarks.Serialization.Comparison;
 /// Compares Orleans deserialization performance against other popular serializers for struct types.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 [BenchmarkCategory("Serialization")]
 //[DisassemblyDiagnoser(recursiveDepth: 4)]

test/Benchmarks/Serialization/Comparison/StructSerializeBenchmark.cs

@@ -19,6 +19,9 @@ namespace Benchmarks.Serialization.Comparison;
 /// Compares Orleans serialization performance against other popular serializers for struct types.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 [BenchmarkCategory("Serialization")]
 [PayloadSizeColumn]

test/Benchmarks/Serialization/ComplexTypeBenchmarks.cs

@@ -19,6 +19,9 @@ namespace Benchmarks.Serialization;
 /// Benchmarks Orleans serialization performance for complex object graphs with circular references.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 [MemoryDiagnoser]
 public class ComplexTypeBenchmarks

test/Benchmarks/Serialization/MegaGraphBenchmark.cs

@@ -16,6 +16,9 @@ namespace Benchmarks.Serialization;
 /// Benchmarks Orleans serialization performance for very large object graphs with hundreds of thousands of items.
 /// </summary>
 [Trait("Category", "Benchmark")]
+[TestSuite("Benchmark")]
+[TestProvider("None")]
+[TestArea("Serialization")]
 [Config(typeof(BenchmarkConfig))]
 public class MegaGraphBenchmark
 {
@@ -63,4 +66,4 @@ public int Serialize()
         writer.Dispose();
         return writer.Position;
     }
-}
+}