Migrate sqlserver and azure-sql-edge to Testcontainers

docker-compose.yml

@@ -57,14 +57,6 @@ services:
     ports:
     - "6379"
 
-  sqledge_arm64:
-    image: mcr.microsoft.com/azure-sql-edge:latest
-    ports:
-    - "1433"
-    environment:
-    - ACCEPT_EULA=Y
-    - SA_PASSWORD=Strong!Passw0rd
-
   cosmosdb-emulator_arm64:
     image: mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview
     command: ["--protocol", "https"]
@@ -219,15 +211,6 @@ services:
     ports:
     - "127.0.0.1:3407:3306"
 
-  sqlserver:
-    image: mcr.microsoft.com/mssql/server:latest
-    profiles: ["group1"]
-    ports:
-    - "127.0.0.1:1433:1433"
-    environment:
-    - ACCEPT_EULA=Y
-    - SA_PASSWORD=Strong!Passw0rd
-
   sqledge:
     image: mcr.microsoft.com/azure-sql-edge:latest
     profiles: ["group2"]
@@ -485,7 +468,6 @@ services:
       - ELASTICSEARCH7_HOST=elasticsearch7:9200
       - ELASTICSEARCH6_HOST=elasticsearch6:9200
       - ELASTICSEARCH5_HOST=elasticsearch5:9200
-      - SQLSERVER_CONNECTION_STRING=Server=sqlserver;User=sa;Password=Strong!Passw0rd;TrustServerCertificate=true
       - POSTGRES_HOST=postgres
       - MYSQL_HOST=mysql
       - MYSQL_PORT=3306
@@ -700,7 +682,6 @@ services:
       - stackexchangeredis
       - stackexchangeredis-replica
       - stackexchangeredis-single
-      - sqlserver
       - postgres
       - mysql
       - mysql57
@@ -711,7 +692,7 @@ services:
       - test-agent
     environment:
       - TIMEOUT_LENGTH=120
-    command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 sqlserver:1433 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 couchbase:11210 test-agent:8126 test-agent:4317 test-agent:4318
+    command: servicestackredis:6379 stackexchangeredis:6379 stackexchangeredis-replica:6379 stackexchangeredis-single:6379 postgres:5432 mysql:3306 mysql57:3306 rabbitmq:5672 kafka-broker:9092 kafka-zookeeper:2181 couchbase:11210 test-agent:8126 test-agent:4317 test-agent:4318
 
   StartDependencies.Group2:
     image: andrewlock/wait-for-dependencies
@@ -742,6 +723,7 @@ services:
     command: dotnet /build/bin/Debug/_build.dll RunIntegrationTests
     volumes:
       - ./:/project
+      - /var/run/docker.sock:/var/run/docker.sock
     cap_add:
       - SYS_PTRACE
     environment:
@@ -767,7 +749,6 @@ services:
       - ELASTICSEARCH7_HOST=elasticsearch7_arm64:9200
       - ELASTICSEARCH6_HOST=elasticsearch7_arm64:9200
       - ELASTICSEARCH5_HOST=elasticsearch7_arm64:9200
-      - SQLSERVER_CONNECTION_STRING=Server=sqledge_arm64;User=sa;Password=Strong!Passw0rd;TrustServerCertificate=true
       - POSTGRES_HOST=postgres_arm64
       - MYSQL_HOST=mysql_arm64
       - MYSQL_PORT=3306
@@ -808,7 +789,6 @@ services:
       - stackexchangeredis_arm64-replica
       - stackexchangeredis_arm64-single
       - elasticsearch7_arm64
-      - sqledge_arm64
       - mongo_arm64
       - postgres_arm64
       - mysql_arm64
@@ -825,7 +805,6 @@ services:
       - stackexchangeredis_arm64-replica
       - stackexchangeredis_arm64-single
       - elasticsearch7_arm64
-      - sqledge_arm64
       - mongo_arm64
       - postgres_arm64
       - mysql_arm64
@@ -835,7 +814,7 @@ services:
       - cosmosdb-emulator_arm64
     environment:
       - TIMEOUT_LENGTH=120
-    command: servicestackredis_arm64:6379 stackexchangeredis_arm64:6379 stackexchangeredis_arm64-replica:6379 stackexchangeredis_arm64-single:6379 elasticsearch7_arm64:9200 sqledge_arm64:1433 mongo_arm64:27017 postgres_arm64:5432 mysql_arm64:3306 rabbitmq_arm64:5672 localstack_arm64:4566 test-agent:8126 test-agent:4317 test-agent:4318 cosmosdb-emulator_arm64:8081
+    command: servicestackredis_arm64:6379 stackexchangeredis_arm64:6379 stackexchangeredis_arm64-replica:6379 stackexchangeredis_arm64-single:6379 elasticsearch7_arm64:9200 mongo_arm64:27017 postgres_arm64:5432 mysql_arm64:3306 rabbitmq_arm64:5672 localstack_arm64:4566 test-agent:8126 test-agent:4317 test-agent:4318 cosmosdb-emulator_arm64:8081
 
   IntegrationTests.ARM64.Debugger:
     build:
@@ -911,15 +890,14 @@ services:
       - stackexchangeredis_osx_arm64-replica
       - stackexchangeredis_osx_arm64-single
       - elasticsearch7_osx_arm64
-      - sqledge_osx_arm64
       - mongo_osx_arm64
       - postgres_osx_arm64
       - mysql_osx_arm64
       - rabbitmq_osx_arm64
       - localstack_osx_arm64
     environment:
       - TIMEOUT_LENGTH=120
-    command: servicestackredis_osx_arm64:6379 stackexchangeredis_osx_arm64:6379 stackexchangeredis_osx_arm64-replica:6379 stackexchangeredis_osx_arm64-single:6379 elasticsearch7_osx_arm64:9200 sqledge_osx_arm64:1433 mongo_osx_arm64:27017 postgres_osx_arm64:5432 mysql_osx_arm64:3306 rabbitmq_osx_arm64:5672 localstack_osx_arm64:4566
+    command: servicestackredis_osx_arm64:6379 stackexchangeredis_osx_arm64:6379 stackexchangeredis_osx_arm64-replica:6379 stackexchangeredis_osx_arm64-single:6379 elasticsearch7_osx_arm64:9200 mongo_osx_arm64:27017 postgres_osx_arm64:5432 mysql_osx_arm64:3306 rabbitmq_osx_arm64:5672 localstack_osx_arm64:4566
 
   # OSX ARM64 dependencies
 
@@ -1003,14 +981,18 @@ services:
     ports:
       - "6391:6379"
 
-  sqledge_osx_arm64:
-    image: mcr.microsoft.com/azure-sql-edge:latest
-    ports:
-      - "1433:1433"
-    environment:
-      - ACCEPT_EULA=Y
-      - SA_PASSWORD=Strong!Passw0rd
+  # Testcontainers pre-pull
+  # These service definitions exist only to pre-pull container images into the CI cache.
+  # They are not started as live services; the actual containers are managed by Testcontainers.
 
   # keep syncronized image version with tracer\test\Datadog.Trace.TestHelpers.AutoInstrumentation\Containers\AerospikeFixture.cs
   aerospike:
     image: aerospike/aerospike-server:6.2.0.6
+
+  # Used on x86_64; keep synchronized with tracer\test\Datadog.Trace.TestHelpers.AutoInstrumentation\Containers\SqlServerFixture.cs
+  sqlserver:
+    image: mcr.microsoft.com/mssql/server:2022-latest
+
+  # Used on arm64 (no native mssql/server image); keep synchronized with tracer\test\Datadog.Trace.TestHelpers.AutoInstrumentation\Containers\SqlServerFixture.cs
+  azure-sql-edge:
+    image: mcr.microsoft.com/azure-sql-edge:latest

tracer/build/_build/Build.Steps.cs

@@ -1842,6 +1842,7 @@ void CompileSamplesThatDependOnDatadogTrace()
                     // Don't apply a custom filter to these tests, they should all be able to be run
                     .When(!string.IsNullOrWhiteSpace(AddAreaFilter(Filter)), c => c.SetFilter(AddAreaFilter(Filter)))
                     .When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
+                    .When(IsWin, o => o.SetProcessEnvironmentVariable("SQLSERVER_CONNECTION_STRING", @"Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Connection Timeout=60"))
                     .When(CodeCoverageEnabled, ConfigureCodeCoverage)
                     .CombineWith(parallelJobs, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
@@ -1864,6 +1865,7 @@ void CompileSamplesThatDependOnDatadogTrace()
                     .SetLogsDirectory(TestLogsDirectory)
                     .When(!string.IsNullOrWhiteSpace(filter), c => c.SetFilter(filter))
                     .When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
+                    .When(IsWin, o => o.SetProcessEnvironmentVariable("SQLSERVER_CONNECTION_STRING", @"Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Connection Timeout=60"))
                     .When(CodeCoverageEnabled, ConfigureCodeCoverage)
                     .CombineWith(ClrProfilerIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))
@@ -2011,6 +2013,7 @@ private string AddAreaFilter(string filter)
                     .SetIsDebugRun(isDebugRun)
                     .SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
                     .SetLogsDirectory(TestLogsDirectory)
+                    .SetProcessEnvironmentVariable("SQLSERVER_CONNECTION_STRING", @"Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Connection Timeout=60")
                     .When(CodeCoverageEnabled, ConfigureCodeCoverage)
                     .CombineWith(ClrProfilerIntegrationTests, (s, project) => s
                         .EnableTrxLogOutput(GetResultsDirectory(project))

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/MicrosoftDataSqlClientTests.cs

@@ -10,6 +10,7 @@
 using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.TestHelpers;
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
 using FluentAssertions;
 using Xunit;
 using Xunit.Abstractions;
@@ -18,12 +19,14 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.AdoNet
 {
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "1")]
+    [Collection(SqlServerCollection.Name)]
     public class MicrosoftDataSqlClientTests : TracingIntegrationTest
     {
-        public MicrosoftDataSqlClientTests(ITestOutputHelper output)
+        public MicrosoftDataSqlClientTests(ITestOutputHelper output, SqlServerFixture sqlServerFixture)
             : base("Microsoft.Data.SqlClient", output)
         {
             SetServiceVersion("1.0.0");
+            ConfigureContainers(sqlServerFixture);
         }
 
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) => span.IsSqlClient(metadataSchemaVersion);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/MicrosoftDataSqliteTests.cs

@@ -73,10 +73,8 @@ public async Task SubmitsTraces(
             settings.AddRegexScrubber(new Regex("Sqlite-Test-[a-zA-Z0-9]{32}"), "System-Data-SqlClient-Test-GUID");
             settings.AddSimpleScrubber("out.host: localhost", "out.host: sqlserver");
             settings.AddSimpleScrubber("out.host: (localdb)\\MSSQLLocalDB", "out.host: sqlserver");
-            settings.AddSimpleScrubber("out.host: sqledge_arm64", "out.host: sqlserver");
             settings.AddSimpleScrubber("peer.service: localhost", "peer.service: sqlserver");
             settings.AddSimpleScrubber("peer.service: (localdb)\\MSSQLLocalDB", "peer.service: sqlserver");
-            settings.AddSimpleScrubber("peer.service: sqledge_arm64", "peer.service: sqlserver");
             settings.AddRegexScrubber(new Regex("dd.instrumentation.time_ms: \\d+.\\d+"), "dd.instrumentation.time_ms: 123.456");
 
             var fileName = nameof(MicrosoftDataSqliteTests);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/SystemDataSqlClientTests.cs

@@ -12,6 +12,7 @@
 using Datadog.Trace.Configuration;
 using Datadog.Trace.ExtensionMethods;
 using Datadog.Trace.TestHelpers;
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
 using FluentAssertions;
 using VerifyXunit;
 using Xunit;
@@ -22,12 +23,14 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.AdoNet
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "1")]
     [UsesVerify]
+    [Collection(SqlServerCollection.Name)]
     public class SystemDataSqlClientTests : TracingIntegrationTest
     {
-        public SystemDataSqlClientTests(ITestOutputHelper output)
+        public SystemDataSqlClientTests(ITestOutputHelper output, SqlServerFixture sqlServerFixture)
             : base("SqlServer", output)
         {
             SetServiceVersion("1.0.0");
+            ConfigureContainers(sqlServerFixture);
         }
 
         public override Result ValidateIntegrationSpan(MockSpan span, string metadataSchemaVersion) => span.IsSqlClient(metadataSchemaVersion);
@@ -88,10 +91,10 @@ public async Task SubmitsTraces(
             settings.AddRegexScrubber(new Regex("System-Data-SqlClient-Test-[a-zA-Z0-9]{32}"), "System-Data-SqlClient-Test-GUID");
             settings.AddSimpleScrubber("out.host: localhost", "out.host: sqlserver");
             settings.AddSimpleScrubber("out.host: (localdb)\\MSSQLLocalDB", "out.host: sqlserver");
-            settings.AddSimpleScrubber("out.host: sqledge_arm64", "out.host: sqlserver");
+            settings.AddRegexScrubber(new Regex(@"out\.host: [\d.]+,\d+"), "out.host: sqlserver");
             settings.AddSimpleScrubber("peer.service: localhost", "peer.service: sqlserver");
             settings.AddSimpleScrubber("peer.service: (localdb)\\MSSQLLocalDB", "peer.service: sqlserver");
-            settings.AddSimpleScrubber("peer.service: sqledge_arm64", "peer.service: sqlserver");
+            settings.AddRegexScrubber(new Regex(@"peer\.service: [\d.]+,\d+"), "peer.service: sqlserver");
             settings.AddRegexScrubber(new Regex("dd.instrumentation.time_ms: \\d+.\\d+"), "dd.instrumentation.time_ms: 123.456");
 
             var fileName = nameof(SystemDataSqlClientTests);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/SystemDataSqliteTests.cs

@@ -81,10 +81,8 @@ private async Task RunTest(string metadataSchemaVersion)
             settings.AddRegexScrubber(new Regex("SQLite-Test-[a-zA-Z0-9]{32}"), "System-Data-SqlClient-Test-GUID");
             settings.AddSimpleScrubber("out.host: localhost", "out.host: sqlserver");
             settings.AddSimpleScrubber("out.host: (localdb)\\MSSQLLocalDB", "out.host: sqlserver");
-            settings.AddSimpleScrubber("out.host: sqledge_arm64", "out.host: sqlserver");
             settings.AddSimpleScrubber("peer.service: localhost", "peer.service: sqlserver");
             settings.AddSimpleScrubber("peer.service: (localdb)\\MSSQLLocalDB", "peer.service: sqlserver");
-            settings.AddSimpleScrubber("peer.service: sqledge_arm64", "peer.service: sqlserver");
             settings.AddRegexScrubber(new Regex("dd.instrumentation.time_ms: \\d+.\\d+"), "dd.instrumentation.time_ms: 123.456");
 
             var fileName = nameof(SystemDataSqliteTests);

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AerospikeTests.cs

@@ -7,6 +7,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.TestHelpers;
 using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
@@ -21,7 +22,8 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "2")]
     [UsesVerify]
-    public class AerospikeTests : TracingIntegrationTest, IClassFixture<AerospikeFixture>
+    [Collection(AerospikeCollection.Name)]
+    public class AerospikeTests : TracingIntegrationTest
     {
         public AerospikeTests(ITestOutputHelper output, AerospikeFixture aerospikeFixture)
             : base("Aerospike", output)

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Datadog.Trace.ClrProfiler.IntegrationTests.csproj

@@ -27,7 +27,7 @@
 
     <PackageReference Include="System.Collections.Immutable" Version="6.0.0" />
 
-    <PackageReference Include="Testcontainers" Version="3.6.0" />
+    <PackageReference Include="Testcontainers" Version="4.11.0" />
     <PackageReference Include="StrongNamer" Version="0.2.5" />  
   </ItemGroup>
 

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/ContainersCollection.cs

@@ -0,0 +1,27 @@
+// <copyright file="ContainersCollection.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+#pragma warning disable SA1649 // File name should match first type name (this will just store all the classes)
+#pragma warning disable SA1402 // File may only contain a single type (this will just store all the classes)
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
+using Xunit;
+
+namespace Datadog.Trace.ClrProfiler.IntegrationTests.Helpers
+{
+    [CollectionDefinition(Name)]
+
+    public class AerospikeCollection : ICollectionFixture<AerospikeFixture>
+    {
+        public const string Name = "Aerospike";
+    }
+
+    [CollectionDefinition(Name)]
+    public class SqlServerCollection : ICollectionFixture<SqlServerFixture>
+    {
+        public const string Name = "SqlServer";
+    }
+}
+
+#pragma warning restore SA1649 // File name should match first type name
+#pragma warning restore SA1402 // File may only contain a single type

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/SmokeTests/ServiceBusMinimalMassTransitTest.cs

@@ -4,18 +4,26 @@
 // </copyright>
 
 using System.Threading.Tasks;
+using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers;
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Datadog.Trace.ClrProfiler.IntegrationTests.SmokeTests
 {
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "2")]
+    [Collection(SqlServerCollection.Name)]
     public class ServiceBusMinimalMassTransitTest : SmokeTestBase
     {
-        public ServiceBusMinimalMassTransitTest(ITestOutputHelper output)
+        public ServiceBusMinimalMassTransitTest(ITestOutputHelper output, SqlServerFixture sqlServerFixture)
             : base(output, "ServiceBus.Minimal.MassTransit", maxTestRunSeconds: 60)
         {
+            foreach (var kvp in sqlServerFixture.GetEnvironmentVariables())
+            {
+                SetEnvironmentVariable(kvp.Key, kvp.Value);
+            }
+
             AssumeSuccessOnTimeout = true;
         }
 

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/SmokeTests/ServiceBusMinimalNServiceBusTest.cs

@@ -4,18 +4,25 @@
 // </copyright>
 
 using System.Threading.Tasks;
+using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers;
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Datadog.Trace.ClrProfiler.IntegrationTests.SmokeTests
 {
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "2")]
+    [Collection(SqlServerCollection.Name)]
     public class ServiceBusMinimalNServiceBusTest : SmokeTestBase
     {
-        public ServiceBusMinimalNServiceBusTest(ITestOutputHelper output)
+        public ServiceBusMinimalNServiceBusTest(ITestOutputHelper output, SqlServerFixture sqlServerFixture)
             : base(output, "ServiceBus.Minimal.NServiceBus", maxTestRunSeconds: 90)
         {
+            foreach (var kvp in sqlServerFixture.GetEnvironmentVariables())
+            {
+                SetEnvironmentVariable(kvp.Key, kvp.Value);
+            }
         }
 
         [SkippableFact]

tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/SmokeTests/ServiceBusMinimalRebusTest.cs

@@ -4,18 +4,25 @@
 // </copyright>
 
 using System.Threading.Tasks;
+using Datadog.Trace.ClrProfiler.IntegrationTests.Helpers;
+using Datadog.Trace.TestHelpers.AutoInstrumentation.Containers;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Datadog.Trace.ClrProfiler.IntegrationTests.SmokeTests
 {
     [Trait("RequiresDockerDependency", "true")]
     [Trait("DockerGroup", "2")]
+    [Collection(SqlServerCollection.Name)]
     public class ServiceBusMinimalRebusTest : SmokeTestBase
     {
-        public ServiceBusMinimalRebusTest(ITestOutputHelper output)
+        public ServiceBusMinimalRebusTest(ITestOutputHelper output, SqlServerFixture sqlServerFixture)
             : base(output, "ServiceBus.Minimal.Rebus", maxTestRunSeconds: 90)
         {
+            foreach (var kvp in sqlServerFixture.GetEnvironmentVariables())
+            {
+                SetEnvironmentVariable(kvp.Key, kvp.Value);
+            }
         }
 
         [SkippableFact]