[Backport 3.x] Use Testcontainers for integration tests (#2033)

.github/workflows/test-integration-unreleased.yml

@@ -90,7 +90,7 @@ jobs:
       - name: Run Integration Test
         run: |
           cd opensearch-java
-          ./gradlew clean integrationTest -Dhttps=false
+          ./gradlew clean integrationTest -Dhttps=false -Dtests.opensearch.testcontainers.enabled=false
 
       - name: Upload Reports
         if: failure()

.github/workflows/test-integration.yml

@@ -40,27 +40,8 @@ jobs:
           java-version: ${{ matrix.entry.java }}
           distribution: 'temurin'
           cache: 'gradle'
-      - name: Run Docker
-        run: |
-          echo "PASSWORD=admin" >> $GITHUB_ENV
-          docker info
-          docker compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=${{ matrix.entry.opensearch_version }}
-          docker compose --project-directory .ci/opensearch up -d
-          sleep 60
-
-      - name: Sets password (new versions)
-        run: |
-          VERSION_COMPONENTS=(${OPENSEARCH_VERSION//./ })
-          MAJOR_VERSION=${VERSION_COMPONENTS[0]}
-          MINOR_VERSION=${VERSION_COMPONENTS[1]}
-          if (( $MAJOR_VERSION > 2 || ( $MAJOR_VERSION == 2 && $MINOR_VERSION >= 12 ) )); then
-              echo "PASSWORD=0_aD^min_0" >> $GITHUB_ENV
-          fi
-        env:
-          OPENSEARCH_VERSION: ${{ matrix.entry.opensearch_version }}
-
       - name: Run Integration Test
-        run: ./gradlew clean integrationTest -Dpassword=${{ env.PASSWORD }}
+        run: ./gradlew clean integrationTest -Dtests.opensearch.version=${{ matrix.entry.opensearch_version }}
 
       - name: Upload Reports
         if: failure()
@@ -69,7 +50,3 @@ jobs:
           name: test-reports-os${{ matrix.entry.opensearch_version }}-java${{ matrix.entry.java }}
           path: java-client/build/reports/
           retention-days: 7
-
-      - name: Stop Docker
-        run: |
-          docker compose --project-directory .ci/opensearch down

CHANGELOG.md

@@ -3,6 +3,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ## [Unreleased 3.x]
 ### Added
+- Run Java client integration tests with a Testcontainers-managed OpenSearch instance by default ([#2033](https://github.com/opensearch-project/opensearch-java/pull/2033))
 - Added `equals()` and `hashCode()` implementations to `FieldValue` ([#1998](https://github.com/opensearch-project/opensearch-java/pull/1998))
 - Add document lifecycle guide and runnable sample ([#2017](https://github.com/opensearch-project/opensearch-java/pull/2017))
 

DEVELOPER_GUIDE.md

@@ -52,17 +52,28 @@ To run unit tests for the java-client:
 
 #### Integration Tests
 
-To run integration tests for the java-client, start an OpenSearch cluster using docker and pass the OpenSearch version:
+To run integration tests for the java-client:
 
 ```
-docker-compose --project-directory .ci/opensearch build --build-arg OPENSEARCH_VERSION=1.3.0
-docker-compose --project-directory .ci/opensearch up -d
+./gradlew clean integrationTest
 ```
 
-Run integration tests after starting OpenSearch cluster:
+By default, the integration test task starts a single OpenSearch test container for the test JVM. To test against a specific OpenSearch image version, pass the OpenSearch version:
 
 ```
-./gradlew clean integrationTest
+./gradlew clean integrationTest -Dtests.opensearch.version=3.2.0
+```
+
+To pass the full official OpenSearch image name, use:
+
+```
+./gradlew clean integrationTest -Dtests.opensearch.image=opensearchproject/opensearch:3.2.0
+```
+
+To run against an already running cluster, disable the test container and pass the cluster endpoint if it is not `localhost:9200`:
+
+```
+./gradlew clean integrationTest -Dtests.opensearch.testcontainers.enabled=false -Dtests.rest.cluster=localhost:9200
 ```
 
 #### AWS Transport Integration Tests

java-client/build.gradle.kts

@@ -143,6 +143,9 @@ tasks.build {
     dependsOn("spotlessJavaCheck")
 }
 
+val opensearchVersion = "3.5.0"
+val opensearchDockerVersion = opensearchVersion.removeSuffix("-SNAPSHOT")
+
 tasks.test {
     systemProperty("tests.security.manager", "false")
 
@@ -168,6 +171,16 @@ val integrationTest = task<Test>("integrationTest") {
     systemProperty("https", System.getProperty("https", "true"))
     systemProperty("user", System.getProperty("user", "admin"))
     systemProperty("password", System.getProperty("password", "admin"))
+    systemProperty(
+        "tests.opensearch.testcontainers.enabled",
+        System.getProperty("tests.opensearch.testcontainers.enabled", "true")
+    )
+    systemProperty(
+        "tests.opensearch.version",
+        System.getProperty("tests.opensearch.version", opensearchDockerVersion)
+    )
+    System.getProperty("tests.rest.cluster")?.let { systemProperty("tests.rest.cluster", it) }
+    System.getProperty("tests.opensearch.image")?.let { systemProperty("tests.opensearch.image", it) }
     systemProperty("tests.awsSdk2support.domainHost",
             System.getProperty("tests.awsSdk2support.domainHost", null))
     systemProperty("tests.awsSdk2support.serviceName",
@@ -176,8 +189,6 @@ val integrationTest = task<Test>("integrationTest") {
             System.getProperty("tests.awsSdk2support.domainRegion", "us-east-1"))
 }
 
-val opensearchVersion = "3.5.0"
-
 dependencies {
     val jacksonVersion = "2.21.2"
     val jacksonDatabindVersion = "2.21.2"
@@ -377,6 +388,7 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
       compileClasspath += sourceSets.main.get().output + sourceSets.test.get().output
       runtimeClasspath += sourceSets.main.get().output + sourceSets.test.get().output
       srcDir("src/test/java11")
+      srcDir("src/test/java21")
     }
   }
 
@@ -387,6 +399,8 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
     testImplementation("org.opensearch.test", "framework", opensearchVersion) {
       exclude(group = "org.hamcrest")
     }
+    testImplementation("org.opensearch:opensearch-testcontainers:4.1.0")
+    testImplementation("org.testcontainers:testcontainers:2.0.4")
   }
 
   tasks.named<JavaCompile>("compileJava21Java") {
@@ -408,4 +422,4 @@ if (runtimeJavaVersion >= JavaVersion.VERSION_21) {
     testClassesDirs += java21.output.classesDirs
     classpath = sourceSets["java21"].runtimeClasspath
  }
-}
+}

java-client/src/test/java11/org/opensearch/client/opensearch/integTest/OpenSearchJavaClientTestCase.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.client.opensearch.integTest;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -30,6 +31,7 @@
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
+import org.junit.ClassRule;
 import org.opensearch.Version;
 import org.opensearch.client.RestClient;
 import org.opensearch.client.RestClientBuilder;
@@ -45,6 +47,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.test.rest.OpenSearchRestTestCase;
 
+@ThreadLeakFilters(filters = TestcontainersThreadFilter.class)
 public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCase implements OpenSearchTransportSupport {
     private static final List<String> systemIndices = List.of(
         ".opensearch-observability",
@@ -59,6 +62,11 @@ public abstract class OpenSearchJavaClientTestCase extends OpenSearchRestTestCas
     private static TreeSet<Version> nodeVersions;
     private static List<HttpHost> clusterHosts;
 
+    // The integration tests run through JUnit 4 (RandomizedRunner), so @ClassRule is the pre/post
+    // lifecycle hook; the rule starts a single container shared by the whole test JVM.
+    @ClassRule
+    public static final OpenSearchTestContainerRule testContainer = new OpenSearchTestContainerRule();
+
     @Before
     public void initJavaClient() throws IOException {
         if (javaClient == null) {

java-client/src/test/java11/org/opensearch/client/opensearch/integTest/TestcontainersThreadFilter.java

@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.client.opensearch.integTest;
+
+import com.carrotsearch.randomizedtesting.ThreadFilter;
+
+public final class TestcontainersThreadFilter implements ThreadFilter {
+    @Override
+    public boolean reject(Thread thread) {
+        String name = thread.getName();
+        // Testcontainers owns these helper threads and Ryuk cleans them up after the JVM exits.
+        return "testcontainers-ryuk".equals(name) || name.startsWith("testcontainers-pull-watchdog-") || name.startsWith("ducttape-");
+    }
+}

java-client/src/test/java21/org/opensearch/client/opensearch/integTest/OpenSearchTestContainerRule.java

@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.client.opensearch.integTest;
+
+import java.net.URI;
+import org.junit.rules.ExternalResource;
+import org.opensearch.testcontainers.OpenSearchContainer;
+import org.opensearch.testcontainers.OpenSearchDockerImage;
+
+/**
+ * Starts one OpenSearch test container for the whole test JVM and points the integration tests at
+ * it through system properties. The container is shared by every test class, so it is left running
+ * after each class; Testcontainers' Ryuk reaper removes it once the JVM exits.
+ */
+final class OpenSearchTestContainerRule extends ExternalResource {
+    private static final String ENABLED_PROPERTY = "tests.opensearch.testcontainers.enabled";
+    private static final String VERSION_PROPERTY = "tests.opensearch.version";
+    private static final String IMAGE_PROPERTY = "tests.opensearch.image";
+    private static final String CLUSTER_PROPERTY = "tests.rest.cluster";
+    private static final String HTTPS_PROPERTY = "https";
+    private static final String USER_PROPERTY = "user";
+    private static final String PASSWORD_PROPERTY = "password";
+
+    private static final String DEFAULT_ADMIN_PASSWORD = "admin";
+
+    private static OpenSearchContainer<?> container;
+
+    @Override
+    protected void before() {
+        startIfNeeded();
+    }
+
+    private static void startIfNeeded() {
+        if (hasText(System.getProperty(CLUSTER_PROPERTY)) || !testcontainersEnabled()) {
+            return;
+        }
+
+        if (container == null) {
+            OpenSearchContainer<?> openSearch = createContainer();
+            openSearch.start();
+            container = openSearch;
+        }
+
+        // getHttpHostAddress() is scheme-prefixed, but tests.rest.cluster expects host:port.
+        URI httpHostAddress = URI.create(container.getHttpHostAddress());
+        System.setProperty(CLUSTER_PROPERTY, httpHostAddress.getHost() + ":" + httpHostAddress.getPort());
+        System.setProperty(HTTPS_PROPERTY, Boolean.toString(container.isSecurityEnabled()));
+        System.setProperty(USER_PROPERTY, container.getUsername());
+        System.setProperty(PASSWORD_PROPERTY, container.getPassword());
+    }
+
+    private static OpenSearchContainer<?> createContainer() {
+        String image = System.getProperty(IMAGE_PROPERTY);
+        OpenSearchContainer<?> openSearch = hasText(image)
+            ? new OpenSearchContainer<>(image)
+            : new OpenSearchContainer<>(OpenSearchDockerImage.ofVersion(requiredVersion()));
+
+        // Disk watermarks must stay disabled; constrained disks otherwise trip index_create_block_exception.
+        openSearch.withSecurityEnabled().withEnv("cluster.routing.allocation.disk.threshold_enabled", "false");
+
+        // The container has no password setter; OPENSEARCH_INITIAL_ADMIN_PASSWORD is its supported
+        // input and getPassword() reflects it. Only forward a non-default override: when the env is
+        // unset, the container substitutes its own strong default on images >= 2.12.
+        String configuredPassword = System.getProperty(PASSWORD_PROPERTY);
+        if (hasText(configuredPassword) && !DEFAULT_ADMIN_PASSWORD.equals(configuredPassword)) {
+            openSearch.withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", configuredPassword);
+        }
+
+        return openSearch;
+    }
+
+    private static String requiredVersion() {
+        String version = System.getProperty(VERSION_PROPERTY);
+        if (!hasText(version)) {
+            throw new IllegalStateException("Missing " + VERSION_PROPERTY + " for OpenSearch Testcontainers image");
+        }
+        return version;
+    }
+
+    private static boolean testcontainersEnabled() {
+        return Boolean.parseBoolean(System.getProperty(ENABLED_PROPERTY, "true"));
+    }
+
+    private static boolean hasText(String value) {
+        return value != null && !value.isBlank();
+    }
+}