Complete CRC64-NVME parity for Netty and RDMA

README.md

@@ -138,7 +138,7 @@ For the full CLI reference (all flags, workload types, distributed options, RDMA
 - **Multi-Workload Support** -- write, read, list, mixed, and mock workloads out of the box, with S3 Tables (Iceberg) benchmarks.
 - **Pluggable S3 Storage Drivers** -- choose the backend that fits your target: `default` (Netty), `aws` (AWS SDK v2), or `rdma` (hardware-accelerated). Select with `--s3-driver`.
 - **S3 Multipart Upload** -- upload large objects in parallel parts with automatic abort on failure, per-part retry (up to 3 attempts), and per-part checksum support. Enable with `--part-size`.
-- **S3 Checksum Validation** -- compute and send checksums on write requests with `--checksum` (`crc32`, `crc32c`, `sha1`, `sha256`). When combined with multipart upload, checksums are applied per part. Supported by both the Netty and AWS SDK drivers.
+- **S3 Checksum Validation** -- compute and send checksums on write requests with `--checksum` (`crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`). When combined with multipart upload, checksums are applied per part. Supported by the Netty, AWS SDK, and RDMA drivers.
 - **Data Compressibility & Deduplication Controls** -- shape generated object data for storage-efficiency benchmarks with `--object-data-compressibility` (0-100% target compressibility) and `--object-data-dedupable=false` (per-4KB anti-dedupe stamping).
 - **Interactive & Headless** -- flip between a terminal UI for live monitoring and headless mode for CI/CD.
 - **Distributed Runs** – preflight checks, node orchestration, and attachment support are built into the CLI.

cli/README.md

@@ -359,7 +359,7 @@ Executes a benchmark test with the specified workload type.
 - `--threads, -t`: Number of parallel client threads (default: 1)
 - `--object-size, -o`: Size of each object (e.g., 1MB, 256KB, 4GB)
 - `--part-size`: Enable S3 multipart upload with the given part size (e.g., 5MB, 64MB). When set, `load.batch.size` is forced to `1`. The engine automatically retries individual parts (up to 3 times) and aborts incomplete uploads on failure. Per-part checksums are applied when checksum is enabled. See [`SPT_SYNTAX.md`](docs/SPT_SYNTAX.md) for details
-- `--checksum`: Enable S3 checksum validation with the specified algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`. When used with `--part-size`, checksums are applied per part. `crc64-nvme` currently requires `--s3-driver aws`. (env: `SPT_CHECKSUM`)
+- `--checksum`: Enable S3 checksum validation with the specified algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`. When used with `--part-size`, checksums are applied per part. (env: `SPT_CHECKSUM`)
 - `--object-data-compressibility`: Target compressibility percentage for generated object data, 0-100 (default: 0 = fully random). Each 4KB chunk is split into random and zero-filled portions according to the percentage. (env: `SPT_OBJECT_DATA_COMPRESSIBILITY`)
 - `--object-data-dedupable`: Whether generated data remains dedupe-friendly (default: true). Set `false` to stamp every 4KB with a unique object-id + offset header that defeats inline deduplication. Incompatible with `--items-file` / file-based data input. (env: `SPT_OBJECT_DATA_DEDUPABLE`)
 - `--cleanup`: Delete all created objects after test completion

cli/cmd/run.go

@@ -1174,9 +1174,6 @@ func buildScenarioParams(workloadType string, cmd *cobra.Command) (scenario.Para
 		default:
 			return params, fmt.Errorf("invalid --checksum value %q: must be one of: crc32, crc32c, sha1, sha256, crc64-nvme", checksumAlgo)
 		}
-		if checksumAlgo == scenario.ChecksumCRC64NVME && s3Driver != scenario.S3DriverAws {
-			return params, fmt.Errorf("invalid --checksum value %q for --s3-driver %q: crc64-nvme requires --s3-driver aws", checksumAlgo, s3Driver)
-		}
 		params.Checksum = checksumAlgo
 	}
 

cli/cmd/run_scenario_test.go

@@ -747,28 +747,28 @@ func TestBuildScenarioParams_S3DriverFlag(t *testing.T) {
 		}
 	})
 
-	t.Run("--checksum crc64-nvme with default driver fails fast", func(t *testing.T) {
+	t.Run("--checksum crc64-nvme with default driver is accepted", func(t *testing.T) {
 		cmd := newCmd()
 		_ = cmd.Flags().Set("checksum", "crc64-nvme")
-		_, err := buildScenarioParams("mock", cmd)
-		if err == nil {
-			t.Fatal("expected validation error")
+		p, err := buildScenarioParams("mock", cmd)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
 		}
-		if !strings.Contains(err.Error(), "requires --s3-driver aws") {
-			t.Errorf("expected aws conflict message, got: %v", err)
+		if p.Checksum != "crc64-nvme" {
+			t.Errorf("Checksum = %q, want %q", p.Checksum, "crc64-nvme")
 		}
 	})
 
-	t.Run("--checksum crc64-nvme with rdma fails fast", func(t *testing.T) {
+	t.Run("--checksum crc64-nvme with rdma is accepted", func(t *testing.T) {
 		cmd := newCmd()
 		_ = cmd.Flags().Set("s3-driver", "rdma")
 		_ = cmd.Flags().Set("checksum", "crc64-nvme")
-		_, err := buildScenarioParams("mock", cmd)
-		if err == nil {
-			t.Fatal("expected validation error")
+		p, err := buildScenarioParams("mock", cmd)
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
 		}
-		if !strings.Contains(err.Error(), "requires --s3-driver aws") {
-			t.Errorf("expected aws conflict message, got: %v", err)
+		if p.Checksum != "crc64-nvme" {
+			t.Errorf("Checksum = %q, want %q", p.Checksum, "crc64-nvme")
 		}
 	})
 

cli/docs/SPT_SYNTAX.md

@@ -40,7 +40,7 @@ You can use these variables to avoid repeating sensitive or commonly used parame
 - **Docker:** `SPT_SKIP_IMAGE_PULL` (skip pulling the engine image)
 - **Engine tuning:** `SPT_SERVICE_THREADS` (virtual-thread carrier parallelism)
 - **Multipart upload:** `SPT_PART_SIZE` (part size, e.g. `64MB`)
-- **Checksum:** `SPT_CHECKSUM` (algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`; `crc64-nvme` requires `SPT_S3_DRIVER=aws` or `--s3-driver aws`)
+- **Checksum:** `SPT_CHECKSUM` (algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`)
 - **Data shaping:** `SPT_OBJECT_DATA_COMPRESSIBILITY` (0-100, default 0), `SPT_OBJECT_DATA_DEDUPABLE` (true/false, default true)
 - **Storage driver:** `SPT_S3_DRIVER` (driver backend: `default`, `aws`, `rdma`)
 - **RDMA:** `SPT_RDMA_ENABLED`, `RDMA_LOCAL_IP`, `RDMA_DEVICE`, `RDMA_LOG_LEVEL`, `RDMA_THRESHOLD_BYTES`, `RDMA_TIMEOUT_MS`, `RDMA_FALLBACK_ENABLED`
@@ -97,7 +97,7 @@ Required for S3 workloads, optional/ignored for `mock`.
 | `--object-count` | `-n` | `0` | Fixed number of objects to process |
 | `--duration` | `-d` | `""` | Fixed time duration (e.g., `5m`, `1h`) |
 | `--seed-objects` | | `2500` | Objects to pre-create for `read` benchmarks |
-| `--checksum` | | `""` | Enable S3 checksum validation with the specified algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`. Omit to disable checksums. When set with `--part-size`, checksums are applied per part. `crc64-nvme` currently requires `--s3-driver aws`. (env: `SPT_CHECKSUM`) |
+| `--checksum` | | `""` | Enable S3 checksum validation with the specified algorithm: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`. Omit to disable checksums. When set with `--part-size`, checksums are applied per part. (env: `SPT_CHECKSUM`) |
 | `--object-data-compressibility` | | `0` | Target compressibility percentage for generated object data (0-100). Each 4KB chunk is split into random and zero-filled portions. 0 = fully random, 100 = fully compressible. (env: `SPT_OBJECT_DATA_COMPRESSIBILITY`) |
 | `--object-data-dedupable` | | `true` | Whether generated data remains dedupe-friendly. Set `false` to stamp every 4KB with a 16-byte object-id + offset header that practically eliminates inline deduplication. Incompatible with file-based data input. (env: `SPT_OBJECT_DATA_DEDUPABLE`) |
 | `--save-items` | | `false` | Save `items.csv` listing created objects (`write` only) |
@@ -324,7 +324,7 @@ When `--part-size` is set, each object goes through a four-phase lifecycle:
 
 **Checksums:**
 
-When the engine's checksum feature is enabled (`storage-checksum-enabled=true` in defaults or scenario config), checksums are computed and sent for **each individual part upload**, not just the final object. Supported algorithms: `md5`, `crc32`, `crc32c`, `sha1`, `sha256`.
+When the engine's checksum feature is enabled (`storage-checksum-enabled=true` in defaults or scenario config), checksums are computed and sent for **each individual part upload**, not just the final object. Supported algorithms: `md5`, `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`.
 
 **Results artifacts:**
 
@@ -517,7 +517,7 @@ spt run write \
     --checksum sha256
 ```
 
-Supported algorithms: `crc32`, `crc32c`, `sha1`, `sha256`. The flag works with both the default Netty driver and the AWS SDK driver (`--s3-driver aws`).
+Supported algorithms: `crc32`, `crc32c`, `sha1`, `sha256`, `crc64-nvme`. The flag works with the default Netty driver, the AWS SDK driver (`--s3-driver aws`), and the RDMA driver (`--s3-driver rdma` / `--use-rdma`).
 
 ### Data Compressibility & Deduplication
 

engine/extensions/storage-drivers/implementations/s3-rdma/src/test/java/com/dell/spt/storage/driver/coop/netty/http/s3/rdma/S3RdmaChecksumParityTest.java

@@ -0,0 +1,208 @@
+package com.dell.spt.storage.driver.coop.netty.http.s3.rdma;
+
+import com.dell.spt.base.data.DataInput;
+import com.dell.spt.base.item.DataItem;
+import com.dell.spt.base.item.Item;
+import com.dell.spt.base.item.op.OpType;
+import com.dell.spt.base.item.op.Operation;
+import com.dell.spt.base.item.op.OperationImpl;
+import com.dell.spt.base.storage.Credential;
+import com.dell.spt.base.env.Extension;
+import com.github.akurilov.commons.collection.TreeUtil;
+import com.github.akurilov.commons.system.SizeInBytes;
+import com.github.akurilov.confuse.Config;
+import com.github.akurilov.confuse.SchemaProvider;
+import com.github.akurilov.confuse.impl.BasicConfig;
+import io.netty.handler.codec.http.DefaultHttpHeaders;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaders;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import java.util.zip.CRC32;
+import java.util.zip.CRC32C;
+import java.util.zip.Checksum;
+
+import static com.dell.spt.base.Constants.APP_NAME;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class S3RdmaChecksumParityTest {
+
+	private static final Credential TEST_CRED = Credential.getInstance("user1", "u5QtPuQx+W5nrrQQEg7nArBqSgC8qLiDt2RhQthb");
+
+	private static Config baseConfig(final boolean checksumEnabled, final String checksumAlg, final String host) {
+		try {
+			final List<Map<String, Object>> configSchemas = Extension
+							.load(Thread.currentThread().getContextClassLoader())
+							.stream()
+							.map(Extension::schemaProvider)
+							.filter(Objects::nonNull)
+							.map(sp -> {
+								try {
+									return sp.schema();
+								} catch (Exception e) {
+									throw new RuntimeException(e);
+								}
+							})
+							.filter(Objects::nonNull)
+							.collect(Collectors.toList());
+			SchemaProvider
+							.resolve(APP_NAME, Thread.currentThread().getContextClassLoader())
+							.stream()
+							.findFirst()
+							.ifPresent(configSchemas::add);
+			final Map<String, Object> configSchema = TreeUtil.reduceForest(configSchemas);
+			final Config config = new BasicConfig("-", configSchema);
+			config.val("load-batch-size", 1024);
+			config.val("storage-driver-limit-concurrency", 0);
+			config.val("storage-driver-threads", 0);
+			config.val("storage-driver-limit-queue-input", 1024);
+			config.val("storage-net-transport", "nio");
+			config.val("storage-net-reuseAddr", true);
+			config.val("storage-net-bindBacklogSize", 0);
+			config.val("storage-net-keepAlive", true);
+			config.val("storage-net-rcvBuf", 0);
+			config.val("storage-net-sndBuf", 0);
+			config.val("storage-net-ssl-enabled", false);
+			config.val("storage-net-ssl-protocols", List.of());
+			config.val("storage-net-ssl-provider", "OPENSSL");
+			config.val("storage-net-tcpNoDelay", false);
+			config.val("storage-net-interestOpQueued", false);
+			config.val("storage-net-writeSpinCount", 1);
+			config.val("storage-net-linger", 0);
+			config.val("storage-net-timeoutMilliSec", 0);
+			config.val("storage-net-ioRatio", 50);
+			config.val("storage-net-node-addrs", List.of(host));
+			config.val("storage-net-node-port", 9024);
+			config.val("storage-net-node-connAttemptsLimit", 0);
+			config.val("storage-net-http-headers", new HashMap<String, String>() {
+				{
+					put("Date", "#{date:formatNowRfc1123()}%{date:formatNowRfc1123()}");
+				}
+			});
+			config.val("storage-net-http-read-metadata-only", false);
+			config.val("storage-net-http-max-chunk-size", 65536);
+			config.val("storage-net-http-uri-args", Map.of());
+			config.val("storage-object-fsAccess", true);
+			config.val("storage-object-tagging-enabled", false);
+			config.val("storage-object-tagging-tags", Map.of());
+			config.val("storage-object-versioning", false);
+			config.val("storage-auth-uid", TEST_CRED.getUid());
+			config.val("storage-auth-token", null);
+			config.val("storage-auth-secret", TEST_CRED.getSecret());
+			config.val("storage-auth-version", 4);
+			config.val("storage-checksum-enabled", checksumEnabled);
+			if (checksumEnabled) {
+				config.val("storage-checksum-algorithm", checksumAlg);
+			}
+			config.val("storage-rdma-enabled", true);
+			config.val("storage-rdma-thresholdBytes", 0L);
+			config.val("storage-rdma-fallback", true);
+			config.val("storage-rdma-device", "auto");
+			config.val("storage-rdma-localIp", "");
+			config.val("storage-rdma-logLevel", "WARN");
+			config.val("storage-rdma-timeoutMs", 30000L);
+			return config;
+		} catch (Throwable t) {
+			throw new RuntimeException(t);
+		}
+	}
+
+	private static DataItem mockDataItemFromBytes(final byte[] payload) throws Exception {
+		final DataItem dataItem = Mockito.mock(DataItem.class);
+		final int[] readOffset = new int[]{0
+		};
+		Mockito.when(dataItem.size()).thenReturn((long) payload.length);
+		Mockito.doAnswer(invocation -> {
+			final ByteBuffer dst = invocation.getArgument(0);
+			if (readOffset[0] >= payload.length) {
+				return 0;
+			}
+			final int n = Math.min(dst.remaining(), payload.length - readOffset[0]);
+			dst.put(payload, readOffset[0], n);
+			readOffset[0] += n;
+			return n;
+		}).when(dataItem).read(Mockito.any(ByteBuffer.class));
+		Mockito.doAnswer(invocation -> {
+			readOffset[0] = 0;
+			return null;
+		}).when(dataItem).reset();
+		return dataItem;
+	}
+
+	private static String checksumHeaderName(final String algorithm) {
+		if ("md5".equals(algorithm)) {
+			return HttpHeaderNames.CONTENT_MD5.toString();
+		}
+		if ("crc64-nvme".equals(algorithm)) {
+			return "x-amz-checksum-crc64nvme";
+		}
+		return "x-amz-checksum-" + algorithm;
+	}
+
+	private static String checksumHeaderFor(final String algorithm, final byte[] payload) throws Exception {
+		final Config cfg = baseConfig(true, algorithm, "s3.us-east-1.amazonaws.com:443");
+		final TestS3RdmaDriver drv = new TestS3RdmaDriver(cfg);
+		final DataItem dataItem = mockDataItemFromBytes(payload);
+		final Operation<Item> op = new OperationImpl<>(1, OpType.CREATE, dataItem, null, "/bucket", TEST_CRED);
+		final HttpHeaders headers = new DefaultHttpHeaders();
+		drv.applyChecksumForTest(headers, op);
+		return headers.get(checksumHeaderName(algorithm));
+	}
+
+	private static String reference32BitChecksum(final Checksum checksum, final byte[] payload) {
+		checksum.reset();
+		checksum.update(payload, 0, payload.length);
+		final byte[] checksumBytes = ByteBuffer.allocate(Integer.BYTES).putInt((int) (checksum.getValue() & 0xFFFFFFFFL)).array();
+		return Base64.getEncoder().encodeToString(checksumBytes);
+	}
+
+	private static class TestS3RdmaDriver extends S3RdmaStorageDriver<Item, Operation<Item>> {
+
+		TestS3RdmaDriver(final Config cfg) throws Exception {
+			super("test-s3-rdma",
+							DataInput.instance(null, "7a42d9c483244167", new SizeInBytes("64KB"), 16, false, 0.0, true),
+							cfg.configVal("storage"), false, cfg.intVal("load-batch-size"));
+		}
+
+		void applyChecksumForTest(final HttpHeaders headers, final Operation<Item> op) {
+			super.applyChecksum(headers, op);
+		}
+	}
+
+	@Test
+	void applyChecksum_crc32_knownVector_123456789_matchesReference() throws Exception {
+		final byte[] payload = "123456789".getBytes(StandardCharsets.UTF_8);
+		final String expected = reference32BitChecksum(new CRC32(), payload);
+		assertEquals(expected, checksumHeaderFor("crc32", payload));
+	}
+
+	@Test
+	void applyChecksum_crc32c_knownVector_123456789_matchesReference() throws Exception {
+		final byte[] payload = "123456789".getBytes(StandardCharsets.UTF_8);
+		final String expected = reference32BitChecksum(new CRC32C(), payload);
+		assertEquals(expected, checksumHeaderFor("crc32c", payload));
+	}
+
+	@Test
+	void applyChecksum_crc64nvme_knownVector_123456789_matchesReference() throws Exception {
+		final byte[] payload = "123456789".getBytes(StandardCharsets.UTF_8);
+		final String expected = "rosUhgp5mIg=";
+		assertEquals(expected, checksumHeaderFor("crc64-nvme", payload));
+	}
+
+	@Test
+	void applyChecksum_crc64nvme_emptyPayload_matchesReference() throws Exception {
+		final byte[] payload = new byte[0];
+		final String expected = "AAAAAAAAAAA=";
+		assertEquals(expected, checksumHeaderFor("crc64-nvme", payload));
+	}
+}

engine/extensions/storage-drivers/implementations/s3/build.gradle

@@ -34,6 +34,7 @@ dependencies {
 	implementation(
 		libs.netty.handler,
 		libs.netty.codec.http,
+		libs.aws.crt,
 	)
 
 	testImplementation(libs.mockito.core)

engine/extensions/storage-drivers/implementations/s3/src/main/java/com/dell/spt/storage/driver/coop/netty/http/s3/S3ResponseHandler.java

@@ -44,12 +44,10 @@ public final class S3ResponseHandler<I extends Item, O extends Operation<I>>
 	private final String checksumHeader; // e.g. "x-amz-checksum-crc32c", or null if disabled
 
 	public S3ResponseHandler(final S3StorageDriver<I, O> driver, final boolean verifyFlag,
-					final boolean versioningEnabled, final String checksumAlgorithm) {
+					final boolean versioningEnabled, final String checksumHeader) {
 		super(driver, verifyFlag);
 		this.versioningEnabled = versioningEnabled;
-		this.checksumHeader = checksumAlgorithm != null
-						? S3Api.AMZ_CHECKSUM_PREFIX + checksumAlgorithm.toLowerCase(java.util.Locale.ROOT)
-						: null;
+		this.checksumHeader = checksumHeader;
 	}
 
 	@Override