From ce4e4f67e86be2b53b2828b19a585e6ccf19c9c9 Mon Sep 17 00:00:00 2001 From: xdk-amz Date: Tue, 9 Dec 2025 13:09:32 -0800 Subject: [PATCH 1/5] Core: Migrate from rustls-pemfile usage to rustls-pki-types (cherry pick 34d035677b9c774ebc363d2f05901da2d5c2b85e) --- glide-core/Cargo.toml | 2 ++ glide-core/redis-rs/redis/Cargo.toml | 3 +- glide-core/redis-rs/redis/src/tls.rs | 43 +++++++++++++++++----------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/glide-core/Cargo.toml b/glide-core/Cargo.toml index 0f13059b6dd..60fb2c3b84b 100644 --- a/glide-core/Cargo.toml +++ b/glide-core/Cargo.toml @@ -20,6 +20,8 @@ redis = { path = "./redis-rs/redis", features = [ "cluster", "cluster-async", ] } +rustls = { version = "0.23", features = ["aws-lc-rs"] } +rustls-pki-types = "1.9" telemetrylib = { path = "./telemetry" } tokio = { version = "1", features = ["macros", "time"] } logger_core = { path = "../logger_core" } diff --git a/glide-core/redis-rs/redis/Cargo.toml b/glide-core/redis-rs/redis/Cargo.toml index b29ebc16623..d1ec3125170 100644 --- a/glide-core/redis-rs/redis/Cargo.toml +++ b/glide-core/redis-rs/redis/Cargo.toml @@ -66,8 +66,7 @@ tokio-retry2 = { version = "0.5", features = ["jitter"], optional = true } rustls = { version = "0.23", features = ["aws-lc-rs"] } rustls-platform-verifier = { version = "0.6", default-features = false } tokio-rustls = { version = "0.26", default-features = false } -rustls-pemfile = { version = "2" } -rustls-pki-types = { version = "1" } +rustls-pki-types = { version = "1.9" } # Only needed for bignum Support num-bigint = "0.4" diff --git a/glide-core/redis-rs/redis/src/tls.rs b/glide-core/redis-rs/redis/src/tls.rs index 6886efb836f..edf7d215c34 100644 --- a/glide-core/redis-rs/redis/src/tls.rs +++ b/glide-core/redis-rs/redis/src/tls.rs @@ -1,6 +1,7 @@ -use std::io::{BufRead, Error, ErrorKind as IOErrorKind}; +use std::io::{Error, ErrorKind as IOErrorKind}; use rustls::RootCertStore; +use rustls_pki_types::pem::PemObject; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use crate::{Client, ConnectionAddr, ConnectionInfo, ErrorKind, RedisError, RedisResult}; @@ -69,18 +70,22 @@ pub(crate) fn retrieve_tls_certificates( client_key, }) = client_tls { - let buf = &mut client_cert.as_slice() as &mut dyn BufRead; - let certs = rustls_pemfile::certs(buf); - let client_cert_chain = certs.collect::, _>>()?; - - let client_key = - rustls_pemfile::private_key(&mut client_key.as_slice() as &mut dyn BufRead)? - .ok_or_else(|| { - Error::new( - IOErrorKind::Other, - "Unable to extract private key from PEM file", - ) - })?; + // Parse certificates using rustls-pki-types v1.9.0+ API + let certs = CertificateDer::pem_slice_iter(&client_cert); + let client_cert_chain = certs.collect::, _>>().map_err(|e| { + Error::new( + IOErrorKind::Other, + format!("Failed to parse certificate: {}", e), + ) + })?; + + // Parse private key using rustls-pki-types v1.9.0+ API + let client_key = PrivateKeyDer::from_pem_slice(&client_key).map_err(|e| { + Error::new( + IOErrorKind::Other, + format!("Failed to parse private key: {}", e), + ) + })?; Some(ClientTlsParams { client_cert_chain, @@ -91,11 +96,17 @@ pub(crate) fn retrieve_tls_certificates( }; let root_cert_store = if let Some(root_cert) = root_cert { - let buf = &mut root_cert.as_slice() as &mut dyn BufRead; - let certs = rustls_pemfile::certs(buf); + // Parse root certificates using rustls-pki-types v1.9.0+ API + let certs = CertificateDer::pem_slice_iter(&root_cert); let mut root_cert_store = RootCertStore::empty(); for result in certs { - if root_cert_store.add(result?.to_owned()).is_err() { + let cert = result.map_err(|e| { + Error::new( + IOErrorKind::Other, + format!("Failed to parse root certificate: {}", e), + ) + })?; + if root_cert_store.add(cert.to_owned()).is_err() { return Err( Error::new(IOErrorKind::Other, "Unable to parse TLS trust anchors").into(), ); From 75be5579a5f0fab8ad12f20fe2a290486a0c8d7a Mon Sep 17 00:00:00 2001 From: affonsov <67347924+affonsov@users.noreply.github.com> Date: Fri, 24 Oct 2025 12:54:38 -0700 Subject: [PATCH 2/5] Fix: LOLWUT Tests and Add Version 9 Support (#4907) * Fix lolwut test Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> * fixing lolwut tests Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> * fix java lint Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> * fix lolwut bug on python Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> * fix lint Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> --------- Signed-off-by: affonsov <67347924+affonsov@users.noreply.github.com> --- go/integTest/cluster_commands_test.go | 46 +++++++++++++++++++ go/integTest/standalone_commands_test.go | 28 +++++++++++ .../test/java/glide/cluster/CommandTests.java | 18 ++++++++ .../java/glide/standalone/CommandTests.java | 23 +++++++++- node/tests/GlideClient.test.ts | 21 +++++++++ node/tests/GlideClusterClient.test.ts | 27 +++++++++++ .../glide/async_commands/cluster_commands.py | 2 +- .../async_commands/standalone_commands.py | 2 +- .../glide_shared/commands/batch.py | 2 +- .../sync_commands/cluster_commands.py | 2 +- .../sync_commands/standalone_commands.py | 2 +- python/tests/async_tests/test_async_client.py | 26 +++++++++++ python/tests/sync_tests/test_sync_client.py | 26 +++++++++++ 13 files changed, 218 insertions(+), 7 deletions(-) diff --git a/go/integTest/cluster_commands_test.go b/go/integTest/cluster_commands_test.go index 0d0081b8ed1..3c9441d9241 100644 --- a/go/integTest/cluster_commands_test.go +++ b/go/integTest/cluster_commands_test.go @@ -1222,6 +1222,52 @@ func (suite *GlideTestSuite) TestLolwutWithOptions_WithRandomRoute() { "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, singleValue) } +func (suite *GlideTestSuite) TestLolwutWithOptions_Version9_AllNodes() { + client := suite.defaultClusterClient() + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if suite.serverVersion >= "9.0.0" { + options := options.ClusterLolwutOptions{ + LolwutOptions: &options.LolwutOptions{ + Version: 9, + Args: []int{30, 4}, + }, + RouteOption: &options.RouteOption{Route: config.AllNodes}, + } + result, err := client.LolwutWithOptions(context.Background(), options) + assert.NoError(suite.T(), err) + assert.True(suite.T(), result.IsMultiValue()) + multiValue := result.MultiValue() + for _, value := range multiValue { + hasVer := strings.Contains(value, "ver") + hasVersion := strings.Contains(value, suite.serverVersion) + assert.True(suite.T(), hasVer && hasVersion, + "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, value) + } + } +} + +func (suite *GlideTestSuite) TestLolwutWithOptions_Version9_RandomNode() { + client := suite.defaultClusterClient() + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if suite.serverVersion >= "9.0.0" { + options := options.ClusterLolwutOptions{ + LolwutOptions: &options.LolwutOptions{ + Version: 9, + Args: []int{40, 20, 1, 2}, + }, + RouteOption: &options.RouteOption{Route: config.RandomRoute}, + } + result, err := client.LolwutWithOptions(context.Background(), options) + assert.NoError(suite.T(), err) + assert.True(suite.T(), result.IsSingleValue()) + singleValue := result.SingleValue() + hasVer := strings.Contains(singleValue, "ver") + hasVersion := strings.Contains(singleValue, suite.serverVersion) + assert.True(suite.T(), hasVer && hasVersion, + "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, singleValue) + } +} + func (suite *GlideTestSuite) TestClientIdCluster() { client := suite.defaultClusterClient() t := suite.T() diff --git a/go/integTest/standalone_commands_test.go b/go/integTest/standalone_commands_test.go index d25f6d08e21..45e4880af83 100644 --- a/go/integTest/standalone_commands_test.go +++ b/go/integTest/standalone_commands_test.go @@ -814,6 +814,34 @@ func (suite *GlideTestSuite) TestLolwutWithOptions_EmptyArgs() { "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, res) } +func (suite *GlideTestSuite) TestLolwutWithOptions_Version9_TwoParams() { + client := suite.defaultClient() + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if suite.serverVersion >= "9.0.0" { + opts := options.NewLolwutOptions(9).SetArgs([]int{30, 4}) + res, err := client.LolwutWithOptions(context.Background(), *opts) + assert.NoError(suite.T(), err) + hasVer := strings.Contains(res, "ver") + hasVersion := strings.Contains(res, suite.serverVersion) + assert.True(suite.T(), hasVer && hasVersion, + "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, res) + } +} + +func (suite *GlideTestSuite) TestLolwutWithOptions_Version9_FourParams() { + client := suite.defaultClient() + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if suite.serverVersion >= "9.0.0" { + opts := options.NewLolwutOptions(9).SetArgs([]int{40, 20, 1, 2}) + res, err := client.LolwutWithOptions(context.Background(), *opts) + assert.NoError(suite.T(), err) + hasVer := strings.Contains(res, "ver") + hasVersion := strings.Contains(res, suite.serverVersion) + assert.True(suite.T(), hasVer && hasVersion, + "Expected output to contain 'ver' and version '%s', got: %s", suite.serverVersion, res) + } +} + func (suite *GlideTestSuite) TestClientId() { client := suite.defaultClient() result, err := client.ClientId(context.Background()) diff --git a/java/integTest/src/test/java/glide/cluster/CommandTests.java b/java/integTest/src/test/java/glide/cluster/CommandTests.java index d385cf4d561..a28cfce6a38 100644 --- a/java/integTest/src/test/java/glide/cluster/CommandTests.java +++ b/java/integTest/src/test/java/glide/cluster/CommandTests.java @@ -883,6 +883,24 @@ public void lolwut_lolwut(GlideClusterClient clusterClient) { clusterResponse.getSingleValue().contains("ver") && clusterResponse.getSingleValue().contains(SERVER_VERSION.toString()), "Expected LOLWUT output to contain version string"); + + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if (SERVER_VERSION.isGreaterThanOrEqualTo("9.0.0")) { + // Test with version 9 and 2 parameters on all nodes + clusterResponse = clusterClient.lolwut(9, new int[] {30, 4}, ALL_NODES).get(); + for (var nodeResponse : clusterResponse.getMultiValue().values()) { + assertTrue( + nodeResponse.contains("ver") && nodeResponse.contains(SERVER_VERSION.toString()), + "Expected LOLWUT output to contain version string"); + } + + // Test with version 9 and 4 parameters on random node + clusterResponse = clusterClient.lolwut(9, new int[] {40, 20, 1, 2}, RANDOM).get(); + assertTrue( + clusterResponse.getSingleValue().contains("ver") + && clusterResponse.getSingleValue().contains(SERVER_VERSION.toString()), + "Expected LOLWUT output to contain version string"); + } } @ParameterizedTest diff --git a/java/integTest/src/test/java/glide/standalone/CommandTests.java b/java/integTest/src/test/java/glide/standalone/CommandTests.java index 5f71b817d90..d1f42247ce6 100644 --- a/java/integTest/src/test/java/glide/standalone/CommandTests.java +++ b/java/integTest/src/test/java/glide/standalone/CommandTests.java @@ -446,9 +446,9 @@ public void lolwut_lolwut(GlideClient regularClient) { response.contains("ver") && response.contains(SERVER_VERSION.toString()), "Expected LOLWUT output to contain version string"); - response = regularClient.lolwut(new int[] {30, 4, 4}).get(); + response = regularClient.lolwut(5, new int[] {30, 4, 4}).get(); System.out.printf( - "%nLOLWUT standalone client standard response with params 30 4 4%n%s%n", response); + "%nLOLWUT standalone client ver 5 response with params 30 4 4%n%s%n", response); assertTrue( response.contains("ver") && response.contains(SERVER_VERSION.toString()), "Expected LOLWUT output to contain version string"); @@ -465,6 +465,25 @@ public void lolwut_lolwut(GlideClient regularClient) { assertTrue( response.contains("ver") && response.contains(SERVER_VERSION.toString()), "Expected LOLWUT output to contain version string"); + + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if (SERVER_VERSION.isGreaterThanOrEqualTo("9.0.0")) { + // Test with version 9 and 2 parameters (columns, rows) + response = regularClient.lolwut(9, new int[] {30, 4}).get(); + System.out.printf( + "%nLOLWUT standalone client ver 9 response with params 30 4%n%s%n", response); + assertTrue( + response.contains("ver") && response.contains(SERVER_VERSION.toString()), + "Expected LOLWUT output to contain version string"); + + // Test with version 9 and 4 parameters (columns, rows, real, imaginary) + response = regularClient.lolwut(9, new int[] {40, 20, 1, 2}).get(); + System.out.printf( + "%nLOLWUT standalone client ver 9 response with params 40 20 1 2%n%s%n", response); + assertTrue( + response.contains("ver") && response.contains(SERVER_VERSION.toString()), + "Expected LOLWUT output to contain version string"); + } } @ParameterizedTest diff --git a/node/tests/GlideClient.test.ts b/node/tests/GlideClient.test.ts index ae1ee588547..1663a7004f6 100644 --- a/node/tests/GlideClient.test.ts +++ b/node/tests/GlideClient.test.ts @@ -613,6 +613,27 @@ describe("GlideClient", () => { result5.includes("ver") && result5.includes(serverVersion), ).toBe(true); + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if (cluster.checkIfServerVersionLessThan("9.0.0") === false) { + // Test with version 9 and 2 parameters (columns, rows) + const result6 = await client.lolwut({ + version: 9, + parameters: [30, 4], + }); + expect( + result6.includes("ver") && result6.includes(serverVersion), + ).toBe(true); + + // Test with version 9 and 4 parameters (columns, rows, real, imaginary) + const result7 = await client.lolwut({ + version: 9, + parameters: [40, 20, 1, 2], + }); + expect( + result7.includes("ver") && result7.includes(serverVersion), + ).toBe(true); + } + // batch tests for (const isAtomic of [true, false]) { const batch = new Batch(isAtomic); diff --git a/node/tests/GlideClusterClient.test.ts b/node/tests/GlideClusterClient.test.ts index 02775c86c28..2c910c9c89d 100644 --- a/node/tests/GlideClusterClient.test.ts +++ b/node/tests/GlideClusterClient.test.ts @@ -790,6 +790,33 @@ describe("GlideClusterClient", () => { result4Str.includes(serverVersion), ).toBe(true); + // Test LOLWUT version 9 (available in Valkey 9.0.0+) + if (cluster.checkIfServerVersionLessThan("9.0.0") === false) { + // Test with version 9 and 2 parameters on all nodes + const result5 = await client.lolwut({ + version: 9, + parameters: [30, 4], + route: "allNodes", + }); + const result5Str = intoString(result5); + expect( + result5Str.includes("ver") && + result5Str.includes(serverVersion), + ).toBe(true); + + // Test with version 9 and 4 parameters on random node + const result6 = await client.lolwut({ + version: 9, + parameters: [40, 20, 1, 2], + route: "randomNode", + }); + const result6Str = intoString(result6); + expect( + result6Str.includes("ver") && + result6Str.includes(serverVersion), + ).toBe(true); + } + // batch tests for (const isAtomic of [true, false]) { const batch = new ClusterBatch(isAtomic); diff --git a/python/glide-async/python/glide/async_commands/cluster_commands.py b/python/glide-async/python/glide/async_commands/cluster_commands.py index 20e60bbae52..83437afa586 100644 --- a/python/glide-async/python/glide/async_commands/cluster_commands.py +++ b/python/glide-async/python/glide/async_commands/cluster_commands.py @@ -1156,7 +1156,7 @@ async def lolwut( args.extend(["VERSION", str(version)]) if parameters: for var in parameters: - args.extend(str(var)) + args.append(str(var)) return cast( TClusterResponse[bytes], await self._execute_command(RequestType.Lolwut, args, route), diff --git a/python/glide-async/python/glide/async_commands/standalone_commands.py b/python/glide-async/python/glide/async_commands/standalone_commands.py index 9954161e8a5..513f2096930 100644 --- a/python/glide-async/python/glide/async_commands/standalone_commands.py +++ b/python/glide-async/python/glide/async_commands/standalone_commands.py @@ -767,7 +767,7 @@ async def lolwut( args.extend(["VERSION", str(version)]) if parameters: for var in parameters: - args.extend(str(var)) + args.append(str(var)) return cast( bytes, await self._execute_command(RequestType.Lolwut, args), diff --git a/python/glide-shared/glide_shared/commands/batch.py b/python/glide-shared/glide_shared/commands/batch.py index 5e0a851ac46..9a127bc1d30 100644 --- a/python/glide-shared/glide_shared/commands/batch.py +++ b/python/glide-shared/glide_shared/commands/batch.py @@ -5091,7 +5091,7 @@ def lolwut( args.extend(["VERSION", str(version)]) if parameters: for var in parameters: - args.extend(str(var)) + args.append(str(var)) return self.append_command(RequestType.Lolwut, args) def random_key(self: TBatch) -> TBatch: diff --git a/python/glide-sync/glide_sync/sync_commands/cluster_commands.py b/python/glide-sync/glide_sync/sync_commands/cluster_commands.py index 7a47cdf9d54..24bb62c8ffe 100644 --- a/python/glide-sync/glide_sync/sync_commands/cluster_commands.py +++ b/python/glide-sync/glide_sync/sync_commands/cluster_commands.py @@ -1041,7 +1041,7 @@ def lolwut( args.extend(["VERSION", str(version)]) if parameters: for var in parameters: - args.extend(str(var)) + args.append(str(var)) return cast( TClusterResponse[bytes], self._execute_command(RequestType.Lolwut, args, route), diff --git a/python/glide-sync/glide_sync/sync_commands/standalone_commands.py b/python/glide-sync/glide_sync/sync_commands/standalone_commands.py index 8d318cd34d4..4718c10cb17 100644 --- a/python/glide-sync/glide_sync/sync_commands/standalone_commands.py +++ b/python/glide-sync/glide_sync/sync_commands/standalone_commands.py @@ -741,7 +741,7 @@ def lolwut( args.extend(["VERSION", str(version)]) if parameters: for var in parameters: - args.extend(str(var)) + args.append(str(var)) return cast( bytes, self._execute_command(RequestType.Lolwut, args), diff --git a/python/tests/async_tests/test_async_client.py b/python/tests/async_tests/test_async_client.py index 07e06e9f7e1..80dc72fec03 100644 --- a/python/tests/async_tests/test_async_client.py +++ b/python/tests/async_tests/test_async_client.py @@ -9330,6 +9330,17 @@ async def test_lolwut(self, glide_client: TGlideClient): result = await glide_client.lolwut(5, [30, 4, 4]) assert b"ver" in result and server_version_bytes in result + # Test LOLWUT version 9 (available in Valkey 9.0.0+) + min_version = "9.0.0" + if await check_if_server_version_lt(glide_client, min_version) is False: + # Test with version 9 and 2 parameters (columns, rows) + result = await glide_client.lolwut(9, [30, 4]) + assert b"ver" in result and server_version_bytes in result + + # Test with version 9 and 4 parameters (columns, rows, real, imaginary) + result = await glide_client.lolwut(9, [40, 20, 1, 2]) + assert b"ver" in result and server_version_bytes in result + if isinstance(glide_client, GlideClusterClient): # test with multi-node route result = await glide_client.lolwut(route=AllNodes()) @@ -9355,6 +9366,21 @@ async def test_lolwut(self, glide_client: TGlideClient): assert isinstance(result, bytes) assert b"ver" in result and server_version_bytes in result + # Test LOLWUT version 9 with cluster routes (available in Valkey 9.0.0+) + if await check_if_server_version_lt(glide_client, min_version) is False: + # Test with version 9 and 2 parameters on all nodes + result = await glide_client.lolwut(9, [30, 4], AllNodes()) + assert isinstance(result, dict) + result_decoded = cast(dict, convert_bytes_to_string_object(result)) + assert result_decoded is not None + for node_result in result_decoded.values(): + assert "ver" in node_result and server_version in node_result + + # Test with version 9 and 4 parameters on random node + result = await glide_client.lolwut(9, [40, 20, 1, 2], RandomNode()) + assert isinstance(result, bytes) + assert b"ver" in result and server_version_bytes in result + @pytest.mark.parametrize("cluster_mode", [True]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) async def test_cluster_client_random_key(self, glide_client: GlideClusterClient): diff --git a/python/tests/sync_tests/test_sync_client.py b/python/tests/sync_tests/test_sync_client.py index 612702d565b..a9931345637 100644 --- a/python/tests/sync_tests/test_sync_client.py +++ b/python/tests/sync_tests/test_sync_client.py @@ -9282,6 +9282,17 @@ def test_sync_lolwut(self, glide_sync_client: TGlideClient): result = glide_sync_client.lolwut(5, [30, 4, 4]) assert b"ver" in result and server_version_bytes in result + # Test LOLWUT version 9 (available in Valkey 9.0.0+) + min_version = "9.0.0" + if sync_check_if_server_version_lt(glide_sync_client, min_version) is False: + # Test with version 9 and 2 parameters (columns, rows) + result = glide_sync_client.lolwut(9, [30, 4]) + assert b"ver" in result and server_version_bytes in result + + # Test with version 9 and 4 parameters (columns, rows, real, imaginary) + result = glide_sync_client.lolwut(9, [40, 20, 1, 2]) + assert b"ver" in result and server_version_bytes in result + if isinstance(glide_sync_client, GlideClusterClient): # test with multi-node route result = glide_sync_client.lolwut(route=AllNodes()) @@ -9307,6 +9318,21 @@ def test_sync_lolwut(self, glide_sync_client: TGlideClient): assert isinstance(result, bytes) assert b"ver" in result and server_version_bytes in result + # Test LOLWUT version 9 with cluster routes (available in Valkey 9.0.0+) + if sync_check_if_server_version_lt(glide_sync_client, min_version) is False: + # Test with version 9 and 2 parameters on all nodes + result = glide_sync_client.lolwut(9, [30, 4], AllNodes()) + assert isinstance(result, dict) + result_decoded = cast(dict, convert_bytes_to_string_object(result)) + assert result_decoded is not None + for node_result in result_decoded.values(): + assert "ver" in node_result and server_version in node_result + + # Test with version 9 and 4 parameters on random node + result = glide_sync_client.lolwut(9, [40, 20, 1, 2], RandomNode()) + assert isinstance(result, bytes) + assert b"ver" in result and server_version_bytes in result + @pytest.mark.parametrize("cluster_mode", [True]) @pytest.mark.parametrize("protocol", [ProtocolVersion.RESP2, ProtocolVersion.RESP3]) def test_sync_cluster_client_random_key( From 7f61ec784c27386b4faa09ba2086873b1bfb3608 Mon Sep 17 00:00:00 2001 From: Noel Hudson Date: Tue, 3 Feb 2026 15:01:15 +0000 Subject: [PATCH 3/5] Initial preparation of SNCR build script --- .github/workflows/npm-cd.yml | 22 +++++++++---------- java/DEVELOPER.md | 4 ++-- java/build.gradle | 2 +- utils/release-candidate-testing/node/index.js | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index f55787c32be..a0e9c6fecb1 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -64,7 +64,7 @@ jobs: if: steps.event-check.outputs.event_name == 'pull_request' shell: bash run: | - RELEASE="255.255.255" + RELEASE="255.255.255-sncr" SHOULD_PUB="false" echo "release_version=$RELEASE" >> $GITHUB_OUTPUT echo "should_publish=$SHOULD_PUB" >> $GITHUB_OUTPUT @@ -153,18 +153,18 @@ jobs: export PLATFORM_MATRIX=$(jq 'map( select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["npm"]))) | .runner = ( - if (.CD_RUNNER != null) - then .CD_RUNNER - elif (.RUNNER != null and (.RUNNER | type != "array")) then .RUNNER + if (.CD_RUNNER != null) + then .CD_RUNNER + elif (.RUNNER != null and (.RUNNER | type != "array")) then .RUNNER else "ubuntu-latest" end ) | .build_type = ( - if (.TARGET | contains("musl")) - then "musl" - elif (.TARGET | contains("gnu")) - then "gnu" - else "mac" + if (.TARGET | contains("musl")) + then "musl" + elif (.TARGET | contains("gnu")) + then "gnu" + else "mac" end ) | if .RUNNER == "macos13" then .["test-runner"] = "macos13" else . end @@ -504,7 +504,7 @@ jobs: if [[ "${{ matrix.build_type }}" == "gnu" ]]; then echo "Installing Valkey on GNU/Linux..." sudo apt update - sudo apt install -y valkey || { + sudo apt install -y valkey || { echo "Valkey not found in default repos, trying alternative source..." curl -fsSL https://packages.redis.io/valkey/setup-valkey | sudo bash - sudo apt install -y valkey || { @@ -627,7 +627,7 @@ jobs: elif [[ "$build_type" == "gnu" ]]; then additional_suffix="-gnu" fi - + package_name="@valkey/valkey-glide-${os}-${arch}${additional_suffix}" attempt_deprecate "${package_name}@${VERSION}" "${DEPRECATION_MESSAGE}" else diff --git a/java/DEVELOPER.md b/java/DEVELOPER.md index f31d67aed1c..00146688ac4 100644 --- a/java/DEVELOPER.md +++ b/java/DEVELOPER.md @@ -178,7 +178,7 @@ Some troubleshooting issues: An example app (`glide.examples.ExamplesApp`) is available under [examples project](../examples/java). To run the ExamplesApp against a local build of valkey-glide client, you can publish your JAR to local Maven repository as a dependency. -To publish to local maven run (default version `255.255.255`): +To publish to local maven run (default version `255.255.255-sncr`): ```bash # Run from the `examples/java` folder @@ -193,7 +193,7 @@ repositories { } dependencies { // Update to use version defined in the previous step - implementation group: 'io.valkey', name: 'valkey-glide', version: '255.255.255' + implementation group: 'io.valkey', name: 'valkey-glide', version: '255.255.255-sncr' } ``` diff --git a/java/build.gradle b/java/build.gradle index 846b00c0437..c67b3d51e08 100644 --- a/java/build.gradle +++ b/java/build.gradle @@ -76,7 +76,7 @@ subprojects { } ext { - defaultReleaseVersion = "255.255.255" + defaultReleaseVersion = "255.255.255-sncr" failedTests = [] } diff --git a/utils/release-candidate-testing/node/index.js b/utils/release-candidate-testing/node/index.js index 450ed0e308d..991f1ba6f7c 100644 --- a/utils/release-candidate-testing/node/index.js +++ b/utils/release-candidate-testing/node/index.js @@ -68,7 +68,7 @@ async function closeClientAndCluster(client, Cluster) { async function getServerVersion(addresses, clusterMode) { // General version for those tests - return "255.255.255"; + return "255.255.255-sncr"; } async function clusterTests() { From c7f5bd96ec3d7b0ecf3335e6622aee98b9eb5078 Mon Sep 17 00:00:00 2001 From: Noel Hudson Date: Tue, 3 Feb 2026 15:05:42 +0000 Subject: [PATCH 4/5] Added support for loading valkey glide runtime from classpath, removed signing, updated CD workflow to work with SNCR tags and also perform a release --- .github/workflows/java-cd.yml | 288 +++++------------- java/client/build.gradle | 47 ++- .../java/glide/ffi/resolvers/NativeUtils.java | 139 +++++++-- java/jedis-compatibility/build.gradle | 11 - 4 files changed, 222 insertions(+), 263 deletions(-) diff --git a/.github/workflows/java-cd.yml b/.github/workflows/java-cd.yml index efb9da41d5d..a1b40024456 100644 --- a/.github/workflows/java-cd.yml +++ b/.github/workflows/java-cd.yml @@ -3,16 +3,15 @@ name: Java Prepare Deployment on: push: tags: - - "v*.*" + - "*-sncr*" workflow_dispatch: inputs: version: - description: "The release version of GLIDE, formatted as *.*.* or *.*.*-rc*" + description: "The release version of GLIDE, formatted as *.*.*-sncr or *.*.*-sncr-rc*" required: true - maven_publish: - description: "Publish to Maven Central" - required: true - type: boolean + +env: + DRY_RUN: false # set to true for a test run without creating release concurrency: group: java-cd-${{ github.head_ref || github.ref }} @@ -35,13 +34,25 @@ jobs: id: load-platform-matrix shell: bash run: | - # Filter entries with maven in PACKAGE_MANAGERS and replace "ephemeral" with "persistent" in RUNNER export PLATFORM_MATRIX=$(jq 'map( - select(.PACKAGE_MANAGERS != null and (.PACKAGE_MANAGERS | contains(["maven"]))) + select( + (.PACKAGE_MANAGERS != null) and (.PACKAGE_MANAGERS | contains(["maven"])) + and ( + ( (.RUNNER | type == "array") and (.RUNNER[1] != "Windows") ) + or + ( (.RUNNER | type != "array") and ((.RUNNER | test("(?i)windows")) | not) ) + ) + ) | .RUNNER = ( - if (.RUNNER | type == "array") - then (.RUNNER | map(if . == "ephemeral" then "persistent" else . end)) - else (if .RUNNER == "ephemeral" then "persistent" else .RUNNER end) + if (.RUNNER | type == "array") then + # Map Linux ARM64 ephemeral to GitHub-hosted runner + if (.RUNNER[0]=="self-hosted" and .RUNNER[1]=="Linux" and .RUNNER[2]=="ARM64") then + "ubuntu-24.04-arm" + else + (.RUNNER | map(if . == "ephemeral" then "persistent" else . end)) + end + else + if .RUNNER == "ephemeral" then "persistent" else .RUNNER end end ) )' < .github/json_matrices/build-matrix.json | jq -c .) @@ -59,7 +70,7 @@ jobs: if ${{ github.event_name == 'workflow_dispatch' }}; then R_VERSION="${{ env.INPUT_VERSION }}" else - R_VERSION=${GITHUB_REF:11} + R_VERSION=${GITHUB_REF:10} fi echo "RELEASE_VERSION=${R_VERSION}" >> $GITHUB_ENV echo "Release version detected: $R_VERSION" @@ -69,7 +80,7 @@ jobs: create-binaries-to-publish: needs: [set-release-version, load-platform-matrix] - if: github.repository_owner == 'valkey-io' + if: github.repository_owner == 'synchronoss' timeout-minutes: 35 env: JAVA_VERSION: "11" @@ -84,6 +95,8 @@ jobs: steps: - name: Setup self-hosted runner access + if: runner.os != 'Windows' + shell: bash run: | GHA_HOME=/home/ubuntu/actions-runner/_work/valkey-glide if [ -d $GHA_HOME ]; then @@ -106,6 +119,7 @@ jobs: os: ${{ matrix.host.OS }} target: ${{ matrix.host.CD_TARGET || matrix.host.TARGET }} github-token: ${{ secrets.GITHUB_TOKEN }} + language: java - name: Run rustup with correct target if: ${{ contains(matrix.host.TARGET, 'musl') }} @@ -129,52 +143,45 @@ jobs: ${{ runner.os }}-gradle-cd- ${{ runner.os }}-gradle- - - name: Create secret key ring file for all Java submodules - working-directory: java/client + - name: Build java client + working-directory: java + shell: bash + env: + GLIDE_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} run: | - # Decode the provided base64 GPG key into the client module - echo "$SECRING_GPG" | base64 --decode > ./secring.gpg - - # Copy the key ring file into the jedis-compatibility module which also performs signing - if [ -d ../jedis-compatibility ]; then - cp ./secring.gpg ../jedis-compatibility/secring.gpg + # Determine gradle command based on OS + if [[ "${{ matrix.host.OS }}" == "windows" ]]; then + GRADLE_CMD="./gradlew.bat" + # TODO: Remove -x :integTest:test when Windows integration tests are implemented + EXTRA_ARGS="-x :integTest:test" else - echo "jedis-compatibility module directory not found" >&2 - exit 1 + GRADLE_CMD="./gradlew" + EXTRA_ARGS="" fi - echo "Listing key ring files to verify presence:" - ls -l ./secring.gpg ../jedis-compatibility/secring.gpg - env: - SECRING_GPG: ${{ secrets.SECRING_GPG }} - - - name: Build java client - working-directory: java - run: | if [[ "${{ matrix.host.TARGET }}" == *"musl"* ]]; then # Build and publish client first - ./gradlew --build-cache :client:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ + $GRADLE_CMD --build-cache :client:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} \ - -Ptarget=${{ matrix.host.TARGET }} - + -Ptarget=${{ matrix.host.TARGET }} $EXTRA_ARGS + # Then build jedis-compatibility - ./gradlew --build-cache :jedis-compatibility:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ + $GRADLE_CMD --build-cache :jedis-compatibility:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} \ - -Ptarget=${{ matrix.host.TARGET }} + -Ptarget=${{ matrix.host.TARGET }} $EXTRA_ARGS else # Build and publish client first - ./gradlew --build-cache :client:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ - -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} - + $GRADLE_CMD --build-cache :client:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ + -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} $EXTRA_ARGS + # Then build jedis-compatibility - ./gradlew --build-cache :jedis-compatibility:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ - -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} + $GRADLE_CMD --build-cache :jedis-compatibility:publishToMavenLocal -Psigning.secretKeyRingFile=secring.gpg \ + -Psigning.password="${{ secrets.GPG_PASSWORD }}" -Psigning.keyId=${{ secrets.GPG_KEY_ID }} $EXTRA_ARGS fi - env: - GLIDE_RELEASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Bundle JAR working-directory: java + shell: bash run: | # Bundle client JAR src_folder=~/.m2/repository/io/valkey/valkey-glide/${{ env.RELEASE_VERSION }} @@ -201,12 +208,9 @@ jobs: java/bundle*.jar java/jedis-bundle*.jar - publish-to-maven-central-deployment: - if: ${{ inputs.maven_publish == true || github.event_name == 'push' }} + package-release-artifacts: needs: [set-release-version, create-binaries-to-publish] runs-on: ubuntu-latest - outputs: - DEPLOYMENT_ID: ${{ steps.maven-deployment.outputs.DEPLOYMENT_ID }} env: RELEASE_VERSION: ${{ needs.set-release-version.outputs.RELEASE_VERSION }} steps: @@ -219,189 +223,57 @@ jobs: cd maven-files for file in $(find ../. -name "*.jar"); do jar xf "$file" ; done - - name: Generate sha1 and md5 files for all Maven files - run: | - cd maven-files - for i in *.jar *.pom *.module; do md5sum $i | cut -d ' ' -f 1 > $i.md5; done - for i in *.jar *.pom *.module; do sha1sum $i | cut -d ' ' -f 1 > $i.sha1; done - - - name: Move files to the correct directory tree + - name: Move files to correct directory tree run: | mkdir -p build/io/valkey/valkey-glide/${{ env.RELEASE_VERSION }} mkdir -p build/io/valkey/valkey-glide-jedis-compatibility/${{ env.RELEASE_VERSION }} - # Move jedis-compatibility files first cp -a maven-files/valkey-glide-jedis-compatibility* build/io/valkey/valkey-glide-jedis-compatibility/${{ env.RELEASE_VERSION }} - # Move client files (exclude jedis-compatibility files) - find maven-files -name "valkey-glide-*" ! -name "*jedis-compatibility*" -exec cp {} build/io/valkey/valkey-glide/${{ env.RELEASE_VERSION }}/ \; + find maven-files -name "valkey-glide-*" ! -name "*jedis-compatibility*" \ + -exec cp {} build/io/valkey/valkey-glide/${{ env.RELEASE_VERSION }}/ \; - rm -rf build/io/valkey/valkey-glide/${{ env.RELEASE_VERSION }}/META-INF - rm -rf build/io/valkey/valkey-glide-jedis-compatibility/${{ env.RELEASE_VERSION }}/META-INF cd build - zip -r ../build io + zip -r ../build.zip io - - name: Upload bundle to CI artifacts + - name: Upload final build.zip artifact uses: actions/upload-artifact@v4 with: name: valkey-${{ env.RELEASE_VERSION }} - path: | - build + path: build.zip - - name: Publish to Maven Central - id: maven-deployment - run: | - export DEPLOYMENT_ID=`curl --request POST \ - -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ - --form bundle=@build.zip \ - https://central.sonatype.com/api/v1/publisher/upload | tail -n 1` - echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_ENV - echo "DEPLOYMENT_ID=$DEPLOYMENT_ID" >> $GITHUB_OUTPUT - echo Uploaded to Maven deployment with deployment ID $DEPLOYMENT_ID. Will be released if smoke tests pass and approved for release. - - - name: Check status of deployment - run: | - for ((retries = 0; retries < 20; retries++)); do - sleep 5 - export DEPLOYMENT_STATUS=`curl --request POST \ - -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ - "https://central.sonatype.com/api/v1/publisher/status?id=${{ env.DEPLOYMENT_ID }}" \ - | jq '.deploymentState'` - - if [[ $DEPLOYMENT_STATUS == ""\"VALIDATED"\"" ]]; then exit 0; fi - done - - curl --request POST \ - -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ - "https://central.sonatype.com/api/v1/publisher/status?id=${{ env.DEPLOYMENT_ID }}" \ - | jq - echo "Deployment ${{ env.DEPLOYMENT_ID }} was unsuccessful with status $DEPLOYMENT_STATUS" - exit 1 - - test-deployment-on-all-architectures: - needs: - [ - set-release-version, - load-platform-matrix, - publish-to-maven-central-deployment, - ] + release: + needs: [set-release-version, package-release-artifacts] + runs-on: ubuntu-latest env: - JAVA_VERSION: "11" RELEASE_VERSION: ${{ needs.set-release-version.outputs.RELEASE_VERSION }} - strategy: - # Run all jobs - fail-fast: false - matrix: - host: ${{ fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX) }} - exclude: - - host: - TARGET: aarch64-unknown-linux-musl - runs-on: ${{ matrix.host.test-runner || matrix.host.runner }} - container: - image: ${{ matrix.host.IMAGE || ''}} - options: ${{ join(' -q ', matrix.host.CONTAINER_OPTIONS) }} # adding `-q` to bypass empty options + permissions: + contents: write + actions: read steps: - - name: Setup self-hosted runner access - if: ${{matrix.host.TARGET == 'aarch64-unknown-linux-gnu' }} - run: sudo chown -R $USER:$USER /home/ubuntu/action-runner-ilia/_work/valkey-glide - - name: Checkout uses: actions/checkout@v4 - - name: Set up JDK - uses: actions/setup-java@v4 - with: - distribution: "temurin" - java-version: ${{ env.JAVA_VERSION }} - - - name: Install git and Java for musl - if: ${{ contains(matrix.host.TARGET, 'musl') }} - run: | - # Set environment variable to indicate container environment for Gradle - echo "GLIDE_CONTAINER_BUILD=true" >> $GITHUB_ENV - apk add openjdk11 git bash - export JAVA_HOME=/usr/lib/jvm/java-11-openjdk - # Create gradle user home and disable auto-download - mkdir -p ~/.gradle - echo "org.gradle.java.installations.auto-download=false" >> ~/.gradle/gradle.properties - echo JAVA_HOME=/usr/lib/jvm/java-11-openjdk >> $GITHUB_ENV - env: - JAVA_HOME: ${{ env.JAVA_HOME }} - - - name: Install shared software dependencies - uses: ./.github/workflows/install-shared-dependencies + - name: Download all artifacts + uses: actions/download-artifact@v4 with: - os: ${{ matrix.host.OS }} - engine-version: "7.2" - target: ${{ matrix.host.TARGET }} - github-token: ${{ secrets.GITHUB_TOKEN }} + path: artifacts + name: valkey-${{ env.RELEASE_VERSION }} - - name: Install protoc (protobuf) - uses: arduino/setup-protoc@v3 - with: - version: "29.1" - repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: List downloaded artifacts + run: ls -R artifacts - - name: Cache Gradle dependencies - uses: actions/cache@v4 + - name: Create GitHub Release + if: env.DRY_RUN != 'true' + uses: softprops/action-gh-release@v2 with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - key: ${{ runner.os }}-gradle-test-cd-${{ hashFiles('java/**/*.gradle*', 'java/**/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle-test-cd- - ${{ runner.os }}-gradle-cd- - ${{ runner.os }}-gradle- - - - name: Start standalone Valkey server - working-directory: utils - id: port - run: | - PORT=$(python3 ./cluster_manager.py start -r 0 2>&1 | grep CLUSTER_NODES | cut -d = -f 2 | cut -d , -f 1 | cut -d : -f 2) - echo "PORT=$PORT" >> $GITHUB_OUTPUT + tag_name: ${{ env.RELEASE_VERSION }} + name: Release ${{ env.RELEASE_VERSION }} + generate_release_notes: true + files: artifacts/**/* - - name: Test deployment - working-directory: java - env: - PORT: ${{ steps.port.outputs.PORT }} - run: | - export ORG_GRADLE_PROJECT_centralManualTestingAuthHeaderName="Authorization" - export ORG_GRADLE_PROJECT_centralManualTestingAuthHeaderValue="Bearer $(echo "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" | base64)" - export GLIDE_RELEASE_VERSION=${{ env.RELEASE_VERSION }} - ./gradlew --build-cache :benchmarks:run --args="--minimal --clients glide --port ${{ env.PORT }}" - - publish-release-to-maven: - if: ${{ inputs.maven_publish == true || github.event_name == 'push' }} - needs: - [ - publish-to-maven-central-deployment, - test-deployment-on-all-architectures, - ] - runs-on: ubuntu-latest - environment: AWS_ACTIONS - env: - DEPLOYMENT_ID: ${{ needs.publish-to-maven-central-deployment.outputs.DEPLOYMENT_ID }} - steps: - - name: Publish to Maven - run: | - curl --request POST \ - -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ - "https://central.sonatype.com/api/v1/publisher/deployment/${{ env.DEPLOYMENT_ID }}" - - drop-deployment-if-validation-fails: - if: ${{ failure() }} - needs: - [ - publish-to-maven-central-deployment, - test-deployment-on-all-architectures, - ] - runs-on: ubuntu-latest - env: - DEPLOYMENT_ID: ${{ needs.publish-to-maven-central-deployment.outputs.DEPLOYMENT_ID }} - steps: - - name: Drop deployment if validation fails + - name: DRY_RUN info + if: env.DRY_RUN == 'true' run: | - curl --request DELETE \ - -u "${{ secrets.CENTRAL_TOKEN_USERNAME }}:${{ secrets.CENTRAL_TOKEN_PASSWORD }}" \ - "https://central.sonatype.com/api/v1/publisher/deployment/${{ env.DEPLOYMENT_ID }}" + echo "DRY_RUN enabled. Would have created release ${{ env.RELEASE_VERSION }}" + echo "Artifacts are ready at artifacts/***" diff --git a/java/client/build.gradle b/java/client/build.gradle index d5c4172fb64..36873543575 100644 --- a/java/client/build.gradle +++ b/java/client/build.gradle @@ -3,7 +3,6 @@ import java.nio.file.Paths plugins { id 'java-library' id 'maven-publish' - id 'signing' id 'io.freefair.lombok' version '8.6' id 'com.github.spotbugs' version '6.0.18' id 'com.google.osdetector' version '1.7.3' @@ -54,7 +53,7 @@ ext { } // osdetector returns 'aarch_64', but rust triplet has 'aarch64' - arch = osdetector.arch == 'aarch_64' ? 'aarch64' : osdetector.arch; + arch = osdetector.arch == 'aarch_64' ? 'aarch64' : osdetector.arch } tasks.register('protobuf', Exec) { @@ -178,19 +177,45 @@ compileJava.dependsOn('protobuf') clean.dependsOn('cleanProtobuf', 'cleanRust') tasks.register('copyNativeLib', Copy) { + println "start copyNativeLib" + def os = osdetector.os def target + def libGlideLocation if (project.hasProperty('target')) { target = project.target - from "${projectDir}/../target/${target}/release/" - } else if (osdetector.os == 'linux' && osdetector.release.id != 'alpine') { - from "${projectDir}/../target/${arch}-unknown-linux-gnu/release/" - } else if (osdetector.os == 'linux' && osdetector.release.id == 'alpine') { - from "${projectDir}/../target/${arch}-unknown-linux-musl/release/" + libGlideLocation = "${projectDir}/../target/${target}/release/" + } else if (os == 'linux' && osdetector.release.id != 'alpine') { + libGlideLocation = "${projectDir}/../target/${arch}-unknown-linux-gnu/release/" + } else if (os == 'linux' && osdetector.release.id == 'alpine') { + libGlideLocation = "${projectDir}/../target/${arch}-unknown-linux-musl/release/" } else { - from "${projectDir}/../target/release/" + libGlideLocation = "${projectDir}/../target/release/" } - include "*.dylib", "*.so" + println "copying source libglide for os/arch ${os}/${arch} from ${libGlideLocation} with target: ${target}" + + from libGlideLocation + include "*.dylib", "*.so", "*.dll" into sourceSets.main.output.resourcesDir + + // Rename libglide_rs. → libglide_rs-. + rename { fileName -> + def matcher = (fileName =~ /^libglide_rs\.(.+)$/) + if (matcher.matches()) { + def newLibGlideName = "libglide_rs-${osdetector.classifier}.${matcher[0][1]}" + println "renaming libglide to ${newLibGlideName}" + return newLibGlideName + } + return fileName + } + + // Delete the original libglide_rs. + doLast { + // Remove original libglide_rs.* files + fileTree(destinationDir).matching { + include "libglide_rs.*" + }.each { it.delete() } + } + println "finish copyNativeLib" } delombok.dependsOn('compileJava') @@ -283,10 +308,6 @@ tasks.withType(Sign) { onlyIf("isReleaseVersion is set") { isReleaseVersion } } -signing { - sign publishing.publications -} - tasks.withType(Test) { testLogging { exceptionFormat "full" diff --git a/java/client/src/main/java/glide/ffi/resolvers/NativeUtils.java b/java/client/src/main/java/glide/ffi/resolvers/NativeUtils.java index c799c67672f..86a421b87f9 100644 --- a/java/client/src/main/java/glide/ffi/resolvers/NativeUtils.java +++ b/java/client/src/main/java/glide/ffi/resolvers/NativeUtils.java @@ -2,23 +2,45 @@ package glide.ffi.resolvers; import java.io.*; -import java.nio.file.FileSystemNotFoundException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.ProviderNotFoundException; -import java.nio.file.StandardCopyOption; +import java.net.URL; +import java.nio.file.*; +import java.util.Locale; +import java.util.logging.Level; /** - * A simple library class which helps with loading dynamic libraries stored in the JAR archive. - * These libraries usually contain implementation of some methods in native code (using JNI - Java - * Native Interface). + * A modified version of {@code NativeUtils} from the valkey-glide project. This utility class + * facilitates loading native libraries packaged within JAR archives. * - * @see https://raw.githubusercontent.com/adamheinrich/native-utils/master/src/main/java/cz/adamh/utils/NativeUtils.java - * @see https://github.com/adamheinrich/native-utils + *

This version of {@code NativeUtils} assumes that the {@code libglide_rs.} files are named + * in the format {@code libglide_rs--.}. For example, instead of {@code + * libglide_rs.so}, it could be {@code libglide_rs-linux-x86_64.so}. + * + *

The following runtime libraries are supported for discovery: + * + *

    + *
  • {@code libglide_rs-osx-aarch_64.dylib} + *
  • {@code libglide_rs-osx-x86_64.dylib} + *
  • {@code libglide_rs-linux-aarch_64.so} + *
  • {@code libglide_rs-linux-x86_64.so} + *
+ * + *

Original sources: + * + *

*/ -public class NativeUtils { +public final class NativeUtils { + + private static final java.util.logging.Logger logger = + java.util.logging.Logger.getLogger(NativeUtils.class.getName()); /** * The minimum length a prefix for a file has to have according to {@link @@ -26,33 +48,56 @@ public class NativeUtils { */ private static final int MIN_PREFIX_LENGTH = 3; + /** Temporary directory to store the native runtime when loading. */ public static final String NATIVE_FOLDER_PATH_PREFIX = "nativeutils"; /** Temporary directory which will contain the dynamic library files. */ private static File temporaryDir; + /** Track if the Glide library has already been loaded */ + private static volatile boolean glideLibLoaded = false; + + /** The native runtime filename for macOS (arm). */ + private static final String LIB_OSX_AARCH_64 = "libglide_rs-osx-aarch_64.dylib"; + + /** The native runtime filename for macOS (x86). */ + private static final String LIB_OSX_X86_64 = "libglide_rs-osx-x86_64.dylib"; + + /** The native runtime filename for Linux (arm). */ + private static final String LIB_LINUX_AARCH_64 = "libglide_rs-linux-aarch_64.so"; + + /** The native runtime filename for Linux (x86). */ + private static final String LIB_LINUX_X86_64 = "libglide_rs-linux-x86_64.so"; + /** Private constructor - this class will never be instanced */ private NativeUtils() {} - public static void loadGlideLib() { - String glideLib = "/libglide_rs"; + public static synchronized void loadGlideLib() { + // Check if already loaded to avoid multiple loads + if (glideLibLoaded) { + return; + } try { - String osName = System.getProperty("os.name").toLowerCase(); - if (osName.contains("mac")) { - NativeUtils.loadLibraryFromJar(glideLib + ".dylib"); - } else if (osName.contains("linux")) { - NativeUtils.loadLibraryFromJar(glideLib + ".so"); - } else { - throw new UnsupportedOperationException( - "OS not supported. Glide is only available on Mac OS and Linux systems."); - } + logClassInfo(); + String libName = "/" + determineLibName(); + NativeUtils.loadLibraryFromJar(libName); + glideLibLoaded = true; } catch (java.io.IOException e) { e.printStackTrace(); } } + /** Logs the location that NativeUtils was loaded from. Useful for debugging. */ + private static void logClassInfo() { + Class clazz = NativeUtils.class; + URL location = clazz.getProtectionDomain().getCodeSource().getLocation(); + log( + Level.FINE, + String.format("Using NativeUtils class: %s from %s", clazz.getName(), location)); + } + /** - * Loads library from current JAR archive + * Loads library from current the classpath. * *

The file from JAR is copied into system temporary directory and then loaded. The temporary * file is deleted after exiting. Method uses String as filename because the pathname is @@ -65,7 +110,7 @@ public static void loadGlideLib() { * @throws IllegalArgumentException If the path is not absolute or if the filename is shorter than * MIN_PREFIX_LENGTH (restriction of {@link File#createTempFile(java.lang.String, * java.lang.String)}). - * @throws FileNotFoundException If the file could not be found inside the JAR. + * @throws FileNotFoundException If the file could not be found on the classpath. */ public static void loadLibraryFromJar(String path) throws IOException { @@ -92,21 +137,24 @@ public static void loadLibraryFromJar(String path) throws IOException { File temp = new File(temporaryDir, filename); try (InputStream is = NativeUtils.class.getResourceAsStream(path)) { + if (is == null) { + cleanupTempFile(temp); + throw new FileNotFoundException("File " + path + " was not found inside JAR."); + } Files.copy(is, temp.toPath(), StandardCopyOption.REPLACE_EXISTING); } catch (IOException e) { - temp.delete(); + cleanupTempFile(temp); throw e; - } catch (NullPointerException e) { - temp.delete(); - throw new FileNotFoundException("File " + path + " was not found inside JAR."); } try { + log(Level.FINE, "Loading native library: " + temp.getName()); System.load(temp.getAbsolutePath()); + log(Level.INFO, "Successfully loaded native library: " + temp.getName()); } finally { if (isPosixCompliant()) { // Assume POSIX compliant file system, can be deleted after loading - temp.delete(); + cleanupTempFile(temp); } else { // Assume non-POSIX, and don't delete until last file descriptor closed temp.deleteOnExit(); @@ -114,6 +162,29 @@ public static void loadLibraryFromJar(String path) throws IOException { } } + private static String determineLibName() { + String os = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + String arch = System.getProperty("os.arch", "").toLowerCase(Locale.ROOT); + boolean isArm = arch.contains("aarch") || arch.contains("arm"); + final String libName; + if (os.contains("mac")) { + libName = isArm ? LIB_OSX_AARCH_64 : LIB_OSX_X86_64; + } else if (os.contains("linux")) { + libName = isArm ? LIB_LINUX_AARCH_64 : LIB_LINUX_X86_64; + } else { + throw new UnsupportedOperationException( + "OS not supported. Glide is only available on Mac OS and Linux systems."); + } + log(Level.FINE, "Determined native library name: " + libName); + return libName; + } + + private static void cleanupTempFile(File temp) { + if (!temp.delete() && temp.exists()) { + temp.deleteOnExit(); + } + } + private static boolean isPosixCompliant() { try { return FileSystems.getDefault().supportedFileAttributeViews().contains("posix"); @@ -131,4 +202,10 @@ private static File createTempDirectory(String prefix) throws IOException { return generatedDir; } + + private static void log(Level level, String message) { + if (logger.isLoggable(level)) { + logger.log(level, String.format("[NativeUtils] %s", message)); + } + } } diff --git a/java/jedis-compatibility/build.gradle b/java/jedis-compatibility/build.gradle index 1a9f21030c3..c410eb86b08 100644 --- a/java/jedis-compatibility/build.gradle +++ b/java/jedis-compatibility/build.gradle @@ -1,7 +1,6 @@ plugins { id 'java-library' id 'maven-publish' - id 'signing' id 'io.freefair.lombok' version '8.6' id 'com.google.osdetector' version '1.7.3' id 'com.gradleup.shadow' version '8.3.8' @@ -161,13 +160,3 @@ tasks.withType(GenerateModuleMetadata) { } publishMavenJavaPublicationToMavenLocal.dependsOn jar, shadowJar - -tasks.withType(Sign) { - def releaseVersion = System.getenv("GLIDE_RELEASE_VERSION") ?: project.ext.defaultReleaseVersion; - def isReleaseVersion = !releaseVersion.endsWith("SNAPSHOT") && releaseVersion != project.ext.defaultReleaseVersion; - onlyIf("isReleaseVersion is set") { isReleaseVersion } -} - -signing { - sign publishing.publications -} From de7e1fb013d2160a5395cff4ec8c8dbca84cec4f Mon Sep 17 00:00:00 2001 From: Noel Hudson Date: Thu, 5 Feb 2026 12:39:34 +0000 Subject: [PATCH 5/5] Addressing some Rust linting issues --- CHANGELOG.md | 10 +++++++++- deny.toml | 4 ++++ glide-core/redis-rs/redis/src/cluster_async/mod.rs | 13 +++++++------ glide-core/redis-rs/redis/src/connection.rs | 1 + glide-core/tests/utilities/mod.rs | 1 + 5 files changed, 22 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4fe820b3b5..9109f36869c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## 2.1.1-sncr + +* JAVA: Multi-platform support with explicitly named libglide_rs runtimes and custom NativeUtils +* Rust: Add RUSTSEC-2025-0141 to ignore list in deny.toml ([#5137](https://github.com/valkey-io/valkey-glide/pull/5137)) +* Rust: Add RUSTSEC-2025-0124 to ignore list in deny.toml +* Rust: Migrate from rustls-pemfile usage to rustls-pki-types ([#5037](https://github.com/valkey-io/valkey-glide/pull/5037)) +* CORE: LOLWUT Tests and Add Version 9 Support ([#4907](https://github.com/valkey-io/valkey-glide/pull/4907)) + ## 2.1.1 #### Changes @@ -15,7 +23,7 @@ * PYTHON: Add MOVE command support for cluster clients ([#4751](https://github.com/valkey-io/valkey-glide/pull/4751)) * JAVA: Add cluster support for Move command ([#4749])(https://github.com/valkey-io/valkey-glide/pull/4749) -#### Fixes +#### Fixes * CORE: Fix SELECT Command Database Persistence Across Reconnections ([#4764](https://github.com/valkey-io/valkey-glide/issues/#4764)) * Rust: Updates the `install-rust-and-protoc` action to explicitly include the `rustfmt` and `clippy` components. ([#4816](https://github.com/valkey-io/valkey-glide/issues/4816)) diff --git a/deny.toml b/deny.toml index 509da0aec11..915dae2c442 100644 --- a/deny.toml +++ b/deny.toml @@ -25,6 +25,10 @@ ignore = [ # suppress this validation until #3226 not resolved # https://github.com/valkey-io/valkey-glide/issues/3226 "RUSTSEC-2025-0007", + # bincode 1.3.3 is declared complete and used only in dev tooling + "RUSTSEC-2025-0141", + # The `rand_os` crate is deprecated and no longer actively maintained, as `OsRng` is now part of `rand_core`. + "RUSTSEC-2025-0124" ] # Threshold for security vulnerabilities, any vulnerability with a CVSS score # lower than the range specified will be ignored. Note that ignored advisories diff --git a/glide-core/redis-rs/redis/src/cluster_async/mod.rs b/glide-core/redis-rs/redis/src/cluster_async/mod.rs index 6b7b0e0ea32..e236b1c34d1 100644 --- a/glide-core/redis-rs/redis/src/cluster_async/mod.rs +++ b/glide-core/redis-rs/redis/src/cluster_async/mod.rs @@ -280,9 +280,9 @@ where } /// Send commands in `pipeline` to the given `route`. If `route` is [None], it will be computed from `pipeline`. - /// - `pipeline_retry_strategy`: Configures retry behavior for pipeline commands. - /// - `retry_server_error`: If `true`, retries commands on server errors (may cause reordering). - /// - `retry_connection_error`: If `true`, retries on connection errors (may lead to duplicate executions). + /// - `pipeline_retry_strategy`: Configures retry behavior for pipeline commands. + /// - `retry_server_error`: If `true`, retries commands on server errors (may cause reordering). + /// - `retry_connection_error`: If `true`, retries on connection errors (may lead to duplicate executions). /// TODO: add wiki link. pub async fn route_pipeline<'a>( &'a mut self, @@ -609,6 +609,7 @@ pub(crate) enum InternalSingleNodeRouting { }, } +#[allow(clippy::derivable_impls)] impl Default for InternalSingleNodeRouting { fn default() -> Self { Self::Random @@ -653,9 +654,9 @@ enum CmdArg { count: usize, route: Option>, sub_pipeline: bool, - /// Configures retry behavior for pipeline commands. - /// - `retry_server_error`: If `true`, retries commands on server errors (may cause reordering). - /// - `retry_connection_error`: If `true`, retries on connection errors (may lead to duplicate executions). + /// Configures retry behavior for pipeline commands. + /// - `retry_server_error`: If `true`, retries commands on server errors (may cause reordering). + /// - `retry_connection_error`: If `true`, retries on connection errors (may lead to duplicate executions). pipeline_retry_strategy: PipelineRetryStrategy, }, ClusterScan { diff --git a/glide-core/redis-rs/redis/src/connection.rs b/glide-core/redis-rs/redis/src/connection.rs index 1a1992c6c30..bc57fe33ff4 100644 --- a/glide-core/redis-rs/redis/src/connection.rs +++ b/glide-core/redis-rs/redis/src/connection.rs @@ -977,6 +977,7 @@ fn setup_connection( } } + #[allow(clippy::unnecessary_unwrap)] if connection_info.client_name.is_some() { match cmd("CLIENT") .arg("SETNAME") diff --git a/glide-core/tests/utilities/mod.rs b/glide-core/tests/utilities/mod.rs index 20ab97e98fa..2dbccbccebd 100644 --- a/glide-core/tests/utilities/mod.rs +++ b/glide-core/tests/utilities/mod.rs @@ -567,6 +567,7 @@ fn set_connection_info_to_connection_request( connection_request: &mut connection_request::ConnectionRequest, ) { connection_request.protocol = convert_to_protobuf_protocol(connection_info.protocol).into(); + #[allow(clippy::unnecessary_unwrap)] if connection_info.password.is_some() { connection_request.authentication_info = protobuf::MessageField(Some(Box::new(AuthenticationInfo {