From 0042b54412687fbfbb9d210e2d46268bacf77c2b Mon Sep 17 00:00:00 2001 From: Toddr Bot Date: Tue, 17 Mar 2026 08:03:54 +0000 Subject: [PATCH 1/3] fix: differentiate encrypt vs sign padding in rsa_crypt() on OpenSSL 3.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On OpenSSL 3.x, rsa_crypt() is shared by encrypt/decrypt (which use EVP_PKEY_encrypt/decrypt) and private_encrypt/public_decrypt (which use EVP_PKEY_sign/verify_recover). The code unconditionally forced OAEP padding for all non-NO_PADDING modes, but OAEP is only valid for encryption — not for sign/verify_recover operations. This meant private_encrypt() and public_decrypt() were broken with any padding except NO_PADDING. The fix replaces the unused `int public` parameter with `int is_encrypt` to distinguish the two operation types: - Encryption path (encrypt/decrypt): forces OAEP, rejects PSS - Sign path (private_encrypt/public_decrypt): passes through PKCS1 and NO_PADDING; rejects OAEP and PSS with clear error messages Co-Authored-By: Claude Opus 4.6 --- MANIFEST | 1 + RSA.pm | 5 ++- RSA.xs | 36 ++++++++++++------ t/private_encrypt.t | 93 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 12 deletions(-) create mode 100644 t/private_encrypt.t diff --git a/MANIFEST b/MANIFEST index 5cdbdc8..0944001 100644 --- a/MANIFEST +++ b/MANIFEST @@ -23,6 +23,7 @@ t/key_lifecycle.t t/keygen.t t/padding.t t/private_crypt.t +t/private_encrypt.t t/pss_auto_promote.t t/rsa.t t/sig_die.t diff --git a/RSA.pm b/RSA.pm index 362201f..50d3c0e 100644 --- a/RSA.pm +++ b/RSA.pm @@ -329,11 +329,14 @@ Decrypt a binary "string". Croaks if the key is public only. =item private_encrypt Encrypt a binary "string" using the private key. Croaks if the key is -public only. +public only. On OpenSSL 3.x, only C and +C are supported; OAEP and PSS will croak. =item public_decrypt Decrypt a binary "string" using the public (portion of the) key. +On OpenSSL 3.x, only C and C +are supported; OAEP and PSS will croak. =item sign diff --git a/RSA.xs b/RSA.xs index fc98463..20d91ac 100644 --- a/RSA.xs +++ b/RSA.xs @@ -453,26 +453,40 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from, #if OPENSSL_VERSION_NUMBER >= 0x30000000L - if(p_rsa->padding == RSA_PKCS1_PSS_PADDING) { - croak("PKCS#1 v2.1 RSA-PSS cannot be used for encryption operations call \"use_pkcs1_oaep_padding\" instead."); - } - EVP_PKEY_CTX *ctx = NULL; int error = 0; int crypt_pad; + if (is_encrypt) { + /* Encryption path: OAEP is the only safe padding for encrypt/decrypt. + PSS is for signatures only. */ + if(p_rsa->padding == RSA_PKCS1_PSS_PADDING) { + croak("PKCS#1 v2.1 RSA-PSS cannot be used for encryption operations call \"use_pkcs1_oaep_padding\" instead."); + } + crypt_pad = p_rsa->padding; + if (p_rsa->padding != RSA_NO_PADDING) { + crypt_pad = RSA_PKCS1_OAEP_PADDING; + } + } else { + /* Sign/verify_recover path (private_encrypt / public_decrypt): + these are low-level RSA operations that respect the user's + padding choice. OAEP and PSS are not valid here. */ + if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) { + croak("OAEP padding is not supported for private_encrypt/public_decrypt. " + "Call use_no_padding() or use_pkcs1_padding() first."); + } + if (p_rsa->padding == RSA_PKCS1_PSS_PADDING) { + croak("PSS padding with private_encrypt/public_decrypt is not supported. " + "Use sign()/verify() for PSS signatures."); + } + crypt_pad = p_rsa->padding; + } + ctx = EVP_PKEY_CTX_new_from_pkey(NULL, (EVP_PKEY* )p_rsa->rsa, NULL); THROW(ctx); THROW(init_crypt(ctx) == 1); - /* After the PKCS1 and PSS guards above, the only reachable padding - values here are RSA_NO_PADDING and RSA_PKCS1_OAEP_PADDING (for - encrypt/decrypt) or RSA_PKCS1_PADDING (for private_encrypt/public_decrypt). */ - crypt_pad = p_rsa->padding; - if (is_encrypt && p_rsa->padding != RSA_NO_PADDING) { - crypt_pad = RSA_PKCS1_OAEP_PADDING; - } THROW(EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_pad) > 0); THROW(p_crypt(ctx, NULL, &to_length, from, from_length) == 1); Newx(to, to_length, UNSIGNED_CHAR); diff --git a/t/private_encrypt.t b/t/private_encrypt.t new file mode 100644 index 0000000..2701af9 --- /dev/null +++ b/t/private_encrypt.t @@ -0,0 +1,93 @@ +use strict; +use warnings; +use Test::More; + +use Crypt::OpenSSL::Random; +use Crypt::OpenSSL::RSA; +use Crypt::OpenSSL::Guess qw(openssl_version); + +my ($major) = openssl_version(); + +plan tests => 10; + +Crypt::OpenSSL::Random::random_seed("OpenSSL needs at least 32 bytes."); +Crypt::OpenSSL::RSA->import_random_seed(); + +my $rsa = Crypt::OpenSSL::RSA->generate_key(2048); +my $key_size = $rsa->size(); + +# --- NO_PADDING: private_encrypt / public_decrypt roundtrip --- + +$rsa->use_no_padding(); +my $data = "\0" x ($key_size - 11) . "Hello World"; +my $enc = $rsa->private_encrypt($data); +ok(defined $enc, "private_encrypt with no_padding succeeds"); +my $dec = $rsa->public_decrypt($enc); +is($dec, $data, "public_decrypt(private_encrypt(data)) round-trips with no_padding"); + +# --- OAEP: should croak for private_encrypt --- + +$rsa->use_pkcs1_oaep_padding(); +eval { $rsa->private_encrypt("test") }; +if ($major ge '3') { + like($@, qr/OAEP padding is not supported for private_encrypt/, + "private_encrypt with OAEP croaks with clear message on OpenSSL 3.x"); +} else { + # Pre-3.x uses RSA_private_encrypt which handles padding differently + ok(1, "private_encrypt with OAEP behavior on pre-3.x (skipped)"); +} + +# --- OAEP: should croak for public_decrypt --- + +eval { $rsa->public_decrypt("test" x 64) }; +if ($major ge '3') { + like($@, qr/OAEP padding is not supported for private_encrypt\/public_decrypt/, + "public_decrypt with OAEP croaks with clear message on OpenSSL 3.x"); +} else { + ok(1, "public_decrypt with OAEP behavior on pre-3.x (skipped)"); +} + +# --- PSS: should croak for private_encrypt --- + +$rsa->use_pkcs1_pss_padding(); +eval { $rsa->private_encrypt("test") }; +if ($major ge '3') { + like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/, + "private_encrypt with PSS croaks with clear message on OpenSSL 3.x"); +} else { + ok(1, "private_encrypt with PSS behavior on pre-3.x (skipped)"); +} + +# --- PSS: should croak for public_decrypt --- + +eval { $rsa->public_decrypt("test" x 64) }; +if ($major ge '3') { + like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/, + "public_decrypt with PSS croaks with clear message on OpenSSL 3.x"); +} else { + ok(1, "public_decrypt with PSS behavior on pre-3.x (skipped)"); +} + +# --- Encryption operations still work correctly --- + +$rsa->use_pkcs1_oaep_padding(); +my $plaintext = "Hello World"; +my $ciphertext = $rsa->encrypt($plaintext); +ok(defined $ciphertext, "encrypt with OAEP still works"); +is($rsa->decrypt($ciphertext), $plaintext, "decrypt with OAEP round-trips"); + +# --- PSS still croaks for encrypt --- + +$rsa->use_pkcs1_pss_padding(); +eval { $rsa->encrypt($plaintext) }; +like($@, qr/RSA-PSS cannot be used for encryption/, + "encrypt with PSS still croaks"); + +# --- Public key cannot private_encrypt --- + +my $pub_key_string = $rsa->get_public_key_string(); +my $rsa_pub = Crypt::OpenSSL::RSA->new_public_key($pub_key_string); +$rsa_pub->use_no_padding(); +eval { $rsa_pub->private_encrypt("\0" x $key_size) }; +like($@, qr/Public keys cannot private_encrypt/, + "public key private_encrypt croaks"); From 5b7083ded2570fc11cb4078b8b44922864ee56ae Mon Sep 17 00:00:00 2001 From: Toddr Bot Date: Wed, 18 Mar 2026 01:09:24 +0000 Subject: [PATCH 2/3] rebase: apply review feedback on #118 --- t/private_encrypt.t | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/t/private_encrypt.t b/t/private_encrypt.t index 2701af9..2c46687 100644 --- a/t/private_encrypt.t +++ b/t/private_encrypt.t @@ -80,8 +80,12 @@ is($rsa->decrypt($ciphertext), $plaintext, "decrypt with OAEP round-trips"); $rsa->use_pkcs1_pss_padding(); eval { $rsa->encrypt($plaintext) }; -like($@, qr/RSA-PSS cannot be used for encryption/, - "encrypt with PSS still croaks"); +if ($major ge '3') { + like($@, qr/RSA-PSS cannot be used for encryption/, + "encrypt with PSS still croaks on OpenSSL 3.x"); +} else { + ok($@, "encrypt with PSS fails on pre-3.x"); +} # --- Public key cannot private_encrypt --- From 141a216ae1823f63fc2e8e25eaa1458bb7cbef9b Mon Sep 17 00:00:00 2001 From: Toddr Bot Date: Sat, 21 Mar 2026 02:43:22 +0000 Subject: [PATCH 3/3] rebase: apply review feedback on #118 - **Moved PSS-for-encrypt guard before `#if OPENSSL_VERSION_NUMBER >= 0x30000000L`** so it fires on all OpenSSL versions, not just 3.x. On pre-3.x smokers, `encrypt()` with PSS was falling through to the legacy RSA code path which produced a generic "unknown padding type" error instead of the friendly croak message. This matches how the Marvin attack (PKCS#1 v1.5) guard is already placed before the ifdef. - **Simplified test 9 in `t/private_encrypt.t`** to use `like()` unconditionally instead of branching on OpenSSL version, since the PSS croak now fires on all versions. --- RSA.xs | 10 +++++----- t/private_encrypt.t | 8 ++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/RSA.xs b/RSA.xs index 20d91ac..3725410 100644 --- a/RSA.xs +++ b/RSA.xs @@ -451,6 +451,10 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from, "Use use_pkcs1_oaep_padding() for encryption, or use_pkcs1_padding() with sign()/verify()."); } + if(is_encrypt && p_rsa->padding == RSA_PKCS1_PSS_PADDING) { + croak("PKCS#1 v2.1 RSA-PSS cannot be used for encryption operations call \"use_pkcs1_oaep_padding\" instead."); + } + #if OPENSSL_VERSION_NUMBER >= 0x30000000L EVP_PKEY_CTX *ctx = NULL; @@ -458,11 +462,7 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from, int crypt_pad; if (is_encrypt) { - /* Encryption path: OAEP is the only safe padding for encrypt/decrypt. - PSS is for signatures only. */ - if(p_rsa->padding == RSA_PKCS1_PSS_PADDING) { - croak("PKCS#1 v2.1 RSA-PSS cannot be used for encryption operations call \"use_pkcs1_oaep_padding\" instead."); - } + /* Encryption path: OAEP is the only safe padding for encrypt/decrypt. */ crypt_pad = p_rsa->padding; if (p_rsa->padding != RSA_NO_PADDING) { crypt_pad = RSA_PKCS1_OAEP_PADDING; diff --git a/t/private_encrypt.t b/t/private_encrypt.t index 2c46687..2701af9 100644 --- a/t/private_encrypt.t +++ b/t/private_encrypt.t @@ -80,12 +80,8 @@ is($rsa->decrypt($ciphertext), $plaintext, "decrypt with OAEP round-trips"); $rsa->use_pkcs1_pss_padding(); eval { $rsa->encrypt($plaintext) }; -if ($major ge '3') { - like($@, qr/RSA-PSS cannot be used for encryption/, - "encrypt with PSS still croaks on OpenSSL 3.x"); -} else { - ok($@, "encrypt with PSS fails on pre-3.x"); -} +like($@, qr/RSA-PSS cannot be used for encryption/, + "encrypt with PSS still croaks"); # --- Public key cannot private_encrypt ---