fix: private_encrypt/public_decrypt broken on OpenSSL 3.x

MANIFEST

@@ -23,6 +23,7 @@ t/key_lifecycle.t
 t/keygen.t
 t/padding.t
 t/private_crypt.t
+t/private_encrypt.t
 t/pss_auto_promote.t
 t/rsa.t
 t/sig_die.t

RSA.pm

@@ -329,11 +329,14 @@ Decrypt a binary "string".  Croaks if the key is public only.
 =item private_encrypt
 
 Encrypt a binary "string" using the private key.  Croaks if the key is
-public only.
+public only.  On OpenSSL 3.x, only C<use_no_padding> and
+C<use_pkcs1_padding> are supported; OAEP and PSS will croak.
 
 =item public_decrypt
 
 Decrypt a binary "string" using the public (portion of the) key.
+On OpenSSL 3.x, only C<use_no_padding> and C<use_pkcs1_padding>
+are supported; OAEP and PSS will croak.
 
 =item sign
 

RSA.xs

@@ -451,28 +451,42 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from,
               "Use use_pkcs1_oaep_padding() for encryption, or use_pkcs1_padding() with sign()/verify().");
     }
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
-
-    if(p_rsa->padding == RSA_PKCS1_PSS_PADDING) {
+    if(is_encrypt && p_rsa->padding == RSA_PKCS1_PSS_PADDING) {
         croak("PKCS#1 v2.1 RSA-PSS cannot be used for encryption operations call \"use_pkcs1_oaep_padding\" instead.");
     }
 
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+
     EVP_PKEY_CTX *ctx = NULL;
     int error = 0;
     int crypt_pad;
 
+    if (is_encrypt) {
+        /* Encryption path: OAEP is the only safe padding for encrypt/decrypt. */
+        crypt_pad = p_rsa->padding;
+        if (p_rsa->padding != RSA_NO_PADDING) {
+            crypt_pad = RSA_PKCS1_OAEP_PADDING;
+        }
+    } else {
+        /* Sign/verify_recover path (private_encrypt / public_decrypt):
+           these are low-level RSA operations that respect the user's
+           padding choice.  OAEP and PSS are not valid here. */
+        if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
+            croak("OAEP padding is not supported for private_encrypt/public_decrypt. "
+                  "Call use_no_padding() or use_pkcs1_padding() first.");
+        }
+        if (p_rsa->padding == RSA_PKCS1_PSS_PADDING) {
+            croak("PSS padding with private_encrypt/public_decrypt is not supported. "
+                  "Use sign()/verify() for PSS signatures.");
+        }
+        crypt_pad = p_rsa->padding;
+    }
+
     ctx = EVP_PKEY_CTX_new_from_pkey(NULL, (EVP_PKEY* )p_rsa->rsa, NULL);
 
     THROW(ctx);
 
     THROW(init_crypt(ctx) == 1);
-    /* After the PKCS1 and PSS guards above, the only reachable padding
-       values here are RSA_NO_PADDING and RSA_PKCS1_OAEP_PADDING (for
-       encrypt/decrypt) or RSA_PKCS1_PADDING (for private_encrypt/public_decrypt). */
-    crypt_pad = p_rsa->padding;
-    if (is_encrypt && p_rsa->padding != RSA_NO_PADDING) {
-        crypt_pad = RSA_PKCS1_OAEP_PADDING;
-    }
     THROW(EVP_PKEY_CTX_set_rsa_padding(ctx, crypt_pad) > 0);
     THROW(p_crypt(ctx, NULL, &to_length, from, from_length) == 1);
     Newx(to, to_length, UNSIGNED_CHAR);

t/private_encrypt.t

@@ -0,0 +1,93 @@
+use strict;
+use warnings;
+use Test::More;
+
+use Crypt::OpenSSL::Random;
+use Crypt::OpenSSL::RSA;
+use Crypt::OpenSSL::Guess qw(openssl_version);
+
+my ($major) = openssl_version();
+
+plan tests => 10;
+
+Crypt::OpenSSL::Random::random_seed("OpenSSL needs at least 32 bytes.");
+Crypt::OpenSSL::RSA->import_random_seed();
+
+my $rsa = Crypt::OpenSSL::RSA->generate_key(2048);
+my $key_size = $rsa->size();
+
+# --- NO_PADDING: private_encrypt / public_decrypt roundtrip ---
+
+$rsa->use_no_padding();
+my $data = "\0" x ($key_size - 11) . "Hello World";
+my $enc = $rsa->private_encrypt($data);
+ok(defined $enc, "private_encrypt with no_padding succeeds");
+my $dec = $rsa->public_decrypt($enc);
+is($dec, $data, "public_decrypt(private_encrypt(data)) round-trips with no_padding");
+
+# --- OAEP: should croak for private_encrypt ---
+
+$rsa->use_pkcs1_oaep_padding();
+eval { $rsa->private_encrypt("test") };
+if ($major ge '3') {
+    like($@, qr/OAEP padding is not supported for private_encrypt/,
+         "private_encrypt with OAEP croaks with clear message on OpenSSL 3.x");
+} else {
+    # Pre-3.x uses RSA_private_encrypt which handles padding differently
+    ok(1, "private_encrypt with OAEP behavior on pre-3.x (skipped)");
+}
+
+# --- OAEP: should croak for public_decrypt ---
+
+eval { $rsa->public_decrypt("test" x 64) };
+if ($major ge '3') {
+    like($@, qr/OAEP padding is not supported for private_encrypt\/public_decrypt/,
+         "public_decrypt with OAEP croaks with clear message on OpenSSL 3.x");
+} else {
+    ok(1, "public_decrypt with OAEP behavior on pre-3.x (skipped)");
+}
+
+# --- PSS: should croak for private_encrypt ---
+
+$rsa->use_pkcs1_pss_padding();
+eval { $rsa->private_encrypt("test") };
+if ($major ge '3') {
+    like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
+         "private_encrypt with PSS croaks with clear message on OpenSSL 3.x");
+} else {
+    ok(1, "private_encrypt with PSS behavior on pre-3.x (skipped)");
+}
+
+# --- PSS: should croak for public_decrypt ---
+
+eval { $rsa->public_decrypt("test" x 64) };
+if ($major ge '3') {
+    like($@, qr/PSS padding with private_encrypt\/public_decrypt is not supported/,
+         "public_decrypt with PSS croaks with clear message on OpenSSL 3.x");
+} else {
+    ok(1, "public_decrypt with PSS behavior on pre-3.x (skipped)");
+}
+
+# --- Encryption operations still work correctly ---
+
+$rsa->use_pkcs1_oaep_padding();
+my $plaintext = "Hello World";
+my $ciphertext = $rsa->encrypt($plaintext);
+ok(defined $ciphertext, "encrypt with OAEP still works");
+is($rsa->decrypt($ciphertext), $plaintext, "decrypt with OAEP round-trips");
+
+# --- PSS still croaks for encrypt ---
+
+$rsa->use_pkcs1_pss_padding();
+eval { $rsa->encrypt($plaintext) };
+like($@, qr/RSA-PSS cannot be used for encryption/,
+     "encrypt with PSS still croaks");
+
+# --- Public key cannot private_encrypt ---
+
+my $pub_key_string = $rsa->get_public_key_string();
+my $rsa_pub = Crypt::OpenSSL::RSA->new_public_key($pub_key_string);
+$rsa_pub->use_no_padding();
+eval { $rsa_pub->private_encrypt("\0" x $key_size) };
+like($@, qr/Public keys cannot private_encrypt/,
+     "public key private_encrypt croaks");