Fix RSA_METHOD parameter mismatch and use-after-free in EVP_PKEY_derive_set_peer

Custom RSA_METHOD callbacks (sign_raw, verify_raw, encrypt, decrypt)
were receiving max_out (output buffer size) as their first parameter
instead of in_len (input data length). This caused callbacks to read
past the input buffer by up to (RSA_size - actual_input_length) bytes.
Fixed all 4 dispatch sites to pass in_len, matching OpenSSL's flen
semantics for RSA_private_encrypt/RSA_public_decrypt/etc.

EVP_PKEY_derive_set_peer had a use-after-free when called with the same
peer key already set and a refcount of 1: EVP_PKEY_free freed the key
before EVP_PKEY_up_ref took a new reference. Reordered to up_ref before
free, and added proper cleanup in the ctrl failure path.

Author: dkostic

crypto/fipsmodule/evp/evp_ctx.c

@@ -366,17 +366,20 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) {
     return 0;
   }
 
+  // Take a reference to the new peer before freeing the old one, in case
+  // peer == ctx->peerkey and the caller holds the sole reference.
+  EVP_PKEY_up_ref(peer);
   EVP_PKEY_free(ctx->peerkey);
   ctx->peerkey = peer;
 
   ret = ctx->pmeth->ctrl(ctx, EVP_PKEY_CTRL_PEER_KEY, 1, peer);
 
   if (ret <= 0) {
+    EVP_PKEY_free(ctx->peerkey);
     ctx->peerkey = NULL;
     return 0;
   }
 
-  EVP_PKEY_up_ref(peer);
   return 1;
 }
 

crypto/fipsmodule/rsa/rsa.c

@@ -549,12 +549,12 @@ static int rsa_sign_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
     // and expect an |out_len| parameter. To remain compatible with this new
     // paradigm and OpenSSL, we initialize |out_len| based on the return value
     // here.
-    if (max_out > INT_MAX) {
+    if (in_len > INT_MAX) {
       OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
       *out_len = 0;
       return 0;
     }
-    int ret = rsa->meth->sign_raw((int)max_out, in, out, rsa, padding);
+    int ret = rsa->meth->sign_raw((int)in_len, in, out, rsa, padding);
     if(ret < 0) {
       *out_len = 0;
       return 0;

crypto/fipsmodule/rsa/rsa_impl.c

@@ -360,7 +360,7 @@ int rsa_verify_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
                                 size_t max_out, const uint8_t *in,
                                 size_t in_len, int padding) {
   if(rsa->meth && rsa->meth->verify_raw) {
-    if (max_out > INT_MAX) {
+    if (in_len > INT_MAX) {
       OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
       *out_len = 0;
       return 0;
@@ -372,7 +372,7 @@ int rsa_verify_raw_no_self_test(RSA *rsa, size_t *out_len, uint8_t *out,
     // and expect an |out_len| parameter. To remain compatible with this new
     // paradigm and OpenSSL, we initialize |out_len| based on the return value
     // here.
-    int ret = rsa->meth->verify_raw((int)max_out, in, out, rsa, padding);
+    int ret = rsa->meth->verify_raw((int)in_len, in, out, rsa, padding);
     if(ret < 0) {
       *out_len = 0;
       return 0;

crypto/rsa_extra/rsa_crypt.c

@@ -338,12 +338,12 @@ int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
     // expect an |out_len| parameter. To remain compatible with this new
     // paradigm and OpenSSL, we initialize |out_len| based on the return value
     // here.
-    if (max_out > INT_MAX) {
+    if (in_len > INT_MAX) {
       OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
       *out_len = 0;
       return 0;
     }
-    int ret = rsa->meth->encrypt((int)max_out, in, out, rsa, padding);
+    int ret = rsa->meth->encrypt((int)in_len, in, out, rsa, padding);
     if(ret < 0) {
       *out_len = 0;
       return 0;
@@ -515,12 +515,12 @@ int RSA_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
     // functions like |RSA_decrypt| diverge from this paradigm and expect
     // an |out_len| parameter. To remain compatible with this new paradigm and
     // OpenSSL, we initialize |out_len| based on the return value here.
-    if (max_out > INT_MAX) {
+    if (in_len > INT_MAX) {
       OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
       *out_len = 0;
       return 0;
     }
-    int ret = rsa->meth->decrypt((int)max_out, in, out, rsa, padding);
+    int ret = rsa->meth->decrypt((int)in_len, in, out, rsa, padding);
     if(ret < 0) {
       *out_len = 0;
       return 0;