feat: Expose operations for RSA flags

This commit adds the functions needed to manipulated the flags on the
RSA object. These flags are not used in normal SSL as far as I know, but
are used by custom providers/engines to change some functionality. The
functions I've added are as follows:

- RSA_test_flags
- RSA_set_flags
- RSA_clear_flags

Since these operations are not available on OpenSSL 1.0.2 or earlier,
I've also added shims that allow one to directly manipulate the "flags"
variable on these older versions.

This patch is made on behalf of Marvell Technology Inc.

Signed-off-by: John Nunley <[email protected]>

Author: John Nunley

openssl-sys/src/handwritten/rsa.rs

@@ -45,6 +45,13 @@ extern "C" {
         iqmp: *mut *const BIGNUM,
     );
 
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_test_flags(r: *const RSA, flags: c_int) -> c_int;
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_set_flags(r: *mut RSA, flags: c_int);
+    #[cfg(any(ossl110, libressl273))]
+    pub fn RSA_clear_flags(r: *mut RSA, flags: c_int);
+
     #[cfg(not(ossl110))]
     pub fn RSA_generate_key(
         modsz: c_int,

openssl/src/rsa.rs

@@ -91,6 +91,26 @@ impl<T> ToOwned for RsaRef<T> {
     }
 }
 
+impl<T> Rsa<T> {
+    /// Sets the RSA flags on the object.
+    #[corresponds(RSA_set_flags)]
+    #[cfg(not(boringssl))]
+    pub fn set_flags(&mut self, flags: i32) {
+        unsafe {
+            RSA_set_flags(self.as_ptr(), flags);
+        }
+    }
+
+    /// Clears the RSA flags on the object.
+    #[corresponds(RSA_set_flags)]
+    #[cfg(not(boringssl))]
+    pub fn clear_flags(&mut self, flags: i32) {
+        unsafe {
+            RSA_clear_flags(self.as_ptr(), flags);
+        }
+    }
+}
+
 impl<T> RsaRef<T>
 where
     T: HasPrivate,
@@ -366,6 +386,15 @@ where
             BigNumRef::from_const_ptr(e)
         }
     }
+
+    /// Tells if the provided set of RSA flags are set.
+    ///
+    /// This function returns the union of all flags that were set on the RSA object.
+    #[corresponds(RSA_test_flags)]
+    #[cfg(not(boringssl))]
+    pub fn test_flags(&self, flags: i32) -> i32 {
+        unsafe { RSA_test_flags(self.as_ptr(), flags) }
+    }
 }
 
 impl Rsa<Public> {
@@ -584,11 +613,32 @@ impl<T> fmt::Debug for Rsa<T> {
     }
 }
 
+cfg_if! {
+    if #[cfg(any(ossl110, libressl273))] {
+        use ffi::{RSA_test_flags, RSA_set_flags, RSA_clear_flags};
+    } else {
+        #[allow(bad_style)]
+        unsafe fn RSA_test_flags(r: *const ffi::RSA, flags: c_int) -> c_int {
+            (*r).flags & flags
+        }
+
+        #[allow(bad_style)]
+        unsafe fn RSA_set_flags(r: *mut ffi::RSA, flags: c_int) {
+            (*r).flags |= flags;
+        }
+
+        #[allow(bad_style)]
+        unsafe fn RSA_clear_flags(r: *mut ffi::RSA, flags: c_int) {
+            (*r).flags &= !flags;
+        }
+    }
+}
+
 cfg_if! {
     if #[cfg(any(ossl110, libressl273, boringssl))] {
         use ffi::{
             RSA_get0_key, RSA_get0_factors, RSA_get0_crt_params, RSA_set0_key, RSA_set0_factors,
-            RSA_set0_crt_params,
+            RSA_set0_crt_params
         };
     } else {
         #[allow(bad_style)]