Avoid checking RestrictedSecurity profile hash during jar verification

If the process of verifying a jar is started before the
RestrictedSecurity profile is loaded, the hash calculation
is triggered as part of it leading to a nested jar verification
and a subsequent error.

To avoid that, the hash calulation of a profile is skipped if
triggered by a jar verification process and is performed later
in the loading process.

Signed-off-by: Kostas Tsiounis <[email protected]>

Author: KostasTsiounis

closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java

@@ -47,6 +47,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import sun.security.jca.ProviderList;
+import sun.security.jca.Providers;
 import sun.security.util.Debug;
 
 /**
@@ -79,6 +81,8 @@ public final class RestrictedSecurity {
 
     private static RestrictedSecurityProperties restricts;
 
+    private static boolean profileHashChecked = false;
+
     private static final Set<String> unmodifiableProperties = new HashSet<>();
 
     private static final Map<String, List<String>> supportedPlatformsNSS = new HashMap<>();
@@ -202,6 +206,11 @@ public static String getRandomProvider() {
             printStackTraceAndExit(
                     "Restricted security mode secure random provider can only be used when restricted security mode is enabled.");
         }
+
+        if (!profileHashChecked) {
+            profileHashChecked = true;
+            checkHashValues();
+        }
         return restricts.jdkSecureRandomProvider;
     }
 
@@ -218,6 +227,10 @@ public static String getRandomAlgorithm() {
             printStackTraceAndExit(
                     "Restricted security mode secure random algorithm can only be used when restricted security mode is enabled.");
         }
+        if (!profileHashChecked) {
+            profileHashChecked = true;
+            checkHashValues();
+        }
         return restricts.jdkSecureRandomAlgorithm;
     }
 
@@ -231,6 +244,10 @@ public static String getRandomAlgorithm() {
      */
     public static boolean isFIPSEnabled() {
         if (securityEnabled) {
+            if (!profileHashChecked) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             return isFIPSEnabled;
         }
         return false;
@@ -244,6 +261,10 @@ public static boolean isFIPSEnabled() {
      */
     public static boolean isServiceAllowed(Service service) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             return restricts.isRestrictedServiceAllowed(service, true);
         }
         return true;
@@ -257,6 +278,10 @@ public static boolean isServiceAllowed(Service service) {
      */
     public static boolean canServiceBeRegistered(Service service) {
         if (securityEnabled) {
+            if (!profileHashChecked) {
+                checkHashValues();
+                profileHashChecked = true;
+            }
             return restricts.isRestrictedServiceAllowed(service, false);
         }
         return true;
@@ -270,6 +295,10 @@ public static boolean canServiceBeRegistered(Service service) {
      */
     public static boolean isProviderAllowed(String providerName) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             // Remove argument, e.g. -NSS-FIPS, if present.
             int pos = providerName.indexOf('-');
             if (pos >= 0) {
@@ -289,6 +318,10 @@ public static boolean isProviderAllowed(String providerName) {
      */
     public static boolean isProviderAllowed(Class<?> providerClazz) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             String providerClassName = providerClazz.getName();
 
             // Check if the specified class extends java.security.Provider.
@@ -378,6 +411,18 @@ private static void getProfileID(Properties props) {
         }
     }
 
+    private static boolean isJarVerifierinStackTrace() {
+        StackTraceElement[] elements = Thread.currentThread().getStackTrace();
+        for (int i = 1; i < elements.length; i++) {
+            StackTraceElement stackTraceElement = elements[i];
+            if ("java.util.jar.JarVerifier".equals(stackTraceElement.getClassName())
+                && "java.base".equals(stackTraceElement.getModuleName())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static void checkIfKnownProfileSupported() {
         if (profileID.contains("NSS") && !isNSSSupported) {
             printStackTraceAndExit("NSS RestrictedSecurity profiles are not supported"
@@ -516,6 +561,7 @@ public static boolean configure(Properties props) {
             }
             printStackTraceAndExit(e);
         }
+
         return securityEnabled;
     }
 

src/java.base/share/classes/sun/security/jca/Providers.java

@@ -111,7 +111,7 @@ private Providers() {
         // triggers a getInstance() call (although that should not happen)
         providerList = ProviderList.EMPTY;
         providerList = ProviderList.fromSecurityProperties();
-        RestrictedSecurity.checkHashValues();
+        //RestrictedSecurity.checkHashValues();
     }
 
     // Return Sun provider.