Avoid checking RestrictedSecurity profile hash during jar verification

If the process of verifying a jar is started before the
RestrictedSecurity profile is loaded, the hash calculation
is triggered as part of it leading to a nested jar verification
and a subsequent error.

To avoid that, the hash calulation of a profile is skipped if
triggered by a jar verification process and is performed later
in the loading process.

Signed-off-by: Kostas Tsiounis <[email protected]>

Author: KostasTsiounis

closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java

@@ -79,6 +79,8 @@ public final class RestrictedSecurity {
 
     private static RestrictedSecurityProperties restricts;
 
+    private static boolean profileHashChecked = false;
+
     private static final Set<String> unmodifiableProperties = new HashSet<>();
 
     private static final Map<String, List<String>> supportedPlatformsNSS = new HashMap<>();
@@ -244,6 +246,10 @@ public static boolean isFIPSEnabled() {
      */
     public static boolean isServiceAllowed(Service service) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             return restricts.isRestrictedServiceAllowed(service, true);
         }
         return true;
@@ -257,6 +263,10 @@ public static boolean isServiceAllowed(Service service) {
      */
     public static boolean canServiceBeRegistered(Service service) {
         if (securityEnabled) {
+            if (!profileHashChecked) {
+                checkHashValues();
+                profileHashChecked = true;
+            }
             return restricts.isRestrictedServiceAllowed(service, false);
         }
         return true;
@@ -270,6 +280,10 @@ public static boolean canServiceBeRegistered(Service service) {
      */
     public static boolean isProviderAllowed(String providerName) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             // Remove argument, e.g. -NSS-FIPS, if present.
             int pos = providerName.indexOf('-');
             if (pos >= 0) {
@@ -289,6 +303,10 @@ public static boolean isProviderAllowed(String providerName) {
      */
     public static boolean isProviderAllowed(Class<?> providerClazz) {
         if (securityEnabled) {
+            if (!(profileHashChecked || isJarVerifierinStackTrace())) {
+                profileHashChecked = true;
+                checkHashValues();
+            }
             String providerClassName = providerClazz.getName();
 
             // Check if the specified class extends java.security.Provider.
@@ -378,6 +396,18 @@ private static void getProfileID(Properties props) {
         }
     }
 
+    private static boolean isJarVerifierinStackTrace() {
+        StackTraceElement[] elements = Thread.currentThread().getStackTrace();
+        for (int i = 1; i < elements.length; i++) {
+            StackTraceElement stackTraceElement = elements[i];
+            if ("java.util.jar.JarVerifier".equals(stackTraceElement.getClassName())
+                && "java.base".equals(stackTraceElement.getModuleName())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static void checkIfKnownProfileSupported() {
         if (profileID.contains("NSS") && !isNSSSupported) {
             printStackTraceAndExit("NSS RestrictedSecurity profiles are not supported"

src/java.base/share/classes/sun/security/jca/Providers.java

@@ -25,7 +25,7 @@
 
 /*
  * ===========================================================================
- * (c) Copyright IBM Corp. 2024, 2024 All Rights Reserved
+ * (c) Copyright IBM Corp. 2024, 2025 All Rights Reserved
  * ===========================================================================
  */
 
@@ -111,7 +111,6 @@ private Providers() {
         // triggers a getInstance() call (although that should not happen)
         providerList = ProviderList.EMPTY;
         providerList = ProviderList.fromSecurityProperties();
-        RestrictedSecurity.checkHashValues();
     }
 
     // Return Sun provider.