Avoid checking RestrictedSecurity profile hash during jar verification

If the process of verifying a jar is started before the
RestrictedSecurity profile is loaded, the hash calculation
is triggered as part of it leading to a nested jar verification
and a subsequent error.

To avoid that, the hash calulation of a profile is skipped if
triggered by a jar verification process and is performed later
in the loading process.

Signed-off-by: Kostas Tsiounis <[email protected]>

Author: KostasTsiounis

closed/src/java.base/share/classes/openj9/internal/security/RestrictedSecurity.java

@@ -172,17 +172,30 @@ private RestrictedSecurity() {
         super();
     }
 
+    private static boolean isJarVerifierInStackTrace() {
+        java.util.function.Predicate<Class<?>> isJarVerifier =
+                clazz -> "java.util.jar.JarVerifier".equals(clazz.getName())
+                      && "java.base".equals(clazz.getModule().getName());
+
+        java.util.function.Function<Stream<StackWalker.StackFrame>, Boolean> matcher =
+                stream -> stream.map(StackWalker.StackFrame::getDeclaringClass)
+                                .anyMatch(isJarVerifier);
+
+        return StackWalker.getInstance(StackWalker.Option.RETAIN_CLASS_REFERENCE).walk(matcher);
+    }
+
     /**
      * Check loaded profiles' hash values.
      *
      * In order to avoid unintentional changes in profiles and incentivize
      * extending profiles, instead of altering them, a digest of the profile
      * is calculated and compared to the expected value.
      */
-    public static void checkHashValues() {
-        if (profileParser != null) {
-            profileParser.checkHashValues();
+    private static void checkHashValues() {
+        ProfileParser parser = profileParser;
+        if ((parser != null) && !isJarVerifierInStackTrace()) {
             profileParser = null;
+            parser.checkHashValues();
         }
     }
 
@@ -255,6 +268,7 @@ public static boolean isFIPSEnabled() {
      */
     public static boolean isServiceAllowed(Service service) {
         if (securityEnabled) {
+            checkHashValues();
             return restricts.isRestrictedServiceAllowed(service, true);
         }
         return true;
@@ -268,6 +282,7 @@ public static boolean isServiceAllowed(Service service) {
      */
     public static boolean canServiceBeRegistered(Service service) {
         if (securityEnabled) {
+            checkHashValues();
             return restricts.isRestrictedServiceAllowed(service, false);
         }
         return true;
@@ -281,6 +296,7 @@ public static boolean canServiceBeRegistered(Service service) {
      */
     public static boolean isProviderAllowed(String providerName) {
         if (securityEnabled) {
+            checkHashValues();
             // Remove argument, e.g. -NSS-FIPS, if present.
             int pos = providerName.indexOf('-');
             if (pos >= 0) {
@@ -300,6 +316,7 @@ public static boolean isProviderAllowed(String providerName) {
      */
     public static boolean isProviderAllowed(Class<?> providerClazz) {
         if (securityEnabled) {
+            checkHashValues();
             String providerClassName = providerClazz.getName();
 
             // Check if the specified class extends java.security.Provider.

src/java.base/share/classes/sun/security/jca/Providers.java

@@ -25,7 +25,7 @@
 
 /*
  * ===========================================================================
- * (c) Copyright IBM Corp. 2024, 2024 All Rights Reserved
+ * (c) Copyright IBM Corp. 2024, 2025 All Rights Reserved
  * ===========================================================================
  */
 
@@ -110,7 +110,6 @@ private Providers() {
         // triggers a getInstance() call (although that should not happen)
         providerList = ProviderList.EMPTY;
         providerList = ProviderList.fromSecurityProperties();
-        RestrictedSecurity.checkHashValues();
     }
 
     // Return Sun provider.