fix: publish reload state atomically, skip store failures

A failed reload no longer leaves new ACME clients with the old
certificate cache: both are published only after loading succeeds.
Store failures in the renewal loop are logged and retried next cycle
instead of burning attempts as CA rejections.

Author: NiccoMlt

carapace-server/src/main/java/org/carapaceproxy/server/certificates/DynamicCertificatesManager.java

@@ -67,6 +67,7 @@
 import org.carapaceproxy.cluster.GroupMembershipHandler;
 import org.carapaceproxy.configstore.CertificateData;
 import org.carapaceproxy.configstore.ConfigurationStore;
+import org.carapaceproxy.configstore.ConfigurationStoreException;
 import org.carapaceproxy.core.HttpProxyServer;
 import org.carapaceproxy.core.RuntimeServerConfiguration;
 import org.carapaceproxy.server.config.ConfigurationNotValidException;
@@ -180,9 +181,11 @@ public synchronized void reloadConfiguration(RuntimeServerConfiguration configur
                     loadOrCreateAcmeUserKeyPair(provider.name()), provider.url(), provider.kid(), provider.hmac()
             ));
         }
-        acmeClients = Map.copyOf(clients);
         domainsCheckerIPAddresses = configuration.getDomainsCheckerIPAddresses();
-        loadCertificates(configuration.getCertificates());
+        final var newCertificates = loadCertificates(configuration.getCertificates());
+        // published together once loading succeeded: a failed reload must not leave new clients with the old cache
+        acmeClients = Map.copyOf(clients);
+        this.certificates = newCertificates;
         period = configuration.getDynamicCertificatesManagerPeriod();
         if (scheduledFuture != null) {
             scheduledFuture.cancel(true);
@@ -219,7 +222,7 @@ private ACMEClient acmeClientFor(CertificateData cert) {
         return client;
     }
 
-    private void loadCertificates(Map<String, SSLCertificateConfiguration> certificates) throws ConfigurationNotValidException {
+    private Map<String, CertificateData> loadCertificates(Map<String, SSLCertificateConfiguration> certificates) throws ConfigurationNotValidException {
         try {
             final var _certificates = new ConcurrentHashMap<String, CertificateData>();
             for (Entry<String, SSLCertificateConfiguration> e : certificates.entrySet()) {
@@ -238,7 +241,7 @@ private void loadCertificates(Map<String, SSLCertificateConfiguration> certifica
                     ));
                 }
             }
-            this.certificates = _certificates; // only certificates/domains specified in the config have to be managed.
+            return _certificates; // only certificates/domains specified in the config have to be managed.
         } catch (GeneralSecurityException | MalformedURLException e) {
             throw new DynamicCertificatesManagerException("Unable to load dynamic certificates configuration.", e);
         }
@@ -392,6 +395,9 @@ private void certificatesLifecycle() {
                     store.saveCertificate(cert);
                     flushCache = true;
                 }
+            } catch (ConfigurationStoreException ex) {
+                // a store failure is infra-transient: retried at the next cycle, not counted as a CA rejection
+                LOG.error("Store failure while handling dynamic certificate for domain {}", domain, ex);
             } catch (AcmeException | IOException | GeneralSecurityException | RuntimeException ex) {
                 // RuntimeException included on purpose, as an escaped one would silently cancel the scheduled task;
                 // this would kill the renewal loop for every certificate

carapace-server/src/test/java/org/carapaceproxy/server/certificates/CertificatesTest.java

@@ -492,6 +492,7 @@ public void testFailedConfigurationApplyRestoresCertificateData() throws Excepti
         CertificateData restored = store.loadCertificateForDomain("localhost2");
         assertEquals(previous.getChain(), restored.getChain());
         assertEquals(previous.getState(), restored.getState());
+        assertEquals(previous.getProvider(), restored.getProvider());
     }
 
     @Test

carapace-server/src/test/java/org/carapaceproxy/server/certificates/DynamicCertificatesManagerTest.java

@@ -61,6 +61,7 @@
 import org.carapaceproxy.cluster.impl.NullGroupMembershipHandler;
 import org.carapaceproxy.configstore.CertificateData;
 import org.carapaceproxy.configstore.ConfigurationStore;
+import org.carapaceproxy.configstore.ConfigurationStoreException;
 import org.carapaceproxy.configstore.PropertiesConfigurationStore;
 import org.carapaceproxy.core.HttpProxyServer;
 import org.carapaceproxy.core.Listeners;
@@ -736,19 +737,20 @@ public void testCertificateProviderRouting() throws Exception {
     }
 
     @Test
-    @Parameters({"stale", "transient"})
+    @Parameters({"stale", "transient", "store"})
     public void testPendingOrderPollFailure(String failureCase) throws Exception {
         // ACME mocking: the pending order cannot be polled back from the CA
         ACMEClient ac = mock(ACMEClient.class);
         Login login = mock(Login.class);
         Order order = mock(Order.class);
         when(login.bindOrder(any())).thenReturn(order);
         when(ac.getLogin()).thenReturn(login);
-        doThrow(failureCase.equals("transient")
-                ? new AcmeNetworkException(new IOException("connection reset"))
-                // e.g., the order belongs to a different CA after a provider change
-                : new AcmeException("unknown order")
-        ).when(ac).checkResponseForOrder(any());
+        doThrow(switch (failureCase) {
+            case "transient" -> new AcmeNetworkException(new IOException("connection reset"));
+            case "store" -> new ConfigurationStoreException(new IOException("db down"));
+            // e.g., the order belongs to a different CA after a provider change
+            default -> new AcmeException("unknown order");
+        }).when(ac).checkResponseForOrder(any());
 
         HttpProxyServer parent = mock(HttpProxyServer.class);
         when(parent.getListeners()).thenReturn(mock(Listeners.class));
@@ -783,7 +785,7 @@ public void testPendingOrderPollFailure(String failureCase) throws Exception {
             man.run();
             assertCertificateState(domain, WAITING, 1, man);
         } else {
-            // state untouched and nothing persisted, will be retried at the next cycle
+            // transient ACME and store failures alike: state untouched and nothing persisted, retried at the next cycle
             assertCertificateState(domain, ORDERING, 0, man);
             verify(store, never()).saveCertificate(any());
         }