diff --git a/src/charm.py b/src/charm.py
index a39e49b..23e95c9 100755
--- a/src/charm.py
+++ b/src/charm.py
@@ -34,6 +34,7 @@ def _core_v1_api():
 
 class ConcourseWebOperatorCharm(CharmBase):
     _authed = False
+    _restart_required = False
     _stored = StoredState()
 
     def __init__(self, *args):
@@ -71,8 +72,6 @@ def _on_peer_relation_changed(self, event):
         # defined on the peer relation, and if not, and we're the leader, create
         # them.
 
-        # We're crea
-
         # First, check if they already exist on disk, if so, exit. We're
         # checking in the charm container and we'll "mirror" content from here
         # into the workload container.
@@ -107,6 +106,9 @@ def _on_peer_relation_changed(self, event):
                 keys_found = False
         if keys_found:
             logger.info("Found all concourse keys via the relation.")
+            # Trigger our config changed hook again.
+            self._restart_required = True
+            self.on.config_changed.emit()
             return
 
         # Assuming they're not available on the relation, we need to generate
@@ -158,6 +160,10 @@ def _on_peer_relation_changed(self, event):
         event.relation.data[self.app]["CONCOURSE_TSA_HOST_KEY_PUB"] = tsa_host_key_pub_data
         logger.info("Set CONCOURSE_TSA_HOST_KEY_PUB on the concourse-web peer relation.")
 
+        # Trigger our config changed hook again.
+        self._restart_required = True
+        self.on.config_changed.emit()
+
     def _get_concourse_binary_path(self):
         container = self.unit.get_container("concourse-web")
         with NamedTemporaryFile(delete=False) as temp:
@@ -283,6 +289,9 @@ def _on_concourse_worker_relation_changed(self, event):
             with open(self._concourse_key_locations["CONCOURSE_TSA_AUTHORIZED_KEYS"], "w") as authorized_keys:
                 authorized_keys.write("\n".join(self._stored.concourse_worker_pub_keys))
                 logger.info("Updated CONCOURSE_TSA_AUTHORIZED_KEYS file")
+            # Trigger our config changed hook again.
+            self._restart_required = True
+            self.on.config_changed.emit()
 
     def _on_concourse_worker_relation_broken(self, _):
         self._stored.concourse_worker = False
@@ -378,12 +387,13 @@ def _on_config_changed(self, event):
             return
         # Update our ingress definition if appropriate.
         self.ingress.update_config(self.ingress_config)
-        if services != layer["services"]:
+        if self._restart_required or services != layer["services"]:
             container.add_layer("concourse-web", layer, combine=True)
             logger.info("Added updated layer to concourse")
             if container.get_service("concourse-web").is_running():
                 container.stop("concourse-web")
             container.start("concourse-web")
+            self._restart_required = False
             logger.info("Restarted concourse-web service")
         self.unit.status = ActiveStatus()