refactor: split self.spawn_dependee_queue into subfunctions

Author: Michael Hammann

supervisor/supervisord.py

@@ -269,7 +269,8 @@ def runforever(self):
             for group in pgroups:
                 group.transition(self)
 
-            self.spawn_dependee_queue()
+            self._spawn_dependee_queue()
+            self._handle_spawn_timeout()
 
             self.reap()
             self.handle_signal()
@@ -346,71 +347,85 @@ def handle_signal(self):
     def get_state(self):
         return self.options.mood
 
-    def spawn_dependee_queue(self):
+    def _spawn_dependee_queue(self):
         """
-        check for processes that are sheduled but not started
-        if the queue is well ordered just pop and spawn all that are ready, 
-        Currently ordering is not considered when adding to queue - 
-        therefore just iterate over queue and spawn all ready processes.
+        Iterate over processes that are not started but added to 
+        process_spawn_dict. Spawn all processes which are ready
+        (All dependees RUNNING or process without dependees)
         """
         if self.process_spawn_dict:
             for process_name, process_object in list(self.process_spawn_dict.items()):
                 if process_object.config.depends_on is not None:
                     if any([dependee.state is ProcessStates.FATAL for dependee in
                             process_object.config.depends_on.values()]):
-                        for process_name, process_object in self.process_spawn_dict.items():
-                            process_object.record_spawnerr(
-                                'Dependee process did not start - set FATAL state for {}'
-                                .format(process_name))
-                            process_object.change_state(ProcessStates.FATAL)
-                        self.process_spawn_set = set()
-                        self.process_spawn_dict = dict()
+                        self._set_fatal_state_and_empty_queue()
                         break
                     if all([dependee.state is ProcessStates.RUNNING for dependee in
                             process_object.config.depends_on.values()]):
-                        self.process_started_dict[process_name] = process_object
-                        del self.process_spawn_dict[process_name]
-                        process_object.spawn(self)
-                        process_object.notify_timer = 5
+                        self._spawn_process_from_process_dict()
                 else:
-                    self.process_started_dict[process_name] = process_object
-                    del self.process_spawn_dict[process_name]
-                    process_object.spawn(self)
-                    process_object.notify_timer = 5
+                    self._spawn_process_from_process_dict()
+
+    def _spawn_process_from_process_dict(self, process_name, process_object):
+        self.process_started_dict[process_name] = process_object
+        del self.process_spawn_dict[process_name]
+        process_object.spawn(self)
+        process_object.notify_timer = 5
+
+    def _set_fatal_state_and_empty_queue(self):
+        for process_name, process_object in self.process_spawn_dict.items():
+            process_object.record_spawnerr(
+                'Dependee process did not start - set FATAL state for {}'
+                .format(process_name))
+            process_object.change_state(ProcessStates.FATAL)
+        self.process_spawn_set = set()
+        self.process_spawn_dict = dict()
 
+    def _handle_spawn_timeout(self):
+        """
+        Log info message each 5 seconds if some process is waiting on a dependee
+        Timeout if a process needs longer than spawn_timeout (default=60 seconds)
+        to reach RUNNING
+        """
         # check if any of the processes that was started did not make it and remove RUNNING ones.
         if self.process_started_dict:
-            for started_process_name, started_process_object in list(self.process_started_dict.items()):
-                # if started_process in RUNNING - remove from set
-                if started_process_object.state is ProcessStates.RUNNING:
-                    del self.process_started_dict[started_process_name]
+            for process_name, process_object in list(self.process_started_dict.items()):
+                if process_object.state is ProcessStates.RUNNING:
+                    del self.process_started_dict[process_name]
                 # handle timeout error.
-                elif (time.time() - started_process_object.laststart) >= started_process_object.config.spawn_timeout:
-                    msg = (
-                        "timeout: dependee process {} in {} state did not reach RUNNING within {} seconds, dependees {} are not spawned"
-                            .format(started_process_name,
-                                    getProcessStateDescription(started_process_object.state),
-                                    started_process_object.config.spawn_timeout,
-                                    [process for process in self.process_spawn_dict.keys()]))
-                    started_process_object.config.options.logger.warn(msg)
-                    started_process_object.record_spawnerr(
-                        'timeout: Process {} did not reach RUNNING state within {} seconds'.format(
-                        started_process_name, started_process_object.config.spawn_timeout))
-                    started_process_object.change_state(ProcessStates.FATAL)
-                    for process_name, process_object in self.process_spawn_dict.items():
-                        process_object.record_spawnerr(
-                            'Dependee process did not start - set FATAL state for {}'.format(
-                            process_name))
-                        process_object.change_state(ProcessStates.FATAL)
-                    self.process_spawn_dict = dict()
-                    self.process_started_dict = dict()
+                elif (time.time() - process_object.laststart) >= process_object.config.spawn_timeout:
+                    self._timeout_process(process_name, process_object)
                 # notify user about waiting
-                elif (time.time() - started_process_object.laststart) >= started_process_object.notify_timer:
-                    started_process_object.notify_timer += 5
-                    msg = ("waiting for dependee process {} in {} state to be RUNNING"
-                            .format(started_process_name,
-                                    getProcessStateDescription(started_process_object.state)))
-                    started_process_object.config.options.logger.info(msg)
+                elif (time.time() - process_object.laststart) >= process_object.notify_timer:
+                    self._notfiy_user_about_waiting(process_name, process_object)
+
+    def _timeout_process(self, process_name, process_object):
+        msg = ("""timeout: dependee process {} in {} did not reach RUNNING
+               within {} seconds, dependees {} are not spawned"""
+                .format(process_name,
+                        getProcessStateDescription(process_object.state),
+                        process_object.config.spawn_timeout,
+                        [process for process in self.process_spawn_dict.keys()]))
+        process_object.config.options.logger.warn(msg)
+        process_object.record_spawnerr(
+            'timeout: Process {} did not reach RUNNING state within {} seconds'
+            .format(process_name,
+                    process_object.config.spawn_timeout))
+        process_object.change_state(ProcessStates.FATAL)
+        for process_name, process_object in self.process_spawn_dict.items():
+            process_object.record_spawnerr(
+                'Dependee process did not start - set FATAL state for {}'
+                .format(process_name))
+            process_object.change_state(ProcessStates.FATAL)
+        self.process_spawn_dict = dict()
+        self.process_started_dict = dict()
+
+    def _notfiy_user_about_waiting(self, process_name, process_object):
+        process_object.notify_timer += 5
+        msg = ("waiting for dependee process {} in {} state to be RUNNING"
+                .format(process_name,
+                        getProcessStateDescription(process_object.state)))
+        process_object.config.options.logger.info(msg)
 
 def timeslice(period, when):
     return int(when - (when % period))