Handle race where process exits immediately before killing it

See also 5b1f3dc

Closes Supervisor#1387
Closes Supervisor#975
Closes Supervisor#688
Closes Supervisor#536

Author: mnaberez

CHANGES.rst

@@ -1,6 +1,9 @@
 4.2.2.dev0 (Next Release)
 -------------------------
 
+- Fixed a bug where ``supervisord`` could crash if a subprocess exited
+  immediately before trying to kill it.
+
 - Fixed a bug where the ``stdout_syslog`` and ``stderr_syslog`` options
   of a ``[program:x]`` section could not be used unless file logging for
   the same program had also been configured.  The file and syslog options

supervisor/process.py

@@ -403,7 +403,8 @@ def give_up(self):
         self.change_state(ProcessStates.FATAL)
 
     def kill(self, sig):
-        """Send a signal to the subprocess.  This may or may not kill it.
+        """Send a signal to the subprocess with the intention to kill
+        it (to make it exit).  This may or may not actually kill it.
 
         Return None if the signal was sent, or an error message string
         if an error occurred or if the subprocess is not running.
@@ -463,7 +464,17 @@ def kill(self, sig):
             pid = -self.pid
 
         try:
-            options.kill(pid, sig)
+            try:
+                options.kill(pid, sig)
+            except OSError as exc:
+                if exc.errno == errno.ESRCH:
+                    msg = ("unable to signal %s (pid %s), it probably just exited "
+                           "on its own: %s" % (processname, self.pid, str(exc)))
+                    options.logger.debug(msg)
+                    # we could change the state here but we intentionally do
+                    # not.  we will do it during normal SIGCHLD processing.
+                    return None
+                raise
         except:
             tb = traceback.format_exc()
             msg = 'unknown problem killing %s (%s):%s' % (processname,

supervisor/tests/test_process.py

@@ -884,6 +884,31 @@ def test_kill_from_running_error(self):
         self.assertEqual(event1.__class__, events.ProcessStateStoppingEvent)
         self.assertEqual(event2.__class__, events.ProcessStateUnknownEvent)
 
+    def test_kill_from_running_error_ESRCH(self):
+        options = DummyOptions()
+        config = DummyPConfig(options, 'test', '/test')
+        options.kill_exception = OSError(errno.ESRCH,
+                                         os.strerror(errno.ESRCH))
+        instance = self._makeOne(config)
+        L = []
+        from supervisor.states import ProcessStates
+        from supervisor import events
+        events.subscribe(events.ProcessStateEvent, lambda x: L.append(x))
+        instance.pid = 11
+        instance.state = ProcessStates.RUNNING
+        instance.kill(signal.SIGTERM)
+        self.assertEqual(options.logger.data[0], 'killing test (pid 11) with '
+            'signal SIGTERM')
+        self.assertEqual(options.logger.data[1], 'unable to signal test (pid 11), '
+            'it probably just exited on its own: %s' %
+            str(options.kill_exception))
+        self.assertTrue(instance.killing)
+        self.assertEqual(instance.pid, 11) # unchanged
+        self.assertEqual(instance.state, ProcessStates.STOPPING)
+        self.assertEqual(len(L), 1)
+        event1 = L[0]
+        self.assertEqual(event1.__class__, events.ProcessStateStoppingEvent)
+
     def test_kill_from_stopping(self):
         options = DummyOptions()
         config = DummyPConfig(options, 'test', '/test')