recover-stale-client: do not report a reused PID as a leftover

wait_for_exit counted any live PID with the same number as a leftover, so a
reused PID produced a misleading "still present after SIGKILL" message. Compare
the original start time and use the process state only to include the original
process while it is an unreaped zombie.

tests/test_recover_stale_client.py

@@ -151,6 +151,30 @@ class RecoverStaleClientTests(unittest.TestCase):
         self.assertIn('still present after SIGKILL', output)
         self.assertIn('parent PID(s): 0', output)
 
+    def test_reused_pid_is_not_reported_as_a_leftover(self):
+        folder = self.process(12, 'mldr', [CLIENT], self.client_env())
+        start_ticks = recover_stale_client.process_start_ticks(folder)
+        stat = (folder / 'stat').read_text()
+        fields = stat.rsplit(')', 1)[1].split()
+        fields[19] = str(int(fields[19]) + 100)
+        (folder / 'stat').write_text(stat.rsplit(')', 1)[0] + ') ' + ' '.join(fields) + '\n')
+        leftovers = recover_stale_client.wait_for_exit(
+            self.prefix, {12: start_ticks}, proc_root=self.proc,
+            sleep=lambda _seconds: None)
+        self.assertEqual(leftovers, [])
+
+    def test_unreaped_zombie_is_still_a_leftover(self):
+        folder = self.process(12, 'mldr', [CLIENT], self.client_env())
+        start_ticks = recover_stale_client.process_start_ticks(folder)
+        stat = (folder / 'stat').read_text()
+        fields = stat.rsplit(')', 1)[1].split()
+        fields[0] = 'Z'
+        (folder / 'stat').write_text(stat.rsplit(')', 1)[0] + ') ' + ' '.join(fields) + '\n')
+        leftovers = recover_stale_client.wait_for_exit(
+            self.prefix, {12: start_ticks}, proc_root=self.proc,
+            sleep=lambda _seconds: None)
+        self.assertEqual(leftovers, [12])
+
     def test_young_client_is_preserved(self):
         self.process(10, 'darlingserver', ['darlingserver'])
         self.process(12, 'mldr', [CLIENT], self.client_env(),

tools/recover-stale-client.py

@@ -76,6 +76,14 @@ def process_state(proc):
         return None
 
 
+def process_start_ticks(proc):
+    """Field 22 of stat: start time in clock ticks, even for a zombie."""
+    try:
+        return int((proc / 'stat').read_text().rsplit(')', 1)[1].split()[19])
+    except (OSError, ValueError, IndexError):
+        return None
+
+
 def server_running(proc_root=Path('/proc')):
     try:
         entries = list(proc_root.iterdir())
@@ -162,8 +170,8 @@ def client_identity(prefix, pid, proc_root=Path('/proc')):
         if not math.isfinite(uptime) or uptime < 0 or ticks <= 0:
             return None
         # Field 22 follows a comm field that may contain spaces/parentheses.
-        start_ticks = int((proc / 'stat').read_text().rsplit(')', 1)[1].split()[19])
-        if start_ticks < 0 or uptime - start_ticks / ticks < MIN_AGE_SECONDS:
+        start_ticks = process_start_ticks(proc)
+        if start_ticks is None or start_ticks < 0 or uptime - start_ticks / ticks < MIN_AGE_SECONDS:
             return None
         return start_ticks
     except (OSError, ValueError, IndexError):
@@ -223,13 +231,17 @@ def _signal_tree(prefix, identities, order, signum, kill, proc_root):
 
 def wait_for_exit(prefix, identities, proc_root=Path('/proc'), sleep=time.sleep,
                   attempts=5):
-    """Wait briefly for the tree to leave the process table; return leftovers."""
+    """Wait briefly for the tree to leave the process table; return leftovers.
+
+    A PID that was reused by an unrelated process is not a leftover. Only the
+    original process, including while it is an unreaped zombie, counts.
+    """
     leftovers = []
     for attempt in range(attempts):
         leftovers = [
             pid for pid, start_ticks in identities.items()
-            if client_identity(prefix, pid, proc_root) == start_ticks
-            or process_state(proc_root / str(pid)) is not None
+            if process_state(proc_root / str(pid)) is not None
+            and process_start_ticks(proc_root / str(pid)) == start_ticks
         ]
         if not leftovers:
             break