fix(launchd): reap orphaned guest zombies and harden stale-client recovery

Children whose parent dies are reparented to the guest launchd, and its job
manager only reaps PIDs it has a job record for, so every leftover mldr/CEF
process from a killed client tree stayed a zombie until shutdown. Reap zombies
that launchd owns but does not track on a 5 second timer; tracked jobs keep
their exit status for job_reap(). proc_listpids() is not implemented in Darling
yet, so the scan enumerates the guest PID namespace through /proc.

recover-stale-client now signals the stale tree deepest-first so a live parent
can still reap its own children, and prepare.py repairs interrupted checkouts
and missing Git LFS objects instead of publishing pointer text as a library.

docs/TROUBLESHOOTING.md

@@ -157,8 +157,11 @@ the normal launcher start the product again. It recognizes `LeagueClientUx`,
 `Riot Client` and `RiotClientServices` windows, does nothing while a game process
 is running or when a managed client window is visible, and fails closed when
 X11/Xwayland properties are missing or unreadable. This saves the Darling
-bootstrap but not the Riot Client/CEF startup itself. If no window appears
-afterward, use a full runtime shutdown/restart.
+bootstrap but not the Riot Client/CEF startup itself. The stale tree is signalled
+deepest-first so live parents can still reap their own children; anything that is
+left over is reparented to the guest launchd, which reaps untracked zombies
+within a few seconds, so a full runtime restart is not needed to clear them. If no
+window appears afterward, use a full runtime shutdown/restart.
 
 ### Multiple client-in-game helpers
 

patches/darling.patch

@@ -11368,3 +11368,100 @@ index 87db50d12..01a7b2a3e 100644
  	// std::ofstream m_dump;
  };
  
+diff --git a/src/launchd/src/core.c b/src/launchd/src/core.c
+index a68739ff..5c8eb836 100644
+--- a/src/launchd/src/core.c
++++ b/src/launchd/src/core.c
+@@ -463,6 +463,9 @@ static job_t job_mig_intran2(jobmgr_t jm, mach_port_t mport, pid_t upid);
+ static job_t jobmgr_lookup_per_user_context_internal(job_t j, uid_t which_user, mach_port_t *mp);
+ static void job_export_all2(jobmgr_t jm, launch_data_t where);
+ static void jobmgr_callback(void *obj, struct kevent *kev);
++
++static char stray_reap_timer;
++void jobmgr_reap_orphaned_children(void);
+ static void jobmgr_setup_env_from_other_jobs(jobmgr_t jm);
+ static void jobmgr_export_env_from_other_jobs(jobmgr_t jm, launch_data_t dict);
+ static struct machservice *jobmgr_lookup_service(jobmgr_t jm, const char *name, bool check_parent, pid_t target_pid);
+@@ -4312,6 +4315,8 @@ jobmgr_callback(void *obj, struct kevent *kev)
+ 		} else if (kev->ident == (uintptr_t)jm) {
+ 			jobmgr_log(jm, LOG_DEBUG, "Shutdown timer firing.");
+ 			jobmgr_still_alive_with_check(jm);
++		} else if (kev->ident == (uintptr_t)&stray_reap_timer) {
++			jobmgr_reap_orphaned_children();
+ 		} else if (kev->ident == (uintptr_t)&jm->reboot_flags) {
+ 			jobmgr_do_garbage_collection(jm);
+ 		} else if (kev->ident == (uintptr_t)&launchd_runtime_busy_time) {
+@@ -6702,6 +6707,65 @@ jobmgr_do_garbage_collection(jobmgr_t jm)
+ 	return jm;
+ }
+ 
++void
++jobmgr_reap_orphaned_children(void)
++{
++	/* Darling: when a process's parent dies, the kernel reparents it to
++	 * launchd, and the job manager never learns about it, so nothing ever
++	 * waits on it and it lingers as a zombie until shutdown. Reap those
++	 * zombies here, leaving tracked jobs alone so their exit status is still
++	 * available to job_reap().
++	 *
++	 * proc_listpids() is not implemented yet, so enumerate the guest PID
++	 * namespace through /proc. */
++	DIR *dir;
++	struct dirent *de;
++	pid_t self = getpid();
++
++	if (root_jobmgr == NULL) {
++		return;
++	}
++
++	dir = opendir("/proc");
++	if (dir == NULL) {
++		jobmgr_log(root_jobmgr, LOG_DEBUG, "Unable to enumerate /proc: %d: %s", errno, strerror(errno));
++		return;
++	}
++
++	while ((de = readdir(dir)) != NULL) {
++		struct proc_bsdshortinfo proc;
++		char *end = NULL;
++		long value = strtol(de->d_name, &end, 10);
++		pid_t pid;
++		int status;
++
++		if (end == de->d_name || *end != '\0' || value <= 0) {
++			continue;
++		}
++		pid = (pid_t)value;
++
++		memset(&proc, 0, sizeof(proc));
++		if (proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 1, &proc, PROC_PIDT_SHORTBSDINFO_SIZE) == 0) {
++			continue;
++		}
++		if (proc.pbsi_ppid != self) {
++			continue;
++		}
++		if (proc.pbsi_status != SZOMB) {
++			continue;
++		}
++		if (jobmgr_find_by_pid(root_jobmgr, pid, false) != NULL) {
++			continue;
++		}
++
++		if (waitpid(pid, &status, WNOHANG) == pid) {
++			jobmgr_log(root_jobmgr, LOG_DEBUG, "Reaped orphaned zombie child PID %u.", pid);
++		}
++	}
++
++	closedir(dir);
++}
++
+ void
+ jobmgr_kill_stray_children(jobmgr_t jm, pid_t *p, size_t np)
+ {
+@@ -11796,6 +11860,7 @@ jobmgr_init(bool sflag)
+ 		}
+ 	}
+ 	s_no_hang_fd = _fd(s_no_hang_fd);
++	(void)jobmgr_assumes_zero_p(root_jobmgr, kevent_mod((uintptr_t)&stray_reap_timer, EVFILT_TIMER, EV_ADD, NOTE_SECONDS, 5, root_jobmgr));
+ }
+ 
+ size_t

scripts/prepare.py

@@ -7,9 +7,38 @@ sys.dont_write_bytecode=True
 from common import ROOT,render
 p=argparse.ArgumentParser(description=__doc__)
 p.add_argument('--source',type=Path,default=ROOT/'build/darling-src')
+p.add_argument('--jobs',type=int,default=4,help='parallel submodule fetches')
 p.add_argument('--check-only',action='store_true',help='validate against temporary base indexes, without changing existing source')
 a=p.parse_args();source=a.source.resolve();manifest=json.loads((ROOT/'upstream/manifest.json').read_text())
 def run(args,**kwargs):return subprocess.run(args,check=True,**kwargs)
+# The pinned darling-swift .lfsconfig points at the retired git-lfs.darlinghq.org; GitHub still serves the objects.
+LFS_URL='lfs.url=https://github.com/darlinghq/darling-swift.git/info/lfs'
+# Without these, a checkout writes Git LFS pointer text where a real dylib belongs, which links but crashes.
+LFS_FILTERS=['-c','filter.lfs.required=true','-c','filter.lfs.smudge=git-lfs smudge -- %f',
+             '-c','filter.lfs.process=git-lfs filter-process','-c','filter.lfs.clean=git-lfs clean -- %f']
+def submodules(checkout):
+    out=subprocess.run(['git','-C',str(checkout),'submodule','status','--recursive'],text=True,capture_output=True).stdout
+    return [checkout/line.split()[1] for line in out.splitlines() if len(line.split())>1]
+def pointers(repo):
+    """Tracked files still checked out as LFS pointer text instead of their real content."""
+    out=subprocess.run(['git','-C',str(repo),'lfs','ls-files'],text=True,capture_output=True).stdout
+    return [f for f in (line.split(maxsplit=2) for line in out.splitlines()) if len(f)==3 and f[1]=='-']
+def repair(checkout):
+    """An interrupted fetch leaves a submodule with a wiped index or pointer text and 'submodule update'
+    treats both as up to date, so check every worktree before this checkout becomes the source tree."""
+    for repo in [checkout,*submodules(checkout)]:
+        name=repo.relative_to(checkout) or '.'
+        if repo!=checkout and subprocess.run(['git','-C',str(repo),'status','--porcelain'],text=True,
+                                             capture_output=True).stdout.strip():
+            print(f'Repairing the interrupted checkout of {name}',flush=True)
+            run(['git','-C',str(repo),'checkout','-f','HEAD'],stdout=subprocess.DEVNULL)
+        if pointers(repo):
+            print(f'Fetching the Git LFS objects of {name} (pointer text is not a library)',flush=True)
+            run(['git','-C',str(repo),'lfs','install','--local'],stdout=subprocess.DEVNULL)
+            run(['git','-C',str(repo),*(['-c',LFS_URL] if 'swift' in name.parts else []),'lfs','pull'],
+                stdout=subprocess.DEVNULL)
+            if pointers(repo):raise SystemExit(f'{name}: Git LFS objects are still missing; install git-lfs and run this again')
+
 def applies(repo,patch,reverse=False):
     return subprocess.run(['git','-C',str(repo),'apply',*(['--reverse'] if reverse else []),'--check','-'],input=patch,text=True,
                           stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode==0
@@ -37,15 +66,19 @@ if not (source/'.git').exists():
         run(['git','-C',str(checkout),'remote','add','origin',manifest['upstream']])
         run(['git','-C',str(checkout),'fetch','--depth','1','origin',manifest['revision']])
         run(['git','-C',str(checkout),'checkout','--detach','FETCH_HEAD'])
+        if subprocess.run(['git','lfs','version'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode:
+            p.error('Git LFS is required to fetch the pinned sources; install git-lfs first')
+        run(['git','-C',str(checkout),*LFS_FILTERS,'lfs','install','--local'],stdout=subprocess.DEVNULL)
         print('Initializing Darling submodules (large repositories/LFS assets may take a while)...',flush=True)
         for attempt in range(1,4):
             try:
-                run(['git','-C',str(checkout),'-c','lfs.url=https://github.com/darlinghq/darling-swift.git/info/lfs',
-                     'submodule','update','--init','--recursive','--depth','1','--progress'])
+                run(['git','-C',str(checkout),'-c',LFS_URL,*LFS_FILTERS,'submodule','update','--init','--recursive',
+                     '--depth','1','--jobs',str(a.jobs),'--progress'])
                 break
             except subprocess.CalledProcessError:
                 if attempt==3:raise
                 print(f'Submodule fetch interrupted (attempt {attempt}/3). Retrying...',flush=True)
+        repair(checkout)
         checkout.rename(source)
 
 # Never reset someone else's checkout to make a patch apply.

tests/test_recover_stale_client.py

@@ -27,7 +27,7 @@ class RecoverStaleClientTests(unittest.TestCase):
         (self.proc / 'uptime').write_text('1000 0\n')
         self.prefix = self.root / '.darling'
 
-    def process(self, pid, comm, argv, env=None, executable=None, age=120):
+    def process(self, pid, comm, argv, env=None, executable=None, age=120, ppid=0):
         folder = self.proc / str(pid)
         folder.mkdir()
         (folder / 'comm').write_text(comm + '\n')
@@ -35,7 +35,7 @@ class RecoverStaleClientTests(unittest.TestCase):
             (argv[0] if executable is None else executable).encode() + b'\0' +
             b''.join(arg.encode() + b'\0' for arg in argv[1:]))
         ticks = os.sysconf('SC_CLK_TCK')
-        fields = ['S'] + ['0'] * 18 + [str(int((1000 - age) * ticks))]
+        fields = ['S', str(ppid)] + ['0'] * 17 + [str(int((1000 - age) * ticks))]
         (folder / 'stat').write_text(f'{pid} ({comm}) ' + ' '.join(fields) + '\n')
         if env is not None:
             (folder / 'environ').write_bytes(env)
@@ -122,6 +122,35 @@ class RecoverStaleClientTests(unittest.TestCase):
         self.assertEqual(result, 0)
         self.assertEqual(killed, [(12, recover_stale_client.signal.SIGTERM)])
 
+    def test_children_are_terminated_before_their_parents(self):
+        self.process(10, 'darlingserver', ['darlingserver'])
+        self.process(12, 'mldr', [CLIENT], self.client_env())
+        self.process(13, 'mldr', ['/Applications/League of Legends.app/Contents/MacOS/LeagueClientUx'], self.client_env(), ppid=12)
+        self.process(14, 'CrUtilityMain', ['/Applications/Riot Client.app/Contents/Frameworks/CrUtilityMain'], self.client_env(), ppid=13)
+        result, killed, output = self.run_recover()
+        self.assertEqual(result, 0)
+        self.assertEqual(killed, [
+            (14, recover_stale_client.signal.SIGTERM),
+            (13, recover_stale_client.signal.SIGTERM),
+            (12, recover_stale_client.signal.SIGTERM),
+            (14, recover_stale_client.signal.SIGKILL),
+            (13, recover_stale_client.signal.SIGKILL),
+            (12, recover_stale_client.signal.SIGKILL),
+        ])
+        self.assertIn('terminating 3 process(es)', output)
+
+    def test_leftover_processes_are_reported(self):
+        self.process(10, 'darlingserver', ['darlingserver'])
+        self.process(12, 'mldr', [CLIENT], self.client_env())
+
+        def survivor(pid, signum):
+            pass
+
+        result, killed, output = self.run_recover(on_kill=survivor)
+        self.assertEqual(result, 0)
+        self.assertIn('still present after SIGKILL', output)
+        self.assertIn('parent PID(s): 0', output)
+
     def test_young_client_is_preserved(self):
         self.process(10, 'darlingserver', ['darlingserver'])
         self.process(12, 'mldr', [CLIENT], self.client_env(),

tools/recover-stale-client.py

@@ -183,6 +183,61 @@ def client_pids(prefix, proc_root=Path('/proc')):
     return sorted(found)
 
 
+def parent_pid(proc):
+    """PPID from /proc/<pid>/stat, coping with a comm field with spaces."""
+    try:
+        return int((proc / 'stat').read_text().rsplit(')', 1)[1].split()[1])
+    except (OSError, ValueError, IndexError):
+        return None
+
+
+def termination_order(identities, proc_root=Path('/proc')):
+    """Deepest processes first, so a live parent can reap its own children."""
+    depth = {}
+
+    def measure(pid, seen):
+        if pid in depth:
+            return depth[pid]
+        if pid in seen:
+            return 0
+        seen.add(pid)
+        parent = parent_pid(proc_root / str(pid))
+        value = 0 if parent in (None, pid) or parent not in identities else measure(parent, seen) + 1
+        depth[pid] = value
+        return value
+
+    for pid in identities:
+        measure(pid, set())
+    return sorted(identities, key=lambda pid: (-depth[pid], pid))
+
+
+def _signal_tree(prefix, identities, order, signum, kill, proc_root):
+    for pid in order:
+        if client_identity(prefix, pid, proc_root) != identities[pid]:
+            continue
+        try:
+            kill(pid, signum)
+        except OSError:
+            pass
+
+
+def wait_for_exit(prefix, identities, proc_root=Path('/proc'), sleep=time.sleep,
+                  attempts=5):
+    """Wait briefly for the tree to leave the process table; return leftovers."""
+    leftovers = []
+    for attempt in range(attempts):
+        leftovers = [
+            pid for pid, start_ticks in identities.items()
+            if client_identity(prefix, pid, proc_root) == start_ticks
+            or process_state(proc_root / str(pid)) is not None
+        ]
+        if not leftovers:
+            break
+        if attempt != attempts - 1:
+            sleep(1)
+    return leftovers
+
+
 def recover(prefix, proc_root=Path('/proc'), window_check=visible_client_window,
             kill=os.kill, sleep=time.sleep, out=sys.stdout):
     if not server_running(proc_root):
@@ -207,21 +262,17 @@ def recover(prefix, proc_root=Path('/proc'), window_check=visible_client_window,
         return 0
     print('recover-stale-client: stale client tree without a window; '
           'terminating %d process(es)' % len(identities), file=out, flush=True)
-    for pid, start_ticks in identities.items():
-        if client_identity(prefix, pid, proc_root) != start_ticks:
-            continue
-        try:
-            kill(pid, signal.SIGTERM)
-        except OSError:
-            pass
+    order = termination_order(identities, proc_root)
+    _signal_tree(prefix, identities, order, signal.SIGTERM, kill, proc_root)
     sleep(4)
-    for pid, start_ticks in identities.items():
-        if client_identity(prefix, pid, proc_root) != start_ticks:
-            continue
-        try:
-            kill(pid, signal.SIGKILL)
-        except OSError:
-            pass
+    _signal_tree(prefix, identities, order, signal.SIGKILL, kill, proc_root)
+    leftovers = wait_for_exit(prefix, identities, proc_root, sleep)
+    if leftovers:
+        parents = sorted({parent_pid(proc_root / str(pid)) for pid in leftovers} - {None})
+        print('recover-stale-client: %d process(es) still present after SIGKILL '
+              '(parent PID(s): %s); the guest launchd reaps untracked zombies within '
+              'seconds' % (len(leftovers), ', '.join(str(p) for p in parents) or 'unknown'),
+              file=out, flush=True)
     return 0