From 9291d6accd1f0e0c0419e39bb4810735e01e7a41 Mon Sep 17 00:00:00 2001 From: MintMachineHQ Date: Fri, 23 Oct 2026 23:36:20 -0600 Subject: [PATCH] fix(launchd): reap orphaned guest zombies and harden stale-client recovery Children whose parent dies are reparented to the guest launchd, and its job manager only reaps PIDs it has a job record for, so every leftover mldr/CEF process from a killed client tree stayed a zombie until shutdown. Reap zombies that launchd owns but does not track on a 5 second timer; tracked jobs keep their exit status for job_reap(). proc_listpids() is not implemented in Darling yet, so the scan enumerates the guest PID namespace through /proc. recover-stale-client now signals the stale tree deepest-first so a live parent can still reap its own children, and prepare.py repairs interrupted checkouts and missing Git LFS objects instead of publishing pointer text as a library. --- docs/TROUBLESHOOTING.md | 7 ++- patches/darling.patch | 97 ++++++++++++++++++++++++++++++ scripts/prepare.py | 37 +++++++++++- tests/test_recover_stale_client.py | 33 +++++++++- tools/recover-stale-client.py | 79 +++++++++++++++++++----- 5 files changed, 233 insertions(+), 20 deletions(-) mode change 100755 => 100644 scripts/prepare.py mode change 100755 => 100644 tools/recover-stale-client.py diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index f4a5488..456f5f8 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -157,8 +157,11 @@ the normal launcher start the product again. It recognizes `LeagueClientUx`, `Riot Client` and `RiotClientServices` windows, does nothing while a game process is running or when a managed client window is visible, and fails closed when X11/Xwayland properties are missing or unreadable. This saves the Darling -bootstrap but not the Riot Client/CEF startup itself. If no window appears -afterward, use a full runtime shutdown/restart. +bootstrap but not the Riot Client/CEF startup itself. The stale tree is signalled +deepest-first so live parents can still reap their own children; anything that is +left over is reparented to the guest launchd, which reaps untracked zombies +within a few seconds, so a full runtime restart is not needed to clear them. If no +window appears afterward, use a full runtime shutdown/restart. ### Multiple client-in-game helpers diff --git a/patches/darling.patch b/patches/darling.patch index e0d6b94..4c39e9f 100644 --- a/patches/darling.patch +++ b/patches/darling.patch @@ -11368,3 +11368,100 @@ index 87db50d12..01a7b2a3e 100644 // std::ofstream m_dump; }; +diff --git a/src/launchd/src/core.c b/src/launchd/src/core.c +index a68739ff..5c8eb836 100644 +--- a/src/launchd/src/core.c ++++ b/src/launchd/src/core.c +@@ -463,6 +463,9 @@ static job_t job_mig_intran2(jobmgr_t jm, mach_port_t mport, pid_t upid); + static job_t jobmgr_lookup_per_user_context_internal(job_t j, uid_t which_user, mach_port_t *mp); + static void job_export_all2(jobmgr_t jm, launch_data_t where); + static void jobmgr_callback(void *obj, struct kevent *kev); ++ ++static char stray_reap_timer; ++void jobmgr_reap_orphaned_children(void); + static void jobmgr_setup_env_from_other_jobs(jobmgr_t jm); + static void jobmgr_export_env_from_other_jobs(jobmgr_t jm, launch_data_t dict); + static struct machservice *jobmgr_lookup_service(jobmgr_t jm, const char *name, bool check_parent, pid_t target_pid); +@@ -4312,6 +4315,8 @@ jobmgr_callback(void *obj, struct kevent *kev) + } else if (kev->ident == (uintptr_t)jm) { + jobmgr_log(jm, LOG_DEBUG, "Shutdown timer firing."); + jobmgr_still_alive_with_check(jm); ++ } else if (kev->ident == (uintptr_t)&stray_reap_timer) { ++ jobmgr_reap_orphaned_children(); + } else if (kev->ident == (uintptr_t)&jm->reboot_flags) { + jobmgr_do_garbage_collection(jm); + } else if (kev->ident == (uintptr_t)&launchd_runtime_busy_time) { +@@ -6702,6 +6707,65 @@ jobmgr_do_garbage_collection(jobmgr_t jm) + return jm; + } + ++void ++jobmgr_reap_orphaned_children(void) ++{ ++ /* Darling: when a process's parent dies, the kernel reparents it to ++ * launchd, and the job manager never learns about it, so nothing ever ++ * waits on it and it lingers as a zombie until shutdown. Reap those ++ * zombies here, leaving tracked jobs alone so their exit status is still ++ * available to job_reap(). ++ * ++ * proc_listpids() is not implemented yet, so enumerate the guest PID ++ * namespace through /proc. */ ++ DIR *dir; ++ struct dirent *de; ++ pid_t self = getpid(); ++ ++ if (root_jobmgr == NULL) { ++ return; ++ } ++ ++ dir = opendir("/proc"); ++ if (dir == NULL) { ++ jobmgr_log(root_jobmgr, LOG_DEBUG, "Unable to enumerate /proc: %d: %s", errno, strerror(errno)); ++ return; ++ } ++ ++ while ((de = readdir(dir)) != NULL) { ++ struct proc_bsdshortinfo proc; ++ char *end = NULL; ++ long value = strtol(de->d_name, &end, 10); ++ pid_t pid; ++ int status; ++ ++ if (end == de->d_name || *end != '\0' || value <= 0) { ++ continue; ++ } ++ pid = (pid_t)value; ++ ++ memset(&proc, 0, sizeof(proc)); ++ if (proc_pidinfo(pid, PROC_PIDT_SHORTBSDINFO, 1, &proc, PROC_PIDT_SHORTBSDINFO_SIZE) == 0) { ++ continue; ++ } ++ if (proc.pbsi_ppid != self) { ++ continue; ++ } ++ if (proc.pbsi_status != SZOMB) { ++ continue; ++ } ++ if (jobmgr_find_by_pid(root_jobmgr, pid, false) != NULL) { ++ continue; ++ } ++ ++ if (waitpid(pid, &status, WNOHANG) == pid) { ++ jobmgr_log(root_jobmgr, LOG_DEBUG, "Reaped orphaned zombie child PID %u.", pid); ++ } ++ } ++ ++ closedir(dir); ++} ++ + void + jobmgr_kill_stray_children(jobmgr_t jm, pid_t *p, size_t np) + { +@@ -11796,6 +11860,7 @@ jobmgr_init(bool sflag) + } + } + s_no_hang_fd = _fd(s_no_hang_fd); ++ (void)jobmgr_assumes_zero_p(root_jobmgr, kevent_mod((uintptr_t)&stray_reap_timer, EVFILT_TIMER, EV_ADD, NOTE_SECONDS, 5, root_jobmgr)); + } + + size_t diff --git a/scripts/prepare.py b/scripts/prepare.py old mode 100755 new mode 100644 index dca0a83..65665f4 --- a/scripts/prepare.py +++ b/scripts/prepare.py @@ -7,9 +7,38 @@ sys.dont_write_bytecode=True from common import ROOT,render p=argparse.ArgumentParser(description=__doc__) p.add_argument('--source',type=Path,default=ROOT/'build/darling-src') +p.add_argument('--jobs',type=int,default=4,help='parallel submodule fetches') p.add_argument('--check-only',action='store_true',help='validate against temporary base indexes, without changing existing source') a=p.parse_args();source=a.source.resolve();manifest=json.loads((ROOT/'upstream/manifest.json').read_text()) def run(args,**kwargs):return subprocess.run(args,check=True,**kwargs) +# The pinned darling-swift .lfsconfig points at the retired git-lfs.darlinghq.org; GitHub still serves the objects. +LFS_URL='lfs.url=https://github.com/darlinghq/darling-swift.git/info/lfs' +# Without these, a checkout writes Git LFS pointer text where a real dylib belongs, which links but crashes. +LFS_FILTERS=['-c','filter.lfs.required=true','-c','filter.lfs.smudge=git-lfs smudge -- %f', + '-c','filter.lfs.process=git-lfs filter-process','-c','filter.lfs.clean=git-lfs clean -- %f'] +def submodules(checkout): + out=subprocess.run(['git','-C',str(checkout),'submodule','status','--recursive'],text=True,capture_output=True).stdout + return [checkout/line.split()[1] for line in out.splitlines() if len(line.split())>1] +def pointers(repo): + """Tracked files still checked out as LFS pointer text instead of their real content.""" + out=subprocess.run(['git','-C',str(repo),'lfs','ls-files'],text=True,capture_output=True).stdout + return [f for f in (line.split(maxsplit=2) for line in out.splitlines()) if len(f)==3 and f[1]=='-'] +def repair(checkout): + """An interrupted fetch leaves a submodule with a wiped index or pointer text and 'submodule update' + treats both as up to date, so check every worktree before this checkout becomes the source tree.""" + for repo in [checkout,*submodules(checkout)]: + name=repo.relative_to(checkout) or '.' + if repo!=checkout and subprocess.run(['git','-C',str(repo),'status','--porcelain'],text=True, + capture_output=True).stdout.strip(): + print(f'Repairing the interrupted checkout of {name}',flush=True) + run(['git','-C',str(repo),'checkout','-f','HEAD'],stdout=subprocess.DEVNULL) + if pointers(repo): + print(f'Fetching the Git LFS objects of {name} (pointer text is not a library)',flush=True) + run(['git','-C',str(repo),'lfs','install','--local'],stdout=subprocess.DEVNULL) + run(['git','-C',str(repo),*(['-c',LFS_URL] if 'swift' in name.parts else []),'lfs','pull'], + stdout=subprocess.DEVNULL) + if pointers(repo):raise SystemExit(f'{name}: Git LFS objects are still missing; install git-lfs and run this again') + def applies(repo,patch,reverse=False): return subprocess.run(['git','-C',str(repo),'apply',*(['--reverse'] if reverse else []),'--check','-'],input=patch,text=True, stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode==0 @@ -37,15 +66,19 @@ if not (source/'.git').exists(): run(['git','-C',str(checkout),'remote','add','origin',manifest['upstream']]) run(['git','-C',str(checkout),'fetch','--depth','1','origin',manifest['revision']]) run(['git','-C',str(checkout),'checkout','--detach','FETCH_HEAD']) + if subprocess.run(['git','lfs','version'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL).returncode: + p.error('Git LFS is required to fetch the pinned sources; install git-lfs first') + run(['git','-C',str(checkout),*LFS_FILTERS,'lfs','install','--local'],stdout=subprocess.DEVNULL) print('Initializing Darling submodules (large repositories/LFS assets may take a while)...',flush=True) for attempt in range(1,4): try: - run(['git','-C',str(checkout),'-c','lfs.url=https://github.com/darlinghq/darling-swift.git/info/lfs', - 'submodule','update','--init','--recursive','--depth','1','--progress']) + run(['git','-C',str(checkout),'-c',LFS_URL,*LFS_FILTERS,'submodule','update','--init','--recursive', + '--depth','1','--jobs',str(a.jobs),'--progress']) break except subprocess.CalledProcessError: if attempt==3:raise print(f'Submodule fetch interrupted (attempt {attempt}/3). Retrying...',flush=True) + repair(checkout) checkout.rename(source) # Never reset someone else's checkout to make a patch apply. diff --git a/tests/test_recover_stale_client.py b/tests/test_recover_stale_client.py index 14099a0..8351351 100644 --- a/tests/test_recover_stale_client.py +++ b/tests/test_recover_stale_client.py @@ -27,7 +27,7 @@ class RecoverStaleClientTests(unittest.TestCase): (self.proc / 'uptime').write_text('1000 0\n') self.prefix = self.root / '.darling' - def process(self, pid, comm, argv, env=None, executable=None, age=120): + def process(self, pid, comm, argv, env=None, executable=None, age=120, ppid=0): folder = self.proc / str(pid) folder.mkdir() (folder / 'comm').write_text(comm + '\n') @@ -35,7 +35,7 @@ class RecoverStaleClientTests(unittest.TestCase): (argv[0] if executable is None else executable).encode() + b'\0' + b''.join(arg.encode() + b'\0' for arg in argv[1:])) ticks = os.sysconf('SC_CLK_TCK') - fields = ['S'] + ['0'] * 18 + [str(int((1000 - age) * ticks))] + fields = ['S', str(ppid)] + ['0'] * 17 + [str(int((1000 - age) * ticks))] (folder / 'stat').write_text(f'{pid} ({comm}) ' + ' '.join(fields) + '\n') if env is not None: (folder / 'environ').write_bytes(env) @@ -122,6 +122,35 @@ class RecoverStaleClientTests(unittest.TestCase): self.assertEqual(result, 0) self.assertEqual(killed, [(12, recover_stale_client.signal.SIGTERM)]) + def test_children_are_terminated_before_their_parents(self): + self.process(10, 'darlingserver', ['darlingserver']) + self.process(12, 'mldr', [CLIENT], self.client_env()) + self.process(13, 'mldr', ['/Applications/League of Legends.app/Contents/MacOS/LeagueClientUx'], self.client_env(), ppid=12) + self.process(14, 'CrUtilityMain', ['/Applications/Riot Client.app/Contents/Frameworks/CrUtilityMain'], self.client_env(), ppid=13) + result, killed, output = self.run_recover() + self.assertEqual(result, 0) + self.assertEqual(killed, [ + (14, recover_stale_client.signal.SIGTERM), + (13, recover_stale_client.signal.SIGTERM), + (12, recover_stale_client.signal.SIGTERM), + (14, recover_stale_client.signal.SIGKILL), + (13, recover_stale_client.signal.SIGKILL), + (12, recover_stale_client.signal.SIGKILL), + ]) + self.assertIn('terminating 3 process(es)', output) + + def test_leftover_processes_are_reported(self): + self.process(10, 'darlingserver', ['darlingserver']) + self.process(12, 'mldr', [CLIENT], self.client_env()) + + def survivor(pid, signum): + pass + + result, killed, output = self.run_recover(on_kill=survivor) + self.assertEqual(result, 0) + self.assertIn('still present after SIGKILL', output) + self.assertIn('parent PID(s): 0', output) + def test_young_client_is_preserved(self): self.process(10, 'darlingserver', ['darlingserver']) self.process(12, 'mldr', [CLIENT], self.client_env(), diff --git a/tools/recover-stale-client.py b/tools/recover-stale-client.py old mode 100755 new mode 100644 index 66466ce..8c75f9e --- a/tools/recover-stale-client.py +++ b/tools/recover-stale-client.py @@ -183,6 +183,61 @@ def client_pids(prefix, proc_root=Path('/proc')): return sorted(found) +def parent_pid(proc): + """PPID from /proc//stat, coping with a comm field with spaces.""" + try: + return int((proc / 'stat').read_text().rsplit(')', 1)[1].split()[1]) + except (OSError, ValueError, IndexError): + return None + + +def termination_order(identities, proc_root=Path('/proc')): + """Deepest processes first, so a live parent can reap its own children.""" + depth = {} + + def measure(pid, seen): + if pid in depth: + return depth[pid] + if pid in seen: + return 0 + seen.add(pid) + parent = parent_pid(proc_root / str(pid)) + value = 0 if parent in (None, pid) or parent not in identities else measure(parent, seen) + 1 + depth[pid] = value + return value + + for pid in identities: + measure(pid, set()) + return sorted(identities, key=lambda pid: (-depth[pid], pid)) + + +def _signal_tree(prefix, identities, order, signum, kill, proc_root): + for pid in order: + if client_identity(prefix, pid, proc_root) != identities[pid]: + continue + try: + kill(pid, signum) + except OSError: + pass + + +def wait_for_exit(prefix, identities, proc_root=Path('/proc'), sleep=time.sleep, + attempts=5): + """Wait briefly for the tree to leave the process table; return leftovers.""" + leftovers = [] + for attempt in range(attempts): + leftovers = [ + pid for pid, start_ticks in identities.items() + if client_identity(prefix, pid, proc_root) == start_ticks + or process_state(proc_root / str(pid)) is not None + ] + if not leftovers: + break + if attempt != attempts - 1: + sleep(1) + return leftovers + + def recover(prefix, proc_root=Path('/proc'), window_check=visible_client_window, kill=os.kill, sleep=time.sleep, out=sys.stdout): if not server_running(proc_root): @@ -207,21 +262,17 @@ def recover(prefix, proc_root=Path('/proc'), window_check=visible_client_window, return 0 print('recover-stale-client: stale client tree without a window; ' 'terminating %d process(es)' % len(identities), file=out, flush=True) - for pid, start_ticks in identities.items(): - if client_identity(prefix, pid, proc_root) != start_ticks: - continue - try: - kill(pid, signal.SIGTERM) - except OSError: - pass + order = termination_order(identities, proc_root) + _signal_tree(prefix, identities, order, signal.SIGTERM, kill, proc_root) sleep(4) - for pid, start_ticks in identities.items(): - if client_identity(prefix, pid, proc_root) != start_ticks: - continue - try: - kill(pid, signal.SIGKILL) - except OSError: - pass + _signal_tree(prefix, identities, order, signal.SIGKILL, kill, proc_root) + leftovers = wait_for_exit(prefix, identities, proc_root, sleep) + if leftovers: + parents = sorted({parent_pid(proc_root / str(pid)) for pid in leftovers} - {None}) + print('recover-stale-client: %d process(es) still present after SIGKILL ' + '(parent PID(s): %s); the guest launchd reaps untracked zombies within ' + 'seconds' % (len(leftovers), ', '.join(str(p) for p in parents) or 'unknown'), + file=out, flush=True) return 0