import argparse import hashlib import json import os import re import subprocess import sys from concurrent.futures import ThreadPoolExecutor from itertools import combinations from pathlib import Path from urllib.parse import parse_qs, quote, urlencode, urlsplit import nid_names PARENT = 'if .parent then "\\(.parent.owner.login)/\\(.parent.name)" else .nameWithOwner end' EXPORT = re.compile(r"^\+.*\bAPS5_VABI\s+(\w+)\s*\(") NID_DISAMBIG = re.compile(r"_nid_disambig[0-9]+$") DEPENDS = re.compile(r'Depends on:[ \t]*((?:#\d+\b(?:(?:[ \t]*,[ \t]*(?:and[ \t]+)?|[ \t]+and[ \t]+)#\d+\b)*|none\b)?(?:\r?\n[ \t]*[-*][ \t]*#\d+\b[^\r\n]*)*)') PLAIN_DEPENDS = re.compile(r'Depends on[ \t]+(#\d+\b(?:(?:[ \t]*,[ \t]*(?:and[ \t]+)?|[ \t]+and[ \t]+)#\d+\b)*)') HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))?", re.M) MARKER = "" FETCH_REFS = 100 PROGRESS_EVERY = 100 _REPO_ROOTS = {} def git(*args): cwd = Path.cwd() root = _REPO_ROOTS.get(cwd) if root is None: root = subprocess.run( ["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, check=True ).stdout.decode("utf-8", errors="replace").strip() _REPO_ROOTS[cwd] = root args = list(args) if "--" in args: separator = args.index("--") for index in range(separator + 1, len(args)): pathspec = args[index] if pathspec.startswith(("../", "..\\")): absolute = (cwd / pathspec).resolve() try: args[index] = absolute.relative_to(Path(root)).as_posix() except ValueError: pass result = subprocess.run( ["git", "-c", "core.quotePath=false", *args], cwd=root, capture_output=True, ) result.stdout = result.stdout.decode("utf-8", errors="replace") result.stderr = result.stderr.decode("utf-8", errors="replace") result.check_returncode() return result.stdout def own(prs, sha): ancestors = set(git("rev-list", sha).splitlines()) return {n: pr for n, pr in prs.items() if pr["sha"] not in ancestors} def gh(*args): return subprocess.run(["gh", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout def upstream(): return os.environ.get("GITHUB_REPOSITORY") or gh("repo", "view", "--json", "nameWithOwner,parent", "--jq", PARENT).strip() def depends_on(body): body = re.sub(r"", "", body or "", flags=re.S) repo = re.escape(globals().get("REPO", "boykopovar/AnyPS5")) repo_pattern = rf"(?:{repo}|boykopovar/AnyPS5)" body = re.sub(rf"\[([^\]]*)\]\(https?://(?:www\.)?github\.com/{repo_pattern}/pull/(\d+)(?:[#/?][^)\s]*)?\)", r"#\2", body, flags=re.IGNORECASE) body = re.sub(rf"\[([^\]]*)\]\(https?://(?:www\.)?github\.com/(?!{repo_pattern}/)[^/\s]+/[^/\s]+/pull/\d+[^)\s]*\)", "", body, flags=re.IGNORECASE) body = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", body) body = re.sub(rf"https?://(?:www\.)?github\.com/{repo_pattern}/pull/(\d+)\b(?:/[^\s]*)?", r"#\1", body, flags=re.IGNORECASE) match = next((match for match in DEPENDS.finditer(body) if match.group(1).strip()), None) or PLAIN_DEPENDS.search(body) return {int(n) for n in re.findall(r"#(\d+)", match.group(1))} if match else set() def progress(message): print(f"pr_overlap: {message}", file=sys.stderr, flush=True) LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"') NEXT_LINK = re.compile(r'<([^>]+)>\s*;\s*rel="next"', re.IGNORECASE) def page(url): out = gh("api", "--include", url) if not out.startswith("HTTP/"): return "", out blank = re.search(r"\r?\n\r?\n", out) return (out, "") if blank is None else (out[:blank.start()], out[blank.end():]) def json_pages(out): decoder = json.JSONDecoder() offset = 0 pages = [] while offset < len(out): while offset < len(out) and out[offset].isspace(): offset += 1 if offset == len(out): break found, offset = decoder.raw_decode(out, offset) if not isinstance(found, list): raise ValueError("gh api returned a non-array page") pages.append(found) return pages def open_prs(base): query = urlencode({"state": "open", "base": base, "per_page": 100}) url = f"repos/{REPO}/pulls?{query}" head, out = page(f"{url}&page=1") pages = json_pages(out) if not pages: raise ValueError("gh api returned no JSON pages") last = LAST_PAGE.search(head) pushed_past_last = None if last: numbers = range(2, int(last.group(1)) + 1) with ThreadPoolExecutor(max_workers=8) as executor: for out in executor.map(lambda number: page(f"{url}&page={number}")[1], numbers): pages.extend(json_pages(out)) pushed_past_last = numbers.stop elif NEXT_LINK.search(head): current_page = 1 while next_link := NEXT_LINK.search(head): values = parse_qs(urlsplit(next_link.group(1)).query, keep_blank_values=True).get("page", []) if len(values) != 1 or re.fullmatch(r"[0-9]+", values[0]) is None or int(values[0]) <= current_page: raise ValueError("gh api returned an invalid next page link") current_page = int(values[0]) head, out = page(f"{url}&page={current_page}") fetched = json_pages(out) if not fetched: raise ValueError("gh api returned no JSON pages") pages.extend(fetched) pushed_past_last = current_page + 1 if pushed_past_last is not None: while len(pages[-1]) == 100: pages.extend(json_pages(page(f"{url}&page={pushed_past_last}")[1]) or [[]]) pushed_past_last += 1 pull_requests = [pr for found in pages for pr in found] prs = {} for pr in pull_requests: prs[pr["number"]] = { "ref": f"refs/pr/{pr['number']}", "sha": pr["head"]["sha"], "author": pr["user"]["login"], "depends": depends_on(pr["body"]), } progress(f"{len(prs)} open pull requests against {base}") return prs def fetch(base, prs): refspecs = [f"+refs/heads/{base}:refs/pr/base"] + [f"+{pr['sha']}:{pr['ref']}" for pr in prs.values()] for start in range(0, len(refspecs), FETCH_REFS): git("fetch", "--no-tags", "-q", f"https://github.com/{REPO}.git", *refspecs[start:start + FETCH_REFS]) progress(f"fetched {min(start + FETCH_REFS, len(refspecs)) - 1}/{len(prs)} pull request heads") def merge(*args): cwd = Path.cwd() root = _REPO_ROOTS.get(cwd) if root is None: root = subprocess.run( ["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, check=True ).stdout.decode("utf-8", errors="replace").strip() _REPO_ROOTS[cwd] = root result = subprocess.run(["git", "-c", "core.quotePath=false", "merge-tree", "--write-tree", "--name-only", "--no-messages", "-z", *args], cwd=root, capture_output=True) lines = result.stdout.decode("utf-8", errors="replace").rstrip("\0").split("\0") if result.returncode not in (0, 1) or not lines[0]: required = (2, 40) if any(arg.startswith("--merge-base") for arg in args) else (2, 38) found = re.match(rb"git version (\d+)\.(\d+)", subprocess.run(["git", "version"], capture_output=True).stdout) version = (int(found.group(1)), int(found.group(2))) if found else None if version and version < required: raise RuntimeError(f"git merge-tree {' '.join(args)} needs git {required[0]}.{required[1]} or newer; " f"this git is {version[0]}.{version[1]}") error = result.stderr.decode("utf-8", errors="replace").strip() raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {error}") return lines[0], set(lines[1:]) if result.returncode else set() def commit(tree): return git("-c", "user.name=pr_overlap", "-c", "user.email=pr_overlap", "commit-tree", tree, "-p", "refs/pr/base", "-m", "pr_overlap").strip() def exports(diff, prefix="+"): found = set() lines = diff.splitlines() for i, line in enumerate(lines): if not line.startswith(prefix) or line.startswith(prefix * 3): continue m = EXPORT.match(prefix + line[1:]) if not m or m.group(1).endswith(("_nid_no_patch", "_nid_no_patch_cut")): continue text = line[1:] for nxt in lines[i + 1:]: if ";" in text or "{" in text or not nxt.startswith(prefix) or nxt.startswith(prefix * 3): break text += nxt[1:] semi, brace = text.find(";"), text.find("{") if semi < 0 or 0 <= brace < semi: name = m.group(1).removesuffix("_nid_postfix") found.add(NID_DISAMBIG.sub("", name)) return found def changed_source(diff, prefix): files, current, path = [], [], None for line in diff.splitlines(): if line.startswith("diff --git "): if current and path: files.append((path, "\n".join(current))) path = line.split(" b/", 1)[1] if " b/" in line else None current = [] elif line.startswith(prefix) and not line.startswith(prefix * 3): current.append(line[1:]) if current and path: files.append((path, "\n".join(current))) return files def library_for_path(path): parts = Path(path).parts return "/".join(parts[:4]) if len(parts) > 3 and parts[:3] == ("core", "libs", "prx") else "" def source_exports(text, library=""): found = {} for match in nid_names.source_definitions(text): name = nid_names.strip_nid_postfix(match.group(1)) if not name.endswith(("_nid_no_patch", "_nid_no_patch_cut")): found.setdefault((library, nid_names.compute_nid(name)), set()).add(name) for name in nid_names.source_assembly_definitions(text): name = nid_names.strip_nid_postfix(name) if not name.endswith(("_nid_no_patch", "_nid_no_patch_cut")): found.setdefault((library, nid_names.compute_nid(name)), set()).add(name) for nid, name in nid_names.source_aliases(text): found.setdefault((library, nid), set()).add(name) return found def source_changed_exports(diff, prefix): found = {} for path, text in changed_source(diff, prefix): library = library_for_path(path) for key, names in source_exports(text, library).items(): found.setdefault(key, set()).update(names) file_diff = "\n".join(prefix + line for line in text.splitlines()) for name in exports(file_diff, prefix): canonical = nid_names.strip_nid_postfix(name) found.setdefault((library, nid_names.compute_nid(canonical)), set()).add(canonical) return found def export_cache_version(): parsers = (Path(__file__), Path(nid_names.__file__)) return hashlib.sha256(b"".join(path.read_bytes() for path in parsers)).hexdigest() def load_export_cache(path, version): try: cache = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return {} if not isinstance(cache, dict) or cache.get("version") != version or not isinstance(cache.get("files"), dict): return {} return cache["files"] def save_export_cache(path, version, files): temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp") try: temporary.write_text(json.dumps({"version": version, "files": files}), encoding="utf-8") os.replace(temporary, path) except OSError: temporary.unlink(missing_ok=True) def base_sources(root, ref): listing = subprocess.run(["git", "-C", root, "ls-tree", "-r", "-z", ref, "--", "core/libs"], capture_output=True, check=True).stdout.decode("utf-8", errors="replace") sources = [] for entry in filter(None, listing.split("\0")): info, name = entry.split("\t", 1) mode, kind, blob = info.split() path = Path(name) if kind != "blob" or mode == "120000" or path.suffix != ".cpp" or any( part.lower() in {"test", "tests"} for part in path.parts[2:-1]): continue sources.append((blob, name)) return sources def blob_texts(root, blobs): if not blobs: return {} out = subprocess.run(["git", "-C", root, "cat-file", "--batch"], input="".join(f"{blob}\n" for blob in blobs).encode(), capture_output=True, check=True).stdout texts, offset = {}, 0 for blob in blobs: header_end = out.index(b"\n", offset) size = int(out[offset:header_end].split()[2]) texts[blob] = out[header_end + 1:header_end + 1 + size].decode("utf-8", errors="replace") offset = header_end + 2 + size return texts def base_exports(ref): root = git("rev-parse", "--show-toplevel").strip() cache_path = Path(root, git("rev-parse", "--git-common-dir").strip()) / "pr_overlap-exports.json" version = export_cache_version() cached = load_export_cache(cache_path, version) sources = base_sources(root, ref) keys = {(blob, name): f"{blob} {library_for_path(name)}" for blob, name in sources} texts = blob_texts(root, sorted({blob for (blob, name), key in keys.items() if key not in cached})) files = {} found = {} for (blob, name), key in keys.items(): if key not in files: files[key] = cached[key] if key in cached else [ [library, nid, sorted(names)] for (library, nid), names in source_exports(texts[blob], library_for_path(name)).items()] for library, nid, names in files[key]: found.setdefault((library, nid), set()).update(names) older_bases_fit = len(cached) + len(files) <= 4 * len(files) kept = {**cached, **files} if older_bases_fit else files if kept != cached: save_export_cache(cache_path, version, kept) return found def scan(pr): tree, conflicted = merge("refs/pr/base", pr["ref"]) pr["tree"] = None if conflicted else commit(tree) old, new = (git("merge-base", "refs/pr/base", pr["ref"]).strip(), pr["ref"]) if conflicted else ("refs/pr/base", pr["tree"]) pr["files"], pr["added"] = set(), set() changes = git("diff", "--name-status", "--no-renames", "-z", old, new).split("\0") for status, path in zip(changes[::2], changes[1::2]): pr["files"].add(path) if status == "A": pr["added"].add(path) diff = git("diff", "-U0", "--no-color", old, new, "--", "core/libs/prx/*.cpp") pr["exports"] = exports(diff) pr["added_exports"] = source_changed_exports(diff, "+") pr["removed_exports"] = source_changed_exports(diff, "-") def probe(pr): changes = git("diff", "--name-status", "--no-renames", "-z", f"refs/pr/base...{pr['ref']}").split("\0") files, existing = set(), set() for status, path in zip(changes[::2], changes[1::2]): files.add(path) if status != "A": existing.add(path) if not any(path.startswith("core/libs/prx/") and path.endswith(".cpp") for path in files): return files, (set(), {}), existing diff = git("diff", "-U0", "--no-color", f"refs/pr/base...{pr['ref']}", "--", "core/libs/prx/*.cpp") return files, (exports(diff), source_changed_exports(diff, "+")), existing def related(prs, executor): mine = prs[0] scan(mine) others = {n: pr for n, pr in prs.items() if n} base_paths = set(git("ls-tree", "-r", "--name-only", "-z", "refs/pr/base").split("\0")) chosen = {} mine_files = mine["files"] mine_exports = mine["exports"] mine_keys = set(mine.get("added_exports", {})) for index, ((n, pr), (files, found, existing)) in enumerate(zip(others.items(), executor.map(probe, others.values())), 1): found_exports, found_added = found if files & mine_files or (found_exports & mine_exports) or (set(found_added) & mine_keys) or existing - base_paths: chosen[n] = pr else: pr.update(tree=None, files=set(), added=set(), exports=set(), added_exports={}, removed_exports={}) if index % PROGRESS_EVERY == 0 or index == len(others): progress(f"probed {index}/{len(others)} pull requests, {len(chosen)} share files or exports with the branch") return chosen def scan_all(prs, jobs): with ThreadPoolExecutor(max_workers=jobs) as executor: pending = related(prs, executor) if 0 in prs else prs for index, _ in enumerate(executor.map(scan, pending.values()), 1): if index % PROGRESS_EVERY == 0 or index == len(pending): progress(f"merged {index}/{len(pending)} pull requests into the base") def hunks(pr, path): diff = git("diff", "-U0", "--no-color", "refs/pr/base", pr["tree"], "--", path) found = [] for start, count in HUNK.findall(diff): start = max(int(start), 1) found.append((start, start + max(int(count or 1), 1) - 1)) return found def clashes(a, b, path): ranges = sorted((min(s1, s2), max(e1, e2)) for s1, e1 in hunks(a, path) for s2, e2 in hunks(b, path) if s1 <= e2 + 1 and s2 <= e1 + 1) merged = [] for start, end in ranges: if merged and start <= merged[-1][1] + 1: merged[-1] = (merged[-1][0], max(merged[-1][1], end)) else: merged.append((start, end)) return merged def conflicts(a, b): if not (a["tree"] and b["tree"] and a["files"] & b["files"]): return {} files = merge(a["tree"], b["tree"])[1] return {path: [] if path in a["added"] else clashes(a, b, path) for path in files} def dependencies(prs, number): found, pending = set(), [number] while pending: for n in prs[pending.pop()]["depends"]: if n in prs and n not in found: found.add(n) pending.append(n) return found def overlaps(prs, a, b): pa, pb = prs[a], prs[b] da, db = dependencies(prs, a), dependencies(prs, b) if b in da or a in db: return None shared = [prs[n] for n in da & db] found = {"added": (pa["added"] & pb["added"]).difference(*(pr["added"] for pr in shared))} if pa["author"] != pb["author"]: found["conflicts"] = conflicts(pa, pb) same_exports = (pa.get("exports", set()) & pb.get("exports", set())).difference(*(pr.get("exports", set()) for pr in shared)) shared_keys = set().union(*(pr.get("added_exports", {}) for pr in shared)) for key in (set(pa.get("added_exports", {})) & set(pb.get("added_exports", {}))) - shared_keys: same_exports.update(pa["added_exports"][key]) same_exports.update(pb["added_exports"][key]) found["exports"] = same_exports if not (pa["tree"] and pb["tree"]): uncompared = pa.get("files", set()) & pb.get("files", set()) if uncompared: found["uncompared"] = uncompared return found if any(found.values()) else None def code_span(item): item = str(item) runs = re.findall(r"`+", item) delimiter = "`" * (max((len(run) for run in runs), default=0) + 1) if "`" in item: return f"{delimiter} {item} {delimiter}" return f"{delimiter}{item}{delimiter}" def cell(items): return "
".join(code_span(item) for item in sorted(items or ())) def spans(conflicts, blob): return "
".join( f"{code_span(path)} " + ", ".join( f"[L{s}-{e}]({blob}/{quote(path)}#L{s}-L{e})" if s != e else f"[L{s}]({blob}/{quote(path)}#L{s})" for s, e in ranges ) for path, ranges in sorted((conflicts or {}).items()) ) def pull(blob, number): return f'[PR {number}]({blob.split("/blob/", 1)[0].replace("https://github.com/", "https://www.github.com/", 1)}/pull/{number})' def report(rows, blob, main_exports=None): if not rows: if not main_exports: return "" confirmed = {number: found for number, found in rows.items() if found.get("conflicts") or found.get("exports") or found["added"]} uncompared = {number: found["uncompared"] for number, found in rows.items() if found.get("uncompared")} lines = [MARKER] if confirmed: lines.extend([ "Other open pull requests touch the same code. Merging one will break the other, " "or both implement the same thing. If this one builds on them, list them in **Depends on**.", "", "| Pull request | Conflicting files (lines on `main`) | Same exports | Same new files |", "|---|---|---|---|", ]) for number, found in sorted(confirmed.items()): lines.append(f'| {pull(blob, number)} | {spans(found.get("conflicts"), blob)} | {cell(found.get("exports"))} | {cell(found["added"])} |') if main_exports: if confirmed: lines.append("") lines.append("Exports introduced here that are already present in `main`:") lines.append("") lines.append(f"- {cell(main_exports)}") if uncompared: if confirmed: lines.append("") lines.extend([ "Some comparisons could not be completed because a pull request conflicts with the base branch. " "Review these shared files manually before concluding that there is no overlap.", "", "| Pull request | Shared files not compared |", "|---|---|", ]) for number, files in sorted(uncompared.items()): lines.append(f"| {pull(blob, number)} | {cell(files)} |") return "\n".join(lines) + "\n" def write_reports(out, rows, blob, main_exports=None): main_exports = main_exports or {} out.mkdir(parents=True, exist_ok=True) for number, found in rows.items(): (out / f"{number}.md").write_text(report(found, blob, main_exports.get(number)), encoding="utf-8") def main_export_findings(prs, current_exports): findings = {} for number, pr in prs.items(): candidates = set(pr.get("added_exports", {})) - set(pr.get("removed_exports", {})) names = set() for key in candidates & set(current_exports): names.update(pr["added_exports"][key]) names.update(current_exports[key]) if names: findings[number] = names return findings def describe_failure(error): command = " ".join(str(part) for part in error.cmd) message = (error.stderr or "").strip() or "no error output" return f"pr_overlap: {command} failed with exit code {error.returncode}: {message}" def main(argv=None): if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace") if hasattr(sys.stderr, "reconfigure"): sys.stderr.reconfigure(encoding="utf-8", errors="backslashreplace") parser = argparse.ArgumentParser( description="report open pull requests that overlap each other", epilog="The exports found in the base are cached by source blob in pr_overlap-exports.json inside the git " "directory; deleting that file only makes the next run slower.") parser.add_argument("--base", default="main") parser.add_argument("--out", type=Path, help="write .md for every open pull request (empty when clean)") parser.add_argument("--branch", help="check a local branch against the open pull requests instead") parser.add_argument("--repo", help="owner/name; defaults to the upstream of the current checkout") parser.add_argument("--jobs", type=int, default=4, help="number of pull request comparisons to run at once") args = parser.parse_args(argv) if bool(args.out) == bool(args.branch): parser.error("pass exactly one of --out or --branch") if args.jobs < 1: parser.error("--jobs must be at least 1") global REPO try: REPO = args.repo or upstream() prs = open_prs(args.base) fetch(args.base, prs) if args.branch: sha = git("rev-parse", args.branch).strip() prs = own(prs, sha) prs[0] = {"ref": sha, "author": None, "depends": set()} scan_all(prs, args.jobs) current_exports = base_exports("refs/pr/base") rows = {n: {} for n in prs} main_exports = main_export_findings(prs, current_exports) pairs = [(0, n) for n in prs if n] if args.branch else combinations(prs, 2) for a, b in pairs: if found := overlaps(prs, a, b): rows[a][b] = rows[b][a] = found blob = f'https://github.com/{REPO}/blob/{git("rev-parse", "refs/pr/base").strip()}' except subprocess.CalledProcessError as error: raise SystemExit(describe_failure(error)) except ValueError as error: raise SystemExit(f"pr_overlap: {error}") if args.branch: print(report(rows[0], blob, main_exports.get(0)) or "No overlap with open pull requests.", end="") return 0 write_reports(args.out, rows, blob, main_exports) return 0 if __name__ == "__main__": raise SystemExit(main())