mirror of
https://github.com/boykopovar/AnyPS5.git
synced 2026-11-04 13:07:32 +00:00
MarcZX10
8a23eed9d2
pr_overlap.py stopped with "Invalid \uXXXX escape" when GitHub returned an open pull request whose body contains a backslash that starts no valid JSON escape, such as a backslash followed by u00XX. `gh api --jq` fails on the same page, so the whole open-PR list failed and the overlap check could not run. First parse the page as before. Only when that fails with an escape error, double each backslash that does not start a valid escape, reading the text left to right the way a JSON parser does, and parse again. Valid escapes keep their meaning. The tool reads a body only to find Depends on lines, so the only text that changes is the invalid backslash itself. Tests: the malformed-escape test fails on upstream main with that error and passes here; the valid-escape test passes on both.
653 lines
27 KiB
Python
653 lines
27 KiB
Python
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from itertools import combinations
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, quote, urlencode, urlsplit
|
|
|
|
import nid_names
|
|
|
|
PARENT = 'if .parent then "\\(.parent.owner.login)/\\(.parent.name)" else .nameWithOwner end'
|
|
EXPORT = re.compile(r"^\+.*\bAPS5_VABI\s+(\w+)\s*\(")
|
|
NID_DISAMBIG = re.compile(r"_nid_disambig[0-9]+$")
|
|
DEPENDS = re.compile(r'Depends on:[ \t]*((?:#\d+\b(?:(?:[ \t]*,[ \t]*(?:and[ \t]+)?|[ \t]+and[ \t]+)#\d+\b)*|none\b)?(?:\r?\n[ \t]*[-*][ \t]*#\d+\b[^\r\n]*)*)')
|
|
PLAIN_DEPENDS = re.compile(r'Depends on[ \t]+(#\d+\b(?:(?:[ \t]*,[ \t]*(?:and[ \t]+)?|[ \t]+and[ \t]+)#\d+\b)*)')
|
|
HUNK = re.compile(r"^@@ -(\d+)(?:,(\d+))?", re.M)
|
|
MARKER = "<!-- pr-overlap -->"
|
|
FETCH_REFS = 100
|
|
PROGRESS_EVERY = 100
|
|
_REPO_ROOTS = {}
|
|
|
|
|
|
def git(*args):
|
|
cwd = Path.cwd()
|
|
root = _REPO_ROOTS.get(cwd)
|
|
if root is None:
|
|
root = subprocess.run(
|
|
["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, check=True
|
|
).stdout.decode("utf-8", errors="replace").strip()
|
|
_REPO_ROOTS[cwd] = root
|
|
args = list(args)
|
|
if "--" in args:
|
|
separator = args.index("--")
|
|
for index in range(separator + 1, len(args)):
|
|
pathspec = args[index]
|
|
if pathspec.startswith(("../", "..\\")):
|
|
absolute = cwd / pathspec
|
|
try:
|
|
absolute = absolute.resolve()
|
|
except RuntimeError as error:
|
|
raise ValueError(f"cannot resolve path {pathspec!r}: {error}") from error
|
|
except OSError as error:
|
|
raise ValueError(f"cannot resolve path {pathspec!r}: {error}") from error
|
|
if absolute.is_symlink():
|
|
try:
|
|
absolute.stat()
|
|
except OSError as error:
|
|
raise ValueError(f"cannot resolve path {pathspec!r}: {error}") from error
|
|
try:
|
|
args[index] = absolute.relative_to(Path(root)).as_posix()
|
|
except ValueError:
|
|
pass
|
|
result = subprocess.run(
|
|
["git", "-c", "core.quotePath=false", *args],
|
|
cwd=root,
|
|
capture_output=True,
|
|
)
|
|
result.stdout = result.stdout.decode("utf-8", errors="replace")
|
|
result.stderr = result.stderr.decode("utf-8", errors="replace")
|
|
result.check_returncode()
|
|
return result.stdout
|
|
|
|
|
|
def own(prs, sha):
|
|
ancestors = set(git("rev-list", sha).splitlines())
|
|
return {n: pr for n, pr in prs.items() if pr["sha"] not in ancestors}
|
|
|
|
|
|
def gh(*args):
|
|
return subprocess.run(["gh", *args], capture_output=True, text=True, encoding="utf-8", check=True).stdout
|
|
|
|
|
|
def upstream():
|
|
return os.environ.get("GITHUB_REPOSITORY") or gh("repo", "view", "--json", "nameWithOwner,parent", "--jq", PARENT).strip()
|
|
|
|
|
|
def depends_on(body):
|
|
body = re.sub(r"<!--.*?-->", "", body or "", flags=re.S)
|
|
repo = re.escape(globals().get("REPO", "boykopovar/AnyPS5"))
|
|
repo_pattern = rf"(?:{repo}|boykopovar/AnyPS5)"
|
|
body = re.sub(rf"\[([^\]]*)\]\(https?://(?:www\.)?github\.com/{repo_pattern}/pull/(\d+)(?:[#/?][^)\s]*)?\)", r"#\2", body, flags=re.IGNORECASE)
|
|
body = re.sub(rf"\[([^\]]*)\]\(https?://(?:www\.)?github\.com/(?!{repo_pattern}/)[^/\s]+/[^/\s]+/pull/\d+[^)\s]*\)", "", body, flags=re.IGNORECASE)
|
|
body = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", body)
|
|
body = re.sub(rf"https?://(?:www\.)?github\.com/{repo_pattern}/pull/(\d+)\b(?:/[^\s]*)?", r"#\1", body, flags=re.IGNORECASE)
|
|
match = next((match for match in DEPENDS.finditer(body) if match.group(1).strip()), None) or PLAIN_DEPENDS.search(body)
|
|
return {int(n) for n in re.findall(r"#(\d+)", match.group(1))} if match else set()
|
|
|
|
def progress(message):
|
|
print(f"pr_overlap: {message}", file=sys.stderr, flush=True)
|
|
|
|
|
|
LAST_PAGE = re.compile(r'[?&]page=(\d+)>; rel="last"')
|
|
NEXT_LINK = re.compile(r'<([^>]+)>\s*;\s*rel="next"', re.IGNORECASE)
|
|
|
|
|
|
def page(url):
|
|
out = gh("api", "--include", url)
|
|
if not out.startswith("HTTP/"):
|
|
return "", out
|
|
blank = re.search(r"\r?\n\r?\n", out)
|
|
return (out, "") if blank is None else (out[:blank.start()], out[blank.end():])
|
|
|
|
|
|
def decode_pages(out):
|
|
decoder = json.JSONDecoder()
|
|
offset = 0
|
|
pages = []
|
|
while offset < len(out):
|
|
while offset < len(out) and out[offset].isspace():
|
|
offset += 1
|
|
if offset == len(out):
|
|
break
|
|
found, offset = decoder.raw_decode(out, offset)
|
|
if not isinstance(found, list):
|
|
raise ValueError("gh api returned a non-array page")
|
|
pages.append(found)
|
|
return pages
|
|
|
|
|
|
def repair_escapes(text):
|
|
repaired = []
|
|
offset = 0
|
|
while offset < len(text):
|
|
character = text[offset]
|
|
if character != "\\":
|
|
repaired.append(character)
|
|
offset += 1
|
|
continue
|
|
following = text[offset + 1:offset + 2]
|
|
if following in ('"', "\\", "/", "b", "f", "n", "r", "t"):
|
|
repaired.append(text[offset:offset + 2])
|
|
offset += 2
|
|
elif following == "u" and re.fullmatch(r"[0-9a-fA-F]{4}", text[offset + 2:offset + 6]):
|
|
repaired.append(text[offset:offset + 6])
|
|
offset += 6
|
|
else:
|
|
repaired.append("\\\\")
|
|
offset += 1
|
|
return "".join(repaired)
|
|
|
|
|
|
def json_pages(out):
|
|
try:
|
|
return decode_pages(out)
|
|
except json.JSONDecodeError as error:
|
|
if "escape" not in error.msg:
|
|
raise
|
|
return decode_pages(repair_escapes(out))
|
|
|
|
|
|
def open_prs(base):
|
|
query = urlencode({"state": "open", "base": base, "per_page": 100})
|
|
url = f"repos/{REPO}/pulls?{query}"
|
|
head, out = page(f"{url}&page=1")
|
|
pages = json_pages(out)
|
|
if not pages:
|
|
raise ValueError("gh api returned no JSON pages")
|
|
last = LAST_PAGE.search(head)
|
|
pushed_past_last = None
|
|
if last:
|
|
numbers = range(2, int(last.group(1)) + 1)
|
|
with ThreadPoolExecutor(max_workers=8) as executor:
|
|
for out in executor.map(lambda number: page(f"{url}&page={number}")[1], numbers):
|
|
pages.extend(json_pages(out))
|
|
pushed_past_last = numbers.stop
|
|
elif NEXT_LINK.search(head):
|
|
current_page = 1
|
|
while next_link := NEXT_LINK.search(head):
|
|
values = parse_qs(urlsplit(next_link.group(1)).query, keep_blank_values=True).get("page", [])
|
|
if len(values) != 1 or re.fullmatch(r"[0-9]+", values[0]) is None or int(values[0]) <= current_page:
|
|
raise ValueError("gh api returned an invalid next page link")
|
|
current_page = int(values[0])
|
|
head, out = page(f"{url}&page={current_page}")
|
|
fetched = json_pages(out)
|
|
if not fetched:
|
|
raise ValueError("gh api returned no JSON pages")
|
|
pages.extend(fetched)
|
|
pushed_past_last = current_page + 1
|
|
if pushed_past_last is not None:
|
|
while len(pages[-1]) == 100:
|
|
pages.extend(json_pages(page(f"{url}&page={pushed_past_last}")[1]) or [[]])
|
|
pushed_past_last += 1
|
|
pull_requests = [pr for found in pages for pr in found]
|
|
prs = {}
|
|
for pr in pull_requests:
|
|
prs[pr["number"]] = {
|
|
"ref": f"refs/pr/{pr['number']}",
|
|
"sha": pr["head"]["sha"],
|
|
"author": pr["user"]["login"],
|
|
"depends": depends_on(pr["body"]),
|
|
}
|
|
progress(f"{len(prs)} open pull requests against {base}")
|
|
return prs
|
|
|
|
|
|
def fetch(base, prs):
|
|
refspecs = [f"+refs/heads/{base}:refs/pr/base"] + [f"+{pr['sha']}:{pr['ref']}" for pr in prs.values()]
|
|
for start in range(0, len(refspecs), FETCH_REFS):
|
|
git("fetch", "--no-tags", "-q", f"https://github.com/{REPO}.git", *refspecs[start:start + FETCH_REFS])
|
|
progress(f"fetched {min(start + FETCH_REFS, len(refspecs)) - 1}/{len(prs)} pull request heads")
|
|
|
|
|
|
def merge(*args):
|
|
cwd = Path.cwd()
|
|
root = _REPO_ROOTS.get(cwd)
|
|
if root is None:
|
|
root = subprocess.run(
|
|
["git", "rev-parse", "--show-toplevel"], cwd=cwd, capture_output=True, check=True
|
|
).stdout.decode("utf-8", errors="replace").strip()
|
|
_REPO_ROOTS[cwd] = root
|
|
result = subprocess.run(["git", "-c", "core.quotePath=false", "merge-tree", "--write-tree", "--name-only", "--no-messages", "-z", *args],
|
|
cwd=root, capture_output=True)
|
|
lines = result.stdout.decode("utf-8", errors="replace").rstrip("\0").split("\0")
|
|
if result.returncode not in (0, 1) or not lines[0]:
|
|
required = (2, 40) if any(arg.startswith("--merge-base") for arg in args) else (2, 38)
|
|
found = re.match(rb"git version (\d+)\.(\d+)", subprocess.run(["git", "version"], capture_output=True).stdout)
|
|
version = (int(found.group(1)), int(found.group(2))) if found else None
|
|
if version and version < required:
|
|
raise RuntimeError(f"git merge-tree {' '.join(args)} needs git {required[0]}.{required[1]} or newer; "
|
|
f"this git is {version[0]}.{version[1]}")
|
|
error = result.stderr.decode("utf-8", errors="replace").strip()
|
|
raise RuntimeError(f"git merge-tree {' '.join(args)} failed: {error}")
|
|
return lines[0], set(lines[1:]) if result.returncode else set()
|
|
|
|
|
|
def commit(tree):
|
|
return git("-c", "user.name=pr_overlap", "-c", "user.email=pr_overlap", "commit-tree", tree,
|
|
"-p", "refs/pr/base", "-m", "pr_overlap").strip()
|
|
|
|
|
|
def exports(diff, prefix="+"):
|
|
found = set()
|
|
lines = diff.splitlines()
|
|
for i, line in enumerate(lines):
|
|
if not line.startswith(prefix) or line.startswith(prefix * 3):
|
|
continue
|
|
m = EXPORT.match(prefix + line[1:])
|
|
if not m or m.group(1).endswith(("_nid_no_patch", "_nid_no_patch_cut")):
|
|
continue
|
|
text = line[1:]
|
|
for nxt in lines[i + 1:]:
|
|
if ";" in text or "{" in text or not nxt.startswith(prefix) or nxt.startswith(prefix * 3):
|
|
break
|
|
text += nxt[1:]
|
|
semi, brace = text.find(";"), text.find("{")
|
|
if semi < 0 or 0 <= brace < semi:
|
|
name = m.group(1).removesuffix("_nid_postfix")
|
|
found.add(NID_DISAMBIG.sub("", name))
|
|
return found
|
|
|
|
|
|
def changed_source(diff, prefix):
|
|
files, current, path = [], [], None
|
|
for line in diff.splitlines():
|
|
if line.startswith("diff --git "):
|
|
if current and path:
|
|
files.append((path, "\n".join(current)))
|
|
path = line.split(" b/", 1)[1] if " b/" in line else None
|
|
current = []
|
|
elif line.startswith(prefix) and not line.startswith(prefix * 3):
|
|
current.append(line[1:])
|
|
if current and path:
|
|
files.append((path, "\n".join(current)))
|
|
return files
|
|
|
|
|
|
def library_for_path(path):
|
|
parts = Path(path).parts
|
|
return "/".join(parts[:4]) if len(parts) > 3 and parts[:3] == ("core", "libs", "prx") else ""
|
|
|
|
|
|
def source_exports(text, library=""):
|
|
found = {}
|
|
for match in nid_names.source_definitions(text):
|
|
name = nid_names.strip_nid_postfix(match.group(1))
|
|
if not name.endswith(("_nid_no_patch", "_nid_no_patch_cut")):
|
|
found.setdefault((library, nid_names.compute_nid(name)), set()).add(name)
|
|
for name in nid_names.source_assembly_definitions(text):
|
|
name = nid_names.strip_nid_postfix(name)
|
|
if not name.endswith(("_nid_no_patch", "_nid_no_patch_cut")):
|
|
found.setdefault((library, nid_names.compute_nid(name)), set()).add(name)
|
|
for nid, name in nid_names.source_aliases(text):
|
|
found.setdefault((library, nid), set()).add(name)
|
|
return found
|
|
|
|
|
|
def source_changed_exports(diff, prefix):
|
|
found = {}
|
|
for path, text in changed_source(diff, prefix):
|
|
library = library_for_path(path)
|
|
for key, names in source_exports(text, library).items():
|
|
found.setdefault(key, set()).update(names)
|
|
file_diff = "\n".join(prefix + line for line in text.splitlines())
|
|
for name in exports(file_diff, prefix):
|
|
canonical = nid_names.strip_nid_postfix(name)
|
|
found.setdefault((library, nid_names.compute_nid(canonical)), set()).add(canonical)
|
|
return found
|
|
|
|
|
|
def export_cache_version():
|
|
parsers = (Path(__file__), Path(nid_names.__file__))
|
|
return hashlib.sha256(b"".join(path.read_bytes() for path in parsers)).hexdigest()
|
|
|
|
|
|
def load_export_cache(path, version):
|
|
try:
|
|
cache = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return {}
|
|
if not isinstance(cache, dict) or cache.get("version") != version or not isinstance(cache.get("files"), dict):
|
|
return {}
|
|
return cache["files"]
|
|
|
|
|
|
def save_export_cache(path, version, files):
|
|
temporary = path.with_name(f"{path.name}.{os.getpid()}.tmp")
|
|
try:
|
|
temporary.write_text(json.dumps({"version": version, "files": files}), encoding="utf-8")
|
|
os.replace(temporary, path)
|
|
except OSError:
|
|
temporary.unlink(missing_ok=True)
|
|
|
|
|
|
def base_sources(root, ref):
|
|
listing = subprocess.run(["git", "-C", root, "ls-tree", "-r", "-z", ref, "--", "core/libs"],
|
|
capture_output=True, check=True).stdout.decode("utf-8", errors="replace")
|
|
sources = []
|
|
for entry in filter(None, listing.split("\0")):
|
|
info, name = entry.split("\t", 1)
|
|
mode, kind, blob = info.split()
|
|
path = Path(name)
|
|
if kind != "blob" or mode == "120000" or path.suffix != ".cpp" or any(
|
|
part.lower() in {"test", "tests"} for part in path.parts[2:-1]):
|
|
continue
|
|
sources.append((blob, name))
|
|
return sources
|
|
|
|
|
|
def blob_texts(root, blobs):
|
|
if not blobs:
|
|
return {}
|
|
out = subprocess.run(["git", "-C", root, "cat-file", "--batch"], input="".join(f"{blob}\n" for blob in blobs).encode(),
|
|
capture_output=True, check=True).stdout
|
|
texts, offset = {}, 0
|
|
for blob in blobs:
|
|
header_end = out.index(b"\n", offset)
|
|
size = int(out[offset:header_end].split()[2])
|
|
texts[blob] = out[header_end + 1:header_end + 1 + size].decode("utf-8", errors="replace")
|
|
offset = header_end + 2 + size
|
|
return texts
|
|
|
|
|
|
def base_exports(ref):
|
|
root = git("rev-parse", "--show-toplevel").strip()
|
|
cache_path = Path(root, git("rev-parse", "--git-common-dir").strip()) / "pr_overlap-exports.json"
|
|
version = export_cache_version()
|
|
cached = load_export_cache(cache_path, version)
|
|
sources = base_sources(root, ref)
|
|
keys = {(blob, name): f"{blob} {library_for_path(name)}" for blob, name in sources}
|
|
texts = blob_texts(root, sorted({blob for (blob, name), key in keys.items() if key not in cached}))
|
|
files = {}
|
|
found = {}
|
|
for (blob, name), key in keys.items():
|
|
if key not in files:
|
|
files[key] = cached[key] if key in cached else [
|
|
[library, nid, sorted(names)] for (library, nid), names in source_exports(texts[blob], library_for_path(name)).items()]
|
|
for library, nid, names in files[key]:
|
|
found.setdefault((library, nid), set()).update(names)
|
|
older_bases_fit = len(cached) + len(files) <= 4 * len(files)
|
|
kept = {**cached, **files} if older_bases_fit else files
|
|
if kept != cached:
|
|
save_export_cache(cache_path, version, kept)
|
|
return found
|
|
|
|
|
|
def scan(pr):
|
|
tree, conflicted = merge("refs/pr/base", pr["ref"])
|
|
pr["tree"] = None if conflicted else commit(tree)
|
|
old, new = (git("merge-base", "refs/pr/base", pr["ref"]).strip(), pr["ref"]) if conflicted else ("refs/pr/base", pr["tree"])
|
|
pr["files"], pr["added"] = set(), set()
|
|
changes = git("diff", "--name-status", "--no-renames", "-z", old, new).split("\0")
|
|
for status, path in zip(changes[::2], changes[1::2]):
|
|
pr["files"].add(path)
|
|
if status == "A":
|
|
pr["added"].add(path)
|
|
diff = git("diff", "-U0", "--no-color", old, new, "--", "core/libs/prx/*.cpp")
|
|
pr["exports"] = exports(diff)
|
|
pr["added_exports"] = source_changed_exports(diff, "+")
|
|
pr["removed_exports"] = source_changed_exports(diff, "-")
|
|
|
|
|
|
def probe(pr):
|
|
changes = git("diff", "--name-status", "--no-renames", "-z", f"refs/pr/base...{pr['ref']}").split("\0")
|
|
files, existing = set(), set()
|
|
for status, path in zip(changes[::2], changes[1::2]):
|
|
files.add(path)
|
|
if status != "A":
|
|
existing.add(path)
|
|
if not any(path.startswith("core/libs/prx/") and path.endswith(".cpp") for path in files):
|
|
return files, (set(), {}), existing
|
|
diff = git("diff", "-U0", "--no-color", f"refs/pr/base...{pr['ref']}", "--", "core/libs/prx/*.cpp")
|
|
return files, (exports(diff), source_changed_exports(diff, "+")), existing
|
|
|
|
|
|
def related(prs, executor):
|
|
mine = prs[0]
|
|
scan(mine)
|
|
others = {n: pr for n, pr in prs.items() if n}
|
|
base_paths = set(git("ls-tree", "-r", "--name-only", "-z", "refs/pr/base").split("\0"))
|
|
chosen = {}
|
|
mine_files = mine["files"]
|
|
mine_exports = mine["exports"]
|
|
mine_keys = set(mine.get("added_exports", {}))
|
|
for index, ((n, pr), (files, found, existing)) in enumerate(zip(others.items(), executor.map(probe, others.values())), 1):
|
|
found_exports, found_added = found
|
|
if files & mine_files or (found_exports & mine_exports) or (set(found_added) & mine_keys) or existing - base_paths:
|
|
chosen[n] = pr
|
|
else:
|
|
pr.update(tree=None, files=set(), added=set(), exports=set(), added_exports={}, removed_exports={})
|
|
if index % PROGRESS_EVERY == 0 or index == len(others):
|
|
progress(f"probed {index}/{len(others)} pull requests, {len(chosen)} share files or exports with the branch")
|
|
return chosen
|
|
|
|
|
|
def scan_all(prs, jobs):
|
|
with ThreadPoolExecutor(max_workers=jobs) as executor:
|
|
pending = related(prs, executor) if 0 in prs else prs
|
|
for index, _ in enumerate(executor.map(scan, pending.values()), 1):
|
|
if index % PROGRESS_EVERY == 0 or index == len(pending):
|
|
progress(f"merged {index}/{len(pending)} pull requests into the base")
|
|
|
|
|
|
def hunks(pr, path):
|
|
diff = git("diff", "-U0", "--no-color", "refs/pr/base", pr["tree"], "--", path)
|
|
found = []
|
|
for start, count in HUNK.findall(diff):
|
|
start = max(int(start), 1)
|
|
found.append((start, start + max(int(count or 1), 1) - 1))
|
|
return found
|
|
|
|
|
|
def clashes(a, b, path):
|
|
ranges = sorted((min(s1, s2), max(e1, e2)) for s1, e1 in hunks(a, path) for s2, e2 in hunks(b, path)
|
|
if s1 <= e2 + 1 and s2 <= e1 + 1)
|
|
merged = []
|
|
for start, end in ranges:
|
|
if merged and start <= merged[-1][1] + 1:
|
|
merged[-1] = (merged[-1][0], max(merged[-1][1], end))
|
|
else:
|
|
merged.append((start, end))
|
|
return merged
|
|
|
|
|
|
def conflicts(a, b):
|
|
if not (a["tree"] and b["tree"] and a["files"] & b["files"]):
|
|
return {}
|
|
files = merge(a["tree"], b["tree"])[1]
|
|
return {path: [] if path in a["added"] else clashes(a, b, path) for path in files}
|
|
|
|
|
|
def dependencies(prs, number):
|
|
found, pending = set(), [number]
|
|
while pending:
|
|
for n in prs[pending.pop()]["depends"]:
|
|
if n in prs and n not in found:
|
|
found.add(n)
|
|
pending.append(n)
|
|
return found
|
|
|
|
|
|
def overlaps(prs, a, b):
|
|
pa, pb = prs[a], prs[b]
|
|
da, db = dependencies(prs, a), dependencies(prs, b)
|
|
if b in da or a in db:
|
|
return None
|
|
shared = [prs[n] for n in da & db]
|
|
found = {"added": (pa["added"] & pb["added"]).difference(*(pr["added"] for pr in shared))}
|
|
if pa["author"] != pb["author"]:
|
|
found["conflicts"] = conflicts(pa, pb)
|
|
same_exports = (pa.get("exports", set()) & pb.get("exports", set())).difference(*(pr.get("exports", set()) for pr in shared))
|
|
shared_keys = set().union(*(pr.get("added_exports", {}) for pr in shared))
|
|
for key in (set(pa.get("added_exports", {})) & set(pb.get("added_exports", {}))) - shared_keys:
|
|
same_exports.update(pa["added_exports"][key])
|
|
same_exports.update(pb["added_exports"][key])
|
|
found["exports"] = same_exports
|
|
if not (pa["tree"] and pb["tree"]):
|
|
uncompared = pa.get("files", set()) & pb.get("files", set())
|
|
if uncompared:
|
|
found["uncompared"] = uncompared
|
|
return found if any(found.values()) else None
|
|
|
|
|
|
|
|
def code_span(item):
|
|
item = str(item)
|
|
runs = re.findall(r"`+", item)
|
|
delimiter = "`" * (max((len(run) for run in runs), default=0) + 1)
|
|
if "`" in item:
|
|
return f"{delimiter} {item} {delimiter}"
|
|
return f"{delimiter}{item}{delimiter}"
|
|
|
|
|
|
def cell(items):
|
|
return "<br>".join(code_span(item) for item in sorted(items or ()))
|
|
|
|
|
|
|
|
|
|
def spans(conflicts, blob):
|
|
return "<br>".join(
|
|
f"{code_span(path)} " + ", ".join(
|
|
f"[L{s}-{e}]({blob}/{quote(path)}#L{s}-L{e})"
|
|
if s != e else f"[L{s}]({blob}/{quote(path)}#L{s})"
|
|
for s, e in ranges
|
|
)
|
|
for path, ranges in sorted((conflicts or {}).items())
|
|
)
|
|
|
|
|
|
def pull(blob, number):
|
|
return f'[PR {number}]({blob.split("/blob/", 1)[0].replace("https://github.com/", "https://www.github.com/", 1)}/pull/{number})'
|
|
|
|
|
|
def report(rows, blob, main_exports=None):
|
|
if not rows:
|
|
if not main_exports:
|
|
return ""
|
|
confirmed = {number: found for number, found in rows.items()
|
|
if found.get("conflicts") or found.get("exports") or found["added"]}
|
|
uncompared = {number: found["uncompared"] for number, found in rows.items() if found.get("uncompared")}
|
|
lines = [MARKER]
|
|
if confirmed:
|
|
lines.extend([
|
|
"Other open pull requests touch the same code. Merging one will break the other, "
|
|
"or both implement the same thing. If this one builds on them, list them in **Depends on**.",
|
|
"",
|
|
"| Pull request | Conflicting files (lines on `main`) | Same exports | Same new files |",
|
|
"|---|---|---|---|",
|
|
])
|
|
for number, found in sorted(confirmed.items()):
|
|
lines.append(f'| {pull(blob, number)} | {spans(found.get("conflicts"), blob)} | {cell(found.get("exports"))} | {cell(found["added"])} |')
|
|
if main_exports:
|
|
if confirmed:
|
|
lines.append("")
|
|
lines.append("Exports introduced here that are already present in `main`:")
|
|
lines.append("")
|
|
lines.append(f"- {cell(main_exports)}")
|
|
if uncompared:
|
|
if confirmed:
|
|
lines.append("")
|
|
lines.extend([
|
|
"Some comparisons could not be completed because a pull request conflicts with the base branch. "
|
|
"Review these shared files manually before concluding that there is no overlap.",
|
|
"",
|
|
"| Pull request | Shared files not compared |",
|
|
"|---|---|",
|
|
])
|
|
for number, files in sorted(uncompared.items()):
|
|
lines.append(f"| {pull(blob, number)} | {cell(files)} |")
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def write_reports(out, rows, blob, main_exports=None):
|
|
main_exports = main_exports or {}
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
for number, found in rows.items():
|
|
(out / f"{number}.md").write_text(report(found, blob, main_exports.get(number)), encoding="utf-8")
|
|
|
|
|
|
def unverified_overlaps(rows):
|
|
return {number: set(found["uncompared"]) for number, found in rows.items() if found.get("uncompared")}
|
|
|
|
|
|
def main_export_findings(prs, current_exports):
|
|
findings = {}
|
|
for number, pr in prs.items():
|
|
candidates = set(pr.get("added_exports", {})) - set(pr.get("removed_exports", {}))
|
|
names = set()
|
|
for key in candidates & set(current_exports):
|
|
names.update(pr["added_exports"][key])
|
|
names.update(current_exports[key])
|
|
if names:
|
|
findings[number] = names
|
|
return findings
|
|
|
|
|
|
def describe_failure(error):
|
|
command = " ".join(str(part) for part in error.cmd)
|
|
message = (error.stderr or "").strip() or "no error output"
|
|
return f"pr_overlap: {command} failed with exit code {error.returncode}: {message}"
|
|
|
|
|
|
def main(argv=None):
|
|
if hasattr(sys.stdout, "reconfigure"):
|
|
sys.stdout.reconfigure(encoding="utf-8", errors="backslashreplace")
|
|
if hasattr(sys.stderr, "reconfigure"):
|
|
sys.stderr.reconfigure(encoding="utf-8", errors="backslashreplace")
|
|
parser = argparse.ArgumentParser(
|
|
description="report open pull requests that overlap each other",
|
|
epilog="The exports found in the base are cached by source blob in pr_overlap-exports.json inside the git "
|
|
"directory; deleting that file only makes the next run slower.")
|
|
parser.add_argument("--base", default="main")
|
|
parser.add_argument("--out", type=Path, help="write <number>.md for every open pull request (empty when clean)")
|
|
parser.add_argument("--branch", help="check a local branch against the open pull requests instead")
|
|
parser.add_argument("--repo", help="owner/name; defaults to the upstream of the current checkout")
|
|
parser.add_argument("--jobs", type=int, default=4, help="number of pull request comparisons to run at once")
|
|
args = parser.parse_args(argv)
|
|
if bool(args.out) == bool(args.branch):
|
|
parser.error("pass exactly one of --out or --branch")
|
|
if args.jobs < 1:
|
|
parser.error("--jobs must be at least 1")
|
|
global REPO
|
|
try:
|
|
REPO = args.repo or upstream()
|
|
prs = open_prs(args.base)
|
|
fetch(args.base, prs)
|
|
if args.branch:
|
|
sha = git("rev-parse", args.branch).strip()
|
|
prs = own(prs, sha)
|
|
prs[0] = {"ref": sha, "author": None, "depends": set()}
|
|
scan_all(prs, args.jobs)
|
|
current_exports = base_exports("refs/pr/base")
|
|
rows = {n: {} for n in prs}
|
|
main_exports = main_export_findings(prs, current_exports)
|
|
pairs = [(0, n) for n in prs if n] if args.branch else combinations(prs, 2)
|
|
for a, b in pairs:
|
|
if found := overlaps(prs, a, b):
|
|
rows[a][b] = rows[b][a] = found
|
|
blob = f'https://github.com/{REPO}/blob/{git("rev-parse", "refs/pr/base").strip()}'
|
|
except subprocess.CalledProcessError as error:
|
|
raise SystemExit(describe_failure(error))
|
|
except ValueError as error:
|
|
raise SystemExit(f"pr_overlap: {error}")
|
|
if args.branch:
|
|
print(report(rows[0], blob, main_exports.get(0)) or "No overlap with open pull requests.", end="")
|
|
unresolved = unverified_overlaps(rows[0])
|
|
if unresolved:
|
|
for number, files in sorted(unresolved.items()):
|
|
progress(f"cannot confirm overlap with PR {number}: {', '.join(sorted(files))}")
|
|
progress("overlap check failed closed; manually compare every reported PR branch before opening this one")
|
|
return 2
|
|
return 0
|
|
write_reports(args.out, rows, blob, main_exports)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|