AnyPS5/tools/sanitizer_report.py

import argparse
import re
import sys
import xml.etree.ElementTree as ET
from pathlib import Path

FAILED_STATUSES = ("fail", "notrun")
SKIPPED_STATUSES = ("disabled",)
CTEST_SKIP = re.compile(r"SKIP_RETURN_CODE=\d+|SKIP_REGULAR_EXPRESSION_MATCHED", re.IGNORECASE)
SANITIZERS = ("address", "undefined", "thread")
REPORT_LINES = (
    re.compile(r"SUMMARY: \w+Sanitizer: .*"),
    re.compile(r"(?:ERROR|WARNING): \w+Sanitizer: .*"),
    re.compile(r"(?:\S+ )?runtime error: .*"),
)
REPORT_LENGTH = 200


def results(path):
    root = ET.parse(path).getroot()
    if root.tag not in ("testsuite", "testsuites"):
        raise ValueError(f"{path}: expected a JUnit testsuite or testsuites root, got {root.tag!r}")
    found = {}
    for case in root.iter("testcase"):
        name = case.get("name")
        if not name:
            raise ValueError(f"{path}: testcase has no name")
        skipped, status = case.find("skipped"), case.get("status")
        reason = (skipped.get("message") or "").strip() if skipped is not None else ""
        if status in SKIPPED_STATUSES or (skipped is not None and (CTEST_SKIP.fullmatch(reason) or status not in FAILED_STATUSES)):
            outcome = "skipped"
        elif status in FAILED_STATUSES or case.find("failure") is not None or case.find("error") is not None:
            outcome = "failed"
        else:
            outcome = "passed"
        found[name] = outcome
    if not found:
        raise ValueError(f"{path}: the report lists no tests")
    return found


def first_report(output):
    for pattern in REPORT_LINES:
        if match := pattern.search(output):
            line = " ".join(match.group(0).split())
            return line if len(line) <= REPORT_LENGTH else line[:REPORT_LENGTH - 3] + "..."
    return ""


def reports(path):
    found = {}
    for case in ET.parse(path).getroot().iter("testcase"):
        output = "\n".join(case.findtext(channel) or "" for channel in ("system-out", "system-err"))
        if report := first_report(output):
            found[case.get("name")] = report
    return found


def load_known(path, sanitizers):
    known, seen = {}, set()
    for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
        if not line.strip():
            continue
        location = f"{path}:{number}"
        fields = line.split(None, 2)
        if len(fields) < 3:
            raise ValueError(f"{location}: expected a test name, its sanitizers and a reason")
        test, names, reason = fields[0], fields[1].split(","), fields[2].strip()
        if any(name not in SANITIZERS for name in names):
            raise ValueError(f"{location}: sanitizers must be a comma-separated list of {', '.join(SANITIZERS)}")
        for name in names:
            if (test, name) in seen:
                raise ValueError(f"{location}: {test} is already listed for {name}")
            seen.add((test, name))
        if set(names) & set(sanitizers):
            known[test] = reason
    return known


def cell(text):
    return " ".join(text.split()).replace("|", "\\|")


def summarize(title, found, known=None, details=None):
    known, details = known or {}, details or {}
    counts = {outcome: sum(1 for value in found.values() if value == outcome) for outcome in ("passed", "skipped", "failed")}
    failed = sorted(name for name, outcome in found.items() if outcome == "failed" and name not in known)
    expected = sorted(name for name, outcome in found.items() if outcome == "failed" and name in known)
    tally = f"{counts['failed']} failed" + (f" ({len(expected)} known)" if expected else "")
    lines = [f"### {title}", "",
             f"{len(found)} tests: {counts['passed']} passed, {counts['skipped']} skipped, {tally}.", ""]
    if failed and any(name in details for name in failed):
        lines += ["| Failed test | First sanitizer report |", "| - | - |"]
        lines += [f"| `{name}` | {cell(details.get(name, ''))} |" for name in failed] + [""]
    elif failed:
        lines += ["| Failed test |", "| - |"] + [f"| `{name}` |" for name in failed] + [""]
    if expected:
        lines += ["| Known failure | Reason |", "| - | - |"] + [f"| `{name}` | {cell(known[name])} |" for name in expected] + [""]
    recovered = sorted(name for name in known if found.get(name) == "passed")
    if recovered:
        lines += ["| Listed as known but passed | Reason |", "| - | - |"] + [f"| `{name}` | {cell(known[name])} |" for name in recovered] + [""]
    absent = sorted(name for name in known if name not in found)
    if absent:
        lines += ["| Listed as known but not in this run |", "| - |"] + [f"| `{name}` |" for name in absent] + [""]
    return "\n".join(lines) + "\n", bool(failed)


def main(argv=None):
    for stream in (sys.stdout, sys.stderr):
        if hasattr(stream, "reconfigure"):
            stream.reconfigure(encoding="utf-8", errors="backslashreplace")
    parser = argparse.ArgumentParser(description="summarize a sanitizer test run and fail when a test failed")
    parser.add_argument("junit", type=Path, help="the file written by ctest --output-junit")
    parser.add_argument("--title", default="Sanitizer run", help="heading of the summary")
    parser.add_argument("--summary", type=Path, help="also append the summary to this file, such as $GITHUB_STEP_SUMMARY")
    parser.add_argument("--known", type=Path,
                        help="known failures, one per line: test name, comma-separated sanitizers and the reason")
    parser.add_argument("--sanitizer", action="append", choices=SANITIZERS, default=[],
                        help="sanitizer the build used; repeat for several. Selects the --known entries that apply")
    args = parser.parse_args(argv)
    if args.known and not args.sanitizer:
        parser.error("--known needs at least one --sanitizer")
    try:
        known = load_known(args.known, args.sanitizer) if args.known else {}
        text, failing = summarize(args.title, results(args.junit), known, reports(args.junit))
        if args.summary:
            with args.summary.open("a", encoding="utf-8") as summary:
                summary.write(text)
    except (OSError, ET.ParseError, ValueError) as error:
        print(f"sanitizer_report: {error}", file=sys.stderr)
        return 2
    sys.stdout.write(text)
    return 1 if failing else 0


if __name__ == "__main__":
    sys.exit(main())