cleanyourtv/tools/gen_tables.py

#!/usr/bin/env python3
"""Writes the TV tables, the brand line and the counts from each examples/<folder>/tv.json.

    gen_tables.py [--root DIR]                 rewrite README.md, examples/README.md, tools/readme_stats.py
    gen_tables.py --check                      write nothing; exit 1 when a file is out of date
    gen_tables.py --font Fredoka.ttf           also redraw docs/stats.png and docs/card.jpg if a count moved
    gen_tables.py --adopt PATCH                write tv.json for folders that have none, from the table rows
                                               in PATCH (an old-style PR diff), then rewrite as above

Every TV-list PR used to add a row to the same two tables, so each merge made the other open list PRs
conflict. Now a list PR adds only its own folder, with a tv.json beside debloat.txt:

    {
      "tv": "Xiaomi Mi TV 4A (MiTV-MOOQ1)",
      "os": "Android TV 10",
      "brand": "Xiaomi",
      "notes": "What the list disables and keeps, one line of Markdown (links relative to examples/)"
    }

and a maintainer runs this after merging. "tv" and "os" fill both tables, "notes" the examples/README.md
one, "brand" the brand line under the README table (a brand whose every TV says "projector" goes on the
projector line). One optional key: "readme_os", the README's OS cell when it is not "os". Both tables list
the folders in the same order, alphabetical by "tv" (case ignored), so a brand's models sit together.

The tables live between <!-- tv-table:start ... --> and <!-- tv-table:end --> lines, the brand line
between <!-- tv-brands:start ... --> and <!-- tv-brands:end -->; everything outside them is hand-written.
The counts are the card's alt text ("N TV models · B brands · L languages · "), the "| **L languages** |"
feature row and STATS in tools/readme_stats.py. Exit codes: 0 up to date (or written), 1 out of date
(--check), or a count moved and --font could not draw one of the pictures (the tables are written; the
picture is not - redraw it before committing), 2 what it was given cannot be read: a tv.json, a marker, or
the path in --root or --adopt (nothing is written then).
"""
import argparse
import json
import re
import subprocess
import sys
from pathlib import Path

KEYS = ("tv", "os", "brand", "notes")
OPTIONAL = ("readme_os",)
WIDTH = 120  # the brand line wraps before this many characters
README_HEADER = ["| TV | OS |", "|---|---|"]
EXAMPLES_HEADER = ["| Folder | TV | OS | Notes |", "|---|---|---|---|"]
SOURCE = "generated by tools/gen_tables.py from examples/*/tv.json; edit those, not this"
# A Markdown link's destination, to check that the ones in a cell lead somewhere.
LINK = re.compile(r"\]\(([^)\s]+)\)")


class Problem(Exception):
    """A tv.json or a marker the script cannot work with; the message says which and why."""


def folders(root):
    return sorted(p for p in (root / "examples").iterdir() if p.is_dir())


def check_cell(where, key, value):
    if not isinstance(value, str) or not value.strip():
        return [f"{where}: \"{key}\" must be a non-empty string"]
    out = []
    if value != value.strip() or "\n" in value:
        out.append(f"{where}: \"{key}\" must be one line with no space at either end")
    if re.search(r"(?<!\\)\|", value):
        out.append(f"{where}: \"{key}\" has a | that would split the table cell; write \\| instead")
    return out


def load(root, allow_missing=False):
    """[(folder name, tv.json dict)] for every folder in examples/; raises Problem listing every bad file.

    `--adopt` writes the tv.json of a folder that has none, so it reads the checkout with allow_missing:
    a folder without one is not a problem there, while every file that is there is judged by these same
    rules before the patch is applied.
    """
    problems, entries = [], []
    for folder in folders(root):
        where = f"examples/{folder.name}/tv.json"
        path = folder / "tv.json"
        if not path.is_file():
            if allow_missing and not path.exists():
                continue
            what = "is missing" if not path.exists() else "is not a file"
            problems.append(f"{where} {what}: it holds the folder's row in the README tables "
                            f"(see CONTRIBUTING.md, step 4)")
            continue
        try:
            data = json.loads(path.read_text(encoding="utf-8"))
        except ValueError as e:
            problems.append(f"{where}: not valid JSON ({e})")
            continue
        if not isinstance(data, dict):
            problems.append(f"{where}: must be a JSON object")
            continue
        unknown = sorted(set(data) - set(KEYS) - set(OPTIONAL))
        if unknown:
            problems.append(f"{where}: unknown key(s) {unknown}; the keys are {list(KEYS + OPTIONAL)}")
        for key in KEYS:
            problems += check_cell(where, key, data.get(key))
        if "readme_os" in data:
            problems += check_cell(where, "readme_os", data["readme_os"])
        for key in ("tv", "notes"):
            value = data.get(key)
            # A value that is not a string has no link to read, and check_cell has already said it is wrong.
            for target in LINK.findall(value if isinstance(value, str) else ""):
                if re.match(r"^(https?:|mailto:|#)", target):
                    continue
                if not (root / "examples" / target.split("#")[0]).exists():
                    problems.append(f"{where}: the link {target} in \"{key}\" leads nowhere "
                                    f"(links are relative to examples/)")
        entries.append((folder.name, data))
    if problems:
        raise Problem("\n".join(problems))
    return entries


def sort_key(entry):
    """Table order: alphabetical by TV, case ignored; the folder name breaks a tie."""
    name, data = entry
    return data["tv"].casefold(), name


def ordered(entries):
    return sorted(entries, key=sort_key)


def readme_rows(entries):
    return README_HEADER + [f"| {d['tv']} | {d.get('readme_os', d['os'])} |"
                            for _, d in ordered(entries)]


def examples_rows(entries):
    return EXAMPLES_HEADER + [f"| `{n}/` | {d['tv']} | {d['os']} | {d['notes']} |"
                              for n, d in ordered(entries)]


def wrap(items):
    lines, cur = [], ""
    for item in items:
        piece = item + " ·"
        if cur and len(cur) + 1 + len(piece) > WIDTH:
            lines.append(cur)
            cur = piece
        else:
            cur = f"{cur} {piece}" if cur else piece
    return lines + ([cur] if cur else [])


def brands(entries):
    """{brand: [TV cells]}."""
    out = {}
    for _, d in entries:
        out.setdefault(d["brand"], []).append(d["tv"])
    return out


def brand_lines(entries):
    """Bold brands with their count, most lists first; single-list brands; then the projector makers."""
    by = brands(entries)
    projectors = sorted((b for b, tvs in by.items() if all("projector" in t.lower() for t in tvs)),
                        key=str.casefold)
    tv = {b: len(tvs) for b, tvs in by.items() if b not in projectors}
    bold = sorted((b for b, n in tv.items() if n > 1), key=lambda b: (-tv[b], b.casefold()))
    plain = sorted((b for b, n in tv.items() if n == 1), key=str.casefold)
    out = wrap([f"**{b}** {tv[b]}" for b in bold]) + wrap(plain)
    if projectors:
        names = ", ".join(projectors[:-1]) + " and " + projectors[-1] if len(projectors) > 1 else projectors[0]
        out.append(names + " projectors")
    return out


def languages(root):
    """English plus every translated strings.xml of the launcher."""
    return len(list((root / "launcher/app/src/main/res").glob("values-*/strings.xml"))) + 1


def between(text, name, body, path):
    """text with the lines between the `name` markers replaced by body (a list of lines), blank-padded."""
    lines = text.split("\n")
    start = [i for i, l in enumerate(lines) if l.startswith(f"<!-- {name}:start")]
    end = [i for i, l in enumerate(lines) if l.strip() == f"<!-- {name}:end -->"]
    if len(start) != 1 or len(end) != 1 or start[0] > end[0]:
        raise Problem(f"{path}: needs exactly one <!-- {name}:start ... --> line before one <!-- {name}:end --> line")
    return "\n".join(lines[:start[0] + 1] + [""] + body + [""] + lines[end[0]:])


def counts(text, entries, root):
    n, b, l = len(entries), len(brands(entries)), languages(root)
    text = re.sub(r"\b\d+ TV models · \d+ brands · \d+ languages · ",
                  f"{n} TV models · {b} brands · {l} languages · ", text)
    return re.sub(r"^\| \*\*\d+ languages\*\* \|", f"| **{l} languages** |", text, flags=re.M)


def stats(text, entries, root):
    for value, label in ((len(entries), "TV MODELS"), (len(brands(entries)), "BRANDS"),
                         (languages(root), "LANGUAGES")):
        text = re.sub(rf'\("\d+", "{label}"\)', f'("{value}", "{label}")', text)
    return text


OWNED = ("README.md", "examples/README.md", "tools/readme_stats.py")


def render(root, texts=None):
    """{relative path: the text it should have}, for the three files this script owns parts of.

    texts, if given, stands in for what those files hold now (by the same paths)."""
    entries = load(root)
    now = {p: (texts or {}).get(p) or (root / p).read_text(encoding="utf-8") for p in OWNED}
    readme = between(now["README.md"], "tv-brands", brand_lines(entries), "README.md")
    readme = between(readme, "tv-table", readme_rows(entries), "README.md")
    readme = counts(readme, entries, root)
    examples = between(now["examples/README.md"], "tv-table", examples_rows(entries), "examples/README.md")
    stats_py = stats(now["tools/readme_stats.py"], entries, root)
    return {"README.md": readme, "examples/README.md": examples, "tools/readme_stats.py": stats_py}


def stale(root):
    """The files whose text differs from what render() gives."""
    return [p for p, text in render(root).items() if (root / p).read_text(encoding="utf-8") != text]


# --adopt: an old-style PR carries its rows in README.md and examples/README.md instead of a tv.json.
EXAMPLES_ROW = re.compile(r"^\| `([A-Za-z0-9][A-Za-z0-9._-]*)/` \| (.+?) \| (.+?) \| (.+) \|$")
README_ROW = re.compile(r"^\| (.+?) \| (.+?) \|$")


def slug(s):
    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")


def guess_brand(folder, tv, known):
    """The known brand whose name the folder starts with (the longest), else the TV cell's first word."""
    hits = [b for b in known if folder == slug(b) or folder.startswith(slug(b) + "-")
            or folder.startswith(slug(b).replace("-", "") + "-")]
    if hits:
        return max(hits, key=len)
    return re.split(r"[\s,(]", tv)[0]


def write_json(path, data):
    path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")


def adopt(root, patch_text):
    """Writes examples/<folder>/tv.json from the rows an old-style PR added to the tables.

    Reads the added lines of a unified diff (or plain table rows): the examples/README.md row gives tv, os
    and notes, a README row with the same TV gives readme_os when its OS differs. A folder with no tv.json
    gets one. A folder that has one (a PR that edited its row) takes the new tv, os and notes only if the
    row the PR removed is what its tv.json still says; if main has changed the row since, nothing is
    written for it and Problem is raised once the rest are done. Returns [(folder, what happened)].

    The tv.json files already in the checkout are read through load(), so one this script cannot use is
    named in the words --check names it and Problem is raised before the first write: a patch is applied
    whole or not at all, which is what the module's exit-2 line says."""
    lines = patch_text.split("\n")
    added = [l[1:] if l.startswith("+") else l for l in lines if not l.startswith(("-", " "))]
    removed = [l[1:] for l in lines if l.startswith("-")]
    rows = {m.group(1): m.groups()[1:] for m in map(EXAMPLES_ROW.match, added) if m}
    before = {m.group(1): m.groups()[1:] for m in map(EXAMPLES_ROW.match, removed) if m}
    readme_os = {m.group(1): m.group(2) for m in map(README_ROW.match, added) if m}
    # Every file that is there, judged the way the tables are built from it. Reading them with a bare
    # json.loads made a trailing comma a JSONDecodeError traceback and a list at the root an
    # AttributeError, both exit 1 - the code --check keeps for "the tables are out of date" - and a
    # tv.json load() refuses was only found by the render step that follows, after this had written the
    # patch's new folders: the run then said nothing was written when some of it was.
    have = dict(load(root, allow_missing=True))
    known = sorted({d.get("brand", "") for d in have.values()} - {""})
    out, stale_rows = [], []
    for name, (tv, os_cell, notes) in sorted(rows.items()):
        folder = root / "examples" / name
        if not folder.is_dir():
            continue
        if name in have:
            now = tuple(have[name][k] for k in ("tv", "os", "notes"))
            if now == (tv, os_cell, notes):
                continue
            if before.get(name) != now:
                stale_rows.append(f"examples/{name}/tv.json: the row this PR edits has changed on main since; "
                                  f"move the PR's change into tv.json by hand")
                continue
            write_json(folder / "tv.json", dict(have[name], tv=tv, os=os_cell, notes=notes))
            out.append((name, "updated tv, os and notes from the PR's row"))
            continue
        brand = guess_brand(name, tv, known)
        data = {"tv": tv, "os": os_cell, "brand": brand, "notes": notes}
        if readme_os.get(tv, os_cell) != os_cell:
            data["readme_os"] = readme_os[tv]
        write_json(folder / "tv.json", data)
        out.append((name, f"new, brand {brand}" + ("" if brand in known else " (a new brand: check the spelling)")))
    if stale_rows:
        raise Problem("\n".join([f"examples/{n}/tv.json: {what}" for n, what in out] + stale_rows))
    return out


def read_inputs(root, patch):
    """(the patch's text or None, the reason this run cannot start or None).

    Nothing is read or written when this returns a reason. Exit 1 is the code main's "README tables" job
    reads as "the tables are out of date", so a path this script cannot read must not come back as 1: a
    `--root` that is not a checkout (a typo, a worktree that was removed) and an `--adopt` patch that is
    not there both ended in a FileNotFoundError traceback before, which is exit 1 - the one answer that
    says the generated files are stale when in fact nothing was read at all.
    """
    if not root.is_dir():
        return None, f"--root '{root}': not a folder. Nothing was read or written."
    # examples/ is what it reads; the three OWNED files are the parts of the checkout it writes.
    needs = ["examples"] + list(OWNED)
    missing = [f"{n}/" if n == "examples" else n for n in needs if not (root / n).exists()]
    if missing:
        return None, (f"--root '{root}': no {', no '.join(missing)} in it. Point --root at a checkout of this "
                      f"repository: nothing was read or written.")
    if patch is None:
        return None, None
    try:
        return Path(patch).read_text(encoding="utf-8"), None
    except OSError as e:
        # A path that is a directory is OSError too (IsADirectoryError), so this covers a patch folder.
        # The path is quoted so an empty --adopt ('' or a stray pair of quotes) is visible in the line.
        return None, f"--adopt: cannot read '{patch}': {e.strerror}. Nothing was written."


def redraw(root, font):
    """Draw docs/stats.png then docs/card.jpg through the two card scripts; a line saying what stopped, or None.

    This is the one place the script starts two other programs, and each of them answers a font that is not
    Fredoka's variable TTF - the Google Fonts zip renamed, a static single-weight file beside it - or a venv
    without Pillow (requirements.txt has none; docs/workflows/press.md installs requirements-perf.txt) with
    its own line and exit 1. With check=True that line was then followed by a CalledProcessError traceback
    that named the font path as a command, after the tables had already been rewritten: a run that read like
    "nothing was drawn" had changed two files, and the pictures it promised were not there.
    """
    for script, out in (("tools/readme_stats.py", "docs/stats.png"),
                        ("tools/readme_card.py", "docs/card.jpg")):
        done = subprocess.run([sys.executable, script, font, out], cwd=root)
        if done.returncode:
            return (f"{out} was not redrawn: {script} exited {done.returncode} and its line above says why. "
                    f"The tables are written; redraw the picture once that is fixed, with "
                    f"python3 {script} <Fredoka.ttf> {out}, and commit it with the tables.")
    return None


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
    ap.add_argument("--root", default=str(Path(__file__).resolve().parents[1]))
    ap.add_argument("--check", action="store_true", help="write nothing; exit 1 when a file is out of date")
    ap.add_argument("--font", help="Fredoka TTF (variable, from Google Fonts; not in this repo): redraw "
                                   "docs/stats.png and docs/card.jpg when a count changed")
    ap.add_argument("--adopt", metavar="PATCH", help="write missing tv.json files from the rows in PATCH first")
    a = ap.parse_args(argv)
    if a.check and a.adopt is not None:
        ap.error("--check and --adopt cannot be used together")
    if a.font and not Path(a.font).is_file():
        # The scripts this hands the font to exit with their own line, but subprocess.run(check=True) would
        # then turn that into a CalledProcessError traceback, which names neither the file nor where it is.
        ap.error(f"cannot open the font at {a.font}: Fredoka (variable TTF, Google Fonts) is not in this repo")
    root = Path(a.root)
    patch, problem = read_inputs(root, a.adopt)
    if problem:
        print(problem, file=sys.stderr)
        return 2
    try:
        if a.adopt:
            for name, what in adopt(root, patch):
                print(f"examples/{name}/tv.json: {what}")
        if a.check:
            out = stale(root)
            if out:
                print(f"Out of date: {', '.join(out)}. These tables are generated from examples/*/tv.json: "
                      f"a maintainer runs python3 tools/gen_tables.py after merging and commits the result.")
                return 1
            return 0
        changed = []
        for path, text in render(root).items():
            if (root / path).read_text(encoding="utf-8") != text:
                (root / path).write_text(text, encoding="utf-8")
                changed.append(path)
    except Problem as e:
        print(e, file=sys.stderr)
        return 2
    if "tools/readme_stats.py" in changed:
        if a.font:
            # The font was checked from where this runs; the scripts run in the checkout, so hand them the full path.
            font = str(Path(a.font).resolve())
            stopped = redraw(root, font)
            if stopped:
                print(stopped, file=sys.stderr)
                return 1
            changed += ["docs/stats.png", "docs/card.jpg"]
        else:
            print("A count changed: redraw docs/stats.png and docs/card.jpg (--font Fredoka.ttf, or "
                  "tools/readme_stats.py and tools/readme_card.py).")
    print("Updated: " + ", ".join(changed) if changed else "Up to date.")
    return 0


if __name__ == "__main__":
    sys.exit(main())