From a4054de4f60bcd28da4c5303c2193fc727e5b867 Mon Sep 17 00:00:00 2001 From: rootiest Date: Sat, 25 Jul 2026 22:44:28 -0400 Subject: [PATCH] feat(docs): generate Starlight content from the manual tree Add build_manual.build_site(), which walks docs/manual and emits the Astro Starlight content collection: plain pages copied with pipeline-only frontmatter (man/site/manTitle/helpKeywords) stripped, and *-functions/ category files exploded into one page per function plus a CardGrid/LinkCard overview. Writes src/sidebar.json for astro.config.mjs to import. Wires --site alongside the existing --concat flag. Fixes two latent defects found while building the real site against the strict content.config.ts schema: - _split_entries now tracks fenced code blocks (like manualtools.shift_headings does) so a `## ` inside a fence can't be mistaken for an entry boundary. - LinkCard title/description are escaped for JSX attribute context, since shell synopses routinely contain `` angle brackets that would otherwise open unterminated MDX/JSX parsing. Also fixes the generated sidebar shape for the functions category: Starlight 0.39+ dropped support for a bare `autogenerate` sibling of `label` on a top-level group, so the autogenerate config now nests inside `items`. Verified with a full `astro build` (temporarily pointing astro.config.mjs at the generated sidebar.json, then reverted since replacing that config is a later task's deliverable): 120 pages built cleanly, no content-collection/frontmatter/MDX errors. --- docs/build-manual.py | 176 ++++++++++++++++++++++++++++++++++++++++-- docs/verify-manual.py | 21 +++++ 2 files changed, 189 insertions(+), 8 deletions(-) diff --git a/docs/build-manual.py b/docs/build-manual.py index c62d7b3..19dc7e6 100644 --- a/docs/build-manual.py +++ b/docs/build-manual.py @@ -8,6 +8,9 @@ """ import argparse +import json +import re +import shutil import sys from pathlib import Path @@ -45,21 +48,178 @@ def build_concat(root: Path) -> str: return "\n\n".join(chunks) + "\n" +SENTENCE_RE = re.compile(r"^(.+?[.!?])(\s|$)", re.S) +PIPELINE_KEYS = ("man", "site", "manTitle", "helpKeywords") +JSX_ATTR_ESCAPES = ( + ("&", "&"), + ('"', """), + ("<", "<"), + ("{", "{"), +) + + +def _jsx_attr_escape(value: str) -> str: + """Escape a string for safe use inside a quoted JSX attribute value. + + `&` must go first so escaping later characters doesn't double-escape + the ampersands it introduces. `"` closes the attribute early; `<` and + `{` are otherwise-live MDX/JSX syntax that must not be interpreted. + """ + for char, escape in JSX_ATTR_ESCAPES: + value = value.replace(char, escape) + return value + + +def _first_sentence(body: str) -> str: + """Extract a one-line description from the start of an entry body.""" + for line in body.split("\n"): + line = line.strip() + if not line or line.startswith(("#", "```", "|", "-", "*", ">")): + continue + m = SENTENCE_RE.match(line) + return (m.group(1) if m else line)[:160] + return "" + + +def _page_fm(fm: dict) -> dict: + """Strip pipeline-only keys from frontmatter destined for the site.""" + return {k: v for k, v in fm.items() if k not in PIPELINE_KEYS} + + +def _split_entries(body: str) -> tuple[str, list[tuple[str, str]]]: + """Split a category body into (intro, [(entry title, entry body)]). + + Fence-aware: an H2-looking line (`## ...`) inside a fenced code block + (tracked the same way as `manualtools.shift_headings`) is treated as + ordinary body text, not an entry boundary. + """ + lines = body.split("\n") + heading_re = re.compile(r"^## (.+)$") + boundaries: list[tuple[int, str]] = [] + in_fence = False + for i, line in enumerate(lines): + if mt.FENCE_RE.match(line): + in_fence = not in_fence + continue + if not in_fence: + m = heading_re.match(line) + if m: + boundaries.append((i, m.group(1))) + + if not boundaries: + return body.strip(), [] + + intro = "\n".join(lines[: boundaries[0][0]]).strip() + entries = [] + for idx, (line_no, title) in enumerate(boundaries): + start = line_no + 1 + end = boundaries[idx + 1][0] if idx + 1 < len(boundaries) else len(lines) + entry_body = "\n".join(lines[start:end]).strip() + entries.append((title.strip(), entry_body)) + return intro, entries + + +def build_site(root: Path, out: Path) -> list[dict]: + """Write the Starlight content tree. Returns the sidebar structure.""" + if out.exists(): + shutil.rmtree(out) + out.mkdir(parents=True) + + sidebar: list[dict] = [] + for path, _depth in mt.walk(root): + fm, body = mt.parse(path) + if not fm.get("site", True): + continue + + rel = path.relative_to(root) + is_function_dir = rel.parts and rel.parts[0].endswith("-functions") + + if not is_function_dir: + target = out / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(mt.serialize(_page_fm(fm), body)) + if rel.name != "index.md": + sidebar.append({"label": fm["title"], "link": "/" + rel.stem + "/"}) + continue + + # Section 5: category index page keeps its slot; entries explode. + slug_dir = "functions" + if rel.name == "index.md": + target = out / slug_dir / "index.md" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(mt.serialize(_page_fm(fm), body)) + sidebar.append( + { + "label": fm["title"], + "collapsed": True, + # Starlight >=0.39 rejects a bare `autogenerate` sibling + # of `label` on a top-level group (removed in v0.39.0); + # the autogenerate config must be nested inside `items`. + "items": [{"autogenerate": {"directory": slug_dir}}], + } + ) + continue + + category = re.sub(r"^\d+-", "", rel.stem) + cat_dir = out / slug_dir / category + cat_dir.mkdir(parents=True, exist_ok=True) + intro, entries = _split_entries(body) + + cards = [] + for title, entry_body in entries: + entry_slug = re.sub(r"[^\w-]+", "-", title.strip().lower()).strip("-") + desc = _first_sentence(entry_body) + entry_fm = {"title": title} + if desc: + entry_fm["description"] = desc + (cat_dir / f"{entry_slug}.md").write_text( + mt.serialize(entry_fm, entry_body) + ) + href = f"/{slug_dir}/{category}/{entry_slug}/" + safe_title = _jsx_attr_escape(title) + safe_desc = _jsx_attr_escape(desc) + cards.append( + f' " + ) + + overview = ( + "import { CardGrid, LinkCard } from '@astrojs/starlight/components';\n\n" + + (f"{intro}\n\n" if intro else "") + + "\n" + + "\n".join(cards) + + "\n\n" + ) + (cat_dir / "index.mdx").write_text(mt.serialize(_page_fm(fm), overview)) + + return sidebar + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--concat", action="store_true", help="emit the pandoc document") + ap.add_argument("--site", action="store_true", help="emit the Starlight content tree") ap.add_argument("-o", "--output", type=Path, help="write to PATH instead of stdout") args = ap.parse_args() - if not args.concat: - ap.error("nothing to do: pass --concat") + if not (args.concat or args.site): + ap.error("nothing to do: pass --concat and/or --site") - text = build_concat(MANUAL) - if args.output: - args.output.write_text(text) - print(f"wrote {args.output}") - else: - sys.stdout.write(text) + if args.site: + src = DOCS / "site" / "src" + out = src / "content" / "docs" + sidebar = build_site(MANUAL, out) + (src / "sidebar.json").write_text(json.dumps(sidebar, indent=2) + "\n") + print(f"wrote site content to {out} ({len(sidebar)} sidebar entries)") + + if args.concat: + text = build_concat(MANUAL) + if args.output: + args.output.write_text(text) + print(f"wrote {args.output}") + else: + sys.stdout.write(text) return 0 diff --git a/docs/verify-manual.py b/docs/verify-manual.py index 1339a7c..8f0f391 100644 --- a/docs/verify-manual.py +++ b/docs/verify-manual.py @@ -192,6 +192,27 @@ def test_every_index_keyword_resolves(): assert not missing, "unresolvable index keywords:\n " + "\n ".join(missing) +def test_site_build_produces_function_pages(): + import tempfile + + import build_manual + + docs = Path(__file__).parent + with tempfile.TemporaryDirectory() as d: + out = Path(d) + sidebar = build_manual.build_site(docs / "manual", out) + pages = list(out.rglob("*.md*")) + assert (out / "index.md").exists(), "landing page missing" + fn_pages = [p for p in pages if "functions" in p.parts and p.name != "index.mdx"] + assert len(fn_pages) > 80, f"expected >80 function pages, got {len(fn_pages)}" + assert not (out / "00-name.md").exists(), "site:false page was published" + assert sidebar, "sidebar structure is empty" + for page in pages: + fm, _ = mt.parse(page) + assert "manTitle" not in fm, f"{page.name} leaked manTitle into site output" + assert "title" in fm, f"{page.name} has no title" + + TESTS = [v for k, v in sorted(globals().items()) if k.startswith("test_")]