marketplace/skills/page/scripts/page.py

#!/usr/bin/env python3
r"""
page.py — what a PAGE TREE is. The domain layer for wiki artifacts.

Not a command; the module the other page scripts build on. It knows how a
directory of markdown becomes a named, ordered tree of pages, and it knows
NOTHING about any wiki: no Gitea, no `tea`, no logins, no HTTP, no `sub_url`.
The layering rule is mechanically checkable — every import in this directory is
stdlib, and `subprocess` is not among them:

    grep -rh '^import \|^from ' skills/page/scripts/ | sort -u

Delete skills/wiki/ entirely and this layer keeps working: a discussion's
artifacts organized into a tree on this machine are a finished thing, not a
draft waiting for an upload.

    tmp/wiki/claude-skills/tea/          <- a SPACE
      .pages.json                        <- the manifest
      Simple-Chains/
        Ideas.md                          title: Simple Chains/Ideas
        Ideas/
          Chain-core.md                   title: Simple Chains/Ideas/Chain core

A space is a directory holding a page tree and one manifest. The space's name
("claude-skills/tea") is an opaque relative path to this module — it happens to
be an owner/repo pair, and this layer never learns that.

Why a manifest at all
---------------------
Because the wiki's own page identity is not derivable from a file path, and
guessing at it is how you get duplicate pages. The manifest is the record of
what each local file IS, written once at import or pull and never re-derived.

Domain keys in a manifest entry are `title` and `order`. Everything else —
`sub_url`, `sha`, `synced`, `pushed` — is written by the wiki layer, carried
through load/save verbatim, and never read here. That passthrough is what lets
one manifest describe both a local-only tree and a published one without the
domain learning a second vocabulary.

Titles
------
The title is the identity that matters, and `/` inside it is the ONLY
hierarchy there is — the wiki this feeds has no directories. A local path is
derived from the title, never the reverse:

    title  "Simple Chains/Ideas/Chain core"
    path   "Simple-Chains/Ideas/Chain-core.md"

That direction is deliberate. Deriving a title back from a path would have to
undo `-`-for-space, and `02-chain-core` proves it cannot: the dashes there are
real. So a title is chosen ONCE, at import or at pull, and then it is a fact in
the manifest. Renaming is an explicit act, not a side effect of editing a
heading.

Ordering
--------
A leading `NN-` on a file name is sort order and nothing else — it never
reaches the title. `order 0` is special: it is the directory's own page, so
`ideas/00-intro.md` becomes the page "…/Ideas" rather than a child of it.
"""
import hashlib
import json
import os
import re

# --------------------------------------------------------------------------
# where the cache lives
# --------------------------------------------------------------------------
# `<repo root>/tmp/wiki`, absolute, resolved once at import — the same anchoring
# rule the issue store uses, and for the same reason: a script's own location is
# a fact about the installation, cwd is a fact about the last `cd`. Walking up
# from __file__ hands every script in both layers one answer no matter where it
# is invoked from.
#
# The twenty lines below are duplicated from the issue domain rather than
# imported from it. Two domains that do not know about each other is worth more
# than the duplication is worth saving: skills/page must keep working with
# skills/issue deleted, exactly as skills/issue keeps working with skills/sync
# deleted.

STORE_PARTS = ("tmp", "wiki")

# `.git` is a directory in a normal clone and a FILE in a worktree — hence
# exists(), not isdir(). AGENTS.md is the fallback for a plugin copied out of
# git; the agents-sync hook only ever puts one at a repository root.
REPO_MARKERS = (".git", "AGENTS.md")

_HERE = os.path.dirname(os.path.abspath(__file__))

MANIFEST = ".pages.json"

# Written here; read here. Everything else in an entry belongs to the wiki
# layer and is passed through untouched.
DOMAIN_KEYS = ("title", "order", "source")


def repo_root(start):
    """Nearest ancestor of `start` (inclusive) carrying a repo marker, or None."""
    d = os.path.abspath(start)
    while True:
        if any(os.path.exists(os.path.join(d, m)) for m in REPO_MARKERS):
            return d
        parent = os.path.dirname(d)
        if parent == d:
            return None
        d = parent


def store_root(start=None):
    """Absolute path of the wiki cache root.

    `start` overrides the anchor so the resolution can be exercised against a
    scratch tree. Outside a repository, cwd gets a turn, then the historical
    cwd-relative location stands — made absolute so an error can name the
    directory it really looked in."""
    for anchor in ([start] if start is not None else [_HERE, os.getcwd()]):
        root = repo_root(anchor)
        if root:
            return os.path.join(root, *STORE_PARTS)
    return os.path.abspath(os.path.join(*STORE_PARTS))


WIKI_ROOT = store_root()


def space_root(space, root=None):
    """Directory of one space. `space` is an opaque relative path — it may
    contain `/` (it usually does) and is used as typed."""
    return os.path.join(root or WIKI_ROOT, *space.split("/"))


# --------------------------------------------------------------------------
# names, titles, order
# --------------------------------------------------------------------------
# Characters a title may not carry into a path. `/` is absent on purpose: it is
# the hierarchy separator and is split on before this ever applies.
_UNSAFE = re.compile(r'[\\:*?"<>|\x00-\x1f]+')
# Inline code in a heading is markup, not a name: `Inventory — \`P-NN\`` is a
# page called "Inventory — P-NN", and a page list does not render markdown.
_MARKUP = re.compile(r"[`*_]+")
# Dropped from a PATH but kept in a title. An apostrophe in "Don't send to this
# one" belongs in the name and does not belong in something a shell has to
# quote.
_PATH_NOISE = re.compile(r"['‘’\"“”,]+")
_DASHES = re.compile(r"-{2,}")
_ORDER = re.compile(r"^(\d+)[-_. ]+(.*)$")
_HEADING = re.compile(r"^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$")


def order_of(name):
    """The `NN-` sort key on a file or directory name, or None.

    `00-intro.md` -> 0, `02-chain-core.md` -> 2, `handoff.md` -> None. Zero is
    a real answer and not None; callers distinguish them."""
    m = _ORDER.match(strip_ext(name))
    return int(m.group(1)) if m else None


def strip_ext(name):
    stem, ext = os.path.splitext(name)
    return stem if ext.lower() in (".md", ".markdown") else name


def strip_order(name):
    """`02-chain-core` -> `chain-core`; a name that is only digits is left
    alone, because stripping it would leave nothing to call the page."""
    m = _ORDER.match(strip_ext(name))
    return m.group(2) if m and m.group(2) else strip_ext(name)


def title_from_name(name):
    """Fallback title: the file or directory name made readable.

    `02-chain-core.md` -> `Chain core`. Only the first letter is raised —
    title-casing would wreck `Q-01`, `sqlc`, `APNs`, and every other name that
    already knows how it is spelled."""
    t = strip_order(name).replace("_", " ").replace("-", " ").strip()
    t = re.sub(r"\s+", " ", t)
    return t[:1].upper() + t[1:] if t else t


def title_from_body(text):
    """The document's first markdown heading, or None.

    Preferred over the file name because it is what a human wrote for a human:
    `03-q-01-do-we-know-the-chain-participant-by-name.md` opens with
    `## Q-01. Do We Know the Chain Participant by Name`, and there is no
    mechanical route from the first string to the second. Only the first
    heading is consulted, and only before any prose — a heading further down is
    a section, not a name."""
    for line in text.splitlines():
        if not line.strip():
            continue
        m = _HEADING.match(line)
        return m.group(1).strip() if m else None
    return None


def sanitize_title(title):
    """Make a string safe to be one title SEGMENT.

    `/` becomes `-`: a slash inside a heading would silently invent a level of
    hierarchy that the author did not ask for, and inventing structure is worse
    than losing a slash."""
    t = _MARKUP.sub("", _UNSAFE.sub("", title.replace("/", "-")))
    return re.sub(r"\s+", " ", t).strip(" .-") or "untitled"


def join_title(*parts):
    """Join title segments with the hierarchy separator, dropping empties."""
    return "/".join(p for p in parts if p)


def path_segment(segment):
    """One title segment as one path component."""
    s = _PATH_NOISE.sub("", _MARKUP.sub("", _UNSAFE.sub("", segment)))
    s = re.sub(r"\s+", "-", s.replace("/", "-").strip())
    return _DASHES.sub("-", s).strip("-.") or "untitled"


def path_for_title(title):
    """Relative path, inside a space, for a title. Always ends in `.md`."""
    parts = [path_segment(p) for p in title.split("/") if p.strip()]
    if not parts:
        parts = ["untitled"]
    return os.path.join(*parts) + ".md"


# --------------------------------------------------------------------------
# the manifest
# --------------------------------------------------------------------------

def blank_manifest(space):
    return {"space": space, "pages": {}}


def manifest_path(space, root=None):
    return os.path.join(space_root(space, root), MANIFEST)


def load_manifest(space, root=None):
    """The space's manifest, or a blank one.

    A missing manifest and an empty one are the same thing to every caller here
    — but they are NOT the same thing to a caller deciding whether to print
    "no such space". That distinction is `os.path.isdir(space_root(...))`, and
    the commands make it themselves rather than reading it out of a dict."""
    p = manifest_path(space, root)
    if not os.path.isfile(p):
        return blank_manifest(space)
    with open(p, encoding="utf-8") as f:
        m = json.load(f)
    m.setdefault("space", space)
    m.setdefault("pages", {})
    return m


def save_manifest(manifest, root=None):
    """Write the manifest, keys sorted, one page per line-block.

    Sorted and indented because this file lands in a diff every time anything
    syncs, and a diff nobody can read is a diff nobody checks."""
    p = manifest_path(manifest["space"], root)
    os.makedirs(os.path.dirname(p), exist_ok=True)
    ordered = {"space": manifest["space"], "pages": {}}
    for path, e in sorted(manifest.get("pages", {}).items()):
        ordered["pages"][path] = {k: e[k] for k in DOMAIN_KEYS if k in e}
        ordered["pages"][path].update(
            {k: v for k, v in sorted(e.items()) if k not in DOMAIN_KEYS})
    with open(p, "w", encoding="utf-8") as f:
        json.dump(ordered, f, ensure_ascii=False, indent=2, sort_keys=False)
        f.write("\n")
    return p


def entry(title, order=None, source=None, **extra):
    """A manifest entry. Domain keys first, passthrough after — the same
    render order the issue layer uses, for the same reason: it makes a diff of
    the file readable."""
    e = {"title": title}
    if order is not None:
        e["order"] = order
    if source is not None:
        e["source"] = source
    e.update({k: v for k, v in extra.items() if v is not None})
    return e


def find_by_source(manifest, source, prefix=""):
    """(relpath, entry) for the page imported from this source file, or
    (None, None).

    The path is derived from the title, so a retitle moves it — and looking a
    page up by its new path would find nothing, treat it as new, and publish a
    duplicate beside the page it was meant to rename. Source is the one link
    that survives a rename, which is why it is recorded at all.

    Scoped by title prefix, so importing the same directory twice under two
    prefixes gives two independent trees rather than one fighting over itself.
    """
    for path, e in manifest.get("pages", {}).items():
        if e.get("source") != source:
            continue
        if prefix and not (e.get("title", "") == prefix
                           or e.get("title", "").startswith(prefix + "/")):
            continue
        return path, e
    return None, None


def sort_key(relpath, e):
    """Order a tree for display and for an index.

    Directory by directory, `order` first and unnumbered pages after — an
    explicit `NN-` is a decision, its absence is not. Ties break on title so
    the output is stable."""
    d = os.path.dirname(relpath)
    o = e.get("order")
    return (d, 0 if o is not None else 1, o if o is not None else 0,
            e.get("title", relpath))


def sorted_pages(manifest):
    """[(relpath, entry)] in tree order."""
    return sorted(manifest.get("pages", {}).items(),
                  key=lambda kv: sort_key(kv[0], kv[1]))


def by_title(manifest):
    return {e["title"]: (p, e) for p, e in manifest.get("pages", {}).items()
            if e.get("title")}


def children_of(manifest, prefix):
    """Every page at or under a title prefix.

    The wiki this feeds is flat, so "children" is a prefix test on the title
    and nothing more — there is no tree to walk, only a naming convention to
    trust."""
    out = []
    for p, e in sorted_pages(manifest):
        t = e.get("title", "")
        if t == prefix or t.startswith(prefix + "/"):
            out.append((p, e))
    return out


def body_hash(text):
    """sha1 of the exact bytes a page would be published as.

    This is the whole of change detection: a page is worth pushing when what is
    on disk hashes differently from what was pushed last. No timestamps, no
    drift model — the same stance the issue store takes."""
    if isinstance(text, str):
        text = text.encode("utf-8")
    return hashlib.sha1(text).hexdigest()


# --------------------------------------------------------------------------
# importing a directory of markdown
# --------------------------------------------------------------------------

SKIP_DIRS = {".git", ".svn", "__pycache__", "node_modules"}
MD_EXT = (".md", ".markdown")


def walk_markdown(src):
    """Every markdown file under `src`, as paths relative to it, depth first
    and sorted so an import is reproducible."""
    out = []
    for dirpath, dirnames, filenames in os.walk(src):
        dirnames[:] = sorted(d for d in dirnames
                             if d not in SKIP_DIRS and not d.startswith("."))
        rel = os.path.relpath(dirpath, src)
        rel = "" if rel == "." else rel
        for f in sorted(filenames):
            if f.lower().endswith(MD_EXT) and not f.startswith("."):
                out.append(os.path.join(rel, f) if rel else f)
    return out


def title_for_source(relpath, text, prefix=""):
    """The title a source file gets on import.

    Three rules, in this order, and the reference doc spells out why:

    1. `order 0` (`00-intro.md`, or a literal `index`/`readme`) is the page for
       the directory it sits in. Its title comes from the DIRECTORY name, not
       from its own heading — a child's title must extend its parent's exactly,
       and `ideas/00-intro.md` opens with "Ideas for chain business
       requirements", which no child would ever be prefixed by.
    2. Any other file takes its first heading, sanitized.
    3. No heading: the file name, made readable.
    """
    parts = relpath.replace(os.sep, "/").split("/")
    name = parts[-1]
    dirs = [sanitize_title(title_from_name(d)) for d in parts[:-1]]

    stem = strip_ext(name).lower()
    if order_of(name) == 0 or stem in ("index", "readme"):
        # The directory's own page. At the root of the import that is the
        # prefix itself.
        return join_title(prefix, *dirs)

    own = title_from_body(text)
    own = sanitize_title(own) if own else sanitize_title(title_from_name(name))
    return join_title(prefix, *dirs, own)


def plan_import(src, prefix="", read=None):
    """Work out what an import would produce, without writing anything.

    Returns (pages, collisions):
      pages       [{"source", "path", "title", "order", "text"}] in tree order
      collisions  [(path, [title, title, ...])] — two sources landing on one
                  file. Reported, never resolved: the wiki would end up with
                  two pages fighting over one local copy, and picking a winner
                  for the operator is how a discussion loses a document."""
    def default_read(p):
        with open(p, encoding="utf-8") as f:
            return f.read()

    read = read or default_read
    pages, seen = [], {}
    for rel in walk_markdown(src):
        source = os.path.join(src, rel)
        text = read(source)
        title = title_for_source(rel, text, prefix)
        path = path_for_title(title)
        seen.setdefault(path, []).append(title)
        # `source` is kept relative to the import root, not absolute: it is the
        # only durable link between a file on the far side and the page it
        # became, and it has to survive the artifacts directory being moved.
        pages.append({"source": source, "rel": rel.replace(os.sep, "/"),
                      "path": path, "title": title,
                      "order": order_of(os.path.basename(rel)), "text": text})
    pages.sort(key=lambda p: sort_key(p["path"], p))
    collisions = [(p, t) for p, t in sorted(seen.items()) if len(t) > 1]
    return pages, collisions


# --------------------------------------------------------------------------
# rendering
# --------------------------------------------------------------------------

def title_tree(manifest, prefix=""):
    """Group pages into a parent -> children map keyed by title.

    Built from the titles, not from the manifest's path order. Those two
    disagree: on disk `Simple-Chains/System.md` sorts before
    `Simple-Chains/Ideas/Scale.md`, while in the hierarchy Scale is a
    grandchild of Simple Chains and System is a child. Nesting has to follow
    the titles, because the titles are the only hierarchy there is.

    A parent with no page of its own still gets a node: `Simple Chains/Parked`
    can have children while nothing is published at that title, and dropping
    its children because it is missing would hide them entirely."""
    kids, entries = {}, {}
    for _, e in manifest.get("pages", {}).items():
        title = e.get("title")
        if not title:
            continue
        if prefix and not (title == prefix or title.startswith(prefix + "/")):
            continue
        entries[title] = e
        parts = title.split("/")
        # Every ancestor gets a node, so a gap in the chain does not orphan a
        # subtree.
        for i in range(len(parts), 0, -1):
            kids.setdefault("/".join(parts[:i - 1]), set()).add("/".join(parts[:i]))
    return kids, entries


def render_index(manifest, prefix="", heading=None):
    """A table-of-contents page for a space or a subtree.

    Nested markdown list, indented by title depth. The wiki is flat and will
    not draw this for you, so the index IS the navigation.

    Links: a published page is linked by its `sub_url`, which is the only
    address Gitea guarantees. A page that has never been pushed has no sub_url
    yet, so it gets Gitea's own `[[Title]]` wiki-link syntax — which resolves
    the escaping itself, at render time, on the server. Rebuilding the index
    after a push upgrades those links to exact ones."""
    kids, entries = title_tree(manifest, prefix)
    lines = ["# %s" % (heading or prefix or "Contents"), ""]

    def order_key(title):
        e = entries.get(title) or {}
        o = e.get("order")
        return (0 if o is not None else 1, o if o is not None else 0, title)

    def walk(node, depth):
        for child in sorted(kids.get(node, ()), key=order_key):
            e = entries.get(child) or {}
            label = child.split("/")[-1]
            sub = e.get("sub_url")
            link = "[%s](%s)" % (label, sub) if sub else "[[%s|%s]]" % (child, label)
            lines.append("%s- %s" % ("  " * depth, link))
            walk(child, depth + 1)

    walk(prefix, 0)
    lines.append("")
    return "\n".join(lines)


def tree_lines(manifest, mark=None):
    """The space as an ascii tree, for a terminal.

    `mark(relpath, entry)` returns a short state tag shown after the title —
    the wiki layer passes sync state through it, and this module stays unaware
    of what the tags mean."""
    out, last_dir = [], None
    for path, e in sorted_pages(manifest):
        d = os.path.dirname(path)
        if d != last_dir:
            out.append("%s/" % d if d else ".")
            last_dir = d
        tag = mark(path, e) if mark else ""
        out.append("  %-40s %s%s" % (os.path.basename(path),
                                     e.get("title", ""),
                                     ("  " + tag) if tag else ""))
    return out