#!/usr/bin/env python3 """ wiki_pull.py — fetch wiki pages into a local space. wiki_pull.py the whole wiki wiki_pull.py --prefix "Simple Chains" a page and everything under it wiki_pull.py --repo other/repo --space docs from elsewhere, into a named space The wiki is flat, so "a page and its children" is a prefix test on the title, run against one listing call. One GET per page follows. There is no tree endpoint to ask for a subtree, and no way to fetch bodies in bulk. Pulling OVERWRITES the local body — a fetch, not a merge, the same stance the issue store takes. `sha` and `synced` tell you how old your copy is; re-pull when it matters. Nothing tracks drift. Usage: wiki_pull.py [--repo owner/repo] [--prefix TITLE] [--space SPACE] [--dry-run] [--out DIR] """ import argparse import os import sys _HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, _HERE) sys.path.insert(0, os.path.join(_HERE, "..", "..", "sync", "scripts")) sys.path.insert(0, os.path.join(_HERE, "..", "..", "page", "scripts")) import _gitea # noqa: E402 import page # noqa: E402 import wikimap # noqa: E402 def main(): ap = argparse.ArgumentParser() ap.add_argument("--repo", help="owner/repo (default: the repo in CWD)") ap.add_argument("--prefix", default="", help="only titles at or under this") ap.add_argument("--space", help="local space (default: the owner/repo slug)") ap.add_argument("--dry-run", action="store_true") ap.add_argument("--out", help="wiki cache root (default: /tmp/wiki)") a = ap.parse_args() login = _gitea.require_login() base = _gitea.repo_base(a.repo) slug = _gitea.repo_slug(login, a.repo) space = a.space or slug root = a.out or page.WIKI_ROOT space_dir = page.space_root(space, root) listing = _gitea.paginate(login, "%s/wiki/pages" % base) if not isinstance(listing, list): _gitea.die("unexpected listing from %s/wiki/pages" % base) wanted = [p for p in listing if wikimap.matches_prefix(p.get("title") or "", a.prefix)] if not wanted: if a.prefix: _gitea.die("no page at or under %r in %s (%d page(s) in the wiki)" % (a.prefix, slug, len(listing))) _gitea.die("%s has no wiki pages" % slug) manifest = page.load_manifest(space, root) # Two remote titles can land on one local path — Gitea keeps them apart with # its `.-` marker, a filesystem does not. Caught before anything is written, # because the failure mode otherwise is one page silently overwriting # another and the manifest pointing both entries at the survivor. seen = {} for p in wanted: seen.setdefault(page.path_for_title(p["title"]), []).append(p["title"]) clashes = {k: v for k, v in seen.items() if len(v) > 1} for path, titles in sorted(clashes.items()): sys.stderr.write("collision: %s <- %s\n" % (path, " | ".join(titles))) created = not os.path.isdir(space_dir) n = 0 for p in sorted(wanted, key=lambda x: x.get("title") or ""): title = p["title"] relpath = page.path_for_title(title) if relpath in clashes: continue if a.dry_run: print("%-44s %s" % (relpath, title)) n += 1 continue full = _gitea.api(login, wikimap.page_endpoint(base, p["sub_url"])) if not isinstance(full, dict): _gitea.warn("could not read %r; skipped" % title) continue text = wikimap.decode(full) dest = os.path.join(space_dir, relpath) os.makedirs(os.path.dirname(dest), exist_ok=True) with open(dest, "w", encoding="utf-8") as f: f.write(text) # The prior entry is the base so `order` — a local decision the wiki # cannot hold — survives a pull. e = dict(manifest["pages"].get(relpath, {})) e.update(wikimap.from_payload(full)) e["synced"] = _gitea.now_iso() # What is on disk is now exactly what is published, so push has nothing # to do until the file is edited. e["pushed"] = page.body_hash(text) manifest["pages"][relpath] = e print("%-44s %s" % (relpath, title)) n += 1 if a.dry_run: print("\ndry run — %d page(s) would be written to %s" % (n, space_dir)) return 1 if clashes else 0 page.save_manifest(manifest, root) if created: sys.stderr.write("created space %s\n" % space_dir) print("\n%d page(s) from %s -> %s" % (n, slug, space_dir)) if clashes: sys.stderr.write("%d collision(s) skipped — rename them in the wiki\n" % len(clashes)) return 1 return 0 if __name__ == "__main__": sys.exit(main())