marketplace/skills/sync/scripts/pull.py

#!/usr/bin/env python3
"""
pull.py — Gitea issues -> the local store.

Writes flat markdown the domain layer owns and prints a compact index; the raw
API payload never reaches the conversation. An issue already in the store keeps
its slug even when its title changes on the server — identity is the local id,
matched through tmp/issues/.remote.json (and recoverable from the `gitea:`
fields if that file is lost).

Two ways to name what to pull:

    pull.py 42 [17 …]                  by key: 42 | #42 | owner/repo#42 | URL
    pull.py --milestone 6              by filter: whole milestone in ONE request
    pull.py --label type/bug --state all
    pull.py -q sqlc --limit 20

Filter mode costs one request per 50 issues — the list payload already carries
the bodies. Gitea silently ignores an unresolvable `milestones=` filter and
returns the whole backlog, so the milestone is resolved up front and every
issue is re-checked locally. Projects are NOT filterable: the projects API is
not exposed (404 on Gitea 1.26) — use milestones or labels, or the web UI.

Other flags:
    --deps [--depth N]   follow dependencies and pull them too
    --comments           also fetch comments (single issue only)
    --cached             skip issues already on disk instead of refetching
    --repo owner/repo    default: auto-detect from the CWD git remote

Pulling overwrites the local body: it is a fetch, not a merge. Local edits you
have not pushed are lost. Draw the graph afterwards with the domain's own
issue_tree.py — it needs no network.

Login: the operator's pin from .claude/settings.local.json (see /tea:auth).
"""
import argparse
import os
import sys

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path[:0] = [_HERE, os.path.normpath(os.path.join(_HERE, "..", "..", "issue", "scripts"))]

import _gitea  # noqa: E402
import issue  # noqa: E402
import issue_index  # noqa: E402
import map as gmap  # noqa: E402


def id_for(payload, store_ids, remote_map, repo, root):
    """Existing slug for this remote issue, or a fresh unique one. A retitled
    issue keeps the slug it was first pulled under — the map is by number."""
    got = remote_map.get(gmap.remote_key(repo, payload["number"]))
    if got:
        return got
    return issue.unique_id(root, issue.slugify(payload.get("title", "")), taken=store_ids)


def main():
    ap = argparse.ArgumentParser(description="Pull Gitea issues into the local store")
    ap.add_argument("keys", nargs="*", help="issue keys: 42, #42, owner/repo#42, URL")
    ap.add_argument("--milestone", help="pull a whole milestone (id or title)")
    ap.add_argument("--label", action="append", default=[],
                    help="filter by label; repeat for AND")
    ap.add_argument("-q", "--query", help="search text in title/body")
    ap.add_argument("--state", default="open", choices=["open", "closed", "all"],
                    help="filter mode only (default: open)")
    ap.add_argument("--limit", type=int, default=100, help="filter mode cap (default: 100)")
    ap.add_argument("--deps", action="store_true", help="follow dependencies and pull them")
    ap.add_argument("--depth", type=int, default=3, help="max dependency depth (default: 3)")
    ap.add_argument("--comments", action="store_true",
                    help="also fetch comments (single issue only)")
    ap.add_argument("--cached", action="store_true",
                    help="skip issues already on disk instead of refetching")
    ap.add_argument("--repo", help="owner/repo (default: auto-detect from CWD git remote)")
    ap.add_argument("--out", default=issue.ISSUE_ROOT, help="store root (default: tmp/issues)")
    args = ap.parse_args()

    filtered = bool(args.milestone or args.label or args.query)
    if args.keys and filtered:
        _gitea.die("pass issue keys OR filters, not both")
    if not args.keys and not filtered:
        _gitea.die("nothing to pull: pass issue keys, or --milestone / --label / -q")

    root = args.out
    login = _gitea.require_login()

    # ---- which repo ------------------------------------------------------
    repo_arg = args.repo
    if not repo_arg and args.keys:
        repos = {_gitea.parse_key(k)[1] for k in args.keys} - {None}
        if len(repos) > 1:
            _gitea.die("all keys must belong to one repo, got: %s" % ", ".join(sorted(repos)))
        repo_arg = repos.pop() if repos else None
    base = _gitea.repo_base(repo_arg)
    repo = _gitea.repo_slug(login, repo_arg)

    issues = issue.load_all(root)
    remote_map = _gitea.load_map(root) or _gitea.rebuild_map(root, issues)
    store_ids = set(issues)
    number_of_id = {gmap.parse_remote_key(k)[1]: v for k, v in remote_map.items()
                    if gmap.parse_remote_key(k)[0] == repo}

    written, skipped, pending = [], [], []

    # ---- seeds -----------------------------------------------------------
    if filtered:
        payloads, ms_title = _gitea.list_issues(
            login, base, state=args.state, labels=args.label, query=args.query,
            milestone=args.milestone, limit=args.limit)
        if not payloads:
            _gitea.die("no issues match that filter")
        what = []
        if args.milestone:
            what.append("milestone %s" % ms_title)
        what += ["label %s" % l for l in args.label]
        if args.query:
            what.append("q=%r" % args.query)
        sys.stderr.write("%d issue(s) match %s (%s)\n"
                         % (len(payloads), " + ".join(what), args.state))
        queue = [(p, 0) for p in payloads]
        seen_numbers = {p["number"] for p in payloads}
    else:
        numbers = [_gitea.parse_key(k)[0] for k in args.keys]
        queue = [(_gitea.get_issue(login, base, n), 0) for n in numbers]
        seen_numbers = set(numbers)

    if args.comments and len(queue) > 1:
        _gitea.die("--comments works on a single issue; loop over the numbers instead")

    # ---- walk ------------------------------------------------------------
    while queue:
        payload, depth = queue.pop(0)
        number = payload["number"]
        id = id_for(payload, store_ids, remote_map, repo, root)
        store_ids.add(id)
        number_of_id[number] = id

        if args.cached and os.path.isfile(issue.path_of(root, id)):
            skipped.append(id)
        else:
            extra = _gitea.native_deps(login, base, number) if args.deps else []
            iss, unresolved = gmap.from_api(payload, id, repo,
                                            id_for_number=number_of_id,
                                            extra_numbers=extra,
                                            synced=_gitea.now_iso())
            issue.save(root, iss)
            remote_map[gmap.remote_key(repo, number)] = id
            written.append(id)
            pending.append((id, unresolved))

        if args.deps and depth < args.depth:
            child_numbers = (gmap.numbers_in_body(payload.get("body") or "")
                             + _gitea.native_deps(login, base, number))
            for n in child_numbers:
                if n in seen_numbers:
                    continue
                seen_numbers.add(n)
                queue.append((_gitea.get_issue(login, base, n), depth + 1))

    # ---- second pass: dependencies that were not yet known on first write --
    for id, unresolved in pending:
        newly = [number_of_id[n] for n in unresolved
                 if n in number_of_id and number_of_id[n] != id]
        if not newly:
            continue
        iss = issue.load(root, id)
        for slug in newly:
            if slug not in iss.depends:
                iss.depends.append(slug)
        issue.save(root, iss)

    cpath = None
    if args.comments:
        id = written[0] if written else skipped[0]
        _repo, number = gmap.parse_remote_key(issue.load(root, id).extra.get("gitea", ""))
        comments = _gitea.get_comments(login, base, number)
        cpath = os.path.join(root, "%s.comments.md" % id)
        if comments:
            with open(cpath, "w") as f:
                f.write(gmap.render_comments(comments))
        else:
            if os.path.isfile(cpath):
                os.remove(cpath)  # stale file from an earlier pull
            cpath = None

    _gitea.save_map(root, remote_map)
    index_path, _ = issue_index.build(root)

    # Compact output — the only thing that lands in the model's context.
    for id in sorted(set(written) | set(skipped)):
        iss = issue.load(root, id)
        print("%s [%s] %s — %s    %s%s" % (
            id, ", ".join(iss.labels) or "no labels", iss.title, iss.state,
            issue.path_of(root, id), "  (cached)" if id in skipped else ""))
    if cpath:
        print("comments: %s" % cpath)
    print("index: %s" % index_path)
    if args.deps:
        print("graph: run issue_tree.py (offline) to draw it")


if __name__ == "__main__":
    main()