marketplace/skills/sync/scripts/pull.py

#!/usr/bin/env python3
"""
pull.py — Gitea issues -> the local store.

Writes flat markdown the domain layer owns and prints a compact index; the raw
API payload never reaches the conversation. An issue already in the store keeps
its slug even when its title changes on the server — identity is the local id,
matched through tmp/issues/.remote.json (and recoverable from the `gitea:`
fields if that file is lost).

Two ways to name what to pull:

    pull.py 42 [17 …]                  by key: 42 | #42 | owner/repo#42 | URL
    pull.py --milestone 6              by filter: whole milestone in ONE request
    pull.py --label type/bug --state all
    pull.py -q sqlc --limit 20

Filter mode costs one request per 50 issues — the list payload already carries
the bodies. Gitea silently ignores an unresolvable `milestones=` filter and
returns the whole backlog, so the milestone is resolved up front and every
issue is re-checked locally. Projects are NOT filterable: the projects API is
not exposed (404 on Gitea 1.26) — use milestones or labels, or the web UI.

A closed issue is not a unit of work, so filter mode enumerates it but leaves
it out of the store: `--state all` still shows the whole picture, and only
`--state closed` writes one. The limit is on the write, not on the selection —
an issue already on disk is refreshed either way, so the local copy learns it
was closed instead of staying open forever, and the count of the ones left out
goes to stderr. Key mode is exempt: an address is not a bulk read, and
`pull.py 1` fetches a closed issue as it always did.

Comments ride along by default, in both modes and for every issue written:
the thread lands in tmp/issues/<id>.comments.md, beside the issue. It costs
nothing when there is nothing to fetch — the payload already carries the
comment count, so an issue with none makes no request, and a file left over
from an earlier pull is deleted. An absent file therefore means "no comments",
never "not asked for". The thread is pull-only: editing it changes nothing in
Gitea (post with comment.py).

Other flags:
    --deps [--depth N]   follow dependencies and pull them too
    --cached             skip issues already on disk (body AND comments)
    --repo owner/repo    default: auto-detect from the CWD git remote

Pulling overwrites the local body: it is a fetch, not a merge. Local edits you
have not pushed are lost. Draw the graph afterwards with the domain's own
issue_tree.py — it needs no network.

Login: the operator's pin from .claude/settings.local.json (see /tea:auth).
"""
import argparse
import os
import sys

_HERE = os.path.dirname(os.path.abspath(__file__))
sys.path[:0] = [_HERE, os.path.normpath(os.path.join(_HERE, "..", "..", "issue", "scripts"))]

import _gitea  # noqa: E402
import issue  # noqa: E402
import issue_index  # noqa: E402
import map as gmap  # noqa: E402


def id_for(payload, store_ids, remote_map, repo, root):
    """Existing slug for this remote issue, or a fresh unique one. A retitled
    issue keeps the slug it was first pulled under — the map is by number."""
    got = remote_map.get(gmap.remote_key(repo, payload["number"]))
    if got:
        return got
    return issue.unique_id(root, issue.slugify(payload.get("title", "")), taken=store_ids)


def comments_path(root, id):
    """Where an issue's comment thread lives — beside it, under the same slug."""
    return os.path.join(root, "%s.comments.md" % id)


def sync_comments(login, base, root, id, number, count):
    """Bring <id>.comments.md in line with the server; return it, or None when
    the issue has no thread.

    `count` is the payload's own comment count, so an issue with none costs no
    request. A file from an earlier pull is removed when the thread is empty:
    the absence of the file is the answer, not a gap in what was asked for."""
    path = comments_path(root, id)
    comments = _gitea.get_comments(login, base, number) if count else []
    if comments:
        with open(path, "w") as f:
            f.write(gmap.render_comments(comments))
        return path
    if os.path.isfile(path):
        os.remove(path)  # stale thread from an earlier pull
    return None


def main():
    ap = argparse.ArgumentParser(description="Pull Gitea issues into the local store")
    ap.add_argument("keys", nargs="*", help="issue keys: 42, #42, owner/repo#42, URL")
    ap.add_argument("--milestone", help="pull a whole milestone (id or title)")
    ap.add_argument("--label", action="append", default=[],
                    help="filter by label; repeat for AND")
    ap.add_argument("-q", "--query", help="search text in title/body")
    ap.add_argument("--state", default="open", choices=["open", "closed", "all"],
                    help="filter mode only (default: open)")
    ap.add_argument("--limit", type=int, default=100, help="filter mode cap (default: 100)")
    ap.add_argument("--deps", action="store_true", help="follow dependencies and pull them")
    ap.add_argument("--depth", type=int, default=3, help="max dependency depth (default: 3)")
    ap.add_argument("--cached", action="store_true",
                    help="skip issues already on disk instead of refetching")
    ap.add_argument("--repo", help="owner/repo (default: auto-detect from CWD git remote)")
    ap.add_argument("--out", default=issue.ISSUE_ROOT, help="store root (default: tmp/issues)")
    args = ap.parse_args()

    filtered = bool(args.milestone or args.label or args.query)
    if args.keys and filtered:
        _gitea.die("pass issue keys OR filters, not both")
    if not args.keys and not filtered:
        _gitea.die("nothing to pull: pass issue keys, or --milestone / --label / -q")

    root = args.out
    login = _gitea.require_login()

    # ---- which repo ------------------------------------------------------
    repo_arg = args.repo
    if not repo_arg and args.keys:
        repos = {_gitea.parse_key(k)[1] for k in args.keys} - {None}
        if len(repos) > 1:
            _gitea.die("all keys must belong to one repo, got: %s" % ", ".join(sorted(repos)))
        repo_arg = repos.pop() if repos else None
    base = _gitea.repo_base(repo_arg)
    repo = _gitea.repo_slug(login, repo_arg)

    issues = issue.load_all(root)
    remote_map = _gitea.load_map(root) or _gitea.rebuild_map(root, issues)
    store_ids = set(issues)
    number_of_id = {gmap.parse_remote_key(k)[1]: v for k, v in remote_map.items()
                    if gmap.parse_remote_key(k)[0] == repo}

    # A closed issue is not a unit of work: filter mode enumerates it but keeps
    # it out of the store unless the operator named the state. A key is an
    # address, not a bulk read, so key mode is exempt.
    drop_closed = filtered and args.state != "closed"

    written, skipped, dropped, pending = [], [], [], []

    # ---- seeds -----------------------------------------------------------
    if filtered:
        payloads, ms_title = _gitea.list_issues(
            login, base, state=args.state, labels=args.label, query=args.query,
            milestone=args.milestone, limit=args.limit)
        if not payloads:
            _gitea.die("no issues match that filter")
        what = []
        if args.milestone:
            what.append("milestone %s" % ms_title)
        what += ["label %s" % l for l in args.label]
        if args.query:
            what.append("q=%r" % args.query)
        sys.stderr.write("%d issue(s) match %s (%s)\n"
                         % (len(payloads), " + ".join(what), args.state))
        queue = [(p, 0) for p in payloads]
        seen_numbers = {p["number"] for p in payloads}
    else:
        numbers = [_gitea.parse_key(k)[0] for k in args.keys]
        queue = [(_gitea.get_issue(login, base, n), 0) for n in numbers]
        seen_numbers = set(numbers)

    # ---- walk ------------------------------------------------------------
    while queue:
        payload, depth = queue.pop(0)
        number = payload["number"]
        id = id_for(payload, store_ids, remote_map, repo, root)
        stored = os.path.isfile(issue.path_of(root, id))

        # Closed and not already ours: nothing is written and nothing is asked
        # of the server for it, not even its comments. The slug stays unclaimed
        # too, so no other issue ends up pointing `depends:` at a missing file.
        if drop_closed and payload.get("state") == "closed" and not stored:
            dropped.append(number)
        else:
            store_ids.add(id)
            number_of_id[number] = id
            if args.cached and stored:
                skipped.append(id)  # untouched, and not one request spent on it
            else:
                extra = _gitea.native_deps(login, base, number) if args.deps else []
                iss, unresolved = gmap.from_api(payload, id, repo,
                                                id_for_number=number_of_id,
                                                extra_numbers=extra,
                                                synced=_gitea.now_iso())
                issue.save(root, iss)
                sync_comments(login, base, root, id, number, payload.get("comments") or 0)
                remote_map[gmap.remote_key(repo, number)] = id
                written.append(id)
                pending.append((id, unresolved))

        if args.deps and depth < args.depth:
            child_numbers = (gmap.numbers_in_body(payload.get("body") or "")
                             + _gitea.native_deps(login, base, number))
            for n in child_numbers:
                if n in seen_numbers:
                    continue
                seen_numbers.add(n)
                queue.append((_gitea.get_issue(login, base, n), depth + 1))

    # Nothing is dropped in silence — say how many closed ones stayed out.
    if dropped:
        sys.stderr.write("%d closed issue(s) enumerated, not stored"
                         " (--state closed to pull them)\n" % len(dropped))

    # ---- second pass: dependencies that were not yet known on first write --
    for id, unresolved in pending:
        newly = [number_of_id[n] for n in unresolved
                 if n in number_of_id and number_of_id[n] != id]
        if not newly:
            continue
        iss = issue.load(root, id)
        for slug in newly:
            if slug not in iss.depends:
                iss.depends.append(slug)
        issue.save(root, iss)

    _gitea.save_map(root, remote_map)
    index_path, _ = issue_index.build(root)

    # Compact output — the only thing that lands in the model's context. The
    # thread rides on the issue's own line; no file means no comments.
    for id in sorted(set(written) | set(skipped)):
        iss = issue.load(root, id)
        note = "  (cached)" if id in skipped else ""
        cpath = comments_path(root, id)
        if os.path.isfile(cpath):
            note += "  +%s comments: %s" % (iss.extra.get("comments") or "?", cpath)
        print("%s [%s] %s — %s    %s%s" % (
            id, ", ".join(iss.labels) or "no labels", iss.title, iss.state,
            issue.path_of(root, id), note))
    print("index: %s" % index_path)
    if args.deps:
        print("graph: run issue_tree.py (offline) to draw it")


if __name__ == "__main__":
    main()