package cmd import ( "flag" "fmt" "os" "path/filepath" "sort" "strings" "time" sdk "code.gitea.io/sdk/gitea" "git.noodles.cam/claude-skills/marketplace/cli/internal/gitea" "git.noodles.cam/claude-skills/marketplace/cli/internal/issue" "git.noodles.cam/claude-skills/marketplace/cli/internal/mapping" "git.noodles.cam/claude-skills/marketplace/cli/internal/wire" ) func init() { register(&Command{ Name: "pull", Group: GroupSync, Args: "[…]", Short: "fetch issues from the tracker into the local store", Long: `THIS IS HOW A PUSHED ISSUE COMES BACK. ` + "`kettle push`" + ` deletes the local file the moment the tracker confirms the write, so a pull is not a refresh of a copy you kept — it is how the copy comes to exist at all. It lands under the SAME slug it had before, after a rename in the web UI and on a machine that has never seen the issue. Three sources answer "what is this issue called here", in this order: .remote.json the number -> slug ledger; the only one that knows what is on disk right now, so it wins the marker in the tracker-side body; it survives a lost ledger, a fresh clone, another machine, and a retitling the title slugified — where an issue filed in the web UI gets its first local name A marker is taken at its word only when the slug is free; a name already in use is a collision, not an identity, and is uniquified rather than allowed to overwrite somebody else's issue. The marker itself is stripped out of what lands on disk. TWO WAYS TO NAME WHAT TO PULL, and they are not the same operation: kettle pull 42 #43 owner/repo#44 by key — an ADDRESS kettle pull --milestone v0.2 by filter — a QUERY A key fetches an issue in ANY state, because a number is an address and not a question about state. Only filter mode leaves closed issues out — a closed issue is not a unit of work — and only ` + "`--state closed`" + ` puts one in the store. An issue already on disk is refreshed either way, so a local copy learns it was closed instead of staying open forever, and the count that stayed out goes to stderr. ` + "`--limit`" + ` IS ON THE WRITE, NOT ON THE SELECTION. It counts the issues this run puts in the store and never the closed ones it enumerated and threw away, so pages keep coming until the budget is full — and stop the moment it is. A filter that matches almost only closed issues ends in a warning and a short answer rather than a walk of the whole tracker. A PULL RETURNS THE UNIT OF WORK, NOT ONE ROW OF IT. ` + "`depends:`" + ` is filled from the tracker's own dependency graph and every blocker comes down with it, recursively, to --depth. What that costs, stated rather than hidden: one request per issue that lands in the store, plus one per blocker the selection did not already carry. ` + "`--no-deps`" + ` is the way back to one request, and narrows the answer to the one issue you asked for. Dependencies are outside --limit: a blocker is followed because a stored issue named it, not because the filter selected it, so a filtered pull can leave more files behind than its limit — including one from another milestone. The one blocker that does not land is a closed one. PULLING OVERWRITES THE BODY: a fetch, not a merge. Local edits you have not pushed are lost, with exactly one exception — checkbox state. A tick is monotone, so a ` + "`[x]`" + ` on either side wins for any item whose text matches; unticking is not, so untick locally and push. ` + "`--cached`" + ` skips an issue before any of that. Comments ride along: the thread lands beside the issue in .comments.md. It costs no request when the payload says there are none, and a file left over from an earlier pull is deleted — so no file means "no comments", never "not asked for". The thread is pull-only; post with ` + "`kettle comment`" + `.`, Examples: []Example{ {"kettle pull 42", "the issue and everything blocking it, in any state"}, {"kettle pull 42 --no-deps", "just that one issue — one request"}, {"kettle pull owner/repo#42", "an issue in another repository"}, {"kettle pull --milestone v0.2 --limit 20", "20 open issues from a milestone, blockers included"}, {"kettle pull --label type/bug --state all", "every bug; the closed ones are enumerated, not stored"}, }, Setup: func(fs *flag.FlagSet) func([]string) error { milestone := fs.String("milestone", "", "pull a whole milestone (id or title)") var labels stringList fs.Var(&labels, "label", "filter by label; repeat for AND") // Both spellings, because both are what somebody has in hand: `-q` is // what a person types and `--query` is what a script reads back. var query string fs.StringVar(&query, "q", "", "search text in title and body") fs.StringVar(&query, "query", "", "the long spelling of -q") state := fs.String("state", "open", "filter mode only: open, closed or all") limit := fs.Int("limit", 100, "filter mode: how many issues to STORE, not to enumerate") noDeps := fs.Bool("no-deps", false, "do not fill depends: and do not follow blockers") depth := fs.Int("depth", 3, "how deep to follow blockers") cached := fs.Bool("cached", false, "skip issues already on disk instead of refetching") out := storeFlag(fs) return func(args []string) error { filtered := *milestone != "" || len(labels) > 0 || query != "" switch { case len(args) > 0 && filtered: return Fail("pass issue keys OR filters, not both") case len(args) == 0 && !filtered: return Fail("nothing to pull: pass an issue key, or --milestone / --label / -q") case !contains([]string{"open", "closed", "all"}, *state): return Fail("--state %q must be open, closed or all", *state) case *limit < 1: return Fail("--limit must be 1 or more, got %d", *limit) case *depth < 0: return Fail("--depth must be 0 or more, got %d", *depth) } // Keys are parsed before anything is opened: a typo in a key is // not a network problem and must not be reported as one. keys, named, err := pullKeys(args) if err != nil { return err } root, client, err := syncStart(*out) if err != nil { return err } // A key may name its own repository; the project's is the // fallback, never an override. if !named.Zero() { client = client.For(named) } repo := client.Repo() // A first pull into a fresh checkout has to create the store, and // it says so — with an absolute path, so it cannot be a stray // working directory. created, err := issue.CreateStore(root) if err != nil { return err } if created { abs, _ := filepath.Abs(root) fmt.Fprintf(os.Stderr, "created store %s\n", abs) } issues, err := issue.LoadAll(root) if err != nil { return err } ledger := loadLedgerOrFold(root, issues) namer := &pullNamer{root: root, repo: repo, ledger: ledger, taken: map[string]bool{}} for id := range issues { namer.taken[id] = true } // What the ledger already knows, so a `#N` in a body resolves to // a slug this run never fetched. numberOf := map[int]string{} for raw, slug := range ledger { if k, err := wire.ParseKey(raw); err == nil && k.Repo == repo { numberOf[k.Number] = slug } } // A closed issue is not a unit of work: filter mode enumerates it // and keeps it out of the store unless the operator named the // state. A key is an address, not a bulk read, so key mode is // exempt. dropClosed := filtered && *state != "closed" queue, err := pullSeed(client, keys, filtered, gitea.IssueFilter{ State: *state, Labels: labels, Query: query, Milestone: *milestone, Limit: *limit, Keep: func(p *sdk.Issue) bool { return pullLandsInStore(p, dropClosed, namer) }, }) if err != nil { return err } var written, skipped []string var dropped []int type unresolved struct { id string numbers []int } var pending []unresolved seen := map[int]bool{} for _, t := range queue { seen[int(t.payload.Index)] = true } for len(queue) > 0 { task := queue[0] queue = queue[1:] p := task.payload // The SDK spells an issue number `Index`, and int64. It is // an int everywhere on this side of the transport — in the // ledger, in a key, in `depends:` — so it is narrowed once, // here, rather than cast at every use. number := int(p.Index) id, err := namer.idFor(p) if err != nil { return err } stored := pullStored(root, id) // Closed and not already ours: nothing is written and nothing // is asked of the server for it — not its comments, not its // links, and its blockers are not followed. The slug stays // unclaimed too, so no other issue ends up pointing // `depends:` at a file that is not there. if dropClosed && p.State == sdk.StateClosed && !stored { dropped = append(dropped, number) continue } namer.taken[id] = true numberOf[number] = id // The native links, fetched ONCE for the two things they are // for: filling this issue's `depends:` and telling the walk // where to go next. One request per issue that lands in the // store, and only one. var blockers []int if !*noDeps { if blockers, err = pullBlockers(client, number, repo); err != nil { return err } } if *cached && stored { skipped = append(skipped, id) // body and thread unread; only the links cost } else { // The copy already on disk, as it was when this run // started. It contributes its ticked checkboxes and // nothing else. local := "" if prev := issues[id]; prev != nil { local = prev.Body } next, missing := mapping.FromPayload(p, id, repo, mapping.PayloadOptions{ IDForNumber: numberOf, ExtraNumbers: blockers, // The clock is the caller's: mapping is a pure layer // and a package with a clock in it is not one. Synced: time.Now().UTC().Format(time.RFC3339), LocalBody: local, }) if _, err := issue.Save(root, next); err != nil { return err } if _, err := pullSyncComments(client, root, id, number, p.Comments); err != nil { return err } ledger.Set(wire.Key{Repo: repo, Number: number}, id) written = append(written, id) pending = append(pending, unresolved{id, missing}) } if *noDeps || task.depth >= *depth { continue } for _, n := range append(mapping.NumbersInBody(p.Body), blockers...) { if seen[n] { continue } seen[n] = true child, err := client.GetIssue(n) if err != nil { return err } queue = append(queue, pullTask{payload: child, depth: task.depth + 1}) } } // Nothing is dropped in silence. if len(dropped) > 0 { fmt.Fprintf(os.Stderr, "%d closed issue(s) enumerated, not stored"+ " (--state closed to pull them)\n", len(dropped)) } // Second pass: a `#N` that named an issue this run had not written // yet. The first write could not resolve it to a slug; by now the // file it names is on disk. for _, u := range pending { var newly []string for _, n := range u.numbers { if slug := numberOf[n]; slug != "" && slug != u.id { newly = append(newly, slug) } } if len(newly) == 0 { continue } i, err := issue.Load(root, u.id) if err != nil { return err } for _, slug := range newly { if !contains(i.Depends, slug) { i.Depends = append(i.Depends, slug) } } if _, err := issue.Save(root, i); err != nil { return err } } if err := ledger.Save(root); err != nil { return err } indexPath, _, err := issue.BuildIndex(root) if err != nil { return err } return pullReceipt(root, written, skipped, indexPath) } }, }) } // pullTask is one issue to walk, and how far from a seed it was found. type pullTask struct { payload *sdk.Issue depth int } // pullKeys parses the positional arguments and the one repository they may name. // // All of them or none: a run addresses one repository, because the client, the // ledger keys and the `gitea:` field all have to agree about which one. func pullKeys(args []string) ([]wire.Key, wire.Repo, error) { var keys []wire.Key var named wire.Repo for _, a := range args { k, err := wire.ParseKey(a) if err != nil { return nil, wire.Repo{}, err } if !k.Repo.Zero() { if !named.Zero() && named != k.Repo { return nil, wire.Repo{}, Fail("all keys must name one repository, got %s and %s", named, k.Repo) } named = k.Repo } keys = append(keys, k) } return keys, named, nil } // pullSeed is what the walk starts from: the issues a key addresses, or the ones // a filter selected. func pullSeed(c *gitea.Client, keys []wire.Key, filtered bool, f gitea.IssueFilter) ([]pullTask, error) { if !filtered { out := make([]pullTask, 0, len(keys)) for _, k := range keys { p, err := c.GetIssue(k.Number) if err != nil { return nil, err } out = append(out, pullTask{payload: p}) } return out, nil } listing, err := c.ListIssues(f) if err != nil { return nil, err } if len(listing.Issues) == 0 { return nil, Fail("no issues match that filter") } if listing.Warning != "" { fmt.Fprintf(os.Stderr, "warning: %s\n", listing.Warning) } var what []string if listing.Milestone != "" { what = append(what, "milestone "+listing.Milestone) } for _, l := range f.Labels { what = append(what, "label "+l) } if f.Query != "" { what = append(what, fmt.Sprintf("q=%q", f.Query)) } fmt.Fprintf(os.Stderr, "%d issue(s) match %s (%s)\n", len(listing.Issues), strings.Join(what, " + "), f.State) out := make([]pullTask, 0, len(listing.Issues)) for _, p := range listing.Issues { out = append(out, pullTask{payload: p}) } return out, nil } // pullLandsInStore is the --limit predicate: would this payload leave a file in // the store? // // It has to be the same test the walk applies, or the budget is spent on issues // that never land — which is the bug it exists to prevent. A closed issue counts // only when the store already has it (it is refreshed, and that is a write); // anything else counts, including one --cached will skip, because a skipped // issue is still an issue the store holds when the run ends. func pullLandsInStore(p *sdk.Issue, dropClosed bool, namer *pullNamer) bool { if !dropClosed || p.State != sdk.StateClosed { return true } id, err := namer.idFor(p) if err != nil { // An id that cannot be allocated is the walk's failure to report, not a // reason to spend the page budget differently. return true } return pullStored(namer.root, id) } // pullBlockers is the numbers of the issues that block this one, in this // repository. // // A blocker in ANOTHER repository is dropped here, and deliberately: everything // downstream — `depends:`, the number -> slug ledger, the walk's own GETs — reads // a bare number against the repository being pulled, so a foreign number would // either resolve to the wrong issue or invent an edge. The body still names it, // so nothing is lost. func pullBlockers(c *gitea.Client, number int, repo wire.Repo) ([]int, error) { keys, err := c.DependencyKeys(number) if err != nil { return nil, err } var out []int for _, k := range keys { if k.Repo == repo { out = append(out, k.Number) } } return out, nil } // pullSyncComments brings .comments.md in line with the tracker and returns // its path, or "" when the issue has no thread. // // count is the payload's own comment count, so an issue with none costs no // request. A file from an earlier pull is removed when the thread is empty: the // absence of the file is the answer, not a gap in what was asked for. func pullSyncComments(c *gitea.Client, root, id string, number, count int) (string, error) { path := commentsSidecarPath(root, id) var thread []*sdk.Comment if count > 0 { var err error if thread, err = c.ListComments(number); err != nil { return "", err } } if len(thread) > 0 { if err := os.WriteFile(path, []byte(mapping.RenderComments(thread)), 0o644); err != nil { return "", err } return path, nil } if err := os.Remove(path); err != nil && !os.IsNotExist(err) { return "", err } return "", nil } // pullReceipt is the only thing that lands in a reader's context: one line per // issue, the raw payload nowhere. func pullReceipt(root string, written, skipped []string, indexPath string) error { cached := map[string]bool{} all := map[string]bool{} for _, id := range written { all[id] = true } for _, id := range skipped { cached[id], all[id] = true, true } ids := make([]string, 0, len(all)) for id := range all { ids = append(ids, id) } sort.Strings(ids) graph := false for _, id := range ids { i, err := issue.Load(root, id) if err != nil { return err } graph = graph || len(i.Depends) > 0 note := "" if cached[id] { note = " (cached)" } if path := commentsSidecarPath(root, id); pullIsFile(path) { n := i.Extra[mapping.CommentsKey] if n == "" { n = "?" } note += fmt.Sprintf(" +%s comments: %s", n, path) } labels := strings.Join(i.Labels, ", ") if labels == "" { labels = "no labels" } fmt.Printf("%s [%s] %s — %s %s%s\n", id, labels, i.Title, i.State, issue.PathOf(root, id), note) } fmt.Printf("index: %s\n", indexPath) // Worth printing when there is something to draw, not on every run that // could have drawn something. if graph { fmt.Println("graph: run `kettle tree` (offline) to draw it") } return nil } // -------------------------------------------------------------------------- // naming, and the files the sync layer parks beside an issue // -------------------------------------------------------------------------- // pullNamer answers "what is this remote issue called here", and remembers what // it has already handed out so one run cannot name two issues the same thing. type pullNamer struct { root string repo wire.Repo ledger gitea.RemoteMap // taken is every slug the store holds plus every one this run has claimed. taken map[string]bool } // idFor is the slug this remote issue belongs under. Three sources, in order — // see the command's own documentation for why that order and not another. func (n *pullNamer) idFor(p *sdk.Issue) (string, error) { if got := n.ledger.Slug(wire.Key{Repo: n.repo, Number: int(p.Index)}); got != "" { return got, nil } marked := mapping.IDInBody(p.Body) // A marker is an identity only while the name is free. A file of that name // already in the store, or a ledger entry holding it under another number, // makes it a collision — and overwriting somebody else's issue is worse than // allocating a suffix. if marked != "" && !n.taken[marked] && !n.ledgerHolds(marked) { return marked, nil } base := marked if base == "" { base = issue.Slugify(p.Title, 0) } taken := make([]string, 0, len(n.taken)) for id := range n.taken { taken = append(taken, id) } return issue.UniqueID(n.root, base, taken) } func (n *pullNamer) ledgerHolds(slug string) bool { for _, s := range n.ledger { if s == slug { return true } } return false } // loadLedgerOrFold is the number -> slug ledger, with the `gitea:` fields still // on disk folded in when there is no ledger to read. // // A MERGE and never a replacement, which is why the fold only happens when the // file is missing or empty: push deletes the file it has just sent, so the store // is a SUBSET of what the ledger knows and a rebuild from the files alone would // throw away every entry it cannot see. What a fold cannot recover — a // pushed-and-dropped issue whose entry was also lost — is not lost either: the // next pull of that number reads the slug off the marker in the body and writes // the entry back. func loadLedgerOrFold(root string, issues map[string]*issue.Issue) gitea.RemoteMap { m := gitea.LoadRemoteMap(root) if len(m) > 0 { return m } for id, i := range issues { if k, ok := mapping.RemoteKeyOf(i); ok { m.Set(k, id) } } return m } // pullStored reports whether the store already holds this issue. func pullStored(root, id string) bool { return pullIsFile(issue.PathOf(root, id)) } func pullIsFile(path string) bool { fi, err := os.Stat(path) return err == nil && !fi.IsDir() }