marketplace/cli/internal/cmd/pull.go

package cmd

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	sdk "code.gitea.io/sdk/gitea"

	"git.noodles.cam/claude-skills/marketplace/cli/internal/gitea"
	"git.noodles.cam/claude-skills/marketplace/cli/internal/issue"
	"git.noodles.cam/claude-skills/marketplace/cli/internal/mapping"
	"git.noodles.cam/claude-skills/marketplace/cli/internal/wire"
)

func init() {
	register(&Command{
		Name:  "pull",
		Group: GroupSync,
		Args:  "[<key>…]",
		Short: "fetch issues from the tracker into the local store",
		Long: `THIS IS HOW A PUSHED ISSUE COMES BACK. ` + "`kettle push`" + ` deletes the local file the
moment the tracker confirms the write, so a pull is not a refresh of a copy you
kept — it is how the copy comes to exist at all.

It lands under the SAME slug it had before, after a rename in the web UI and on
a machine that has never seen the issue. Three sources answer "what is this
issue called here", in this order:

    .remote.json        the number -> slug ledger; the only one that knows what
                        is on disk right now, so it wins
    <!-- kettle:id … -->  the marker in the tracker-side body; it survives a lost
                        ledger, a fresh clone, another machine, and a retitling
    the title           slugified — where an issue filed in the web UI gets its
                        first local name

A marker is taken at its word only when the slug is free; a name already in use
is a collision, not an identity, and is uniquified rather than allowed to
overwrite somebody else's issue. The marker itself is stripped out of what lands
on disk.

TWO WAYS TO NAME WHAT TO PULL, and they are not the same operation:

    kettle pull 42 #43 owner/repo#44     by key — an ADDRESS
    kettle pull --milestone v0.2         by filter — a QUERY

A key fetches an issue in ANY state, because a number is an address and not a
question about state. Only filter mode leaves closed issues out — a closed issue
is not a unit of work — and only ` + "`--state closed`" + ` puts one in the store. An issue
already on disk is refreshed either way, so a local copy learns it was closed
instead of staying open forever, and the count that stayed out goes to stderr.

` + "`--limit`" + ` IS ON THE WRITE, NOT ON THE SELECTION. It counts the issues this run puts
in the store and never the closed ones it enumerated and threw away, so pages
keep coming until the budget is full — and stop the moment it is. A filter that
matches almost only closed issues ends in a warning and a short answer rather
than a walk of the whole tracker.

A PULL RETURNS THE UNIT OF WORK, NOT ONE ROW OF IT. ` + "`depends:`" + ` is filled from the
tracker's own dependency graph and every blocker comes down with it, recursively,
to --depth. What that costs, stated rather than hidden: one request per issue
that lands in the store, plus one per blocker the selection did not already
carry. ` + "`--no-deps`" + ` is the way back to one request, and narrows the answer to the
one issue you asked for. Dependencies are outside --limit: a blocker is followed
because a stored issue named it, not because the filter selected it, so a
filtered pull can leave more files behind than its limit — including one from
another milestone. The one blocker that does not land is a closed one.

PULLING OVERWRITES THE BODY: a fetch, not a merge. Local edits you have not
pushed are lost, with exactly one exception — checkbox state. A tick is monotone,
so a ` + "`[x]`" + ` on either side wins for any item whose text matches; unticking is not,
so untick locally and push. ` + "`--cached`" + ` skips an issue before any of that.

Comments ride along: the thread lands beside the issue in <id>.comments.md. It
costs no request when the payload says there are none, and a file left over from
an earlier pull is deleted — so no file means "no comments", never "not asked
for". The thread is pull-only; post with ` + "`kettle comment`" + `.`,
		Examples: []Example{
			{"kettle pull 42", "the issue and everything blocking it, in any state"},
			{"kettle pull 42 --no-deps", "just that one issue — one request"},
			{"kettle pull owner/repo#42", "an issue in another repository"},
			{"kettle pull --milestone v0.2 --limit 20", "20 open issues from a milestone, blockers included"},
			{"kettle pull --label type/bug --state all", "every bug; the closed ones are enumerated, not stored"},
		},
		Setup: func(fs *flag.FlagSet) func([]string) error {
			milestone := fs.String("milestone", "", "pull a whole milestone (id or title)")
			var labels stringList
			fs.Var(&labels, "label", "filter by label; repeat for AND")
			// Both spellings, because both are what somebody has in hand: `-q` is
			// what a person types and `--query` is what a script reads back.
			var query string
			fs.StringVar(&query, "q", "", "search text in title and body")
			fs.StringVar(&query, "query", "", "the long spelling of -q")
			state := fs.String("state", "open", "filter mode only: open, closed or all")
			limit := fs.Int("limit", 100, "filter mode: how many issues to STORE, not to enumerate")
			noDeps := fs.Bool("no-deps", false, "do not fill depends: and do not follow blockers")
			depth := fs.Int("depth", 3, "how deep to follow blockers")
			cached := fs.Bool("cached", false, "skip issues already on disk instead of refetching")
			out := storeFlag(fs)

			return func(args []string) error {
				filtered := *milestone != "" || len(labels) > 0 || query != ""
				switch {
				case len(args) > 0 && filtered:
					return Fail("pass issue keys OR filters, not both")
				case len(args) == 0 && !filtered:
					return Fail("nothing to pull: pass an issue key, or --milestone / --label / -q")
				case !contains([]string{"open", "closed", "all"}, *state):
					return Fail("--state %q must be open, closed or all", *state)
				case *limit < 1:
					return Fail("--limit must be 1 or more, got %d", *limit)
				case *depth < 0:
					return Fail("--depth must be 0 or more, got %d", *depth)
				}

				// Keys are parsed before anything is opened: a typo in a key is
				// not a network problem and must not be reported as one.
				keys, named, err := pullKeys(args)
				if err != nil {
					return err
				}

				root, client, err := syncStart(*out)
				if err != nil {
					return err
				}
				// A key may name its own repository; the project's is the
				// fallback, never an override.
				if !named.Zero() {
					client = client.For(named)
				}
				repo := client.Repo()

				// A first pull into a fresh checkout has to create the store, and
				// it says so — with an absolute path, so it cannot be a stray
				// working directory.
				created, err := issue.CreateStore(root)
				if err != nil {
					return err
				}
				if created {
					abs, _ := filepath.Abs(root)
					fmt.Fprintf(os.Stderr, "created store %s\n", abs)
				}

				issues, err := issue.LoadAll(root)
				if err != nil {
					return err
				}
				ledger := loadLedgerOrFold(root, issues)
				namer := &pullNamer{root: root, repo: repo, ledger: ledger, taken: map[string]bool{}}
				for id := range issues {
					namer.taken[id] = true
				}
				// What the ledger already knows, so a `#N` in a body resolves to
				// a slug this run never fetched.
				numberOf := map[int]string{}
				for raw, slug := range ledger {
					if k, err := wire.ParseKey(raw); err == nil && k.Repo == repo {
						numberOf[k.Number] = slug
					}
				}

				// A closed issue is not a unit of work: filter mode enumerates it
				// and keeps it out of the store unless the operator named the
				// state. A key is an address, not a bulk read, so key mode is
				// exempt.
				dropClosed := filtered && *state != "closed"

				queue, err := pullSeed(client, keys, filtered, gitea.IssueFilter{
					State: *state, Labels: labels, Query: query, Milestone: *milestone,
					Limit: *limit,
					Keep: func(p *sdk.Issue) bool {
						return pullLandsInStore(p, dropClosed, namer)
					},
				})
				if err != nil {
					return err
				}

				var written, skipped []string
				var dropped []int
				type unresolved struct {
					id      string
					numbers []int
				}
				var pending []unresolved

				seen := map[int]bool{}
				for _, t := range queue {
					seen[int(t.payload.Index)] = true
				}

				for len(queue) > 0 {
					task := queue[0]
					queue = queue[1:]
					p := task.payload
					// The SDK spells an issue number `Index`, and int64. It is
					// an int everywhere on this side of the transport — in the
					// ledger, in a key, in `depends:` — so it is narrowed once,
					// here, rather than cast at every use.
					number := int(p.Index)
					id, err := namer.idFor(p)
					if err != nil {
						return err
					}
					stored := pullStored(root, id)

					// Closed and not already ours: nothing is written and nothing
					// is asked of the server for it — not its comments, not its
					// links, and its blockers are not followed. The slug stays
					// unclaimed too, so no other issue ends up pointing
					// `depends:` at a file that is not there.
					if dropClosed && p.State == sdk.StateClosed && !stored {
						dropped = append(dropped, number)
						continue
					}

					namer.taken[id] = true
					numberOf[number] = id

					// The native links, fetched ONCE for the two things they are
					// for: filling this issue's `depends:` and telling the walk
					// where to go next. One request per issue that lands in the
					// store, and only one.
					var blockers []int
					if !*noDeps {
						if blockers, err = pullBlockers(client, number, repo); err != nil {
							return err
						}
					}

					if *cached && stored {
						skipped = append(skipped, id) // body and thread unread; only the links cost
					} else {
						// The copy already on disk, as it was when this run
						// started. It contributes its ticked checkboxes and
						// nothing else.
						local := ""
						if prev := issues[id]; prev != nil {
							local = prev.Body
						}
						next, missing := mapping.FromPayload(p, id, repo, mapping.PayloadOptions{
							IDForNumber:  numberOf,
							ExtraNumbers: blockers,
							// The clock is the caller's: mapping is a pure layer
							// and a package with a clock in it is not one.
							Synced:    time.Now().UTC().Format(time.RFC3339),
							LocalBody: local,
						})
						if _, err := issue.Save(root, next); err != nil {
							return err
						}
						if _, err := pullSyncComments(client, root, id, number, p.Comments); err != nil {
							return err
						}
						ledger.Set(wire.Key{Repo: repo, Number: number}, id)
						written = append(written, id)
						pending = append(pending, unresolved{id, missing})
					}

					if *noDeps || task.depth >= *depth {
						continue
					}
					for _, n := range append(mapping.NumbersInBody(p.Body), blockers...) {
						if seen[n] {
							continue
						}
						seen[n] = true
						child, err := client.GetIssue(n)
						if err != nil {
							return err
						}
						queue = append(queue, pullTask{payload: child, depth: task.depth + 1})
					}
				}

				// Nothing is dropped in silence.
				if len(dropped) > 0 {
					fmt.Fprintf(os.Stderr, "%d closed issue(s) enumerated, not stored"+
						" (--state closed to pull them)\n", len(dropped))
				}

				// Second pass: a `#N` that named an issue this run had not written
				// yet. The first write could not resolve it to a slug; by now the
				// file it names is on disk.
				for _, u := range pending {
					var newly []string
					for _, n := range u.numbers {
						if slug := numberOf[n]; slug != "" && slug != u.id {
							newly = append(newly, slug)
						}
					}
					if len(newly) == 0 {
						continue
					}
					i, err := issue.Load(root, u.id)
					if err != nil {
						return err
					}
					for _, slug := range newly {
						if !contains(i.Depends, slug) {
							i.Depends = append(i.Depends, slug)
						}
					}
					if _, err := issue.Save(root, i); err != nil {
						return err
					}
				}

				if err := ledger.Save(root); err != nil {
					return err
				}
				indexPath, _, err := issue.BuildIndex(root)
				if err != nil {
					return err
				}
				return pullReceipt(root, written, skipped, indexPath)
			}
		},
	})
}

// pullTask is one issue to walk, and how far from a seed it was found.
type pullTask struct {
	payload *sdk.Issue
	depth   int
}

// pullKeys parses the positional arguments and the one repository they may name.
//
// All of them or none: a run addresses one repository, because the client, the
// ledger keys and the `gitea:` field all have to agree about which one.
func pullKeys(args []string) ([]wire.Key, wire.Repo, error) {
	var keys []wire.Key
	var named wire.Repo
	for _, a := range args {
		k, err := wire.ParseKey(a)
		if err != nil {
			return nil, wire.Repo{}, err
		}
		if !k.Repo.Zero() {
			if !named.Zero() && named != k.Repo {
				return nil, wire.Repo{}, Fail("all keys must name one repository, got %s and %s", named, k.Repo)
			}
			named = k.Repo
		}
		keys = append(keys, k)
	}
	return keys, named, nil
}

// pullSeed is what the walk starts from: the issues a key addresses, or the ones
// a filter selected.
func pullSeed(c *gitea.Client, keys []wire.Key, filtered bool, f gitea.IssueFilter) ([]pullTask, error) {
	if !filtered {
		out := make([]pullTask, 0, len(keys))
		for _, k := range keys {
			p, err := c.GetIssue(k.Number)
			if err != nil {
				return nil, err
			}
			out = append(out, pullTask{payload: p})
		}
		return out, nil
	}

	listing, err := c.ListIssues(f)
	if err != nil {
		return nil, err
	}
	if len(listing.Issues) == 0 {
		return nil, Fail("no issues match that filter")
	}
	if listing.Warning != "" {
		fmt.Fprintf(os.Stderr, "warning: %s\n", listing.Warning)
	}

	var what []string
	if listing.Milestone != "" {
		what = append(what, "milestone "+listing.Milestone)
	}
	for _, l := range f.Labels {
		what = append(what, "label "+l)
	}
	if f.Query != "" {
		what = append(what, fmt.Sprintf("q=%q", f.Query))
	}
	fmt.Fprintf(os.Stderr, "%d issue(s) match %s (%s)\n",
		len(listing.Issues), strings.Join(what, " + "), f.State)

	out := make([]pullTask, 0, len(listing.Issues))
	for _, p := range listing.Issues {
		out = append(out, pullTask{payload: p})
	}
	return out, nil
}

// pullLandsInStore is the --limit predicate: would this payload leave a file in
// the store?
//
// It has to be the same test the walk applies, or the budget is spent on issues
// that never land — which is the bug it exists to prevent. A closed issue counts
// only when the store already has it (it is refreshed, and that is a write);
// anything else counts, including one --cached will skip, because a skipped
// issue is still an issue the store holds when the run ends.
func pullLandsInStore(p *sdk.Issue, dropClosed bool, namer *pullNamer) bool {
	if !dropClosed || p.State != sdk.StateClosed {
		return true
	}
	id, err := namer.idFor(p)
	if err != nil {
		// An id that cannot be allocated is the walk's failure to report, not a
		// reason to spend the page budget differently.
		return true
	}
	return pullStored(namer.root, id)
}

// pullBlockers is the numbers of the issues that block this one, in this
// repository.
//
// A blocker in ANOTHER repository is dropped here, and deliberately: everything
// downstream — `depends:`, the number -> slug ledger, the walk's own GETs — reads
// a bare number against the repository being pulled, so a foreign number would
// either resolve to the wrong issue or invent an edge. The body still names it,
// so nothing is lost.
func pullBlockers(c *gitea.Client, number int, repo wire.Repo) ([]int, error) {
	keys, err := c.DependencyKeys(number)
	if err != nil {
		return nil, err
	}
	var out []int
	for _, k := range keys {
		if k.Repo == repo {
			out = append(out, k.Number)
		}
	}
	return out, nil
}

// pullSyncComments brings <id>.comments.md in line with the tracker and returns
// its path, or "" when the issue has no thread.
//
// count is the payload's own comment count, so an issue with none costs no
// request. A file from an earlier pull is removed when the thread is empty: the
// absence of the file is the answer, not a gap in what was asked for.
func pullSyncComments(c *gitea.Client, root, id string, number, count int) (string, error) {
	path := commentsSidecarPath(root, id)
	var thread []*sdk.Comment
	if count > 0 {
		var err error
		if thread, err = c.ListComments(number); err != nil {
			return "", err
		}
	}
	if len(thread) > 0 {
		if err := os.WriteFile(path, []byte(mapping.RenderComments(thread)), 0o644); err != nil {
			return "", err
		}
		return path, nil
	}
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		return "", err
	}
	return "", nil
}

// pullReceipt is the only thing that lands in a reader's context: one line per
// issue, the raw payload nowhere.
func pullReceipt(root string, written, skipped []string, indexPath string) error {
	cached := map[string]bool{}
	all := map[string]bool{}
	for _, id := range written {
		all[id] = true
	}
	for _, id := range skipped {
		cached[id], all[id] = true, true
	}
	ids := make([]string, 0, len(all))
	for id := range all {
		ids = append(ids, id)
	}
	sort.Strings(ids)

	graph := false
	for _, id := range ids {
		i, err := issue.Load(root, id)
		if err != nil {
			return err
		}
		graph = graph || len(i.Depends) > 0
		note := ""
		if cached[id] {
			note = "  (cached)"
		}
		if path := commentsSidecarPath(root, id); pullIsFile(path) {
			n := i.Extra[mapping.CommentsKey]
			if n == "" {
				n = "?"
			}
			note += fmt.Sprintf("  +%s comments: %s", n, path)
		}
		labels := strings.Join(i.Labels, ", ")
		if labels == "" {
			labels = "no labels"
		}
		fmt.Printf("%s [%s] %s — %s    %s%s\n",
			id, labels, i.Title, i.State, issue.PathOf(root, id), note)
	}
	fmt.Printf("index: %s\n", indexPath)
	// Worth printing when there is something to draw, not on every run that
	// could have drawn something.
	if graph {
		fmt.Println("graph: run `kettle tree` (offline) to draw it")
	}
	return nil
}

// --------------------------------------------------------------------------
// naming, and the files the sync layer parks beside an issue
// --------------------------------------------------------------------------

// pullNamer answers "what is this remote issue called here", and remembers what
// it has already handed out so one run cannot name two issues the same thing.
type pullNamer struct {
	root   string
	repo   wire.Repo
	ledger gitea.RemoteMap
	// taken is every slug the store holds plus every one this run has claimed.
	taken map[string]bool
}

// idFor is the slug this remote issue belongs under. Three sources, in order —
// see the command's own documentation for why that order and not another.
func (n *pullNamer) idFor(p *sdk.Issue) (string, error) {
	if got := n.ledger.Slug(wire.Key{Repo: n.repo, Number: int(p.Index)}); got != "" {
		return got, nil
	}
	marked := mapping.IDInBody(p.Body)
	// A marker is an identity only while the name is free. A file of that name
	// already in the store, or a ledger entry holding it under another number,
	// makes it a collision — and overwriting somebody else's issue is worse than
	// allocating a suffix.
	if marked != "" && !n.taken[marked] && !n.ledgerHolds(marked) {
		return marked, nil
	}
	base := marked
	if base == "" {
		base = issue.Slugify(p.Title, 0)
	}
	taken := make([]string, 0, len(n.taken))
	for id := range n.taken {
		taken = append(taken, id)
	}
	return issue.UniqueID(n.root, base, taken)
}

func (n *pullNamer) ledgerHolds(slug string) bool {
	for _, s := range n.ledger {
		if s == slug {
			return true
		}
	}
	return false
}

// loadLedgerOrFold is the number -> slug ledger, with the `gitea:` fields still
// on disk folded in when there is no ledger to read.
//
// A MERGE and never a replacement, which is why the fold only happens when the
// file is missing or empty: push deletes the file it has just sent, so the store
// is a SUBSET of what the ledger knows and a rebuild from the files alone would
// throw away every entry it cannot see. What a fold cannot recover — a
// pushed-and-dropped issue whose entry was also lost — is not lost either: the
// next pull of that number reads the slug off the marker in the body and writes
// the entry back.
func loadLedgerOrFold(root string, issues map[string]*issue.Issue) gitea.RemoteMap {
	m := gitea.LoadRemoteMap(root)
	if len(m) > 0 {
		return m
	}
	for id, i := range issues {
		if k, ok := mapping.RemoteKeyOf(i); ok {
			m.Set(k, id)
		}
	}
	return m
}

// pullStored reports whether the store already holds this issue.
func pullStored(root, id string) bool { return pullIsFile(issue.PathOf(root, id)) }

func pullIsFile(path string) bool {
	fi, err := os.Stat(path)
	return err == nil && !fi.IsDir()
}