Merge remote-tracking branch 'origin/main' into fix/worktree-login-pin
This commit is contained in:
naudachu
@@ -38,6 +38,13 @@ import urllib.parse
|
||||
|
||||
REMOTE_MAP = ".remote.json"
|
||||
|
||||
# How far past the ideal page count a `keep`-bounded listing may scan before it
|
||||
# gives up (see list_issues). The ideal is what `limit` would need if every
|
||||
# payload counted; the slack pays for the ones that do not. It is a bound on
|
||||
# requests, deliberately small: "fetch until N are kept" without one is "fetch
|
||||
# the whole tracker" on any repo whose filter matches mostly closed issues.
|
||||
PAGE_SLACK = 4
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# where request bodies land
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -174,17 +181,28 @@ def api(login, endpoint, method="GET", payload=None, payload_name=None,
|
||||
die("`tea api %s` returned non-JSON:\n%s" % (endpoint, body[:500]))
|
||||
|
||||
|
||||
def paginate(login, endpoint, limit=50, max_pages=40, **kw):
|
||||
"""GET a list endpoint page by page; return the concatenated list."""
|
||||
def pages(login, endpoint, limit=50, max_pages=40, **kw):
|
||||
"""GET a list endpoint page by page, yielding each page as it arrives.
|
||||
|
||||
A generator, because a caller whose budget is spent on what it *keeps*
|
||||
cannot be served by a function that fetches everything first: the page after
|
||||
the one that completed the budget must never be requested. Stop consuming
|
||||
and no further request is made."""
|
||||
sep = "&" if "?" in endpoint else "?"
|
||||
out = []
|
||||
for page in range(1, max_pages + 1):
|
||||
batch = api(login, "%s%spage=%d&limit=%d" % (endpoint, sep, page, limit), **kw)
|
||||
if not isinstance(batch, list) or not batch:
|
||||
break
|
||||
out.extend(batch)
|
||||
return
|
||||
yield batch
|
||||
if len(batch) < limit:
|
||||
break
|
||||
return # a short page is the last one
|
||||
|
||||
|
||||
def paginate(login, endpoint, limit=50, max_pages=40, **kw):
|
||||
"""GET a list endpoint page by page; return the concatenated list."""
|
||||
out = []
|
||||
for batch in pages(login, endpoint, limit=limit, max_pages=max_pages, **kw):
|
||||
out.extend(batch)
|
||||
return out
|
||||
|
||||
|
||||
@@ -247,11 +265,31 @@ def matches(payload, milestone_id=None, labels=()):
|
||||
|
||||
|
||||
def list_issues(login, base, state="open", labels=(), query=None,
|
||||
milestone=None, limit=100):
|
||||
milestone=None, limit=100, keep=None):
|
||||
"""Filtered issue payloads. Returns (payloads, milestone_title).
|
||||
|
||||
One request per page, and the payload already carries the issue bodies — a
|
||||
whole milestone costs one call per 50 issues, not one per issue."""
|
||||
whole milestone costs one call per 50 issues, not one per issue.
|
||||
|
||||
`limit` counts the payloads the CALLER cares about, not the ones the server
|
||||
returned. Without `keep` those are the same thing and this behaves as it
|
||||
always did. With it, `keep(payload)` says whether a payload counts, pages
|
||||
keep coming until `limit` of them have, and the returned list carries the
|
||||
ones that did not count too — they were enumerated, and a caller that has
|
||||
something to say about them (pull.py: "N closed, not stored") still can.
|
||||
|
||||
What `keep` means is the caller's business; this module only counts. Two
|
||||
boundaries hold whatever it decides:
|
||||
|
||||
- **Stop at the limit.** The page after the one that completed the budget
|
||||
is not requested — `pages` is a generator and this loop returns out of it.
|
||||
- **Stop at the page budget.** A predicate that rejects everything must not
|
||||
turn a bounded read into a walk of the whole tracker, so a filtered read
|
||||
may scan at most `PAGE_SLACK` times the pages `limit` would need if every
|
||||
payload counted. Hitting that with an unfilled budget is a warning, not a
|
||||
silent short answer: the caller asked for N and is told it got fewer."""
|
||||
if limit < 1:
|
||||
die("--limit must be 1 or more, got %d" % limit)
|
||||
ms_id, ms_title = (None, None)
|
||||
if milestone is not None:
|
||||
ms_id, ms_title = resolve_milestone(login, base, milestone)
|
||||
@@ -266,10 +304,25 @@ def list_issues(login, base, state="open", labels=(), query=None,
|
||||
endpoint = "%s/issues?%s" % (base, urllib.parse.urlencode(params))
|
||||
|
||||
per_page = min(limit, 50)
|
||||
got = paginate(login, endpoint, limit=per_page,
|
||||
max_pages=max(1, -(-limit // per_page)))
|
||||
got = [p for p in got if matches(p, ms_id, labels)]
|
||||
return got[:limit], ms_title
|
||||
ideal = max(1, -(-limit // per_page))
|
||||
budget = ideal if keep is None else ideal * PAGE_SLACK
|
||||
|
||||
got, kept, seen_pages, last_full = [], 0, 0, False
|
||||
for batch in pages(login, endpoint, limit=per_page, max_pages=budget):
|
||||
seen_pages += 1
|
||||
last_full = len(batch) == per_page
|
||||
for p in batch:
|
||||
if not matches(p, ms_id, labels):
|
||||
continue
|
||||
got.append(p)
|
||||
if keep is None or keep(p):
|
||||
kept += 1
|
||||
if kept >= limit:
|
||||
return got, ms_title
|
||||
if keep is not None and seen_pages >= budget and last_full:
|
||||
warn("scanned %d page(s) and stopped %d short of --limit %d — there may"
|
||||
" be more; narrow the filter or raise --limit" % (budget, limit - kept, limit))
|
||||
return got, ms_title
|
||||
|
||||
|
||||
def get_issue(login, base, number):
|
||||
|
||||
+131
-35
@@ -28,11 +28,31 @@ not exposed (404 on Gitea 1.26) — use milestones or labels, or the web UI.
|
||||
|
||||
A closed issue is not a unit of work, so filter mode enumerates it but leaves
|
||||
it out of the store: `--state all` still shows the whole picture, and only
|
||||
`--state closed` writes one. The limit is on the write, not on the selection —
|
||||
an issue already on disk is refreshed either way, so the local copy learns it
|
||||
was closed instead of staying open forever, and the count of the ones left out
|
||||
goes to stderr. Key mode is exempt: an address is not a bulk read, and
|
||||
`pull.py 1` fetches a closed issue as it always did.
|
||||
`--state closed` writes one. An issue already on disk is refreshed either way,
|
||||
so the local copy learns it was closed instead of staying open forever, and the
|
||||
count of the ones left out goes to stderr. Key mode is exempt: an address is not
|
||||
a bulk read, and `pull.py 1` fetches a closed issue as it always did.
|
||||
|
||||
**`--limit` is on the write, not on the selection.** It counts the issues this
|
||||
run puts in the store — written, or left in place by `--cached` — and never the
|
||||
closed ones it enumerated and threw away. `--limit 20` over a milestone whose
|
||||
first 30 issues are closed still writes 20, if 20 open ones are there to write:
|
||||
pages keep coming until the budget is full. Two boundaries keep that honest:
|
||||
|
||||
- Pages stop the moment the budget is full. Never one page more.
|
||||
- A filtered read may scan at most `_gitea.PAGE_SLACK` times the pages the limit
|
||||
would need if nothing were dropped. A filter that matches almost only closed
|
||||
issues therefore ends in a warning and a short answer, not in a walk of the
|
||||
whole tracker. Narrow the filter, or raise `--limit`, which raises the budget
|
||||
with it.
|
||||
- Dependencies are outside the count: a blocker is followed because a stored
|
||||
issue named it, not because the filter selected it. `--limit 20` can
|
||||
therefore leave more than 20 files behind — the budget counts the selection's
|
||||
writes, and the graph is not part of the selection.
|
||||
|
||||
`remote.py` is the deliberate exception, and it is not the same flag twice: it
|
||||
writes nothing at all, so there is no write to bound and its `--limit` means
|
||||
what it says — how many lines to print.
|
||||
|
||||
Comments ride along by default, in both modes and for every issue written:
|
||||
the thread lands in tmp/issues/<id>.comments.md, beside the issue. It costs
|
||||
@@ -42,8 +62,39 @@ from an earlier pull is deleted. An absent file therefore means "no comments",
|
||||
never "not asked for". The thread is pull-only: editing it changes nothing in
|
||||
Gitea (post with comment.py).
|
||||
|
||||
**Dependencies come with every pull.** A pull answers with the whole unit of
|
||||
work — the issue and what blocks it — so `depends:` is filled from Gitea's
|
||||
native dependency graph and every blocker is pulled too, recursively, down to
|
||||
`--depth` (default 3). That graph is the only source there is: `map.from_api`
|
||||
writes slugs into the `## Depends on` prose and never `#N`, so an edge cannot be
|
||||
recovered from the body. `--no-deps` turns off both halves — no `depends:`, no
|
||||
recursion, and no request spent on either. `--deps` is still accepted and now
|
||||
does nothing; it names what already happens.
|
||||
|
||||
What it costs, stated rather than hidden:
|
||||
|
||||
- **One request per issue that lands in the store** — `GET …/issues/{n}/dependencies`,
|
||||
fetched once and used twice, since the same links both fill `depends:` and
|
||||
tell the walk where to go next. A closed issue that filter mode drops costs
|
||||
nothing: nothing was stored, so there is no unit of work to complete.
|
||||
- **One request per blocker the selection did not already carry** — a `GET` for
|
||||
the issue itself, then its own links, and so on until `--depth`.
|
||||
- So `--milestone X` over 50 open issues is one list request + 50 link requests
|
||||
+ one pair for every blocker outside the milestone, where it used to be one
|
||||
request flat. `--no-deps` is the way back to one.
|
||||
|
||||
**In filter mode a blocker the filter did not select still lands in the store,
|
||||
and that is deliberate.** `--milestone X` can leave an issue from milestone Y on
|
||||
disk and `--label` an unlabelled one: a blocker is followed because a stored
|
||||
issue names it, not because it matched. The one blocker that does not land is a
|
||||
closed one — closed is not a unit of work, filter mode drops it the way it drops
|
||||
any other closed issue, and the `depends:` edge to it goes with it, so nothing
|
||||
points at a file that is not there. Key mode has no such rule and stores it.
|
||||
|
||||
Other flags:
|
||||
--deps [--depth N] follow dependencies and pull them too
|
||||
--no-deps do not fill depends:, do not follow blockers
|
||||
--deps accepted, does nothing: it is the default now
|
||||
--depth N how deep to follow blockers (default 3)
|
||||
--cached skip issues already on disk (body AND comments)
|
||||
--repo owner/repo default: auto-detect from the CWD git remote
|
||||
|
||||
@@ -52,7 +103,9 @@ have not pushed are lost — with exactly one exception, checkbox state. A `[x]`
|
||||
on either side wins for any item whose text matches, because a tick is monotone
|
||||
and unioning the two sides is not conflict resolution (gmap.merge_checkbox_state
|
||||
has the rule and its price). `--cached` skips an issue before any of that: it is
|
||||
not read and not merged. Draw the graph afterwards with the domain's own
|
||||
not read and not merged — it still costs its one link request, because a cached
|
||||
issue's blockers can be missing from disk even when it is not (`--cached
|
||||
--no-deps` is the free one). Draw the graph afterwards with the domain's own
|
||||
issue_tree.py — it needs no network.
|
||||
|
||||
Login: the operator's pin from .claude/settings.local.json (see /tea:auth).
|
||||
@@ -98,6 +151,23 @@ def id_for(payload, store_ids, remote_map, repo, root):
|
||||
taken=store_ids)
|
||||
|
||||
|
||||
def lands_in_store(payload, drop_closed, store_ids, remote_map, repo, root):
|
||||
"""Would this payload leave a file in the store? The `--limit` predicate.
|
||||
|
||||
It has to be the same test the walk below applies, or the budget is spent on
|
||||
issues that never land — which is the bug this exists to prevent. So: a
|
||||
closed issue counts only when the store already has it (it is refreshed, and
|
||||
that is a write); anything else counts, including one `--cached` will skip,
|
||||
because a skipped issue is still an issue the store holds when the run ends.
|
||||
|
||||
Cheap in the common case: only a closed payload costs an `id_for`, and that
|
||||
is a lookup plus, at worst, a stat."""
|
||||
if not (drop_closed and payload.get("state") == "closed"):
|
||||
return True
|
||||
id = id_for(payload, store_ids, remote_map, repo, root)
|
||||
return os.path.isfile(issue.path_of(root, id))
|
||||
|
||||
|
||||
def comments_path(root, id):
|
||||
"""Where an issue's comment thread lives — beside it, under the same slug.
|
||||
Named in `_gitea` because push.py has to delete the same file."""
|
||||
@@ -131,8 +201,18 @@ def main():
|
||||
ap.add_argument("-q", "--query", help="search text in title/body")
|
||||
ap.add_argument("--state", default="open", choices=["open", "closed", "all"],
|
||||
help="filter mode only (default: open)")
|
||||
ap.add_argument("--limit", type=int, default=100, help="filter mode cap (default: 100)")
|
||||
ap.add_argument("--deps", action="store_true", help="follow dependencies and pull them")
|
||||
ap.add_argument("--limit", type=int, default=100,
|
||||
help="filter mode: how many issues to STORE, not to enumerate"
|
||||
" (default: 100)")
|
||||
# Dependencies are the default: a pull answers with the unit of work, not
|
||||
# one row of it. `--deps` stays accepted so the calls and command tables
|
||||
# written against the old default keep working — it now sets what is
|
||||
# already set.
|
||||
ap.add_argument("--no-deps", dest="deps", action="store_false",
|
||||
help="do not fill depends: and do not follow blockers")
|
||||
ap.add_argument("--deps", dest="deps", action="store_true",
|
||||
help="accepted, does nothing: dependencies are followed by default")
|
||||
ap.set_defaults(deps=True)
|
||||
ap.add_argument("--depth", type=int, default=3, help="max dependency depth (default: 3)")
|
||||
ap.add_argument("--cached", action="store_true",
|
||||
help="skip issues already on disk instead of refetching")
|
||||
@@ -180,9 +260,14 @@ def main():
|
||||
|
||||
# ---- seeds -----------------------------------------------------------
|
||||
if filtered:
|
||||
# The limit bounds the write, so the transport is told what a write is
|
||||
# and counts those; the closed ones it enumerated on the way come back
|
||||
# in the list anyway, to be reported and dropped below.
|
||||
payloads, ms_title = _gitea.list_issues(
|
||||
login, base, state=args.state, labels=args.label, query=args.query,
|
||||
milestone=args.milestone, limit=args.limit)
|
||||
milestone=args.milestone, limit=args.limit,
|
||||
keep=lambda p: lands_in_store(p, drop_closed, store_ids, remote_map,
|
||||
repo, root))
|
||||
if not payloads:
|
||||
_gitea.die("no issues match that filter")
|
||||
what = []
|
||||
@@ -208,35 +293,42 @@ def main():
|
||||
stored = os.path.isfile(issue.path_of(root, id))
|
||||
|
||||
# Closed and not already ours: nothing is written and nothing is asked
|
||||
# of the server for it, not even its comments. The slug stays unclaimed
|
||||
# too, so no other issue ends up pointing `depends:` at a missing file.
|
||||
# of the server for it — not its comments, not its links, and its own
|
||||
# blockers are not followed. The slug stays unclaimed too, so no other
|
||||
# issue ends up pointing `depends:` at a missing file.
|
||||
if drop_closed and payload.get("state") == "closed" and not stored:
|
||||
dropped.append(number)
|
||||
continue # not stored: no unit of work here, so no links are fetched
|
||||
|
||||
store_ids.add(id)
|
||||
number_of_id[number] = id
|
||||
|
||||
# The native links, fetched ONCE for the two things they are for:
|
||||
# filling this issue's `depends:` and telling the walk where to go next.
|
||||
# One request per issue that lands in the store, and only one — the cost
|
||||
# the docstring quotes is this line.
|
||||
deps = _gitea.native_deps(login, base, number) if args.deps else []
|
||||
|
||||
if args.cached and stored:
|
||||
skipped.append(id) # body and thread unread; only the links cost
|
||||
else:
|
||||
store_ids.add(id)
|
||||
number_of_id[number] = id
|
||||
if args.cached and stored:
|
||||
skipped.append(id) # untouched, unread, and not one request spent
|
||||
else:
|
||||
extra = _gitea.native_deps(login, base, number) if args.deps else []
|
||||
# The copy already on disk, as it was when this run started. It
|
||||
# contributes its ticked checkboxes and nothing else; None when
|
||||
# the store has never seen this issue.
|
||||
prev = issues.get(id)
|
||||
iss, unresolved = gmap.from_api(payload, id, repo,
|
||||
id_for_number=number_of_id,
|
||||
extra_numbers=extra,
|
||||
synced=_gitea.now_iso(),
|
||||
local_body=prev.body if prev else None)
|
||||
issue.save(root, iss)
|
||||
sync_comments(login, base, root, id, number, payload.get("comments") or 0)
|
||||
remote_map[gmap.remote_key(repo, number)] = id
|
||||
written.append(id)
|
||||
pending.append((id, unresolved))
|
||||
# The copy already on disk, as it was when this run started. It
|
||||
# contributes its ticked checkboxes and nothing else; None when
|
||||
# the store has never seen this issue.
|
||||
prev = issues.get(id)
|
||||
iss, unresolved = gmap.from_api(payload, id, repo,
|
||||
id_for_number=number_of_id,
|
||||
extra_numbers=deps,
|
||||
synced=_gitea.now_iso(),
|
||||
local_body=prev.body if prev else None)
|
||||
issue.save(root, iss)
|
||||
sync_comments(login, base, root, id, number, payload.get("comments") or 0)
|
||||
remote_map[gmap.remote_key(repo, number)] = id
|
||||
written.append(id)
|
||||
pending.append((id, unresolved))
|
||||
|
||||
if args.deps and depth < args.depth:
|
||||
child_numbers = (gmap.numbers_in_body(payload.get("body") or "")
|
||||
+ _gitea.native_deps(login, base, number))
|
||||
child_numbers = gmap.numbers_in_body(payload.get("body") or "") + deps
|
||||
for n in child_numbers:
|
||||
if n in seen_numbers:
|
||||
continue
|
||||
@@ -265,8 +357,10 @@ def main():
|
||||
|
||||
# Compact output — the only thing that lands in the model's context. The
|
||||
# thread rides on the issue's own line; no file means no comments.
|
||||
graph = False
|
||||
for id in sorted(set(written) | set(skipped)):
|
||||
iss = issue.load(root, id)
|
||||
graph = graph or bool(iss.depends)
|
||||
note = " (cached)" if id in skipped else ""
|
||||
cpath = comments_path(root, id)
|
||||
if os.path.isfile(cpath):
|
||||
@@ -275,7 +369,9 @@ def main():
|
||||
id, ", ".join(iss.labels) or "no labels", iss.title, iss.state,
|
||||
issue.path_of(root, id), note))
|
||||
print("index: %s" % index_path)
|
||||
if args.deps:
|
||||
# Now that dependencies are the default, the hint is worth printing when
|
||||
# there is something to draw, not on every run that could have drawn it.
|
||||
if graph:
|
||||
print("graph: run issue_tree.py (offline) to draw it")
|
||||
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ way, so nothing is lost, but the tracker shows no edge for it.
|
||||
|
||||
The graph goes up with them. Once an issue has its number, every `depends:`
|
||||
entry that also has one becomes a **native Gitea link** — the same
|
||||
`/dependencies` that `pull.py --deps` reads back, so the tracker shows the
|
||||
`/dependencies` that every `pull.py` reads back, so the tracker shows the
|
||||
blocking panel and refuses to close a blocked issue first. Topological order
|
||||
means the blocker already has its number by then; no second pass is needed.
|
||||
`--update` links whatever appeared in `depends:` since the last push. A link
|
||||
|
||||
@@ -16,6 +16,11 @@ Usage:
|
||||
remote.py [--state open|closed|all] [--label L]… [-q TEXT]
|
||||
[--milestone M] [--limit N] [--repo owner/repo]
|
||||
|
||||
`--limit` here caps the LISTING: N lines out, closed ones among them. That is
|
||||
not what the same flag means to `pull.py`, and the difference is not an
|
||||
oversight — pull.py bounds what it writes, and this command writes nothing, so
|
||||
there is nothing else for a limit to bound. Enumeration is the whole job.
|
||||
|
||||
Login: the operator's pin from .claude/settings.local.json (see /tea:auth).
|
||||
"""
|
||||
import argparse
|
||||
|
||||
Reference in New Issue
Block a user