package cmd_test // `kettle gen` writes documentation an agent reads to invoke this binary, into // files a human also writes prose in. Both halves of that are tested here: what // it produces has to be the same twice over, and what it does NOT own has to // come back byte for byte. import ( "os" "path/filepath" "strings" "testing" ) const ( genOpen = "" genClose = "" ) func TestGenWritesOneFilePerGroupAndIsIdempotent(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "skills") first := mustRun(t, dir, "gen", "skills", "--out", out) for _, group := range []string{"project", "issue", "sync"} { path := filepath.Join(out, group, "SKILL.md") raw, err := os.ReadFile(path) if err != nil { t.Fatalf("%s was not created: %v\n%s", path, err, first.out()) } body := string(raw) // The frontmatter is what makes it a skill at all, and the description // is prose a human tunes — the stub says so and generates nothing. if !strings.HasPrefix(body, "---\nname: "+group+"\n") { t.Errorf("%s has no frontmatter naming the group:\n%s", path, firstLines(body, 5)) } if !strings.Contains(body, genOpen) || !strings.Contains(body, genClose) { t.Errorf("%s was created without the region markers:\n%s", path, body) } // The block has to say what wrote it: the first thing anybody who finds // it will want to do is edit it in place. if !strings.Contains(body, "kettle gen skills") { t.Errorf("%s does not name the command that regenerates it:\n%s", path, body) } } // One command's documentation, end to end: usage line, short, a flag out of // the flag set, and a worked example with its explanation beside it. issues, err := os.ReadFile(filepath.Join(out, "issue", "SKILL.md")) if err != nil { t.Fatal(err) } for _, want := range []string{ "## `kettle evict […]`", "remove closed issues from the local store", "| `--dry-run` | `false` | print what would be removed; touch nothing |", "kettle evict --dry-run", "# print what would go; touch nothing", } { if !strings.Contains(string(issues), want) { t.Errorf("the issue group is missing %q:\n%s", want, issues) } } // Deterministic to the byte: a regeneration of something that has not // changed must produce no diff at all, or every run of a CI step is a // spurious one. before := readAll(t, out) second := mustRun(t, dir, "gen", "skills", "--out", out) if strings.Contains(second.stdout, "updated") { t.Errorf("a second run rewrote a file:\n%s", second.out()) } for path, content := range before { if now := readFile(t, path); now != content { t.Errorf("%s changed on a second run with nothing else changed", path) } } if r := run(t, dir, "gen", "skills", "--out", out, "--check"); r.code != 0 { t.Errorf("--check exited %d on files that were just written:\n%s", r.code, r.out()) } } // The generator owns a region, not a file. Everything outside the markers is // somebody's prose and comes back exactly as it was. func TestGenLeavesHandWrittenProseAlone(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "skills") mustRun(t, dir, "gen", "skills", "--out", out) path := filepath.Join(out, "issue", "SKILL.md") raw := readFile(t, path) start := strings.Index(raw, genOpen) end := strings.Index(raw, genClose) + len(genClose) if start < 0 || end < len(genClose) { t.Fatalf("no region in the generated file:\n%s", raw) } const above = "\n## Identity: the slug\n\nThe file name is the id, and it never changes.\n\n" const below = "\n\n## Layering rule\n\nThis skill must keep working with the sync skill deleted.\n" // A description a human tuned, in the frontmatter the generator must not // touch: it is the only thing that decides whether the skill loads at all. edited := strings.Replace(raw[:start], "description: TODO", "description: Work with this project's issues as units of work", 1) edited += above + raw[start:end] + below if err := os.WriteFile(path, []byte(edited), 0o644); err != nil { t.Fatal(err) } mustRun(t, dir, "gen", "skills", "--out", out) after := readFile(t, path) if after != edited { t.Errorf("a no-op regeneration did not return the file byte for byte:\n--- want ---\n%s\n--- got ---\n%s", edited, after) } // And the prose survives a regeneration that actually rewrites the block. shortened := strings.Replace(after, genClose, "the block was gutted by hand\n"+genClose, 1) if err := os.WriteFile(path, []byte(shortened), 0o644); err != nil { t.Fatal(err) } mustRun(t, dir, "gen", "skills", "--out", out) restored := readFile(t, path) if restored != edited { t.Error("regenerating the block did not restore it, or did not preserve the prose around it") } if !strings.Contains(restored, "description: Work with this project's issues") { t.Errorf("the hand-tuned description was overwritten:\n%s", firstLines(restored, 5)) } if !strings.Contains(restored, above) || !strings.Contains(restored, below) { t.Errorf("hand-written prose outside the markers was lost:\n%s", restored) } } // Clobbering somebody's prose because they forgot a marker is the failure this // whole design exists to prevent. func TestGenNeverOverwritesAFileWithoutMarkers(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "skills") path := filepath.Join(out, "issue", "SKILL.md") if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { t.Fatal(err) } const prose = "---\nname: issue\ndescription: hand written, every word of it\n---\n\n# Everything here is somebody's work\n" if err := os.WriteFile(path, []byte(prose), 0o644); err != nil { t.Fatal(err) } r := mustRun(t, dir, "gen", "skills", "--out", out) if got := readFile(t, path); got != prose { t.Fatalf("a file with no markers was rewritten:\n%s", got) } // Left alone silently is how it drifts unnoticed, so it is reported — and // on stderr, where a warning belongs. if !strings.Contains(r.stderr, path) { t.Errorf("the skipped file was not named on stderr:\n%s", r.out()) } if !strings.Contains(r.stdout, "without a region") { t.Errorf("the receipt did not account for it:\n%s", r.stdout) } // The other groups still got written — one unmanaged file stops nothing. if _, err := os.Stat(filepath.Join(out, "sync", "SKILL.md")); err != nil { t.Error("one file without markers stopped the whole run") } } func TestGenCheckFailsOnAStaleFileAndNamesIt(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "skills") mustRun(t, dir, "gen", "skills", "--out", out) stale := filepath.Join(out, "sync", "SKILL.md") raw := readFile(t, stale) edited := strings.Replace(raw, genClose, "kettle push --thoroughly-renamed-flag\n"+genClose, 1) if err := os.WriteFile(stale, []byte(edited), 0o644); err != nil { t.Fatal(err) } r := run(t, dir, "gen", "skills", "--out", out, "--check") if r.code != 1 { t.Fatalf("--check exited %d, want 1 — this is what a hook or a CI step calls:\n%s", r.code, r.out()) } if !strings.Contains(r.stdout, stale) { t.Errorf("--check did not say which file is out of date:\n%s", r.out()) } // A question about the tree, never an answer written into it. if got := readFile(t, stale); got != edited { t.Error("--check wrote to the file it was asked about") } // A file that is not there at all is out of date too, not a nothing. if err := os.Remove(stale); err != nil { t.Fatal(err) } if r := run(t, dir, "gen", "skills", "--out", out, "--check"); r.code != 1 { t.Errorf("--check exited %d for a missing file, want 1:\n%s", r.code, r.out()) } if _, err := os.Stat(stale); err == nil { t.Error("--check created the file it was asked about") } } func TestGenDryRunWritesNothingAtAll(t *testing.T) { dir := t.TempDir() out := filepath.Join(dir, "skills") fresh := mustRun(t, dir, "gen", "skills", "--out", out, "--dry-run") if !strings.Contains(fresh.stdout, "would create") { t.Errorf("a dry run said nothing about what it would do:\n%s", fresh.out()) } if _, err := os.Stat(out); err == nil { t.Fatal("a dry run created the output directory") } // And on an existing tree: the file is described, never touched. mustRun(t, dir, "gen", "skills", "--out", out) path := filepath.Join(out, "issue", "SKILL.md") edited := strings.Replace(readFile(t, path), genClose, "gutted\n"+genClose, 1) if err := os.WriteFile(path, []byte(edited), 0o644); err != nil { t.Fatal(err) } r := mustRun(t, dir, "gen", "skills", "--out", out, "--dry-run") if !strings.Contains(r.stdout, "would update") || !strings.Contains(r.stdout, "nothing was written") { t.Errorf("the dry run did not report the pending change:\n%s", r.out()) } if got := readFile(t, path); got != edited { t.Error("a dry run rewrote the file") } } func TestGenRefusesAnUnknownTargetAndAMissingOut(t *testing.T) { dir := t.TempDir() if r := run(t, dir, "gen", "skills"); r.code == 0 || !strings.Contains(r.stderr, "--out") { t.Errorf("gen without --out must stop and say so:\n%s", r.out()) } if r := run(t, dir, "gen", "agents", "--out", filepath.Join(dir, "x")); r.code == 0 { t.Errorf("an unknown target must be refused:\n%s", r.out()) } if _, err := os.Stat(filepath.Join(dir, "x")); err == nil { t.Error("the refused run created its output directory anyway") } } func readFile(t *testing.T, path string) string { t.Helper() raw, err := os.ReadFile(path) if err != nil { t.Fatal(err) } return string(raw) } // readAll is every file under root, by path, for a byte-for-byte comparison // after a second run. func readAll(t *testing.T, root string) map[string]string { t.Helper() out := map[string]string{} err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil || info.IsDir() { return err } raw, err := os.ReadFile(path) if err != nil { return err } out[path] = string(raw) return nil }) if err != nil { t.Fatal(err) } return out } func firstLines(s string, n int) string { lines := strings.SplitN(s, "\n", n+1) if len(lines) > n { lines = lines[:n] } return strings.Join(lines, "\n") }