update distill example

soleprint/station/tools/distill/distill-example.json

@@ -1,29 +1,97 @@
 {
-  "_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that. Everything that shapes the run is at the top; 'repos' is just a list of paths. Preview any change with: ./distill.sh list",
+  "_comment": "Template for distill.json, the job distill.sh reads when no repo is named on the command line. Copy this to distill.json (gitignored) and edit that — which repos a machine distils is not a fact about the tool. Every key the file understands appears below, once, with the value it would fall back to if you deleted the line. Preview any change with: ./distill.sh list  — or ./distill.sh check, which resolves every path and ref in about a second.",
 
-  "_command": "tree = a directory per repo. digest = one .md per repo, flattened into a single readable file. both = each repo as a directory AND a .md. list = write nothing, just report what would be kept.",
+  "_command": "tree = a directory per repo, a copy you can open, grep and build. digest = one .md per repo, the whole thing flattened into a single readable file (header, file tree, manifest, then every file under its path). both = each repo comes out as a directory AND a .md — they answer different questions and the selection work is shared, so the second costs only a copy. list = write nothing, just report what would be kept, weighed. check = write nothing and read nothing but git's index: does every path exist, does every ref resolve, is there room. Every other command runs check first anyway.",
   "command": "list",
 
+  "_out": "Where the run writes. Relative to wherever you invoke the script from, so an absolute path is the safer thing to write down; ~ is expanded. Ignored by list and check.",
   "out": "distilled",
 
-  "_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base (it gets a _PARTIAL.md saying so).",
+  "_root": "Where a bare slug in 'repos' resolves, so entries can read \"path\": \"myrepo\" instead of the whole absolute path. Leave it out and slugs resolve against the current directory, or DISTILL_ROOT. Absolute paths never touch it.",
+  "root": "~/wdir",
+
+  "_branch_mode": "full = each branch is a complete, standalone copy. diff = only the files that differ from diff_base, which is far smaller but is a changelist, not the branch (it gets a _PARTIAL.md saying so). diff_base must name one of the refs actually listed, and it is the one that comes out whole. Set per entry to override this default.",
   "branch_mode": "full",
   "diff_base": "main",
 
-  "_filters": "Applied to every repo. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, images, minified, maps). max_bytes skips anything larger and names it in MANIFEST.md.",
-  "exclude": [],
+  "_filters": "Applied to every repo, and overridable per entry. exclude/include are path globs; a pattern with no / also matches basenames at any depth. 'all' keeps the noise (lockfiles, minified, maps, compiled output, caches) that is dropped by default. max_bytes drops anything larger outright, and names it in MANIFEST.md.",
+  "exclude": ["*.ipynb", "fixtures/**"],
   "include": [],
   "all": false,
-  "max_bytes": null,
+  "max_bytes": "2M",
 
-  "_split_tokens": "A digest over this many tokens is written as NAME.md (an index: the tree, and the manifest saying which part holds each file) plus NAME.part-01.md, NAME.part-02.md... holding the files, cut between files. Each part stands on its own. Default 100k; 0 never splits. Can also be set per entry.",
+  "_budget": "Three different answers to 'this is too much to read', and they do not compete. clip_bytes inlines only the head and tail of any file over that size, with a marker saying how much was cut. max_tokens holds each digest to roughly that many tokens by clipping largest-first — one shared ceiling, lowered until the total fits — so the biggest file pays and the hundred small ones that describe the project keep every line. Both of those lose text; split_tokens loses none, it just stops writing one enormous file: over ~N tokens the digest becomes NAME.md (an index holding the tree and the manifest, saying which part each file is in) plus NAME.part-01.md, NAME.part-02.md… cut between files in path order, each part standing on its own. Default 100k; 0 never splits. Clipping is a statement about the document, not the files: the tree copy always has them whole. Numbers take k/M/G, decimal.",
+  "clip_bytes": null,
+  "max_tokens": null,
   "split_tokens": "100k",
+
+  "_with_root": "Only matters when an entry names a subpath. Keeps the repo's top-level files (README, pyproject.toml, package.json) alongside the subtree, so the copy still says which project it is a part of.",
+  "with_root": false,
+
+  "_keep_secrets": ".env files, private keys, keystores and the like are dropped by default and named in MANIFEST.md. 'all' does not bring them back — only this does, and only deliberately.",
+  "keep_secrets": false,
+
+  "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes anything in 'out' this run did not produce, so dropping a repo from the list drops its output too.",
   "skip_unchanged": true,
   "prune": true,
 
-  "_repos": "'path' is absolute, or a name resolved under --root. 'branches' is optional — leave it out for the working tree as it stands, uncommitted changes included. A path may appear more than once.",
+  "_bundle": "Also write _BUNDLE.md: every .md in the destination concatenated, in one document, behind an index. A directory of files is right for grep and for reading; an upload box takes one file. Costs a second copy of the text on disk and nothing else. Note that it concatenates whatever split_tokens produced, parts included — bundle and split are opposite answers to the same question, and you can have both on disk.",
+  "bundle": false,
+
+  "_refs_patch": "Put the full diff, not just the diffstat, in NAME@REFS.md. Answers 'what changed' instead of 'how much'.",
+  "refs_patch": false,
+
+  "_raw_fences": "Write runs of backticks and tildes into the digest as they are, instead of escaping them as ⟪BT3⟫ / ⟪TL3⟫. Escaping is the default because a fence inside a file otherwise closes the fence the digest is being read inside; explode.sh puts the characters back. Turn this off only when nothing is going to render the result as markdown.",
+  "raw_fences": false,
+
+  "_top": "'list' only: how many heavy files and directories to show. 0 for none.",
+  "top": 10,
+
+  "_repos": "One entry per thing you want out. 'path' is absolute, or a name resolved under 'root'. The same path may appear as many times as you like, which is why this is a list rather than a keyed object — one entry per repo could not hold two branches of the same repo. Every filter and budget key above may be repeated inside an entry, where it replaces the top-level value for that entry alone; a command-line option still wins over both.",
   "repos": [
-    { "path": "/path/to/some-repo" },
-    { "path": "/path/to/another-repo", "branches": ["origin/main", "origin/feature/example"] }
+    {
+      "_note": "The plainest entry: the working tree as it stands right now, uncommitted changes included, under every setting from the top of the file.",
+      "path": "/path/to/some-repo"
+    },
+
+    {
+      "_note": "Named refs. Each is read with ls-tree/archive, so nothing is checked out and a dirty working tree is never touched. In full mode each of these is a complete copy. A hash, a tag or HEAD~3 works anywhere a branch name does.",
+      "path": "/path/to/another-repo",
+      "branches": ["origin/main", "v1.4.0", "3f2a9c1"]
+    },
+
+    {
+      "_note": "Many branches. In full mode this would be one complete copy per branch, most of them the same bytes — the right answer for an archive and the wrong one for anything with a context window. So: origin/main whole, and the rest as changelists against it.",
+      "path": "/path/to/busy-repo",
+      "branch_mode": "diff",
+      "diff_base": "origin/main",
+      "branches": ["origin/main", "origin/topic-a", "origin/topic-b"]
+    },
+
+    {
+      "_note": "A subtree, and only it — with the repo's own ignore rules, refs and deltas still applying, because the repo root is found for you. subpath may be a list: 'the part I am working on' is usually the module plus the tests or docs beside it, and two entries for that would be two outputs to read separately. 'name' overrides what the output is called, which is what you want when the same repo appears more than once.",
+      "path": "slug-resolved-under-root",
+      "subpath": ["src/api", "tests/api"],
+      "with_root": true,
+      "name": "api-only"
+    },
+
+    {
+      "_note": "Per-entry filters and budget. One generated file does not get to spend the whole digest; this entry also keeps the lockfiles the top of the file drops, and splits later than everything else.",
+      "path": "/path/to/heavy-repo",
+      "include": ["src/**", "*.md"],
+      "exclude": ["src/generated/**"],
+      "all": true,
+      "max_bytes": "5M",
+      "clip_bytes": "64k",
+      "max_tokens": "80k",
+      "split_tokens": "250k"
+    },
+
+    {
+      "_note": "Kept in the file, out of the run. Better than deleting the entry when it is something you distil occasionally — the paths and the branch list stay written down.",
+      "path": "/path/to/retired-repo",
+      "enabled": false
+    }
   ]
 }