distill updates

soleprint/station/tools/distill/distill-example.json

@@ -15,6 +15,9 @@
   "include": [],
   "all": false,
   "max_bytes": null,
+
+  "_split_tokens": "A digest over this many tokens is written as NAME.md (an index: the tree, and the manifest saying which part holds each file) plus NAME.part-01.md, NAME.part-02.md... holding the files, cut between files. Each part stands on its own. Default 100k; 0 never splits. Can also be set per entry.",
+  "split_tokens": "100k",
   "skip_unchanged": true,
   "prune": true,
 

soleprint/station/tools/distill/distill.sh

@@ -25,7 +25,16 @@
 #   distill.sh digest [opts] -o DEST REPO...   # one concatenated .md per repo
 #   distill.sh both   [opts] -o DEST REPO...   # both, from a single pass
 #   distill.sh list   [opts] REPO...           # what would be kept, weighed
-#   distill.sh [tree|digest|both|list] -c FILE # read the whole job from JSON
+#   distill.sh check  [opts] REPO...           # seconds: will the run work?
+#   distill.sh [tree|digest|both|list|check] -c FILE # read the whole job from JSON
+#
+# check copies nothing. For every entry it confirms the path exists, each branch
+# or commit resolves and each subpath is there at it, and weighs it from git's
+# own index; then that the tools are installed and the output and temp folders
+# have room. It reports every problem rather than the first, so a moved folder
+# or a mistyped hash turns up before a long run, not twenty minutes into it.
+# Every other command runs it first, in a second or so, and stops on a problem
+# before copying anything; --no-check skips that.
 #
 # tree and digest answer different questions. tree gives you files — open them,
 # grep them, build them. digest gives you one document to read or hand over:
@@ -45,7 +54,7 @@
 #     "branch_mode": "full",        // or "diff", against diff_base
 #     "diff_base": "main",
 #     "exclude": [], "include": [], "all": false, "max_bytes": null,
-#     "clip_bytes": null, "max_tokens": null, "with_root": false,
+#     "clip_bytes": null, "max_tokens": null, "split_tokens": null, "with_root": false,
 #     "skip_unchanged": false, "prune": false, "bundle": false,
 #     "raw_fences": false,
 #     "repos": [
@@ -59,7 +68,8 @@
 # rather than a keyed object; one entry per repo could not hold two branches of
 # the same repo. Per-entry keys: path, branches, subpath (a string, or a list
 # of them), name, enabled, branch_mode, diff_base, include, exclude, max_bytes,
-# clip_bytes, max_tokens, with_root — each falling back to the top of the file.
+# clip_bytes, max_tokens, split_tokens, with_root — each falling back to the top
+# of the file.
 # A command-line option overrides both: one repo in the list wanting a tighter
 # budget should say so in its entry, but `--max-tokens 60k` on the command line
 # is a thing someone just typed, and it wins over the whole file.
@@ -106,6 +116,10 @@
 #                   largest-first — one shared size ceiling, lowered until the
 #                   total fits — so the biggest file pays for it and the hundred
 #                   small ones that actually describe the project do not
+#   --split-tokens N  write a digest over ~N tokens as NAME.md, an index with the
+#                   tree and the manifest, plus NAME.part-01.md, NAME.part-02.md…
+#                   holding the files, cut between files in path order. Each
+#                   part stands on its own. Default 100k; 0 never splits
 #   --with-root     with a subpath in play, keep the repo's top-level files too
 #                   (README, pyproject.toml, package.json) so a subtree copy
 #                   still says which project it is a part of
@@ -121,6 +135,7 @@
 #   --refs-patch    put the full diff, not just the diffstat, in NAME@REFS.md
 #   --raw-fences    write runs of backticks and tildes into the digest as they
 #                   are, instead of escaping them as ⟪BT3⟫ / ⟪TL3⟫ (see below)
+#   --no-check      skip the check every other command runs first (see check)
 #   --keep-secrets  include .env, private keys and the like, which are dropped
 #                   by default and are NOT re-included by --all
 #   -n              dry run — say what would happen, write nothing
@@ -248,12 +263,15 @@ lang_for() {
 # leading option is not an error here.
 CMD=""
 case "${1:-}" in
-    tree|digest|both|list) CMD="$1"; shift ;;
+    tree|digest|both|list|check) CMD="$1"; shift ;;
     -h|--help|help)   usage; exit 0 ;;
     "")               usage >&2; exit 1 ;;
     -*)               ;;
-    *)                die "unknown command: $1 (expected tree, digest, both or list)" ;;
+    *)                die "unknown command: $1 (expected tree, digest, both, list or check)" ;;
 esac
+# Kept as given, so the check that runs first sees exactly the same job.
+ARGS=("$@")
+NO_CHECK=""
 
 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 CONFIG=""
@@ -272,6 +290,12 @@ KEEP_NOISE=""
 MAX_BYTES=""
 CLIP_BYTES=""
 MAX_TOKENS=""
+SPLIT_TOKENS=""
+# A digest past this many tokens is written as an index and numbered parts. Big
+# enough that an ordinary repo stays one document; small enough that a part
+# still fits the attachment of a web chat without crowding out the question.
+DEFAULT_SPLIT_TOKENS=100000
+DIGEST_PARTS=1
 WITH_ROOT=""
 TOP_N=10
 TOP_SET=""
@@ -299,6 +323,7 @@ while [ $# -gt 0 ]; do
         --max-bytes) shift; MAX_BYTES="${1:-}" ;;
         --clip-bytes) shift; CLIP_BYTES="${1:-}" ;;
         --max-tokens) shift; MAX_TOKENS="${1:-}" ;;
+        --split-tokens) shift; SPLIT_TOKENS="${1:-}" ;;
         --with-root) WITH_ROOT=1 ;;
         --top)       shift; TOP_N="${1:-}"; TOP_SET=1 ;;
         --all)       KEEP_NOISE=1 ;;
@@ -308,6 +333,7 @@ while [ $# -gt 0 ]; do
         --refs-patch) REFS_PATCH=1 ;;
         --raw-fences) RAW_FENCES=1 ;;
         --keep-secrets) KEEP_SECRETS=1 ;;
+        --no-check)  NO_CHECK=1 ;;
         --skip-unchanged) SKIP_UNCHANGED=1 ;;
         -n)          DRY=1 ;;
         -d)          MIRROR=1 ;;
@@ -353,6 +379,7 @@ normalize_limits() {
     [ -n "$MAX_BYTES" ]  && MAX_BYTES="$(num_arg "$MAX_BYTES" --max-bytes)"
     [ -n "$CLIP_BYTES" ] && CLIP_BYTES="$(num_arg "$CLIP_BYTES" --clip-bytes)"
     [ -n "$MAX_TOKENS" ] && MAX_TOKENS="$(num_arg "$MAX_TOKENS" --max-tokens)"
+    [ -n "$SPLIT_TOKENS" ] && SPLIT_TOKENS="$(num_arg "$SPLIT_TOKENS" --split-tokens)"
     [[ "$TOP_N" =~ ^[0-9]+$ ]] || die "--top wants a plain count, got: $TOP_N"
     return 0
 }
@@ -437,6 +464,7 @@ if [ -n "$CONFIG" ]; then
             max_bytes:  (($e.max_bytes  // $cfg.max_bytes  // "") | tostring),
             clip_bytes: (($e.clip_bytes // $cfg.clip_bytes // "") | tostring),
             max_tokens: (($e.max_tokens // $cfg.max_tokens // "") | tostring),
+            split_tokens: (($e.split_tokens // $cfg.split_tokens // "") | tostring),
             with_root:  (if ($e|has("with_root")) then $e.with_root
                          elif ($cfg|has("with_root")) then $cfg.with_root
                          else false end)
@@ -456,14 +484,14 @@ fi
 [ -d "$ROOT" ] || die "root is not a directory: $ROOT"
 
 case "$CMD" in
-    tree|digest|both|list) ;;
+    tree|digest|both|list|check) ;;
-    *) die "unknown command: $CMD (expected tree, digest, both or list)" ;;
+    *) die "unknown command: $CMD (expected tree, digest, both, list or check)" ;;
 esac
 
-if [ "$CMD" != list ]; then
+if [ "$CMD" != list ] && [ "$CMD" != check ]; then
     [ -n "$DEST" ] || die "an output directory is required for $CMD (-o DEST, or \"out\" in the config)"
 fi
-case "$CMD" in tree|both) ;; *) [ -z "$MIRROR" ] || die "-d only applies to 'tree' or 'both'" ;; esac
+case "$CMD" in tree|both|check) ;; *) [ -z "$MIRROR" ] || die "-d only applies to 'tree' or 'both'" ;; esac
 
 # ── spec parsing ───────────────────────────────────────────────────────────
 # <repo>[@<ref>[,<ref>...]][:<subpath>]
@@ -1025,16 +1053,144 @@ render_tree() {
 # of this?" without guessing.
 write_digest() {
     local staged="$1" out="$2" title="$3" subtitle="$4"
-    local f rel fence lang bytes nfiles lines meta
+    local f rel fence lang bytes nfiles lines meta i
 
     bytes=$(du -sb "$staged" | cut -f1)
     nfiles=$(find "$staged" -type f | wc -l)
+
+    # Every file's section is rendered first, to its own temp file, so the
+    # digest can be cut between files once their sizes are known. Unsplit, the
+    # sections are concatenated in order and the document is what it always was.
+    local sections="$TMP/sections"
+    rm -rf "$sections"; mkdir -p "$sections"
+    : > "$sections.order"
+    i=0
+    while IFS= read -r -d '' f; do
+        rel="${f#$staged/}"
+        is_binary_file "$rel" && continue
+        lang="$(lang_for "$rel")"
+        lines=$(wc -l < "$f")
+        # The body is rendered first and the fence measured on that, so it is
+        # measured on exactly what goes between the fences: escaped or not,
+        # clipped or not. escape_fences also ends the last line, so a file with
+        # no trailing newline cannot weld itself to the closing fence.
+        if is_clipped "$rel" "$staged"; then
+            : > "$TMP/body.raw"
+            clip_render "$f" "$CLIP_T" "$TMP/body.raw"
+            escape_fences < "$TMP/body.raw" > "$TMP/body"
+            meta="_${lines} lines · $(stat -c%s "$f") bytes · CLIPPED — head and tail only_"
+        else
+            escape_fences < "$f" > "$TMP/body"
+            meta="_${lines} lines · $(stat -c%s "$f") bytes_"
+        fi
+        fence="$(fence_for "$TMP/body")"
+        i=$((i + 1))
         {
+            echo "## $rel"
+            echo
+            echo "$meta"
+            echo
+            echo "${fence}${lang}"
+            cat "$TMP/body"
+            [ -s "$TMP/body" ] && [ -n "$(tail -c1 "$TMP/body")" ] && echo
+            echo "$fence"
+            echo
+        } > "$sections/$i"
+        printf '%s\t%s\t%s\n' "$i" "$rel" "$(stat -c%s "$sections/$i")" >> "$sections.order"
+    done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | sed "s|^|$staged/|" | tr '\n' '\0')
+
+    # Parts from an earlier, longer run of this same digest would otherwise sit
+    # beside the new ones looking current.
+    rm -f "${out%.md}".part-[0-9][0-9].md
+    DIGEST_PARTS=1
+
+    local limit total
+    limit=$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} * BYTES_PER_TOKEN ))
+    total=$(awk -F'\t' '{ s += $3 } END { print s + 0 }' "$sections.order")
+
+    if [ "$limit" -eq 0 ] || [ "$total" -le "$limit" ]; then
+        {
+            digest_intro "$title" "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(digest_extras)" full "$staged"
+            digest_manifest "$staged" ""
+            while IFS=$'\t' read -r i _ _; do cat "$sections/$i"; done < "$sections.order"
+            digest_end "$title" "$nfiles" "The manifest above lists"
+        } > "$out"
+        return 0
+    fi
+
+    # Greedy, in path order, so a folder's files stay together and each part
+    # reads as a contiguous stretch of the tree. A single file over the limit
+    # gets a part to itself rather than being split: --max-tokens is what
+    # shortens files.
+    awk -F'\t' -v limit="$limit" '
+        { if (size > 0 && size + $3 > limit) { part++; size = 0 }
+          if (part == 0) part = 1
+          size += $3
+          print $1 "\t" $2 "\t" $3 "\t" part }' "$sections.order" > "$sections.parts"
+    DIGEST_PARTS=$(awk -F'\t' 'END { print $4 }' "$sections.parts")
+
+    local stem base p pfile pfiles pbytes first last
+    stem="${out%.md}"; base="$(basename "$stem")"
+    {
+        digest_intro "$title" "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(digest_extras) · in $DIGEST_PARTS parts" index "$staged"
+        echo "## Parts"
+        echo
+        echo "This document is the index. The files themselves are in $DIGEST_PARTS parts, each"
+        echo "under ~$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} / 1000 ))k tokens, cut between files in path order. Each part stands"
+        echo "on its own; attach the ones the question needs."
+        echo
+        echo "| part | files | bytes | first file | last file |"
+        echo "|---|---:|---:|---|---|"
+        for p in $(seq 1 "$DIGEST_PARTS"); do
+            pfiles=$(awk -F'\t' -v p="$p" '$4 == p' "$sections.parts" | wc -l)
+            pbytes=$(awk -F'\t' -v p="$p" '$4 == p { s += $3 } END { print s + 0 }' "$sections.parts")
+            first=$(awk -F'\t' -v p="$p" '$4 == p { print $2; exit }' "$sections.parts")
+            last=$(awk -F'\t' -v p="$p" '$4 == p { l = $2 } END { print l }' "$sections.parts")
+            printf '| `%s.part-%02d.md` | %d | %s | `%s` | `%s` |\n' \
+                "$base" "$p" "$pfiles" "$(numfmt --to=iec "$pbytes")" "$first" "$last"
+        done
+        echo
+        digest_manifest "$staged" "$sections.parts"
+        echo "## End of $title (index)"
+        echo
+        printf 'The manifest above lists %d files, in %d parts.\n' "$nfiles" "$DIGEST_PARTS"
+    } > "$out"
+
+    for p in $(seq 1 "$DIGEST_PARTS"); do
+        pfile="$(printf '%s.part-%02d.md' "$stem" "$p")"
+        pfiles=$(awk -F'\t' -v p="$p" '$4 == p' "$sections.parts" | wc -l)
+        pbytes=$(awk -F'\t' -v p="$p" '$4 == p { s += $3 } END { print s + 0 }' "$sections.parts")
+        {
+            digest_intro "$title — part $p of $DIGEST_PARTS" \
+                "$subtitle · part $p of $DIGEST_PARTS · $pfiles of $nfiles files · $(numfmt --to=iec "$pbytes")" \
+                part "$staged" "$(basename "$out")"
+            while IFS=$'\t' read -r i _ _ pp; do
+                [ "$pp" = "$p" ] && cat "$sections/$i"
+            done < "$sections.parts"
+            echo "## End of $title — part $p of $DIGEST_PARTS"
+            echo
+            printf 'This part holds %d of the %d files listed in %s; the other parts hold the rest.\n' \
+                "$pfiles" "$nfiles" "$(basename "$out")"
+        } > "$pfile"
+        produced "$pfile"
+    done
+}
+
+# The subtitle's tail: what is listed but not inlined, and what was clipped.
+digest_extras() {
+    [ ${#BINARY_FILES[@]} -gt 0 ] && printf ' · %d binary, listed but not inlined' "${#BINARY_FILES[@]}"
+    [ "$CLIP_N" -gt 0 ] && printf ' · %d clipped to fit ~%dk tokens' "$CLIP_N" "$((DIGEST_BYTES / (BYTES_PER_TOKEN * 1000)))"
+    return 0
+}
+
+# The top of a digest, an index or a part: what the document is and how to read
+# it. The escape notice is in all three, above the first '## ', which is where
+# explode.sh looks for it.
+digest_intro() {
+    local title="$1" subline="$2" kind="$3" staged="$4" index="${5:-}"
     echo "# $title"
     echo
-        echo "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(
+    echo "$subline"
-            [ ${#BINARY_FILES[@]} -gt 0 ] && printf ' · %d binary, listed but not inlined' "${#BINARY_FILES[@]}" || true)$(
-            [ "$CLIP_N" -gt 0 ] && printf ' · %d clipped to fit ~%dk tokens' "$CLIP_N" "$((DIGEST_BYTES / (BYTES_PER_TOKEN * 1000)))" || true)"
     echo
     echo "Each file below opens with a \`## <path>\` heading and is wrapped in a"
     echo "fence longer than any run of backticks inside it, so no file can close"
@@ -1047,6 +1203,12 @@ write_digest() {
         echo "three backticks or three tildes in a row inside file contents."
         echo
     fi
+    if [ "$kind" = part ]; then
+        echo "This is one part of a digest too long for one document. The tree and the"
+        echo "manifest of every file, with the part each one is in, are in $index."
+        echo
+        return 0
+    fi
     if [ "$CLIP_N" -gt 0 ]; then
         # "1 files" reads like a bug in whatever produced the document, and
         # this document is asking to be trusted about its own completeness.
@@ -1061,85 +1223,62 @@ write_digest() {
     echo
     render_tree "$staged"
     echo
+}
+
+# The manifest, and the binary files named rather than silently absent. With a
+# parts map (index\trel\tsize\tpart), each row also says which part it is in.
+digest_manifest() {
+    local staged="$1" parts="$2" rel part=""
     echo "## Manifest"
     echo
+    if [ -n "$parts" ]; then
+        echo "| path | lines | bytes | inlined | part |"
+        echo "|---|---:|---:|---|---:|"
+    else
         echo "| path | lines | bytes | inlined |"
         echo "|---|---:|---:|---|"
-    } > "$out"
+    fi
-
     while IFS= read -r rel; do
+        [ -n "$parts" ] && part=" $(awk -F'\t' -v r="$rel" '$2 == r { print $4; exit }' "$parts") |"
         if is_binary_file "$rel"; then
-            printf '| `%s` | — | %s | no — binary, in the tree copy only |\n' \
+            printf '| `%s` | — | %s | no — binary, in the tree copy only |%s\n' \
-                "$rel" "$(stat -c%s "$staged/$rel")" >> "$out"
+                "$rel" "$(stat -c%s "$staged/$rel")" "${part:+ —  |}"
         elif is_clipped "$rel" "$staged"; then
-            printf '| `%s` | %s | %s | **clipped** to ~%s |\n' "$rel" \
+            printf '| `%s` | %s | %s | **clipped** to ~%s |%s\n' "$rel" \
                 "$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" \
-                "$(numfmt --to=iec "$CLIP_T")" >> "$out"
+                "$(numfmt --to=iec "$CLIP_T")" "$part"
         else
-            printf '| `%s` | %s | %s | full |\n' "$rel" \
+            printf '| `%s` | %s | %s | full |%s\n' "$rel" \
-                "$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" >> "$out"
+                "$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" "$part"
         fi
     done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)
-    echo >> "$out"
+    echo
 
     # Named, not silently absent. Something reading only this file would
     # otherwise have no idea the spreadsheets exist at all.
     if [ ${#BINARY_FILES[@]} -gt 0 ]; then
-        {
         echo "## Binary files (present in the copy, not inlined here)"
         echo
         for rel in ${BINARY_FILES[@]+"${BINARY_FILES[@]}"}; do
             echo "- \`$rel\` ($(numfmt --to=iec "$(stat -c%s "$staged/$rel")"))"
         done
         echo
-        } >> "$out"
-    fi
-
-    while IFS= read -r -d '' f; do
-        rel="${f#$staged/}"
-        is_binary_file "$rel" && continue
-        lang="$(lang_for "$rel")"
-        lines=$(wc -l < "$f")
-        # The body is rendered first and the fence measured on that, so it is
-        # measured on exactly what goes between the fences: escaped or not,
-        # clipped or not. escape_fences also ends the last line, so a file with
-        # no trailing newline cannot weld itself to the closing fence.
-        if is_clipped "$rel" "$staged"; then
-            : > "$TMP/body.raw"
-            clip_render "$f" "$CLIP_T" "$TMP/body.raw"
-            escape_fences < "$TMP/body.raw" > "$TMP/body"
-            meta="_${lines} lines · $(stat -c%s "$f") bytes · CLIPPED — head and tail only_"
-        else
-            escape_fences < "$f" > "$TMP/body"
-            meta="_${lines} lines · $(stat -c%s "$f") bytes_"
     fi
-        fence="$(fence_for "$TMP/body")"
+}
-        {
-            echo "## $rel"
-            echo
-            echo "$meta"
-            echo
-            echo "${fence}${lang}"
-            cat "$TMP/body"
-            [ -s "$TMP/body" ] && [ -n "$(tail -c1 "$TMP/body")" ] && echo
-            echo "$fence"
-            echo
-        } >> "$out"
-    done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | sed "s|^|$staged/|" | tr '\n' '\0')
 
-    # The reader has no other way to know the document did not stop early. The
+# The reader has no other way to know the document did not stop early. The
-    # count has to be exact, including the ways a file can be here but not
+# count has to be exact, including the ways a file can be here but not whole —
-    # whole — a marker claiming everything is complete, next to a clipped file,
+# a marker claiming everything is complete, next to a clipped file, is worse
-    # is worse than no marker.
+# than no marker.
-    {
+digest_end() {
+    local title="$1" nfiles="$2" lead="$3"
     echo "## End of $title"
     echo
-        printf 'The manifest above lists %d files: %d in full' \
+    printf '%s %d files: %d in full' "$lead" \
         "$nfiles" "$(( nfiles - CLIP_N - ${#BINARY_FILES[@]} ))"
     [ "$CLIP_N" -gt 0 ] && printf ', %d clipped (each marked at the cut)' "$CLIP_N"
     [ ${#BINARY_FILES[@]} -gt 0 ] && printf ', %d binary and not inlined' "${#BINARY_FILES[@]}"
     printf '.\n'
-    } >> "$out"
 }
 
 # When several refs of one repo are distilled, ship the comparison too. Whatever
@@ -1212,6 +1351,21 @@ STATE_OLD="$TMP/state.old"
 STATE_NEW="$TMP/state.new"
 PRODUCED=()
 
+# Saved after every entry, not once at the end. A run over a dozen repos and
+# their branches takes long enough to get interrupted, and a record written only
+# on completion meant every interrupted run started again from the first entry
+# and never reached the ones at the bottom of the list. The file on disk is
+# always: what this run has finished, plus the previous run's records for what
+# it has not reached yet. Replaced atomically, so an interruption mid-write
+# leaves the previous version whole.
+save_state() {
+    [ -n "$STATE_FILE" ] || return 0
+    awk -F'\t' 'NR == FNR { done[$1] = 1; print; next } !($1 in done)' \
+        "$STATE_NEW" "$( [ -f "$STATE_OLD" ] && echo "$STATE_OLD" || echo /dev/null )" \
+        > "$STATE_FILE.partial"
+    mv "$STATE_FILE.partial" "$STATE_FILE"
+}
+
 state_lookup() {  # label -> prints the stored record, or nothing
     [ -f "$STATE_OLD" ] || return 0
     grep -F -m1 "$(printf '%s\t' "$1")" "$STATE_OLD" 2>/dev/null || true
@@ -1233,10 +1387,10 @@ fingerprint() {
     else
         src="plain:$(find "$dir" -type f -printf '%P %s %T@\n' 2>/dev/null | LC_ALL=C sort | cksum | cut -d" " -f1)"
     fi
-    printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
+    printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
         "$src" "$sub" "$CMD" "$BASE_REF" "$KEEP_NOISE" "$MAX_BYTES" \
         "${INCLUDES[*]-}" "${EXCLUDES[*]-}" "$MIRROR" \
-        "$CLIP_BYTES" "$MAX_TOKENS" "$WITH_ROOT" "$RAW_FENCES" \
+        "$CLIP_BYTES" "$MAX_TOKENS" "$WITH_ROOT" "$RAW_FENCES" "${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS}" \
         | cksum | cut -d' ' -f1
 }
 
@@ -1341,6 +1495,7 @@ process() {
             TOTAL_TEXT=$((TOTAL_TEXT + p_text))
             printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$p_files" "$p_bytes" \
                 "$(printf '%s' "$p_row" | base64 -w0)" "$p_text" >> "$STATE_NEW"
+            save_state
             return 0
         fi
     fi
@@ -1391,7 +1546,14 @@ process() {
         "$label" "$STAGED" "$(numfmt --to=iec "$bytes")" "$((tokens / 1000))" \
         "$([ "$CLIP_N" -gt 0 ] && printf '  (%d clipped at %s)' "$CLIP_N" "$(numfmt --to=iec "$CLIP_T")" || true)"
 
-    if [ "$CMD" = list ]; then report_weight "$staged"; fi
+    if [ "$CMD" = list ]; then
+        local split_at=$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} ))
+        if [ "$split_at" -gt 0 ] && [ "$tokens" -gt "$split_at" ]; then
+            printf '    digest would be split: ~%sk tokens against %sk per part\n' \
+                "$((tokens / 1000))" "$((split_at / 1000))"
+        fi
+        report_weight "$staged"
+    fi
 
     local dropped=$((DROPPED_NOISE + DROPPED_BIG + DROPPED_GONE + DROPPED_SECRET))
     MANIFEST_ROWS+=("| \`$label\` | $dir | $kind | $STAGED | $dropped | $(numfmt --to=iec "$bytes") | ~$((tokens / 1000))k |")
@@ -1455,6 +1617,7 @@ process() {
             [ -n "$sub" ] && desc="$desc · scope $sub"
             [ -n "$is_delta" ] && desc="$desc · ONLY files differing from $BASE_REF"
             write_digest "$staged" "$DEST/$label.md" "$name" "$desc"
+            [ "$DIGEST_PARTS" -gt 1 ] && printf '    split: %s.md is the index, the files are in %d parts\n' "$label" "$DIGEST_PARTS"
             ;;
     esac
 
@@ -1463,6 +1626,7 @@ process() {
         printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$STAGED" "$bytes" \
             "$(printf '%s' "${MANIFEST_ROWS[-1]}" | base64 -w0)" \
             "$DIGEST_BYTES" >> "$STATE_NEW"
+        save_state
     fi
 }
 
@@ -1508,8 +1672,117 @@ label_for() {
     LABEL="$label"
 }
 
+# ── check: will the run work? ──────────────────────────────────────────────
+CHECK_FAILS=0
+CHECK_BYTES=0
+CHECK_BIGGEST=0
+CHECK_WARN_BYTES=5000000
+
+check_fail() { printf '  FAIL  %s\n' "$1"; CHECK_FAILS=$((CHECK_FAILS + 1)); }
+
+# Everything the run needs that is not the repos themselves. Said once, first.
+check_tools() {
+    local t missing=()
+    for t in git awk sed find sort cksum numfmt base64 stat du df; do
+        command -v "$t" >/dev/null || missing+=("$t")
+    done
+    [ -n "$CONFIG" ] && { command -v jq >/dev/null || missing+=(jq); }
+    command -v rsync >/dev/null || missing+=("rsync (needed for folders that are not git repos)")
+    if [ ${#missing[@]} -gt 0 ]; then
+        for t in "${missing[@]}"; do check_fail "not installed: $t"; done
+    else
+        echo "  ok    tools"
+    fi
+}
+
+check_spec() {
+    local spec="$1" parsed ref sub s size label weigh
+    # parse_spec dies on the first problem; in a subshell that is one line of
+    # report instead of the end of the check.
+    if ! parsed="$( (parse_spec "$spec" && declare -p SPEC_DIR SPEC_SUB SPEC_NAME SPEC_REFS) 2>&1 )"; then
+        check_fail "$(printf '%s' "$parsed" | sed "s/^$SELF: //" | tail -1)"
+        return 0
+    fi
+    eval "$parsed"
+
+    local refs=("${SPEC_REFS[@]}") subs=()
+    [ ${#refs[@]} -gt 0 ] || refs=("")
+    [ -n "$SPEC_SUB" ] && IFS=, read -ra subs <<< "$SPEC_SUB"
+
+    for ref in "${refs[@]}"; do
+        label="$SPEC_NAME${ref:+@$ref}${SPEC_SUB:+:$SPEC_SUB}"
+        local bad=""
+        for sub in ${subs[@]+"${subs[@]}"}; do
+            if [ -n "$ref" ]; then
+                git -C "$SPEC_DIR" cat-file -e "$ref:$sub" 2>/dev/null \
+                    || { check_fail "$label: no '$sub' at $ref"; bad=1; }
+            elif [ ! -e "$SPEC_DIR/$sub" ]; then
+                check_fail "$label: no '$sub' in $SPEC_DIR"; bad=1
+            fi
+        done
+        [ -z "$bad" ] || continue
+
+        # Weighed from what git already knows, so nothing is read or copied: a
+        # ref from its tree, a working tree from HEAD's (uncommitted edits aside).
+        if is_git "$SPEC_DIR"; then
+            weigh="${ref:-HEAD}"
+            if git -C "$SPEC_DIR" rev-parse --verify --quiet "$weigh^{commit}" >/dev/null; then
+                size="$(git -C "$SPEC_DIR" ls-tree -r -l "$weigh" -- ${subs[@]+"${subs[@]}"} \
+                    | awk '$4 ~ /^[0-9]+$/ { s += $4 } END { print s + 0 }')"
+            else
+                size=0   # a repo with no commits yet
+            fi
+        else
+            size=0
+            if [ ${#subs[@]} -gt 0 ]; then
+                for s in "${subs[@]}"; do
+                    size=$((size + $(du -sb "$SPEC_DIR/$s" 2>/dev/null | cut -f1)))
+                done
+            else
+                size="$(du -sb --exclude=.git "$SPEC_DIR" 2>/dev/null | cut -f1)"
+            fi
+        fi
+        CHECK_BYTES=$((CHECK_BYTES + size))
+        [ "$size" -gt "$CHECK_BIGGEST" ] && CHECK_BIGGEST="$size"
+        if [ "$size" -gt "$CHECK_WARN_BYTES" ]; then
+            printf '  ok    %-60s %8s  large: exclude its data or set max_tokens?\n' "$label" "$(numfmt --to=iec "$size")"
+        else
+            printf '  ok    %-60s %8s\n' "$label" "$(numfmt --to=iec "$size")"
+        fi
+    done
+}
+
+# Space is judged against the source sizes, before noise filters: an upper
+# bound. The temp copy holds one entry at a time; the destination all of them,
+# twice over for 'both'.
+check_space() {
+    local where need avail
+    if [ -n "$DEST" ]; then
+        where="$DEST"
+        while [ ! -d "$where" ]; do where="$(dirname "$where")"; done
+        if [ ! -w "$where" ]; then
+            check_fail "cannot write to $where (for $DEST)"
+        else
+            need="$CHECK_BYTES"; [ "$CMD" = both ] && need=$((need * 2))
+            avail=$(( $(df -Pk "$where" | awk 'NR == 2 { print $4 }') * 1024 ))
+            if [ "$avail" -lt "$need" ]; then
+                check_fail "$DEST: up to $(numfmt --to=iec "$need") needed, $(numfmt --to=iec "$avail") free"
+            else
+                echo "  ok    output $DEST: $(numfmt --to=iec "$avail") free for up to $(numfmt --to=iec "$need")"
+            fi
+        fi
+    fi
+    avail=$(( $(df -Pk "$TMP" | awk 'NR == 2 { print $4 }') * 1024 ))
+    if [ "$avail" -lt "$CHECK_BIGGEST" ]; then
+        check_fail "temp $(dirname "$TMP"): the largest entry needs $(numfmt --to=iec "$CHECK_BIGGEST"), $(numfmt --to=iec "$avail") free (set TMPDIR elsewhere)"
+    else
+        echo "  ok    temp $(dirname "$TMP"): $(numfmt --to=iec "$avail") free"
+    fi
+}
+
 run_spec() {
     local spec="$1" override="${2:-}" ref dirty label
+    if [ "$CMD" = check ]; then check_spec "$spec"; return 0; fi
 
     normalize_limits
 
@@ -1544,14 +1817,31 @@ run_spec() {
     fi
 }
 
+if [ "$CMD" = check ]; then
+    echo "checking${CONFIG:+ $CONFIG}"
+    check_tools
+elif [ -z "$NO_CHECK" ]; then
+    # Every entry, before the first slow one: a broken entry near the bottom of
+    # the list otherwise costs every entry above it first. The check copies
+    # nothing, so this costs a second or two; its report is shown only when it
+    # finds something.
+    if check_out="$("$0" check ${ARGS[@]+"${ARGS[@]}"} 2>&1)"; then
+        echo "check: $(printf '%s\n' "$check_out" | tail -1)"
+    else
+        printf '%s\n' "$check_out" | grep -vE '^  ok ' >&2
+        echo "$SELF: stopped before copying anything (--no-check to run anyway)" >&2
+        exit 1
+    fi
+fi
+
 if [ -n "$DRY" ]; then
     echo "dry run — nothing will be written"
     [ "$CMD" != list ] && echo "would write to: $DEST"
 fi
 
-[ "$CMD" = list ] || [ -n "$DRY" ] || mkdir -p "$DEST"
+[ "$CMD" = list ] || [ "$CMD" = check ] || [ -n "$DRY" ] || mkdir -p "$DEST"
 
-if [ "$CMD" != list ] && [ -z "$DRY" ]; then
+if [ "$CMD" != list ] && [ "$CMD" != check ] && [ -z "$DRY" ]; then
     STATE_FILE="$DEST/.distill-state"
     [ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$STATE_OLD"
     : > "$STATE_NEW"
@@ -1573,6 +1863,7 @@ CLI_BASE_REF="$BASE_REF"
 CLI_MAX_BYTES="$MAX_BYTES"
 CLI_CLIP_BYTES="$CLIP_BYTES"
 CLI_MAX_TOKENS="$MAX_TOKENS"
+CLI_SPLIT_TOKENS="$SPLIT_TOKENS"
 CLI_KEEP_NOISE="$KEEP_NOISE"
 CLI_WITH_ROOT="$WITH_ROOT"
 CLI_INCLUDES=(${INCLUDES[@]+"${INCLUDES[@]}"})
@@ -1590,6 +1881,7 @@ if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then
         MAX_BYTES="${CLI_MAX_BYTES:-$(job_value "$job" .max_bytes)}"
         CLIP_BYTES="${CLI_CLIP_BYTES:-$(job_value "$job" .clip_bytes)}"
         MAX_TOKENS="${CLI_MAX_TOKENS:-$(job_value "$job" .max_tokens)}"
+        SPLIT_TOKENS="${CLI_SPLIT_TOKENS:-$(job_value "$job" .split_tokens)}"
 
         if [ -n "$CLI_KEEP_NOISE" ] || [ "$(job_value "$job" .all)" = true ]
         then KEEP_NOISE=1; else KEEP_NOISE=""; fi
@@ -1608,6 +1900,17 @@ if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then
     done < "$JOBS"
 fi
 
+if [ "$CMD" = check ]; then
+    check_space
+    echo
+    if [ "$CHECK_FAILS" -gt 0 ]; then
+        echo "$CHECK_FAILS problem(s): fix them before the real run"
+        exit 1
+    fi
+    echo "all good: up to $(numfmt --to=iec "$CHECK_BYTES") of sources to distill"
+    exit 0
+fi
+
 echo
 printf 'total: %d files, %s, ~%sk tokens\n' \
     "$TOTAL_FILES" "$(numfmt --to=iec "$TOTAL_BYTES")" "$((TOTAL_TEXT / 4000))"
@@ -1653,8 +1956,12 @@ fi
 if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
     while IFS= read -r -d '' entry; do
         keep=""
+        # A digest left alone as unchanged registers only its index, not the
+        # parts beside it, which belong to it all the same.
+        owner="$entry"
+        case "$entry" in *.part-[0-9][0-9].md) owner="${entry%.part-[0-9][0-9].md}.md" ;; esac
         for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
-            [ "$entry" = "$kept" ] && { keep=1; break; }
+            { [ "$entry" = "$kept" ] || [ "$owner" = "$kept" ]; } && { keep=1; break; }
         done
         if [ -z "$keep" ]; then
             echo "  pruned $(basename "$entry")"