mariano/soleprint
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

distill updates

Browse Source
This commit is contained in:
buenosairesam
2026-09-16 23:14:27 -03:00
parent 004b397b94
commit 809a13eebe
2 changed files with 403 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
soleprint/station/tools/distill/distill-example.json
View File

@@ -15,6 +15,9 @@
"include": [],
"all": false,
"max_bytes": null,
"_split_tokens": "A digest over this many tokens is written as NAME.md (an index: the tree, and the manifest saying which part holds each file) plus NAME.part-01.md, NAME.part-02.md... holding the files, cut between files. Each part stands on its own. Default 100k; 0 never splits. Can also be set per entry.",
"split_tokens": "100k",
"skip_unchanged": true,
"prune": true,

445
soleprint/station/tools/distill/distill.sh
View File

@@ -25,7 +25,16 @@
# distill.sh digest [opts] -o DEST REPO... # one concatenated .md per repo
# distill.sh both [opts] -o DEST REPO... # both, from a single pass
# distill.sh list [opts] REPO... # what would be kept, weighed
# distill.sh [tree|digest|both|list] -c FILE # read the whole job from JSON
# distill.sh check [opts] REPO... # seconds: will the run work?
# distill.sh [tree|digest|both|list|check] -c FILE # read the whole job from JSON
#
# check copies nothing. For every entry it confirms the path exists, each branch
# or commit resolves and each subpath is there at it, and weighs it from git's
# own index; then that the tools are installed and the output and temp folders
# have room. It reports every problem rather than the first, so a moved folder
# or a mistyped hash turns up before a long run, not twenty minutes into it.
# Every other command runs it first, in a second or so, and stops on a problem
# before copying anything; --no-check skips that.
#
# tree and digest answer different questions. tree gives you files — open them,
# grep them, build them. digest gives you one document to read or hand over:
@@ -45,7 +54,7 @@
# "branch_mode": "full", // or "diff", against diff_base
# "diff_base": "main",
# "exclude": [], "include": [], "all": false, "max_bytes": null,
# "clip_bytes": null, "max_tokens": null, "with_root": false,
# "clip_bytes": null, "max_tokens": null, "split_tokens": null, "with_root": false,
# "skip_unchanged": false, "prune": false, "bundle": false,
# "raw_fences": false,
# "repos": [
@@ -59,7 +68,8 @@
# rather than a keyed object; one entry per repo could not hold two branches of
# the same repo. Per-entry keys: path, branches, subpath (a string, or a list
# of them), name, enabled, branch_mode, diff_base, include, exclude, max_bytes,
# clip_bytes, max_tokens, with_root — each falling back to the top of the file.
# clip_bytes, max_tokens, split_tokens, with_root — each falling back to the top
# of the file.
# A command-line option overrides both: one repo in the list wanting a tighter
# budget should say so in its entry, but `--max-tokens 60k` on the command line
# is a thing someone just typed, and it wins over the whole file.
@@ -106,6 +116,10 @@
# largest-first — one shared size ceiling, lowered until the
# total fits — so the biggest file pays for it and the hundred
# small ones that actually describe the project do not
# --split-tokens N write a digest over ~N tokens as NAME.md, an index with the
# tree and the manifest, plus NAME.part-01.md, NAME.part-02.md…
# holding the files, cut between files in path order. Each
# part stands on its own. Default 100k; 0 never splits
# --with-root with a subpath in play, keep the repo's top-level files too
# (README, pyproject.toml, package.json) so a subtree copy
# still says which project it is a part of
@@ -121,6 +135,7 @@
# --refs-patch put the full diff, not just the diffstat, in NAME@REFS.md
# --raw-fences write runs of backticks and tildes into the digest as they
# are, instead of escaping them as ⟪BT3⟫ / ⟪TL3⟫ (see below)
# --no-check skip the check every other command runs first (see check)
# --keep-secrets include .env, private keys and the like, which are dropped
# by default and are NOT re-included by --all
# -n dry run — say what would happen, write nothing
@@ -248,12 +263,15 @@ lang_for() {
# leading option is not an error here.
CMD=""
case "${1:-}" in
tree|digest|both|list) CMD="$1"; shift ;;
tree|digest|both|list|check) CMD="$1"; shift ;;
-h|--help|help) usage; exit 0 ;;
"") usage >&2; exit 1 ;;
-*) ;;
*) die "unknown command: $1 (expected tree, digest, both or list)" ;;
*) die "unknown command: $1 (expected tree, digest, both, list or check)" ;;
esac
# Kept as given, so the check that runs first sees exactly the same job.
ARGS=("$@")
NO_CHECK=""
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
CONFIG=""
@@ -272,6 +290,12 @@ KEEP_NOISE=""
MAX_BYTES=""
CLIP_BYTES=""
MAX_TOKENS=""
SPLIT_TOKENS=""
# A digest past this many tokens is written as an index and numbered parts. Big
# enough that an ordinary repo stays one document; small enough that a part
# still fits the attachment of a web chat without crowding out the question.
DEFAULT_SPLIT_TOKENS=100000
DIGEST_PARTS=1
WITH_ROOT=""
TOP_N=10
TOP_SET=""
@@ -299,6 +323,7 @@ while [ $# -gt 0 ]; do
--max-bytes) shift; MAX_BYTES="${1:-}" ;;
--clip-bytes) shift; CLIP_BYTES="${1:-}" ;;
--max-tokens) shift; MAX_TOKENS="${1:-}" ;;
--split-tokens) shift; SPLIT_TOKENS="${1:-}" ;;
--with-root) WITH_ROOT=1 ;;
--top) shift; TOP_N="${1:-}"; TOP_SET=1 ;;
--all) KEEP_NOISE=1 ;;
@@ -308,6 +333,7 @@ while [ $# -gt 0 ]; do
--refs-patch) REFS_PATCH=1 ;;
--raw-fences) RAW_FENCES=1 ;;
--keep-secrets) KEEP_SECRETS=1 ;;
--no-check) NO_CHECK=1 ;;
--skip-unchanged) SKIP_UNCHANGED=1 ;;
-n) DRY=1 ;;
-d) MIRROR=1 ;;
@@ -353,6 +379,7 @@ normalize_limits() {
[ -n "$MAX_BYTES" ] && MAX_BYTES="$(num_arg "$MAX_BYTES" --max-bytes)"
[ -n "$CLIP_BYTES" ] && CLIP_BYTES="$(num_arg "$CLIP_BYTES" --clip-bytes)"
[ -n "$MAX_TOKENS" ] && MAX_TOKENS="$(num_arg "$MAX_TOKENS" --max-tokens)"
[ -n "$SPLIT_TOKENS" ] && SPLIT_TOKENS="$(num_arg "$SPLIT_TOKENS" --split-tokens)"
[[ "$TOP_N" =~ ^[0-9]+$ ]] || die "--top wants a plain count, got: $TOP_N"
return 0
}
@@ -437,6 +464,7 @@ if [ -n "$CONFIG" ]; then
max_bytes: (($e.max_bytes // $cfg.max_bytes // "") | tostring),
clip_bytes: (($e.clip_bytes // $cfg.clip_bytes // "") | tostring),
max_tokens: (($e.max_tokens // $cfg.max_tokens // "") | tostring),
split_tokens: (($e.split_tokens // $cfg.split_tokens // "") | tostring),
with_root: (if ($e|has("with_root")) then $e.with_root
elif ($cfg|has("with_root")) then $cfg.with_root
else false end)
@@ -456,14 +484,14 @@ fi
[ -d "$ROOT" ] || die "root is not a directory: $ROOT"
case "$CMD" in
tree|digest|both|list) ;;
*) die "unknown command: $CMD (expected tree, digest, both or list)" ;;
tree|digest|both|list|check) ;;
*) die "unknown command: $CMD (expected tree, digest, both, list or check)" ;;
esac
if [ "$CMD" != list ]; then
if [ "$CMD" != list ] && [ "$CMD" != check ]; then
[ -n "$DEST" ] || die "an output directory is required for $CMD (-o DEST, or \"out\" in the config)"
fi
case "$CMD" in tree|both) ;; *) [ -z "$MIRROR" ] || die "-d only applies to 'tree' or 'both'" ;; esac
case "$CMD" in tree|both|check) ;; *) [ -z "$MIRROR" ] || die "-d only applies to 'tree' or 'both'" ;; esac
# ── spec parsing ───────────────────────────────────────────────────────────
# <repo>[@<ref>[,<ref>...]][:<subpath>]
@@ -1025,16 +1053,144 @@ render_tree() {
# of this?" without guessing.
write_digest() {
local staged="$1" out="$2" title="$3" subtitle="$4"
local f rel fence lang bytes nfiles lines meta
local f rel fence lang bytes nfiles lines meta i
bytes=$(du -sb "$staged" | cut -f1)
nfiles=$(find "$staged" -type f | wc -l)
# Every file's section is rendered first, to its own temp file, so the
# digest can be cut between files once their sizes are known. Unsplit, the
# sections are concatenated in order and the document is what it always was.
local sections="$TMP/sections"
rm -rf "$sections"; mkdir -p "$sections"
: > "$sections.order"
i=0
while IFS= read -r -d '' f; do
rel="${f#$staged/}"
is_binary_file "$rel" && continue
lang="$(lang_for "$rel")"
lines=$(wc -l < "$f")
# The body is rendered first and the fence measured on that, so it is
# measured on exactly what goes between the fences: escaped or not,
# clipped or not. escape_fences also ends the last line, so a file with
# no trailing newline cannot weld itself to the closing fence.
if is_clipped "$rel" "$staged"; then
: > "$TMP/body.raw"
clip_render "$f" "$CLIP_T" "$TMP/body.raw"
escape_fences < "$TMP/body.raw" > "$TMP/body"
meta="_${lines} lines · $(stat -c%s "$f") bytes · CLIPPED — head and tail only_"
else
escape_fences < "$f" > "$TMP/body"
meta="_${lines} lines · $(stat -c%s "$f") bytes_"
fi
fence="$(fence_for "$TMP/body")"
i=$((i + 1))
{
echo "## $rel"
echo
echo "$meta"
echo
echo "${fence}${lang}"
cat "$TMP/body"
[ -s "$TMP/body" ] && [ -n "$(tail -c1 "$TMP/body")" ] && echo
echo "$fence"
echo
} > "$sections/$i"
printf '%s\t%s\t%s\n' "$i" "$rel" "$(stat -c%s "$sections/$i")" >> "$sections.order"
done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | sed "s|^|$staged/|" | tr '\n' '\0')
# Parts from an earlier, longer run of this same digest would otherwise sit
# beside the new ones looking current.
rm -f "${out%.md}".part-[0-9][0-9].md
DIGEST_PARTS=1
local limit total
limit=$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} * BYTES_PER_TOKEN ))
total=$(awk -F'\t' '{ s += $3 } END { print s + 0 }' "$sections.order")
if [ "$limit" -eq 0 ] || [ "$total" -le "$limit" ]; then
{
digest_intro "$title" "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(digest_extras)" full "$staged"
digest_manifest "$staged" ""
while IFS=$'\t' read -r i _ _; do cat "$sections/$i"; done < "$sections.order"
digest_end "$title" "$nfiles" "The manifest above lists"
} > "$out"
return 0
fi
# Greedy, in path order, so a folder's files stay together and each part
# reads as a contiguous stretch of the tree. A single file over the limit
# gets a part to itself rather than being split: --max-tokens is what
# shortens files.
awk -F'\t' -v limit="$limit" '
{ if (size > 0 && size + $3 > limit) { part++; size = 0 }
if (part == 0) part = 1
size += $3
print $1 "\t" $2 "\t" $3 "\t" part }' "$sections.order" > "$sections.parts"
DIGEST_PARTS=$(awk -F'\t' 'END { print $4 }' "$sections.parts")
local stem base p pfile pfiles pbytes first last
stem="${out%.md}"; base="$(basename "$stem")"
{
digest_intro "$title" "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(digest_extras) · in $DIGEST_PARTS parts" index "$staged"
echo "## Parts"
echo
echo "This document is the index. The files themselves are in $DIGEST_PARTS parts, each"
echo "under ~$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} / 1000 ))k tokens, cut between files in path order. Each part stands"
echo "on its own; attach the ones the question needs."
echo
echo "| part | files | bytes | first file | last file |"
echo "|---|---:|---:|---|---|"
for p in $(seq 1 "$DIGEST_PARTS"); do
pfiles=$(awk -F'\t' -v p="$p" '$4 == p' "$sections.parts" | wc -l)
pbytes=$(awk -F'\t' -v p="$p" '$4 == p { s += $3 } END { print s + 0 }' "$sections.parts")
first=$(awk -F'\t' -v p="$p" '$4 == p { print $2; exit }' "$sections.parts")
last=$(awk -F'\t' -v p="$p" '$4 == p { l = $2 } END { print l }' "$sections.parts")
printf '| `%s.part-%02d.md` | %d | %s | `%s` | `%s` |\n' \
"$base" "$p" "$pfiles" "$(numfmt --to=iec "$pbytes")" "$first" "$last"
done
echo
digest_manifest "$staged" "$sections.parts"
echo "## End of $title (index)"
echo
printf 'The manifest above lists %d files, in %d parts.\n' "$nfiles" "$DIGEST_PARTS"
} > "$out"
for p in $(seq 1 "$DIGEST_PARTS"); do
pfile="$(printf '%s.part-%02d.md' "$stem" "$p")"
pfiles=$(awk -F'\t' -v p="$p" '$4 == p' "$sections.parts" | wc -l)
pbytes=$(awk -F'\t' -v p="$p" '$4 == p { s += $3 } END { print s + 0 }' "$sections.parts")
{
digest_intro "$title — part $p of $DIGEST_PARTS" \
"$subtitle · part $p of $DIGEST_PARTS · $pfiles of $nfiles files · $(numfmt --to=iec "$pbytes")" \
part "$staged" "$(basename "$out")"
while IFS=$'\t' read -r i _ _ pp; do
[ "$pp" = "$p" ] && cat "$sections/$i"
done < "$sections.parts"
echo "## End of $title — part $p of $DIGEST_PARTS"
echo
printf 'This part holds %d of the %d files listed in %s; the other parts hold the rest.\n' \
"$pfiles" "$nfiles" "$(basename "$out")"
} > "$pfile"
produced "$pfile"
done
}
# The subtitle's tail: what is listed but not inlined, and what was clipped.
digest_extras() {
[ ${#BINARY_FILES[@]} -gt 0 ] && printf ' · %d binary, listed but not inlined' "${#BINARY_FILES[@]}"
[ "$CLIP_N" -gt 0 ] && printf ' · %d clipped to fit ~%dk tokens' "$CLIP_N" "$((DIGEST_BYTES / (BYTES_PER_TOKEN * 1000)))"
return 0
}
# The top of a digest, an index or a part: what the document is and how to read
# it. The escape notice is in all three, above the first '## ', which is where
# explode.sh looks for it.
digest_intro() {
local title="$1" subline="$2" kind="$3" staged="$4" index="${5:-}"
echo "# $title"
echo
echo "$subtitle · $nfiles files · $(numfmt --to=iec "$bytes")$(
[ ${#BINARY_FILES[@]} -gt 0 ] && printf ' · %d binary, listed but not inlined' "${#BINARY_FILES[@]}" || true)$(
[ "$CLIP_N" -gt 0 ] && printf ' · %d clipped to fit ~%dk tokens' "$CLIP_N" "$((DIGEST_BYTES / (BYTES_PER_TOKEN * 1000)))" || true)"
echo "$subline"
echo
echo "Each file below opens with a \`## <path>\` heading and is wrapped in a"
echo "fence longer than any run of backticks inside it, so no file can close"
@@ -1047,6 +1203,12 @@ write_digest() {
echo "three backticks or three tildes in a row inside file contents."
echo
fi
if [ "$kind" = part ]; then
echo "This is one part of a digest too long for one document. The tree and the"
echo "manifest of every file, with the part each one is in, are in $index."
echo
return 0
fi
if [ "$CLIP_N" -gt 0 ]; then
# "1 files" reads like a bug in whatever produced the document, and
# this document is asking to be trusted about its own completeness.
@@ -1061,85 +1223,62 @@ write_digest() {
echo
render_tree "$staged"
echo
}
# The manifest, and the binary files named rather than silently absent. With a
# parts map (index\trel\tsize\tpart), each row also says which part it is in.
digest_manifest() {
local staged="$1" parts="$2" rel part=""
echo "## Manifest"
echo
if [ -n "$parts" ]; then
echo "| path | lines | bytes | inlined | part |"
echo "|---|---:|---:|---|---:|"
else
echo "| path | lines | bytes | inlined |"
echo "|---|---:|---:|---|"
} > "$out"
fi
while IFS= read -r rel; do
[ -n "$parts" ] && part=" $(awk -F'\t' -v r="$rel" '$2 == r { print $4; exit }' "$parts") |"
if is_binary_file "$rel"; then
printf '| `%s` | — | %s | no — binary, in the tree copy only |\n' \
"$rel" "$(stat -c%s "$staged/$rel")" >> "$out"
printf '| `%s` | — | %s | no — binary, in the tree copy only |%s\n' \
"$rel" "$(stat -c%s "$staged/$rel")" "${part:+ — |}"
elif is_clipped "$rel" "$staged"; then
printf '| `%s` | %s | %s | **clipped** to ~%s |\n' "$rel" \
printf '| `%s` | %s | %s | **clipped** to ~%s |%s\n' "$rel" \
"$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" \
"$(numfmt --to=iec "$CLIP_T")" >> "$out"
"$(numfmt --to=iec "$CLIP_T")" "$part"
else
printf '| `%s` | %s | %s | full |\n' "$rel" \
"$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" >> "$out"
printf '| `%s` | %s | %s | full |%s\n' "$rel" \
"$(wc -l < "$staged/$rel")" "$(stat -c%s "$staged/$rel")" "$part"
fi
done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort)
echo >> "$out"
echo
# Named, not silently absent. Something reading only this file would
# otherwise have no idea the spreadsheets exist at all.
if [ ${#BINARY_FILES[@]} -gt 0 ]; then
{
echo "## Binary files (present in the copy, not inlined here)"
echo
for rel in ${BINARY_FILES[@]+"${BINARY_FILES[@]}"}; do
echo "- \`$rel\` ($(numfmt --to=iec "$(stat -c%s "$staged/$rel")"))"
done
echo
} >> "$out"
fi
while IFS= read -r -d '' f; do
rel="${f#$staged/}"
is_binary_file "$rel" && continue
lang="$(lang_for "$rel")"
lines=$(wc -l < "$f")
# The body is rendered first and the fence measured on that, so it is
# measured on exactly what goes between the fences: escaped or not,
# clipped or not. escape_fences also ends the last line, so a file with
# no trailing newline cannot weld itself to the closing fence.
if is_clipped "$rel" "$staged"; then
: > "$TMP/body.raw"
clip_render "$f" "$CLIP_T" "$TMP/body.raw"
escape_fences < "$TMP/body.raw" > "$TMP/body"
meta="_${lines} lines · $(stat -c%s "$f") bytes · CLIPPED — head and tail only_"
else
escape_fences < "$f" > "$TMP/body"
meta="_${lines} lines · $(stat -c%s "$f") bytes_"
fi
fence="$(fence_for "$TMP/body")"
{
echo "## $rel"
echo
echo "$meta"
echo
echo "${fence}${lang}"
cat "$TMP/body"
[ -s "$TMP/body" ] && [ -n "$(tail -c1 "$TMP/body")" ] && echo
echo "$fence"
echo
} >> "$out"
done < <(cd "$staged" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | sed "s|^|$staged/|" | tr '\n' '\0')
}
# The reader has no other way to know the document did not stop early. The
# count has to be exact, including the ways a file can be here but not
# whole — a marker claiming everything is complete, next to a clipped file,
# is worse than no marker.
{
# The reader has no other way to know the document did not stop early. The
# count has to be exact, including the ways a file can be here but not whole —
# a marker claiming everything is complete, next to a clipped file, is worse
# than no marker.
digest_end() {
local title="$1" nfiles="$2" lead="$3"
echo "## End of $title"
echo
printf 'The manifest above lists %d files: %d in full' \
printf '%s %d files: %d in full' "$lead" \
"$nfiles" "$(( nfiles - CLIP_N - ${#BINARY_FILES[@]} ))"
[ "$CLIP_N" -gt 0 ] && printf ', %d clipped (each marked at the cut)' "$CLIP_N"
[ ${#BINARY_FILES[@]} -gt 0 ] && printf ', %d binary and not inlined' "${#BINARY_FILES[@]}"
printf '.\n'
} >> "$out"
}
# When several refs of one repo are distilled, ship the comparison too. Whatever
@@ -1212,6 +1351,21 @@ STATE_OLD="$TMP/state.old"
STATE_NEW="$TMP/state.new"
PRODUCED=()
# Saved after every entry, not once at the end. A run over a dozen repos and
# their branches takes long enough to get interrupted, and a record written only
# on completion meant every interrupted run started again from the first entry
# and never reached the ones at the bottom of the list. The file on disk is
# always: what this run has finished, plus the previous run's records for what
# it has not reached yet. Replaced atomically, so an interruption mid-write
# leaves the previous version whole.
save_state() {
[ -n "$STATE_FILE" ] || return 0
awk -F'\t' 'NR == FNR { done[$1] = 1; print; next } !($1 in done)' \
"$STATE_NEW" "$( [ -f "$STATE_OLD" ] && echo "$STATE_OLD" || echo /dev/null )" \
> "$STATE_FILE.partial"
mv "$STATE_FILE.partial" "$STATE_FILE"
}
state_lookup() { # label -> prints the stored record, or nothing
[ -f "$STATE_OLD" ] || return 0
grep -F -m1 "$(printf '%s\t' "$1")" "$STATE_OLD" 2>/dev/null || true
@@ -1233,10 +1387,10 @@ fingerprint() {
else
src="plain:$(find "$dir" -type f -printf '%P %s %T@\n' 2>/dev/null | LC_ALL=C sort | cksum | cut -d" " -f1)"
fi
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s' \
"$src" "$sub" "$CMD" "$BASE_REF" "$KEEP_NOISE" "$MAX_BYTES" \
"${INCLUDES[*]-}" "${EXCLUDES[*]-}" "$MIRROR" \
"$CLIP_BYTES" "$MAX_TOKENS" "$WITH_ROOT" "$RAW_FENCES" \
"$CLIP_BYTES" "$MAX_TOKENS" "$WITH_ROOT" "$RAW_FENCES" "${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS}" \
| cksum | cut -d' ' -f1
}
@@ -1341,6 +1495,7 @@ process() {
TOTAL_TEXT=$((TOTAL_TEXT + p_text))
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$p_files" "$p_bytes" \
"$(printf '%s' "$p_row" | base64 -w0)" "$p_text" >> "$STATE_NEW"
save_state
return 0
fi
fi
@@ -1391,7 +1546,14 @@ process() {
"$label" "$STAGED" "$(numfmt --to=iec "$bytes")" "$((tokens / 1000))" \
"$([ "$CLIP_N" -gt 0 ] && printf ' (%d clipped at %s)' "$CLIP_N" "$(numfmt --to=iec "$CLIP_T")" || true)"
if [ "$CMD" = list ]; then report_weight "$staged"; fi
if [ "$CMD" = list ]; then
local split_at=$(( ${SPLIT_TOKENS:-$DEFAULT_SPLIT_TOKENS} ))
if [ "$split_at" -gt 0 ] && [ "$tokens" -gt "$split_at" ]; then
printf ' digest would be split: ~%sk tokens against %sk per part\n' \
"$((tokens / 1000))" "$((split_at / 1000))"
fi
report_weight "$staged"
fi
local dropped=$((DROPPED_NOISE + DROPPED_BIG + DROPPED_GONE + DROPPED_SECRET))
MANIFEST_ROWS+=("| \`$label\` | $dir | $kind | $STAGED | $dropped | $(numfmt --to=iec "$bytes") | ~$((tokens / 1000))k |")
@@ -1455,6 +1617,7 @@ process() {
[ -n "$sub" ] && desc="$desc · scope $sub"
[ -n "$is_delta" ] && desc="$desc · ONLY files differing from $BASE_REF"
write_digest "$staged" "$DEST/$label.md" "$name" "$desc"
[ "$DIGEST_PARTS" -gt 1 ] && printf ' split: %s.md is the index, the files are in %d parts\n' "$label" "$DIGEST_PARTS"
;;
esac
@@ -1463,6 +1626,7 @@ process() {
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$label" "$fp" "$STAGED" "$bytes" \
"$(printf '%s' "${MANIFEST_ROWS[-1]}" | base64 -w0)" \
"$DIGEST_BYTES" >> "$STATE_NEW"
save_state
fi
}
@@ -1508,8 +1672,117 @@ label_for() {
LABEL="$label"
}
# ── check: will the run work? ──────────────────────────────────────────────
CHECK_FAILS=0
CHECK_BYTES=0
CHECK_BIGGEST=0
CHECK_WARN_BYTES=5000000
check_fail() { printf ' FAIL %s\n' "$1"; CHECK_FAILS=$((CHECK_FAILS + 1)); }
# Everything the run needs that is not the repos themselves. Said once, first.
check_tools() {
local t missing=()
for t in git awk sed find sort cksum numfmt base64 stat du df; do
command -v "$t" >/dev/null || missing+=("$t")
done
[ -n "$CONFIG" ] && { command -v jq >/dev/null || missing+=(jq); }
command -v rsync >/dev/null || missing+=("rsync (needed for folders that are not git repos)")
if [ ${#missing[@]} -gt 0 ]; then
for t in "${missing[@]}"; do check_fail "not installed: $t"; done
else
echo " ok tools"
fi
}
check_spec() {
local spec="$1" parsed ref sub s size label weigh
# parse_spec dies on the first problem; in a subshell that is one line of
# report instead of the end of the check.
if ! parsed="$( (parse_spec "$spec" && declare -p SPEC_DIR SPEC_SUB SPEC_NAME SPEC_REFS) 2>&1 )"; then
check_fail "$(printf '%s' "$parsed" | sed "s/^$SELF: //" | tail -1)"
return 0
fi
eval "$parsed"
local refs=("${SPEC_REFS[@]}") subs=()
[ ${#refs[@]} -gt 0 ] || refs=("")
[ -n "$SPEC_SUB" ] && IFS=, read -ra subs <<< "$SPEC_SUB"
for ref in "${refs[@]}"; do
label="$SPEC_NAME${ref:+@$ref}${SPEC_SUB:+:$SPEC_SUB}"
local bad=""
for sub in ${subs[@]+"${subs[@]}"}; do
if [ -n "$ref" ]; then
git -C "$SPEC_DIR" cat-file -e "$ref:$sub" 2>/dev/null \
|| { check_fail "$label: no '$sub' at $ref"; bad=1; }
elif [ ! -e "$SPEC_DIR/$sub" ]; then
check_fail "$label: no '$sub' in $SPEC_DIR"; bad=1
fi
done
[ -z "$bad" ] || continue
# Weighed from what git already knows, so nothing is read or copied: a
# ref from its tree, a working tree from HEAD's (uncommitted edits aside).
if is_git "$SPEC_DIR"; then
weigh="${ref:-HEAD}"
if git -C "$SPEC_DIR" rev-parse --verify --quiet "$weigh^{commit}" >/dev/null; then
size="$(git -C "$SPEC_DIR" ls-tree -r -l "$weigh" -- ${subs[@]+"${subs[@]}"} \
| awk '$4 ~ /^[0-9]+$/ { s += $4 } END { print s + 0 }')"
else
size=0 # a repo with no commits yet
fi
else
size=0
if [ ${#subs[@]} -gt 0 ]; then
for s in "${subs[@]}"; do
size=$((size + $(du -sb "$SPEC_DIR/$s" 2>/dev/null | cut -f1)))
done
else
size="$(du -sb --exclude=.git "$SPEC_DIR" 2>/dev/null | cut -f1)"
fi
fi
CHECK_BYTES=$((CHECK_BYTES + size))
[ "$size" -gt "$CHECK_BIGGEST" ] && CHECK_BIGGEST="$size"
if [ "$size" -gt "$CHECK_WARN_BYTES" ]; then
printf ' ok %-60s %8s large: exclude its data or set max_tokens?\n' "$label" "$(numfmt --to=iec "$size")"
else
printf ' ok %-60s %8s\n' "$label" "$(numfmt --to=iec "$size")"
fi
done
}
# Space is judged against the source sizes, before noise filters: an upper
# bound. The temp copy holds one entry at a time; the destination all of them,
# twice over for 'both'.
check_space() {
local where need avail
if [ -n "$DEST" ]; then
where="$DEST"
while [ ! -d "$where" ]; do where="$(dirname "$where")"; done
if [ ! -w "$where" ]; then
check_fail "cannot write to $where (for $DEST)"
else
need="$CHECK_BYTES"; [ "$CMD" = both ] && need=$((need * 2))
avail=$(( $(df -Pk "$where" | awk 'NR == 2 { print $4 }') * 1024 ))
if [ "$avail" -lt "$need" ]; then
check_fail "$DEST: up to $(numfmt --to=iec "$need") needed, $(numfmt --to=iec "$avail") free"
else
echo " ok output $DEST: $(numfmt --to=iec "$avail") free for up to $(numfmt --to=iec "$need")"
fi
fi
fi
avail=$(( $(df -Pk "$TMP" | awk 'NR == 2 { print $4 }') * 1024 ))
if [ "$avail" -lt "$CHECK_BIGGEST" ]; then
check_fail "temp $(dirname "$TMP"): the largest entry needs $(numfmt --to=iec "$CHECK_BIGGEST"), $(numfmt --to=iec "$avail") free (set TMPDIR elsewhere)"
else
echo " ok temp $(dirname "$TMP"): $(numfmt --to=iec "$avail") free"
fi
}
run_spec() {
local spec="$1" override="${2:-}" ref dirty label
if [ "$CMD" = check ]; then check_spec "$spec"; return 0; fi
normalize_limits
@@ -1544,14 +1817,31 @@ run_spec() {
fi
}
if [ "$CMD" = check ]; then
echo "checking${CONFIG:+ $CONFIG}"
check_tools
elif [ -z "$NO_CHECK" ]; then
# Every entry, before the first slow one: a broken entry near the bottom of
# the list otherwise costs every entry above it first. The check copies
# nothing, so this costs a second or two; its report is shown only when it
# finds something.
if check_out="$("$0" check ${ARGS[@]+"${ARGS[@]}"} 2>&1)"; then
echo "check: $(printf '%s\n' "$check_out" | tail -1)"
else
printf '%s\n' "$check_out" | grep -vE '^ ok ' >&2
echo "$SELF: stopped before copying anything (--no-check to run anyway)" >&2
exit 1
fi
fi
if [ -n "$DRY" ]; then
echo "dry run — nothing will be written"
[ "$CMD" != list ] && echo "would write to: $DEST"
fi
[ "$CMD" = list ] || [ -n "$DRY" ] || mkdir -p "$DEST"
[ "$CMD" = list ] || [ "$CMD" = check ] || [ -n "$DRY" ] || mkdir -p "$DEST"
if [ "$CMD" != list ] && [ -z "$DRY" ]; then
if [ "$CMD" != list ] && [ "$CMD" != check ] && [ -z "$DRY" ]; then
STATE_FILE="$DEST/.distill-state"
[ -f "$STATE_FILE" ] && cp "$STATE_FILE" "$STATE_OLD"
: > "$STATE_NEW"
@@ -1573,6 +1863,7 @@ CLI_BASE_REF="$BASE_REF"
CLI_MAX_BYTES="$MAX_BYTES"
CLI_CLIP_BYTES="$CLIP_BYTES"
CLI_MAX_TOKENS="$MAX_TOKENS"
CLI_SPLIT_TOKENS="$SPLIT_TOKENS"
CLI_KEEP_NOISE="$KEEP_NOISE"
CLI_WITH_ROOT="$WITH_ROOT"
CLI_INCLUDES=(${INCLUDES[@]+"${INCLUDES[@]}"})
@@ -1590,6 +1881,7 @@ if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then
MAX_BYTES="${CLI_MAX_BYTES:-$(job_value "$job" .max_bytes)}"
CLIP_BYTES="${CLI_CLIP_BYTES:-$(job_value "$job" .clip_bytes)}"
MAX_TOKENS="${CLI_MAX_TOKENS:-$(job_value "$job" .max_tokens)}"
SPLIT_TOKENS="${CLI_SPLIT_TOKENS:-$(job_value "$job" .split_tokens)}"
if [ -n "$CLI_KEEP_NOISE" ] || [ "$(job_value "$job" .all)" = true ]
then KEEP_NOISE=1; else KEEP_NOISE=""; fi
@@ -1608,6 +1900,17 @@ if [ -n "$CONFIG" ] && [ ${#SPECS[@]} -eq 0 ]; then
done < "$JOBS"
fi
if [ "$CMD" = check ]; then
check_space
echo
if [ "$CHECK_FAILS" -gt 0 ]; then
echo "$CHECK_FAILS problem(s): fix them before the real run"
exit 1
fi
echo "all good: up to $(numfmt --to=iec "$CHECK_BYTES") of sources to distill"
exit 0
fi
echo
printf 'total: %d files, %s, ~%sk tokens\n' \
"$TOTAL_FILES" "$(numfmt --to=iec "$TOTAL_BYTES")" "$((TOTAL_TEXT / 4000))"
@@ -1653,8 +1956,12 @@ fi
if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
while IFS= read -r -d '' entry; do
keep=""
# A digest left alone as unchanged registers only its index, not the
# parts beside it, which belong to it all the same.
owner="$entry"
case "$entry" in *.part-[0-9][0-9].md) owner="${entry%.part-[0-9][0-9].md}.md" ;; esac
for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
[ "$entry" = "$kept" ] && { keep=1; break; }
{ [ "$entry" = "$kept" ] || [ "$owner" = "$kept" ]; } && { keep=1; break; }
done
if [ -z "$keep" ]; then
echo " pruned $(basename "$entry")"