From f4b2c15a9f9b3a40d7dbdb530af25c16f0088477 Mon Sep 17 00:00:00 2001 From: buenosaires Date: Tue, 22 Sep 2026 06:22:04 -0300 Subject: [PATCH] update distill --- .../tools/distill/distill-example.json | 2 +- soleprint/station/tools/distill/distill.sh | 55 +++++++++++++++++-- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/soleprint/station/tools/distill/distill-example.json b/soleprint/station/tools/distill/distill-example.json index da019aa..f931036 100644 --- a/soleprint/station/tools/distill/distill-example.json +++ b/soleprint/station/tools/distill/distill-example.json @@ -31,7 +31,7 @@ "_keep_secrets": ".env files, private keys, keystores and the like are dropped by default and named in MANIFEST.md. 'all' does not bring them back — only this does, and only deliberately.", "keep_secrets": false, - "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes anything in 'out' this run did not produce, so dropping a repo from the list drops its output too.", + "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes what an earlier run into 'out' wrote and this run no longer produces, so dropping a repo from the list drops its output too. Anything distill did not write (it keeps the list in out/.distill-owned) is never touched, so 'out' can be a folder shared with other things.", "skip_unchanged": true, "prune": true, diff --git a/soleprint/station/tools/distill/distill.sh b/soleprint/station/tools/distill/distill.sh index 3b53fcb..d41d627 100755 --- a/soleprint/station/tools/distill/distill.sh +++ b/soleprint/station/tools/distill/distill.sh @@ -128,8 +128,12 @@ # --strict refuse a dirty working tree (only affects worktree copies) # --skip-unchanged leave alone anything whose source and settings have not # moved since the last run into this destination -# --prune delete anything in DEST this run did not produce, so -# dropping a repo from the list drops its output too +# --prune delete what an earlier run into DEST wrote and this run no +# longer produces, so dropping a repo from the list drops its +# output too. Only what distill created is ever removed +# (DEST/.distill-owned records it): other things in DEST, and +# folders that existed before a run first synced into them, +# are never touched, so DEST can be a shared folder # --bundle also write DEST/_BUNDLE.md: every digest concatenated into # one document, for anything that takes a single file # --refs-patch put the full diff, not just the diffstat, in NAME@REFS.md @@ -1350,6 +1354,17 @@ STATE_FILE="" STATE_OLD="$TMP/state.old" STATE_NEW="$TMP/state.new" PRODUCED=() +# Which top-level entries of DEST distill wrote, as of the previous run. Prune +# reads this, not the directory: DEST may be a folder shared with things that +# were never distill's, and those must survive a prune however it is set. +# +# Owned means created, not merely written to. An entry that was already there +# the first time a run wrote into it — a folder being synced into, holding files +# of its own — is updated but never claimed, so dropping it from the list can +# never delete what was there before distill. +OWNED_FILE="" +OWNED_OLD="$TMP/owned.old" +EXISTED_BEFORE="$TMP/existed.before" # Saved after every entry, not once at the end. A run over a dozen repos and # their branches takes long enough to get interrupted, and a record written only @@ -1847,6 +1862,10 @@ if [ "$CMD" != list ] && [ "$CMD" != check ] && [ -z "$DRY" ]; then : > "$STATE_NEW" produced "$DEST/MANIFEST.md" produced "$STATE_FILE" + OWNED_FILE="$DEST/.distill-owned" + [ -f "$OWNED_FILE" ] && cp "$OWNED_FILE" "$OWNED_OLD" + find "$DEST" -mindepth 1 -maxdepth 1 -printf '%f\n' > "$EXISTED_BEFORE" + produced "$OWNED_FILE" fi # Repos named on the command line run under the global options, as before. @@ -1950,16 +1969,19 @@ if [ -n "$BUNDLE" ] && [ "$CMD" != tree ] && [ "$CMD" != list ] && [ -z "$DRY" ] fi fi -# Anything at the top of the destination this run did not produce came from a -# previous, longer list. Scoped to depth 1 and to a destination we just wrote -# to: this deletes, so it should never go hunting. -if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then +# Something distill wrote on an earlier run and this run did not produce came +# from a previous, longer list. Only that: an entry distill never wrote is not +# an orphan but someone else's, and DEST may well be a folder shared with them. +# Scoped to depth 1 and to a destination we just wrote to: this deletes, so it +# should never go hunting. +if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ] && [ -f "$OWNED_OLD" ]; then while IFS= read -r -d '' entry; do keep="" # A digest left alone as unchanged registers only its index, not the # parts beside it, which belong to it all the same. owner="$entry" case "$entry" in *.part-[0-9][0-9].md) owner="${entry%.part-[0-9][0-9].md}.md" ;; esac + grep -qxF "$(basename "$owner")" "$OWNED_OLD" || continue for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do { [ "$entry" = "$kept" ] || [ "$owner" = "$kept" ]; } && { keep=1; break; } done @@ -1970,6 +1992,27 @@ if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then done < <(find "$DEST" -mindepth 1 -maxdepth 1 -print0) fi +# Written whether or not prune is on: what this run created, plus whatever an +# earlier run created that is still on disk, so turning prune on later still +# knows everything distill put here. +if [ -n "$OWNED_FILE" ]; then + { + for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do + kept="$(basename "$kept")" + if ! grep -qxF "$kept" "$EXISTED_BEFORE" \ + || { [ -f "$OWNED_OLD" ] && grep -qxF "$kept" "$OWNED_OLD"; }; then + printf '%s\n' "$kept" + fi + done + if [ -f "$OWNED_OLD" ]; then + while IFS= read -r o; do + if [ -n "$o" ] && [ -e "$DEST/$o" ]; then printf '%s\n' "$o"; fi + done < "$OWNED_OLD" + fi + } | LC_ALL=C sort -u > "$OWNED_FILE.partial" + mv "$OWNED_FILE.partial" "$OWNED_FILE" +fi + if [ -n "$SKIP_UNCHANGED" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then mv "$STATE_NEW" "$STATE_FILE" fi