update distill

soleprint/station/tools/distill/distill-example.json

@@ -31,7 +31,7 @@
   "_keep_secrets": ".env files, private keys, keystores and the like are dropped by default and named in MANIFEST.md. 'all' does not bring them back — only this does, and only deliberately.",
   "keep_secrets": false,
 
-  "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes anything in 'out' this run did not produce, so dropping a repo from the list drops its output too.",
+  "_run": "skip_unchanged leaves alone anything whose source and settings have not moved since the last run into this destination. prune deletes what an earlier run into 'out' wrote and this run no longer produces, so dropping a repo from the list drops its output too. Anything distill did not write (it keeps the list in out/.distill-owned) is never touched, so 'out' can be a folder shared with other things.",
   "skip_unchanged": true,
   "prune": true,
 

soleprint/station/tools/distill/distill.sh

@@ -128,8 +128,12 @@
 #   --strict        refuse a dirty working tree (only affects worktree copies)
 #   --skip-unchanged  leave alone anything whose source and settings have not
 #                   moved since the last run into this destination
-#   --prune         delete anything in DEST this run did not produce, so
+#   --prune         delete what an earlier run into DEST wrote and this run no
-#                   dropping a repo from the list drops its output too
+#                   longer produces, so dropping a repo from the list drops its
+#                   output too. Only what distill created is ever removed
+#                   (DEST/.distill-owned records it): other things in DEST, and
+#                   folders that existed before a run first synced into them,
+#                   are never touched, so DEST can be a shared folder
 #   --bundle        also write DEST/_BUNDLE.md: every digest concatenated into
 #                   one document, for anything that takes a single file
 #   --refs-patch    put the full diff, not just the diffstat, in NAME@REFS.md
@@ -1350,6 +1354,17 @@ STATE_FILE=""
 STATE_OLD="$TMP/state.old"
 STATE_NEW="$TMP/state.new"
 PRODUCED=()
+# Which top-level entries of DEST distill wrote, as of the previous run. Prune
+# reads this, not the directory: DEST may be a folder shared with things that
+# were never distill's, and those must survive a prune however it is set.
+#
+# Owned means created, not merely written to. An entry that was already there
+# the first time a run wrote into it — a folder being synced into, holding files
+# of its own — is updated but never claimed, so dropping it from the list can
+# never delete what was there before distill.
+OWNED_FILE=""
+OWNED_OLD="$TMP/owned.old"
+EXISTED_BEFORE="$TMP/existed.before"
 
 # Saved after every entry, not once at the end. A run over a dozen repos and
 # their branches takes long enough to get interrupted, and a record written only
@@ -1847,6 +1862,10 @@ if [ "$CMD" != list ] && [ "$CMD" != check ] && [ -z "$DRY" ]; then
     : > "$STATE_NEW"
     produced "$DEST/MANIFEST.md"
     produced "$STATE_FILE"
+    OWNED_FILE="$DEST/.distill-owned"
+    [ -f "$OWNED_FILE" ] && cp "$OWNED_FILE" "$OWNED_OLD"
+    find "$DEST" -mindepth 1 -maxdepth 1 -printf '%f\n' > "$EXISTED_BEFORE"
+    produced "$OWNED_FILE"
 fi
 
 # Repos named on the command line run under the global options, as before.
@@ -1950,16 +1969,19 @@ if [ -n "$BUNDLE" ] && [ "$CMD" != tree ] && [ "$CMD" != list ] && [ -z "$DRY" ]
     fi
 fi
 
-# Anything at the top of the destination this run did not produce came from a
+# Something distill wrote on an earlier run and this run did not produce came
-# previous, longer list. Scoped to depth 1 and to a destination we just wrote
+# from a previous, longer list. Only that: an entry distill never wrote is not
-# to: this deletes, so it should never go hunting.
+# an orphan but someone else's, and DEST may well be a folder shared with them.
-if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
+# Scoped to depth 1 and to a destination we just wrote to: this deletes, so it
+# should never go hunting.
+if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ] && [ -f "$OWNED_OLD" ]; then
     while IFS= read -r -d '' entry; do
         keep=""
         # A digest left alone as unchanged registers only its index, not the
         # parts beside it, which belong to it all the same.
         owner="$entry"
         case "$entry" in *.part-[0-9][0-9].md) owner="${entry%.part-[0-9][0-9].md}.md" ;; esac
+        grep -qxF "$(basename "$owner")" "$OWNED_OLD" || continue
         for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
             { [ "$entry" = "$kept" ] || [ "$owner" = "$kept" ]; } && { keep=1; break; }
         done
@@ -1970,6 +1992,27 @@ if [ -n "$PRUNE" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
     done < <(find "$DEST" -mindepth 1 -maxdepth 1 -print0)
 fi
 
+# Written whether or not prune is on: what this run created, plus whatever an
+# earlier run created that is still on disk, so turning prune on later still
+# knows everything distill put here.
+if [ -n "$OWNED_FILE" ]; then
+    {
+        for kept in ${PRODUCED[@]+"${PRODUCED[@]}"}; do
+            kept="$(basename "$kept")"
+            if ! grep -qxF "$kept" "$EXISTED_BEFORE" \
+               || { [ -f "$OWNED_OLD" ] && grep -qxF "$kept" "$OWNED_OLD"; }; then
+                printf '%s\n' "$kept"
+            fi
+        done
+        if [ -f "$OWNED_OLD" ]; then
+            while IFS= read -r o; do
+                if [ -n "$o" ] && [ -e "$DEST/$o" ]; then printf '%s\n' "$o"; fi
+            done < "$OWNED_OLD"
+        fi
+    } | LC_ALL=C sort -u > "$OWNED_FILE.partial"
+    mv "$OWNED_FILE.partial" "$OWNED_FILE"
+fi
+
 if [ -n "$SKIP_UNCHANGED" ] && [ "$CMD" != list ] && [ -z "$DRY" ]; then
     mv "$STATE_NEW" "$STATE_FILE"
 fi