815 lines
39 KiB
Bash
Executable File
815 lines
39 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Explode one file back into the tree of files it describes.
|
|
#
|
|
# The inverse of distill.sh's digest: something hands you a single text
|
|
# file with many files inside it, each introduced by its path, and you want the
|
|
# directory back.
|
|
#
|
|
# Three layouts are understood, picked automatically. Prefer the first if you
|
|
# control what writes the file:
|
|
#
|
|
# @@ FILE: pkg/models/domain.py explicit open and close. Nothing has to be
|
|
# <the file> counted or inferred, and a block that is
|
|
# @@ END FILE: pkg/models/domain.py never closed is an error rather than a
|
|
# file quietly missing its tail. The path
|
|
# after END is optional; when it is there it
|
|
# has to match, so two files merged into one
|
|
# block fail instead of gluing together.
|
|
#
|
|
# === FILE: pkg/models/domain.py the same thing with '===' instead of '@@'.
|
|
# <the file> Still read, but do not ask for it: see the
|
|
# === END note on markdown below.
|
|
#
|
|
# === ./pkg/models/domain.py a marker line, then the file, until the
|
|
# <the file> next marker or the end
|
|
#
|
|
# ## pkg/models/domain.py distill.sh's own digest: a heading, then
|
|
# ```python a fenced block. The fence may be longer
|
|
# <the file> than three backticks, and the closing one
|
|
# ``` has to match it exactly.
|
|
#
|
|
# Usage:
|
|
# explode.sh [opts] FILE
|
|
#
|
|
# Options:
|
|
# -o DEST where to write the tree (default: the current directory)
|
|
# --list print what the file contains and write nothing
|
|
# -n same as --list
|
|
# --force overwrite files that already exist
|
|
# --raw leave ⟪BT3⟫-style escapes as they are (see below)
|
|
# --format F fenced | marker | digest | auto (default: auto)
|
|
# --contract print the instructions to hand to whatever generates the file
|
|
# --check F test a reply to the word "check" (see below), write nothing
|
|
# --selftest check this copy of the script against known input and exit
|
|
#
|
|
# Examples:
|
|
# explode.sh --list bundle.txt
|
|
# explode.sh -o ./restored bundle.txt
|
|
# explode.sh -o ./restored --force repo.md
|
|
# explode.sh --contract > /tmp/contract.txt # attach this, do not paste it
|
|
# explode.sh -o ~/wdir ~/Downloads/reply.md # one reply, every project
|
|
#
|
|
# Why the explicit form is worth asking for: a writer that emits plain three-
|
|
# backtick fences truncates any file that itself contains a fence — every README
|
|
# with a shell example — and does it silently, because the nested fence looks
|
|
# exactly like the closing one. distill.sh avoids that by making its fences
|
|
# longer than anything inside the file, but nothing else will bother.
|
|
#
|
|
# Ask for '@@', not '==='. A chat box renders markdown before the model sees the
|
|
# message, and '===' alone on a line directly under text is setext syntax for a
|
|
# level-one heading — so the format spec you paste gets swallowed and rendered as
|
|
# a title, and the model is told nothing. '---' is worse (heading AND horizontal
|
|
# rule), '##' is a heading, backticks open a fence. '@@' has no meaning in
|
|
# markdown at all, which is the whole reason to use it. Both are parsed here, so
|
|
# nothing already written stops working.
|
|
#
|
|
# The other half of that: put the spec in an attached file rather than the
|
|
# message body. Attachments are not rendered. '--contract' prints the exact text
|
|
# to attach, so the wording cannot drift from the parser that reads the reply
|
|
# back.
|
|
#
|
|
# One reply, several projects. A thread produces one file however many repos it
|
|
# touched, so --contract asks for paths that start with the project name. Point
|
|
# -o at the directory those projects are siblings in and each file lands in its
|
|
# own worktree — no table to maintain, and a new worktree needs no change here.
|
|
#
|
|
# Two limits worth knowing. A file whose last line has no trailing newline comes
|
|
# back with one: the digest has to put a newline before the closing fence, so the
|
|
# distinction is not in the input to recover. And in marker layout a line
|
|
# starting with "=== " inside a file's own content is indistinguishable from a
|
|
# real marker — there are no fences to say otherwise. The digest layout has no
|
|
# such ambiguity, which is the reason to prefer it when something else is
|
|
# generating the file.
|
|
#
|
|
# Escapes. distill.sh writes every run of three or more backticks inside a file
|
|
# as ⟪BTn⟫, tildes as ⟪TLn⟫, and a literal ⟪ as ⟪LQ⟫, and --contract asks for the
|
|
# same in the reply, because a real ``` inside a file closes the fence the chat
|
|
# UI renders it in and the rest of the reply turns into markdown soup. They are
|
|
# turned back into the characters on the way out: always for the @@ and marker
|
|
# layouts, and for a digest only when its header says it was escaped. --raw
|
|
# turns that off.
|
|
#
|
|
# Fences the model adds anyway. Asked for bare blocks, a model still wraps
|
|
# things: all of it in one fence, which puts the fence lines between blocks
|
|
# where they are ignored as prose, or each file in its own, which puts them
|
|
# inside. A block whose first line opens a fence and whose last line closes one
|
|
# has both dropped. Only the pair: a README can end on a closing fence, but one
|
|
# that also starts on an opening fence is not a README anyone writes. Take the
|
|
# reply from the "copy response" button, not by selecting the rendered text —
|
|
# the button gives the markdown as written.
|
|
#
|
|
# The format check. A long reply that turns out unusable costs the whole wait.
|
|
# The contract defines a six-line check file that exercises what breaks: a code
|
|
# fence that has to be escaped, a tab, straight quotes. Send the model the single
|
|
# word "check", save its reply, and run explode.sh --check on it: it unpacks the
|
|
# reply through the real parser, compares the result byte for byte, and says what
|
|
# went wrong — escapes skipped, tabs turned into spaces by the copy, curly quotes,
|
|
# missing wrapper — with a correction to paste back. Seconds, not a lost answer.
|
|
#
|
|
# Paths come out of a text file, so they are treated as untrusted: anything
|
|
# absolute, or reaching upward with .., is refused and nothing is written. A
|
|
# file that describes /etc/cron.d/x is not a file you want to expand blindly.
|
|
set -euo pipefail
|
|
|
|
SELF="$(basename "$0")"
|
|
usage() { awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"; }
|
|
die() { echo "$SELF: $*" >&2; exit 1; }
|
|
|
|
DEST="."
|
|
LIST=""
|
|
FORCE=""
|
|
RAW=""
|
|
FORMAT="auto"
|
|
SRC=""
|
|
SELFTEST=""
|
|
CONTRACT=""
|
|
CHECK=""
|
|
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
-o) shift; DEST="${1:-}" ;;
|
|
--list|-n) LIST=1 ;;
|
|
--force) FORCE=1 ;;
|
|
--raw) RAW=1 ;;
|
|
--format) shift; FORMAT="${1:-}" ;;
|
|
--contract) CONTRACT=1 ;;
|
|
--check) CHECK=1 ;;
|
|
--selftest) SELFTEST=1 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
-*) die "unknown option: $1" ;;
|
|
*) [ -z "$SRC" ] && SRC="$1" || die "one input file at a time (got '$SRC' and '$1')" ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
# ── the contract ───────────────────────────────────────────────────────────
|
|
# One copy of the wording, printed rather than remembered, so what you ask for
|
|
# and what this parses cannot drift apart. Attach it; do not paste it into the
|
|
# message body, where markdown gets a say first.
|
|
contract() {
|
|
cat <<'CONTRACT'
|
|
INSTRUCTIONS FOR THIS CONVERSATION
|
|
|
|
Read this whole file before replying, and follow it in every reply of this
|
|
conversation. Your replies are unpacked into files by a script. A reply that
|
|
does not follow the format below cannot be unpacked and is thrown away, however
|
|
good the code in it is.
|
|
|
|
|
|
HOW TO WORK
|
|
|
|
1. Everything you need is in the attached files. Do not search the web and do
|
|
not cite sources. If something you need is missing, ask for it and stop.
|
|
2. Do not restate the task, do not announce what you are about to do, and do
|
|
not summarise what you did. Files first. Notes after, three sentences at most.
|
|
3. If the request is ambiguous, ask one short question and stop. Do not write
|
|
code for a guess.
|
|
4. Replies have an output length limit, and a reply that runs out in the middle
|
|
of a file loses that file. So when the change touches more than 3 files, or
|
|
the files together are longer than about 400 lines, first reply with only a
|
|
plan: the files grouped into numbered batches, one line per file saying what
|
|
changes in it. A batch is at most 5 files and about 400 lines; a file longer
|
|
than that is a batch on its own. Then wait for "go".
|
|
5. Send one batch per reply, in order. Right after the opening ~~~~~~~~ line,
|
|
write a line that is exactly @@ BATCH n OF m. If it is not the last batch,
|
|
end the reply with a line that is exactly @@ MORE and wait for "continue".
|
|
Never start a file you cannot finish in the same reply.
|
|
|
|
|
|
OUTPUT FORMAT
|
|
|
|
Return every file you changed or created in full, one after another, in exactly
|
|
this shape:
|
|
|
|
~~~~~~~~
|
|
@@ FILE: <project>/relative/path/to/file.py
|
|
<the complete contents of the file>
|
|
@@ END FILE: <project>/relative/path/to/file.py
|
|
~~~~~~~~
|
|
|
|
Rules:
|
|
|
|
- One @@ FILE: line per file, and after its last line an @@ END FILE: line
|
|
repeating the same path.
|
|
- Put every block inside ONE fenced block, opened by a line of eight tildes
|
|
(~~~~~~~~) before the first @@ FILE: and closed by the same line after the
|
|
last @@ END. That is the only fence in the whole reply: none around
|
|
individual files, none inside them.
|
|
- Inside file contents, never write three or more backticks in a row, or three
|
|
or more tildes in a row. Write them as ⟪BTn⟫ and ⟪TLn⟫, n being how many:
|
|
⟪BT3⟫ for three backticks, ⟪BT4⟫ for four, ⟪TL3⟫ for three tildes. Write a
|
|
literal ⟪ as ⟪LQ⟫. Files you were given already use these escapes; copy
|
|
them through verbatim. They are turned back into the real characters when
|
|
the reply is unpacked; a real run of backticks breaks the reply.
|
|
- Keep tabs as tabs and quotes as straight quotes (" and '). Never replace them
|
|
with spaces or typographic quotes.
|
|
- Start every path with the project it belongs to, spelled exactly as the
|
|
heading of the document it came from, then the path relative to that
|
|
project's root. The prefix is the only thing that says which file goes
|
|
where, so it is never optional and never abbreviated.
|
|
- No leading ./ or /.
|
|
- Between @@ FILE: and @@ END, emit the file verbatim apart from those
|
|
escapes. Do not add line numbers, and do not elide anything as "unchanged"
|
|
or "...". A partial file is worse than no file.
|
|
- Return whole files only. No diffs, no patches, no hunks.
|
|
- If a file's own content has a line starting with @@, say so in your notes.
|
|
|
|
|
|
EXAMPLE
|
|
|
|
A complete reply returning one file whose content has a code block:
|
|
|
|
~~~~~~~~
|
|
@@ FILE: myproject/docs/usage.md
|
|
# Usage
|
|
|
|
⟪BT3⟫bash
|
|
make run
|
|
⟪BT3⟫
|
|
@@ END FILE: myproject/docs/usage.md
|
|
~~~~~~~~
|
|
Added the usage page.
|
|
|
|
|
|
FORMAT CHECK
|
|
|
|
When my whole message is the single word check, reply with nothing but one
|
|
file in the output format above, and no notes. Its path is _check/check.md
|
|
(no project prefix for this one), and its content is exactly these six lines:
|
|
|
|
line 1: # check
|
|
line 2: three backticks immediately followed by the word bash
|
|
line 3: echo "ok"
|
|
line 4: three backticks
|
|
line 5: one tab character, then the word indented
|
|
line 6: done
|
|
CONTRACT
|
|
}
|
|
|
|
if [ -n "$CONTRACT" ]; then contract; exit 0; fi
|
|
|
|
# ── the format check ───────────────────────────────────────────────────────
|
|
# What the contract's FORMAT CHECK asks for, as bytes once unpacked. Every line
|
|
# is there to catch one way replies break: line 2 and 4 need the escape, line 3
|
|
# straight quotes, line 5 a real tab.
|
|
CHECK_WANT='# check\n```bash\necho "ok"\n```\n\tindented\ndone\n'
|
|
|
|
check_reply() {
|
|
local src="$1" t problems="" warnings="" advice="" n
|
|
t="$(mktemp -d)"; trap 'rm -rf "$t"' RETURN
|
|
printf "$CHECK_WANT" > "$t/want"
|
|
tr -d '\r' < "$src" > "$t/reply"
|
|
|
|
fail() { problems="$problems FAIL $1"$'\n'; advice="$advice- $2"$'\n'; }
|
|
warn() { warnings="$warnings warn $1"$'\n'; advice="$advice- $2"$'\n'; }
|
|
|
|
if ! grep -qE '^@@ +FILE: +_check/check\.md[ \t]*$' "$t/reply"; then
|
|
fail "no '@@ FILE: _check/check.md' line" \
|
|
"Reply to check with the file _check/check.md in the output format, and nothing else."
|
|
else
|
|
awk '/^@@ +FILE: +_check\/check\.md[ \t]*$/ { on = 1; next }
|
|
on && /^@@ +END/ { exit }
|
|
on { print }' "$t/reply" > "$t/raw"
|
|
if ! grep -qE '^@@ +END FILE: +_check/check\.md[ \t]*$' "$t/reply"; then
|
|
fail "the block is not closed with '@@ END FILE: _check/check.md'" \
|
|
"Close every file with an @@ END FILE: line repeating its path."
|
|
fi
|
|
if grep -q '```' "$t/raw"; then
|
|
fail "real backticks inside the file" \
|
|
"Inside file contents, write three backticks as ⟪BT3⟫, never as the characters."
|
|
elif ! grep -q '⟪BT3⟫' "$t/raw"; then
|
|
fail "no ⟪BT3⟫ where the code fence goes" \
|
|
"Line 2 of the check is ⟪BT3⟫bash and line 4 is ⟪BT3⟫."
|
|
fi
|
|
if grep -q '[“”‘’]' "$t/raw"; then
|
|
fail "typographic quotes instead of straight ones" \
|
|
"Use straight quotes (\" and ') in file contents."
|
|
fi
|
|
if grep -qE '^ +indented' "$t/raw"; then
|
|
fail "the tab on line 5 arrived as spaces" \
|
|
"Keep tab characters as tabs. (If the model did write a tab, the copy converted it: use the copy-response button.)"
|
|
fi
|
|
if "$0" -o "$t/out" "$t/reply" > "$t/log" 2>&1 && [ -f "$t/out/_check/check.md" ]; then
|
|
if ! cmp -s "$t/want" "$t/out/_check/check.md"; then
|
|
fail "unpacked, but not the six expected lines:" \
|
|
"The check file is exactly six lines: # check / ⟪BT3⟫bash / echo \"ok\" / ⟪BT3⟫ / a tab then indented / done."
|
|
problems="$problems$(diff <(sed -n l "$t/want") <(sed -n l "$t/out/_check/check.md") | sed 's/^/ /' || true)"$'\n'
|
|
fi
|
|
else
|
|
fail "explode could not unpack it: $(grep -v '^format:' "$t/log" | head -1)" \
|
|
"Follow the output format exactly: markers on their own lines, nothing between the blocks."
|
|
fi
|
|
fi
|
|
|
|
n=$(grep -c '^~~~~~~~[ \t]*$' "$t/reply" || true)
|
|
if [ "$n" -lt 2 ]; then
|
|
warn "no ~~~~~~~~ lines around the block (unpacks, but the chat will render it badly)" \
|
|
"Put all the blocks between two lines of eight tildes, and use no other fence."
|
|
fi
|
|
n=$(awk '/^~~~~~~~[ \t]*$/ { inside = !inside; next }
|
|
!inside && /^@@ +FILE:/ { block = 1 }
|
|
!inside && !block && NF { c++ }
|
|
!inside && /^@@ +END/ { block = 0 }
|
|
END { print c + 0 }' "$t/reply")
|
|
if [ "$n" -gt 0 ]; then
|
|
warn "$n line(s) of notes outside the block" \
|
|
"For check, reply with the file only: no notes before or after."
|
|
fi
|
|
|
|
if [ -z "$problems" ]; then
|
|
echo " ok escapes, tab, quotes and markers all came through"
|
|
[ -n "$warnings" ] && printf '%s' "$warnings"
|
|
echo
|
|
echo "format check passed"
|
|
return 0
|
|
fi
|
|
printf '%s' "$problems" "$warnings"
|
|
echo
|
|
echo "format check FAILED. Send this back to the model:"
|
|
echo
|
|
echo "The format check failed. Re-read the instructions file, fix these, and reply to check again:"
|
|
printf '%s' "$advice"
|
|
return 1
|
|
}
|
|
|
|
# ── self-test ──────────────────────────────────────────────────────────────
|
|
# So a copy of this script on another machine can be checked without any real
|
|
# input, and without asking whether it is the version that knows a given format.
|
|
# Every case here is one that has actually gone wrong.
|
|
selftest() {
|
|
local t rc=0 got want
|
|
t="$(mktemp -d)"; trap 'rm -rf "$t"' RETURN
|
|
|
|
check() { # name, expected, actual
|
|
if [ "$2" = "$3" ]; then printf ' ok %s\n' "$1"
|
|
else printf ' FAIL %s\n expected: %s\n got: %s\n' "$1" "$2" "$3"; rc=1
|
|
fi
|
|
}
|
|
|
|
# A file containing a fence and a stray === line: the two things that break
|
|
# naive parsers.
|
|
cat > "$t/a.txt" <<'FIXTURE'
|
|
Here is the code.
|
|
|
|
=== FILE: pkg/core/client.py
|
|
class Client:
|
|
pass
|
|
=== END
|
|
|
|
=== FILE: README.md
|
|
# proj
|
|
|
|
```bash
|
|
pip install proj
|
|
```
|
|
|
|
A line that says === not a marker
|
|
=== END
|
|
FIXTURE
|
|
"$0" -o "$t/a" "$t/a.txt" >/dev/null 2>&1 || true
|
|
check "fenced: file count" "2" "$(find "$t/a" -type f 2>/dev/null | wc -l)"
|
|
check "fenced: nested fences" "2" "$(grep -c '```' "$t/a/README.md" 2>/dev/null || echo 0)"
|
|
check "fenced: === in content" "1" "$(grep -c 'not a marker' "$t/a/README.md" 2>/dev/null || echo 0)"
|
|
check "fenced: subdirectory" "class Client:" "$(head -1 "$t/a/pkg/core/client.py" 2>/dev/null)"
|
|
check "fenced: prose skipped" "0" "$(find "$t/a" -name 'Here*' 2>/dev/null | wc -l)"
|
|
|
|
# An unclosed block must be refused, not written short.
|
|
printf '=== FILE: a.py\nx = 1\n=== END\n\n=== FILE: b.py\ny = 2\n' > "$t/b.txt"
|
|
"$0" -o "$t/b" "$t/b.txt" >/dev/null 2>&1 || true
|
|
check "unterminated: refused" "1" "$([ -e "$t/b" ] && echo 0 || echo 1)"
|
|
|
|
# Paths out of a text file are untrusted.
|
|
printf '=== FILE: ../escape.py\nx\n=== END\n' > "$t/c.txt"
|
|
"$0" -o "$t/c" "$t/c.txt" >/dev/null 2>&1 || true
|
|
check "traversal: refused" "1" "$([ -e "$t/c" ] && echo 0 || echo 1)"
|
|
|
|
# The stale-copy failure: explicit format read by the marker parser.
|
|
"$0" --format marker -o "$t/d" "$t/a.txt" >/dev/null 2>&1 || true
|
|
check "wrong parser: refused" "1" "$([ -e "$t/d" ] && echo 0 || echo 1)"
|
|
|
|
# The other two layouts still work.
|
|
# The @@ markers, which are the ones to ask a chat model for.
|
|
printf '@@ FILE: pkg/a.py\nx = 1\n@@ END\n@@ FILE: b.md\n# t\n\n```sh\nls\n```\n@@ END\n' > "$t/g.txt"
|
|
"$0" -o "$t/g" "$t/g.txt" >/dev/null 2>&1 || true
|
|
check "at-markers: file count" "2" "$(find "$t/g" -type f 2>/dev/null | wc -l)"
|
|
check "at-markers: fenced body" "2" "$(grep -c '```' "$t/g/b.md" 2>/dev/null || echo 0)"
|
|
printf '@@ FILE: a.py\nx = 1\n' > "$t/h.txt"
|
|
"$0" -o "$t/h" "$t/h.txt" >/dev/null 2>&1 || true
|
|
check "at-markers: unterminated" "1" "$([ -e "$t/h" ] && echo 0 || echo 1)"
|
|
|
|
# One reply, several projects: the prefix is just the first directory.
|
|
printf '@@ FILE: projA/a.py\nx = 1\n@@ END\n@@ FILE: projB/deep/b.py\ny = 2\n@@ END\n' > "$t/i.txt"
|
|
"$0" -o "$t/i" "$t/i.txt" >/dev/null 2>&1 || true
|
|
check "multi-project: first" "x = 1" "$(cat "$t/i/projA/a.py" 2>/dev/null)"
|
|
check "multi-project: nested" "y = 2" "$(cat "$t/i/projB/deep/b.py" 2>/dev/null)"
|
|
|
|
printf '=== ./x/y.py\nz = 1\n' > "$t/e.txt"
|
|
"$0" -o "$t/e" "$t/e.txt" >/dev/null 2>&1 || true
|
|
check "marker layout" "z = 1" "$(cat "$t/e/x/y.py" 2>/dev/null)"
|
|
|
|
printf '# d\n\n## x/y.py\n\n```python\nz = 1\n```\n' > "$t/f.txt"
|
|
"$0" -o "$t/f" "$t/f.txt" >/dev/null 2>&1 || true
|
|
check "digest layout" "z = 1" "$(cat "$t/f/x/y.py" 2>/dev/null)"
|
|
|
|
# What distill actually writes: a metadata line between the heading and the
|
|
# fence, and prose sections whose heading is followed by no fence at all.
|
|
printf '# d\n\n## Tree\n\nx/\n y.py\n\n## x/y.py\n\n_1 lines · 6 bytes_\n\n```python\nz = 1\n```\n' > "$t/j.txt"
|
|
"$0" -o "$t/j" "$t/j.txt" >/dev/null 2>&1 || true
|
|
check "digest: metadata line" "z = 1" "$(cat "$t/j/x/y.py" 2>/dev/null)"
|
|
check "digest: prose skipped" "1" "$(find "$t/j" -type f 2>/dev/null | wc -l)"
|
|
|
|
# A clipped file is head and tail only. Writing it would truncate the real
|
|
# one, so the whole run is refused.
|
|
printf '# d\n\n## x/y.py\n\n_900 lines · 60000 bytes · CLIPPED — head and tail only_\n\n```python\nz = 1\n```\n' > "$t/k.txt"
|
|
"$0" -o "$t/k" "$t/k.txt" >/dev/null 2>&1 || true
|
|
check "digest: clipped refused" "1" "$([ -e "$t/k" ] && echo 0 || echo 1)"
|
|
|
|
# Escapes come back as the characters, and an escaped ⟪ as itself — not as
|
|
# the backticks its escaped spelling would otherwise decode to.
|
|
printf '@@ FILE: r.md\n⟪BT3⟫sh\nls ⟪TL4⟫\n⟪BT3⟫\nsee ⟪LQ⟫BT3⟫\n@@ END FILE: r.md\n' > "$t/l.txt"
|
|
"$0" -o "$t/l" "$t/l.txt" >/dev/null 2>&1 || true
|
|
check "escapes: backticks" '```sh' "$(sed -n 1p "$t/l/r.md" 2>/dev/null)"
|
|
check "escapes: tildes" 'ls ~~~~' "$(sed -n 2p "$t/l/r.md" 2>/dev/null)"
|
|
check "escapes: literal" 'see ⟪BT3⟫' "$(sed -n 4p "$t/l/r.md" 2>/dev/null)"
|
|
"$0" --raw -o "$t/l2" "$t/l.txt" >/dev/null 2>&1 || true
|
|
check "escapes: --raw" '⟪BT3⟫sh' "$(sed -n 1p "$t/l2/r.md" 2>/dev/null)"
|
|
|
|
# What the model does anyway: one fence around everything, and a fence
|
|
# around each file inside its block, with a blank line before the END.
|
|
printf 'Done.\n~~~~~~~~\n@@ FILE: a.py\n```python\nx = 1\n```\n\n@@ END FILE: a.py\n@@ FILE: b.md\n# t\n```\n@@ END\n~~~~~~~~\n' > "$t/m.txt"
|
|
"$0" -o "$t/m" "$t/m.txt" >/dev/null 2>&1 || true
|
|
check "wrapped: file count" "2" "$(find "$t/m" -type f 2>/dev/null | wc -l)"
|
|
check "wrapped: fences dropped" "x = 1" "$(cat "$t/m/a.py" 2>/dev/null)"
|
|
check "wrapped: lone fence kept" "2" "$(wc -l < "$t/m/b.md" 2>/dev/null)"
|
|
|
|
# A dropped close glues two files into one block. The named END says so.
|
|
printf '@@ FILE: a.py\nx = 1\n@@ FILE: b.py\ny = 2\n@@ END FILE: b.py\n' > "$t/n.txt"
|
|
"$0" -o "$t/n" "$t/n.txt" >/dev/null 2>&1 || true
|
|
check "named end: mismatch" "1" "$([ -e "$t/n" ] && echo 0 || echo 1)"
|
|
|
|
# A digest is unescaped only when distill said it escaped it.
|
|
printf '# d\n\nInside files ... ⟪BTn⟫ ...\n\n## x.md\n\n```markdown\n⟪BT3⟫\n```\n' > "$t/o.txt"
|
|
"$0" -o "$t/o" "$t/o.txt" >/dev/null 2>&1 || true
|
|
check "digest: escaped" '```' "$(cat "$t/o/x.md" 2>/dev/null)"
|
|
printf '# d\n\n## x.md\n\n````markdown\n⟪BT3⟫\n````\n' > "$t/p.txt"
|
|
"$0" -o "$t/p" "$t/p.txt" >/dev/null 2>&1 || true
|
|
check "digest: not escaped" '⟪BT3⟫' "$(cat "$t/p/x.md" 2>/dev/null)"
|
|
|
|
# The format check: a good reply passes, and each way replies break is named.
|
|
good='~~~~~~~~\n@@ FILE: _check/check.md\n# check\n⟪BT3⟫bash\necho "ok"\n⟪BT3⟫\n\tindented\ndone\n@@ END FILE: _check/check.md\n~~~~~~~~\n'
|
|
printf "$good" > "$t/q1.md"
|
|
check "check: good reply" "0" "$("$0" --check "$t/q1.md" >/dev/null 2>&1; echo $?)"
|
|
printf "$good" | sed 's/⟪BT3⟫/```/' > "$t/q2.md"
|
|
check "check: real backticks" "1" "$("$0" --check "$t/q2.md" 2>&1 | grep -c 'real backticks')"
|
|
printf "$good" | sed 's/^\tindented/ indented/' > "$t/q3.md"
|
|
check "check: tab became spaces" "1" "$("$0" --check "$t/q3.md" 2>&1 | grep -c 'arrived as spaces')"
|
|
printf "$good" | sed 's/"ok"/“ok”/' > "$t/q4.md"
|
|
check "check: curly quotes" "1" "$("$0" --check "$t/q4.md" 2>&1 | grep -c 'typographic quotes')"
|
|
{ echo "Sure! Here is the check."; printf "$good"; } > "$t/q5.md"
|
|
check "check: notes only warn" "0" "$("$0" --check "$t/q5.md" >/dev/null 2>&1; echo $?)"
|
|
printf "$good" | sed '/^done$/d' > "$t/q6.md"
|
|
check "check: wrong content" "1" "$("$0" --check "$t/q6.md" 2>&1 | grep -c 'not the six expected')"
|
|
|
|
# Real replies: raw fences unpack but are reported, and @@ MORE is noticed.
|
|
printf '@@ FILE: p/r.md\n# r\n```sh\nls\n```\ndone\n@@ END FILE: p/r.md\n@@ MORE\n' > "$t/r.txt"
|
|
"$0" -o "$t/r" "$t/r.txt" > "$t/r.log" 2>&1 || true
|
|
check "notes: raw fence reported" "1" "$(grep -c 'real ``` instead' "$t/r.log")"
|
|
check "notes: @@ MORE noticed" "1" "$(grep -c '@@ MORE' "$t/r.log")"
|
|
check "notes: still unpacked" "2" "$(grep -c '```' "$t/r/p/r.md" 2>/dev/null || echo 0)"
|
|
printf '~~~~~~~~\n@@ BATCH 2 OF 3\n@@ FILE: p/b.py\nx = 1\n@@ END FILE: p/b.py\n~~~~~~~~\n@@ MORE\n' > "$t/s.txt"
|
|
"$0" -o "$t/s" "$t/s.txt" > "$t/s.log" 2>&1 || true
|
|
check "notes: batch n of m" "1" "$(grep -c 'batch 2 of 3: say "continue" for batch 3' "$t/s.log")"
|
|
check "batch line is not a file" "x = 1" "$(cat "$t/s/p/b.py" 2>/dev/null)"
|
|
printf '~~~~~~~~\n@@ BATCH 1 OF 2\n@@ FILE: p/c.py\nx = 1\n' > "$t/u.txt"
|
|
"$0" -o "$t/u" "$t/u.txt" > "$t/u.log" 2>&1 || true
|
|
check "cut-off reply explained" "1" "$(grep -c 'length limit' "$t/u.log")"
|
|
|
|
echo
|
|
if [ "$rc" -eq 0 ]; then echo "all checks passed — this copy is current"
|
|
else echo "SOME CHECKS FAILED — this copy is out of date or broken" >&2
|
|
fi
|
|
return "$rc"
|
|
}
|
|
|
|
if [ -n "$SELFTEST" ]; then selftest; exit $?; fi
|
|
|
|
if [ -n "$CHECK" ]; then
|
|
[ -n "$SRC" ] || die "--check needs the saved reply: $SELF --check reply.md"
|
|
[ -f "$SRC" ] || die "no such file: $SRC"
|
|
check_reply "$SRC"; exit $?
|
|
fi
|
|
|
|
[ -n "$SRC" ] || { usage >&2; exit 1; }
|
|
[ -f "$SRC" ] || die "no such file: $SRC"
|
|
case "$FORMAT" in fenced|marker|digest|auto) ;; *) die "--format must be fenced, marker, digest or auto" ;; esac
|
|
|
|
# Which layout is it? Count the two shapes and take the commoner one, rather
|
|
# than trusting the first line that happens to match: a digest of a repo full of
|
|
# markdown will contain plenty of '=== ' inside its own fenced content, and a
|
|
# marker file can quote a '## ' heading just as easily.
|
|
if [ "$FORMAT" = auto ]; then
|
|
n_fenced=$(grep -cE '^(===|@@) +[Ff][Ii][Ll][Ee]: +[^ ]' "$SRC" || true)
|
|
n_marker=$(grep -cE '^=== +\.?/?[^ ]' "$SRC" || true)
|
|
n_marker=$((n_marker - n_fenced - $(grep -cE '^(===|@@) +END([ \t\r]*$|[ \t]+[Ff][Ii][Ll][Ee]:)' "$SRC" || true)))
|
|
[ "$n_marker" -lt 0 ] && n_marker=0
|
|
n_digest=$(grep -cE '^## +[^ ]' "$SRC" || true)
|
|
# Except that distill.sh stamps every file in a digest with an
|
|
# "_N lines · B bytes_" line, and a digest that includes this script — or
|
|
# any file quoting the contract — has '@@ FILE:' lines of its own. The stamp
|
|
# is the stronger signal: nothing else writes it.
|
|
if grep -qE '^_[0-9]+ lines · [0-9]+ bytes' "$SRC"; then
|
|
FORMAT=digest
|
|
elif [ "$n_fenced" -gt 0 ]; then
|
|
FORMAT=fenced
|
|
elif [ "$n_marker" -eq 0 ] && [ "$n_digest" -eq 0 ]; then
|
|
die "found no '=== FILE:' blocks, no '=== path' markers and no '## path' headings in $SRC"
|
|
elif [ "$n_marker" -ge "$n_digest" ]; then FORMAT=marker
|
|
else FORMAT=digest
|
|
fi
|
|
echo "format: $FORMAT"
|
|
fi
|
|
|
|
# ── the parser ─────────────────────────────────────────────────────────────
|
|
# One awk, two modes. In list mode it prints "path<TAB>lines"; otherwise it
|
|
# writes each file under DEST. Reading the whole thing in awk rather than a
|
|
# bash read-loop matters once the input is a few megabytes.
|
|
#
|
|
# In digest mode a heading only opens a file if a fence follows it. distill.sh
|
|
# writes '## Tree' and '## Binary files ...' sections that are prose, and
|
|
# treating those as files would scatter junk through the output.
|
|
#
|
|
# A file's lines are held until its block closes, and only then counted or
|
|
# written. That is what lets a wrapping fence at the end of a block be seen as
|
|
# the last line and dropped.
|
|
#
|
|
# LC_ALL=C: the escapes are multibyte, and matching them as plain bytes behaves
|
|
# the same in every locale and on input that is not valid UTF-8. No {3,} in the
|
|
# regexes: the mawk on Ubuntu 22.04 does not know interval expressions.
|
|
parse() {
|
|
LC_ALL=C awk -v dest="$DEST" -v mode="$1" -v fmt="$FORMAT" -v unesc="$2" '
|
|
function unescape(s, r, m) {
|
|
r = ""
|
|
while (match(s, /⟪(BT[0-9]+|TL[0-9]+|LQ)⟫/)) {
|
|
m = substr(s, RSTART, RLENGTH)
|
|
gsub(/⟪|⟫/, "", m)
|
|
r = r substr(s, 1, RSTART - 1) \
|
|
(m == "LQ" ? "⟪" : repeat(substr(m, 1, 2) == "BT" ? "`" : "~", substr(m, 3) + 0))
|
|
s = substr(s, RSTART + RLENGTH)
|
|
}
|
|
return r s
|
|
}
|
|
function repeat(c, k, r) { r = ""; while (k-- > 0) r = r c; return r }
|
|
function is_open_fence(l) { return l ~ /^[ \t]*(```+|~~~+)[A-Za-z0-9_+.#-]*[ \t\r]*$/ }
|
|
function is_close_fence(l) { return l ~ /^[ \t]*(```+|~~~+)[ \t\r]*$/ }
|
|
function flush( i, first, last, out, d) {
|
|
if (path == "") { n = 0; return }
|
|
first = 1; last = n
|
|
if (fmt == "fenced") {
|
|
# Trailing blank lines do not stop a closing fence counting as
|
|
# the last line; they go with it.
|
|
while (last > 0 && buf[last] ~ /^[ \t\r]*$/) last--
|
|
if (last > 1 && is_open_fence(buf[1]) && is_close_fence(buf[last])) {
|
|
first = 2; last--
|
|
} else last = n
|
|
}
|
|
if (mode == "list") {
|
|
printf "%s\t%d\n", path, last - first + 1
|
|
# Unpacks fine, but it is what breaks the chat rendering, and
|
|
# what the model has to be told to stop doing.
|
|
if (fmt == "fenced")
|
|
for (i = first; i <= last; i++)
|
|
if (buf[i] ~ /```/) { print "RAWFENCE\t" path; break }
|
|
}
|
|
else {
|
|
out = dest "/" path
|
|
d = out; sub(/\/[^\/]*$/, "", d)
|
|
system("mkdir -p \"" d "\"")
|
|
printf "" > out
|
|
for (i = first; i <= last; i++)
|
|
print (unesc ? unescape(buf[i]) : buf[i]) > out
|
|
close(out)
|
|
}
|
|
path = ""; n = 0
|
|
}
|
|
function clean(p) {
|
|
sub(/^\.\//, "", p)
|
|
sub(/[ \t\r]+$/, "", p)
|
|
return p
|
|
}
|
|
function unsafe(p) {
|
|
return (p == "" || p ~ /^\// || p ~ /^[A-Za-z]:/ || p ~ /(^|\/)\.\.(\/|$)/)
|
|
}
|
|
function open_file(p) { path = p; n = 0 }
|
|
|
|
# Explicit open/close. The whole point is that nothing is inferred:
|
|
# content is content until the END line, whatever it looks like.
|
|
fmt == "fenced" && path == "" && /^(===|@@) +[Ff][Ii][Ll][Ee]: +/ {
|
|
p = substr($0, index($0, ":") + 1)
|
|
sub(/^[ \t]+/, "", p)
|
|
p = clean(p)
|
|
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
|
open_file(p)
|
|
next
|
|
}
|
|
fmt == "fenced" && path != "" && /^(===|@@) +END([ \t\r]*$|[ \t]+[Ff][Ii][Ll][Ee]:)/ {
|
|
p = $0
|
|
if (sub(/^(===|@@) +END[ \t]+[Ff][Ii][Ll][Ee]:[ \t]*/, "", p)) {
|
|
p = clean(p)
|
|
# The END names a different file: the model dropped a close
|
|
# somewhere and two files are now one block. Say which.
|
|
if (p != path) { print "MISMATCH\t" path "\t" p; bad = 1 }
|
|
}
|
|
flush()
|
|
next
|
|
}
|
|
fmt == "fenced" && path == "" { next } # anything between blocks is prose
|
|
|
|
fmt == "marker" && /^=== +/ {
|
|
flush()
|
|
p = clean(substr($0, index($0, " ") + 1))
|
|
# "FILE: ./x.py" and "END" are not paths, they are the explicit
|
|
# format being read by the wrong parser. Left alone this writes a
|
|
# directory literally called "FILE: ." and a file called "END",
|
|
# which is what an out-of-date copy of this script did once.
|
|
if (p ~ /^FILE:/ || p == "END") { print "WRONGFMT\t" p; bad = 1; next }
|
|
if (unsafe(p)) { print "UNSAFE\t" p; bad = 1; next }
|
|
open_file(p)
|
|
next
|
|
}
|
|
|
|
# Only when NOT already inside a file. The content of any markdown file
|
|
# in the input is full of "## " headings, and the fence is the only
|
|
# thing that says which ones are structure and which are text. Once a
|
|
# file is open, the matching close fence is the sole way out.
|
|
fmt == "digest" && path == "" && /^## +/ {
|
|
flush()
|
|
pending = clean(substr($0, 4))
|
|
expect = 1
|
|
next
|
|
}
|
|
fmt == "digest" && expect == 1 {
|
|
if ($0 ~ /^[ \t]*$/) next # blank line between the two
|
|
# distill.sh puts an italic "N lines, B bytes" line under each heading,
|
|
# so the fence is no longer the next thing after it. Step over that
|
|
# line rather than reading it as prose — without this, every file in
|
|
# a current digest is skipped and the whole document looks empty.
|
|
if ($0 ~ /^_.*_[ \t]*$/) {
|
|
# Unless it says the file was clipped. A clipped body is head
|
|
# and tail with a marker in between; writing it out would
|
|
# replace a real file with a truncated one that reads complete,
|
|
# which is the exact failure the unterminated check exists for.
|
|
if ($0 ~ /CLIPPED/) {
|
|
print "CLIPPED\t" pending
|
|
bad = 1; expect = 0; pending = ""
|
|
next
|
|
}
|
|
next
|
|
}
|
|
if ($0 ~ /^```+/) { # a fence: this is a file
|
|
match($0, /^`+/)
|
|
fence = substr($0, 1, RLENGTH)
|
|
expect = 0
|
|
if (unsafe(pending)) { print "UNSAFE\t" pending; bad = 1; next }
|
|
open_file(pending)
|
|
next
|
|
}
|
|
expect = 0 # prose section, not a file
|
|
pending = ""
|
|
next
|
|
}
|
|
fmt == "digest" && path != "" && $0 == fence { flush(); next }
|
|
|
|
|
|
{ if (path != "") buf[++n] = $0 }
|
|
|
|
END {
|
|
if (fmt == "fenced" && path != "") {
|
|
print "UNTERMINATED\t" path
|
|
bad = 1
|
|
path = ""
|
|
}
|
|
flush()
|
|
exit (bad ? 3 : 0)
|
|
}
|
|
' "$SRC"
|
|
}
|
|
|
|
# Whether to undo the escapes. A reply written to the contract has them; a
|
|
# digest has them only if distill.sh wrote its notice, which names ⟪BTn⟫. Look
|
|
# for it only in the header, above the first '## ': an unescaped digest of
|
|
# these very scripts has ⟪BTn⟫ all through its file bodies.
|
|
UNESC=1
|
|
[ -n "$RAW" ] && UNESC=0
|
|
if [ "$FORMAT" = digest ] && [ -z "$RAW" ] \
|
|
&& ! sed '/^## /q' "$SRC" | grep -qF '⟪BTn⟫'; then UNESC=0; fi
|
|
|
|
# Validate before writing anything: a refusal after half the tree is on disk is
|
|
# not a refusal.
|
|
# awk exits non-zero when it found something wrong; that is the signal, not a
|
|
# crash, so let it through and report it properly below.
|
|
scan="$(parse list "$UNESC" || true)"
|
|
|
|
refused="$(printf '%s\n' "$scan" | grep '^UNSAFE' || true)"
|
|
if [ -n "$refused" ]; then
|
|
echo "$SELF: refusing — these paths escape the destination:" >&2
|
|
printf '%s\n' "$refused" | sed 's/^UNSAFE\t/ /' >&2
|
|
exit 1
|
|
fi
|
|
|
|
# A block that never closed means the input is malformed, or a file contained
|
|
# the END line. Either way the tail is missing, and a truncated source file that
|
|
# looks complete is the failure this format exists to prevent.
|
|
wrongfmt="$(printf '%s\n' "$scan" | grep '^WRONGFMT' || true)"
|
|
if [ -n "$wrongfmt" ]; then
|
|
echo "$SELF: this file uses 'FILE: path' / 'END' markers, but it was read as" >&2
|
|
echo "the plain marker format, which would create a directory called 'FILE: .'" >&2
|
|
echo "and files called 'END'. Re-run with --format fenced, or update this script." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mismatch="$(printf '%s\n' "$scan" | grep '^MISMATCH' || true)"
|
|
if [ -n "$mismatch" ]; then
|
|
echo "$SELF: refusing — these blocks were closed with another file's name:" >&2
|
|
printf '%s\n' "$mismatch" | awk -F'\t' '{ printf " opened %s, closed %s\n", $2, $3 }' >&2
|
|
echo "a close went missing, so one block holds more than one file" >&2
|
|
echo "tell the model: close every file with an @@ END FILE: line repeating its path, then resend those files" >&2
|
|
exit 1
|
|
fi
|
|
|
|
unterminated="$(printf '%s\n' "$scan" | grep '^UNTERMINATED' || true)"
|
|
if [ -n "$unterminated" ]; then
|
|
echo "$SELF: refusing — this block was never closed with '@@ END' or '=== END':" >&2
|
|
printf '%s\n' "$unterminated" | sed 's/^UNTERMINATED\t/ /' >&2
|
|
echo "the file it describes would be silently truncated" >&2
|
|
echo "the reply most likely hit its length limit mid-file" >&2
|
|
echo "tell the model: that file was cut off; resend it whole, closed with @@ END FILE: and its path, and keep batches smaller" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# distill clips the largest files to fit a budget, and says so. The tree copy
|
|
# beside the digest has them whole, so the fix is to take them from there — not
|
|
# to write out the head and tail under the real name.
|
|
clipped="$(printf '%s\n' "$scan" | grep '^CLIPPED' || true)"
|
|
if [ -n "$clipped" ]; then
|
|
echo "$SELF: refusing — distill clipped these, so the digest has only their" >&2
|
|
echo "head and tail:" >&2
|
|
printf '%s\n' "$clipped" | sed 's/^CLIPPED\t/ /' >&2
|
|
echo "take them from the tree copy instead; writing these would truncate them" >&2
|
|
exit 1
|
|
fi
|
|
|
|
listing="$(printf '%s\n' "$scan" | grep -vE '^(UNSAFE|UNTERMINATED|WRONGFMT|CLIPPED|MISMATCH|RAWFENCE)' || true)"
|
|
|
|
# Not refusals: the reply unpacks. Said anyway, because each is something to
|
|
# tell the model before the next reply rather than discover after it.
|
|
notes() {
|
|
local raw
|
|
raw="$(printf '%s\n' "$scan" | grep '^RAWFENCE' | cut -f2 || true)"
|
|
if [ -n "$raw" ]; then
|
|
echo "note: these came back with real \`\`\` instead of ⟪BT3⟫ (unpacked fine; tell the model):" >&2
|
|
printf '%s\n' "$raw" | sed 's/^/ /' >&2
|
|
fi
|
|
# The last batch line wins: a reply quoting an earlier one is still this batch.
|
|
local batch n m
|
|
batch="$(grep -oE '^@@ +BATCH +[0-9]+ +OF +[0-9]+' "$SRC" | tail -1 || true)"
|
|
if [ -n "$batch" ]; then
|
|
n="$(printf '%s' "$batch" | awk '{ print $3 }')"; m="$(printf '%s' "$batch" | awk '{ print $5 }')"
|
|
if [ "$n" -lt "$m" ]; then
|
|
echo "note: batch $n of $m: say \"continue\" for batch $((n + 1)), and explode that reply too" >&2
|
|
else
|
|
echo "note: batch $n of $m, the last one" >&2
|
|
fi
|
|
elif grep -qE '^@@ +MORE[ \t\r]*$' "$SRC"; then
|
|
echo "note: the reply ends with @@ MORE: say \"continue\" and explode the next reply too" >&2
|
|
fi
|
|
}
|
|
[ -n "$listing" ] || die "no files found in $SRC (format: $FORMAT)"
|
|
count=$(printf '%s\n' "$listing" | grep -c . )
|
|
|
|
if [ -n "$LIST" ]; then
|
|
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %-60s %5d lines\n", $1, $2 }'
|
|
echo "$count files"
|
|
notes
|
|
exit 0
|
|
fi
|
|
|
|
# Existing files are someone's work until proven otherwise.
|
|
if [ -z "$FORCE" ]; then
|
|
clashes=""
|
|
while IFS=$'\t' read -r p _; do
|
|
[ -e "$DEST/$p" ] && clashes="$clashes $p"$'\n'
|
|
done <<< "$listing"
|
|
if [ -n "$clashes" ]; then
|
|
echo "$SELF: these already exist under $DEST:" >&2
|
|
printf '%s' "$clashes" >&2
|
|
echo "re-run with --force to overwrite" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$DEST"
|
|
parse write "$UNESC" >/dev/null
|
|
printf '%s\n' "$listing" | awk -F'\t' '{ printf " %s\n", $1 }'
|
|
echo "wrote $count files to $DEST"
|
|
notes
|