120 lines
3.8 KiB
Bash
Executable File
120 lines
3.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# Batch-extract every document under a directory through process_doc.py,
|
|
# mirroring the input folder structure into the output base. The doocus twin of
|
|
# ctrl/batch.sh (which does the same for meeting videos via process_meeting.py).
|
|
#
|
|
# Recursive and space-safe (paths come off a Windows mount, so they often have
|
|
# spaces). Each document's run folder is auto-created by process_doc.py inside
|
|
# the mirrored output subfolder.
|
|
#
|
|
# Usage:
|
|
# ctrl/batch_docs.sh -i <input-dir> -o <output-dir> [-e "pdf docx ..."] [-n] \
|
|
# [-- <process_doc.py flags>]
|
|
#
|
|
# Examples:
|
|
# # Everything under a downloaded Drive folder, mirrored into docs-output
|
|
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output
|
|
#
|
|
# # Dry run: just show which docs map to which output folders
|
|
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -n
|
|
#
|
|
# # Only pdfs/docx, render page thumbnails
|
|
# ctrl/batch_docs.sh -i "./download" -o docs-output -e "pdf docx" -- --render
|
|
#
|
|
# A doc at <input>/2026/team a/notes.docx produces
|
|
# <output>/2026/team a/<YYYYMMDD-NNN-notes>/...
|
|
# i.e. the run folder lands inside the mirrored subtree.
|
|
set -euo pipefail
|
|
|
|
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
|
cd "$PROJECT_DIR"
|
|
|
|
# python: honor $PYTHON, else prefer python3
|
|
PYTHON="${PYTHON:-}"
|
|
if [ -z "$PYTHON" ]; then
|
|
if command -v python3 >/dev/null 2>&1; then PYTHON=python3; else PYTHON=python; fi
|
|
fi
|
|
|
|
usage() { sed -n '2,26p' "$0"; }
|
|
|
|
INPUT=""
|
|
OUTPUT=""
|
|
EXTS="csv docx html htm jpg jpeg json md txt pdf png pptx xlsx yaml yml mp4"
|
|
DRY=false
|
|
FORWARD=()
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
-i|--input) INPUT="$2"; shift 2 ;;
|
|
-o|--output) OUTPUT="$2"; shift 2 ;;
|
|
-e|--ext) EXTS="$2"; shift 2 ;;
|
|
-n|--dry-run) DRY=true; shift ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
--) shift; FORWARD=("$@"); break ;;
|
|
*) echo "Unknown arg: $1" >&2; usage; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; }
|
|
[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; }
|
|
[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; }
|
|
|
|
# Absolute paths so relative-path math is stable regardless of where we cd'd.
|
|
INPUT="$(realpath "$INPUT")"
|
|
mkdir -p "$OUTPUT"
|
|
OUTPUT="$(realpath "$OUTPUT")"
|
|
|
|
# Build the find extension filter: \( -iname '*.pdf' -o -iname '*.docx' ... \)
|
|
find_expr=()
|
|
for ext in $EXTS; do
|
|
find_expr+=( -iname "*.${ext}" -o )
|
|
done
|
|
unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o
|
|
|
|
echo "Input : $INPUT"
|
|
echo "Output: $OUTPUT"
|
|
echo "Exts : $EXTS"
|
|
[ "$DRY" = true ] && echo "(dry run — nothing will be processed)"
|
|
echo
|
|
|
|
total=0 ok=0 fail=0
|
|
# Process substitution (not a pipe) so counters survive into the summary.
|
|
while IFS= read -r -d '' doc; do
|
|
total=$((total + 1))
|
|
|
|
rel="${doc#"$INPUT"/}" # path relative to the input root
|
|
reldir="$(dirname "$rel")"
|
|
if [ "$reldir" = "." ]; then
|
|
outdir="$OUTPUT" # doc sat directly in the input root
|
|
else
|
|
outdir="$OUTPUT/$reldir" # mirror the subfolder structure
|
|
fi
|
|
|
|
echo "[$total] $rel"
|
|
echo " -> $outdir"
|
|
|
|
if [ "$DRY" = true ]; then
|
|
continue
|
|
fi
|
|
|
|
mkdir -p "$outdir"
|
|
# stdin from /dev/null so any child process can't eat the file list and stall
|
|
# the batch (same guard as ctrl/batch.sh).
|
|
if "$PYTHON" "$PROJECT_DIR/process_doc.py" "$doc" \
|
|
--output-dir "$outdir" "${FORWARD[@]+"${FORWARD[@]}"}" </dev/null; then
|
|
ok=$((ok + 1))
|
|
else
|
|
echo " !! FAILED (continuing)" >&2
|
|
fail=$((fail + 1))
|
|
fi
|
|
echo
|
|
done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z)
|
|
|
|
echo "----------------------------------------"
|
|
if [ "$DRY" = true ]; then
|
|
echo "Found $total document(s)."
|
|
else
|
|
echo "Done. $total document(s): $ok ok, $fail failed."
|
|
fi
|
|
[ "$fail" -eq 0 ]
|