doocus first ver
This commit is contained in:
Mariano Gabriel
119
ctrl/batch_docs.sh
Executable file
119
ctrl/batch_docs.sh
Executable file
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
# Batch-extract every document under a directory through process_doc.py,
|
||||
# mirroring the input folder structure into the output base. The doocus twin of
|
||||
# ctrl/batch.sh (which does the same for meeting videos via process_meeting.py).
|
||||
#
|
||||
# Recursive and space-safe (paths come off a Windows mount, so they often have
|
||||
# spaces). Each document's run folder is auto-created by process_doc.py inside
|
||||
# the mirrored output subfolder.
|
||||
#
|
||||
# Usage:
|
||||
# ctrl/batch_docs.sh -i <input-dir> -o <output-dir> [-e "pdf docx ..."] [-n] \
|
||||
# [-- <process_doc.py flags>]
|
||||
#
|
||||
# Examples:
|
||||
# # Everything under a downloaded Drive folder, mirrored into docs-output
|
||||
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output
|
||||
#
|
||||
# # Dry run: just show which docs map to which output folders
|
||||
# ctrl/batch_docs.sh -i "/mnt/win/drive" -o docs-output -n
|
||||
#
|
||||
# # Only pdfs/docx, render page thumbnails
|
||||
# ctrl/batch_docs.sh -i "./download" -o docs-output -e "pdf docx" -- --render
|
||||
#
|
||||
# A doc at <input>/2026/team a/notes.docx produces
|
||||
# <output>/2026/team a/<YYYYMMDD-NNN-notes>/...
|
||||
# i.e. the run folder lands inside the mirrored subtree.
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$PROJECT_DIR"
|
||||
|
||||
# python: honor $PYTHON, else prefer python3
|
||||
PYTHON="${PYTHON:-}"
|
||||
if [ -z "$PYTHON" ]; then
|
||||
if command -v python3 >/dev/null 2>&1; then PYTHON=python3; else PYTHON=python; fi
|
||||
fi
|
||||
|
||||
usage() { sed -n '2,26p' "$0"; }
|
||||
|
||||
INPUT=""
|
||||
OUTPUT=""
|
||||
EXTS="csv docx html htm jpg jpeg json md txt pdf png pptx xlsx yaml yml mp4"
|
||||
DRY=false
|
||||
FORWARD=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
-i|--input) INPUT="$2"; shift 2 ;;
|
||||
-o|--output) OUTPUT="$2"; shift 2 ;;
|
||||
-e|--ext) EXTS="$2"; shift 2 ;;
|
||||
-n|--dry-run) DRY=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--) shift; FORWARD=("$@"); break ;;
|
||||
*) echo "Unknown arg: $1" >&2; usage; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$INPUT" ] || { echo "ERROR: -i/--input is required" >&2; exit 1; }
|
||||
[ -n "$OUTPUT" ] || { echo "ERROR: -o/--output is required" >&2; exit 1; }
|
||||
[ -d "$INPUT" ] || { echo "ERROR: input dir not found: $INPUT" >&2; exit 1; }
|
||||
|
||||
# Absolute paths so relative-path math is stable regardless of where we cd'd.
|
||||
INPUT="$(realpath "$INPUT")"
|
||||
mkdir -p "$OUTPUT"
|
||||
OUTPUT="$(realpath "$OUTPUT")"
|
||||
|
||||
# Build the find extension filter: \( -iname '*.pdf' -o -iname '*.docx' ... \)
|
||||
find_expr=()
|
||||
for ext in $EXTS; do
|
||||
find_expr+=( -iname "*.${ext}" -o )
|
||||
done
|
||||
unset 'find_expr[${#find_expr[@]}-1]' # drop the trailing -o
|
||||
|
||||
echo "Input : $INPUT"
|
||||
echo "Output: $OUTPUT"
|
||||
echo "Exts : $EXTS"
|
||||
[ "$DRY" = true ] && echo "(dry run — nothing will be processed)"
|
||||
echo
|
||||
|
||||
total=0 ok=0 fail=0
|
||||
# Process substitution (not a pipe) so counters survive into the summary.
|
||||
while IFS= read -r -d '' doc; do
|
||||
total=$((total + 1))
|
||||
|
||||
rel="${doc#"$INPUT"/}" # path relative to the input root
|
||||
reldir="$(dirname "$rel")"
|
||||
if [ "$reldir" = "." ]; then
|
||||
outdir="$OUTPUT" # doc sat directly in the input root
|
||||
else
|
||||
outdir="$OUTPUT/$reldir" # mirror the subfolder structure
|
||||
fi
|
||||
|
||||
echo "[$total] $rel"
|
||||
echo " -> $outdir"
|
||||
|
||||
if [ "$DRY" = true ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir -p "$outdir"
|
||||
# stdin from /dev/null so any child process can't eat the file list and stall
|
||||
# the batch (same guard as ctrl/batch.sh).
|
||||
if "$PYTHON" "$PROJECT_DIR/process_doc.py" "$doc" \
|
||||
--output-dir "$outdir" "${FORWARD[@]+"${FORWARD[@]}"}" </dev/null; then
|
||||
ok=$((ok + 1))
|
||||
else
|
||||
echo " !! FAILED (continuing)" >&2
|
||||
fail=$((fail + 1))
|
||||
fi
|
||||
echo
|
||||
done < <(find "$INPUT" -type f \( "${find_expr[@]}" \) -print0 | sort -z)
|
||||
|
||||
echo "----------------------------------------"
|
||||
if [ "$DRY" = true ]; then
|
||||
echo "Found $total document(s)."
|
||||
else
|
||||
echo "Done. $total document(s): $ok ok, $fail failed."
|
||||
fi
|
||||
[ "$fail" -eq 0 ]
|
||||
70
ctrl/sync.sh
Executable file
70
ctrl/sync.sh
Executable file
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
# Sync (copy) this project to another location, excluding git-ignored files,
|
||||
# the .git directory, and local-only cruft.
|
||||
#
|
||||
# Respects every .gitignore in the tree (root and nested, e.g. ui/.gitignore)
|
||||
# via rsync's per-directory merge filter, so node_modules/, build output,
|
||||
# output/ runs, samples/, local-run.sh, etc. are never copied. Tracked files
|
||||
# (including the .gitignore files themselves) are copied so the destination is
|
||||
# a clean, working copy of the project.
|
||||
#
|
||||
# Usage:
|
||||
# ctrl/sync.sh [-n] [-d] <destination>
|
||||
#
|
||||
# <destination> local path or rsync target
|
||||
# (e.g. ../meetus-copy, /mnt/backup/meetus, host:~/meetus)
|
||||
# -n dry run — list what would change, copy nothing
|
||||
# -d mirror mode — also delete files in <destination> that are
|
||||
# not in the source (destructive; off by default)
|
||||
#
|
||||
# Examples:
|
||||
# ctrl/sync.sh ../meetus-copy
|
||||
# ctrl/sync.sh -n /mnt/backup/meetus # preview
|
||||
# ctrl/sync.sh -d server:~/projects/meetus # exact mirror
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
|
||||
usage() {
|
||||
# Print the header comment block (skip shebang, stop at first code line).
|
||||
awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0"
|
||||
}
|
||||
|
||||
DRY_RUN=""
|
||||
DELETE=""
|
||||
DEST=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-n) DRY_RUN="--dry-run" ;;
|
||||
-d) DELETE="--delete" ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--) shift; break ;;
|
||||
-*) echo "Unknown option: $1" >&2; usage >&2; exit 1 ;;
|
||||
*) if [ -z "$DEST" ]; then DEST="$1"; else echo "Unexpected argument: $1" >&2; exit 1; fi ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Any trailing args after -- are the destination too (support one).
|
||||
if [ -z "$DEST" ] && [ $# -gt 0 ]; then DEST="$1"; fi
|
||||
|
||||
if [ -z "$DEST" ]; then
|
||||
echo "Error: destination required." >&2
|
||||
echo "Usage: ctrl/sync.sh [-n] [-d] <destination>" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Syncing project -> $DEST"
|
||||
[ -n "$DRY_RUN" ] && echo " (dry run: no changes will be made)"
|
||||
[ -n "$DELETE" ] && echo " (mirror mode: extraneous files in destination will be DELETED)"
|
||||
|
||||
# --filter ':- .gitignore' reads a .gitignore in every directory rsync visits
|
||||
# and applies its ignore patterns to that subtree.
|
||||
rsync -ah --info=stats1 $DRY_RUN $DELETE \
|
||||
--exclude='.git/' \
|
||||
--exclude='.DS_Store' \
|
||||
--filter=':- .gitignore' \
|
||||
"$PROJECT_DIR"/ "$DEST"/
|
||||
|
||||
echo "Sync complete."
|
||||
Reference in New Issue
Block a user