45 lines
1.5 KiB
Bash
Executable File
45 lines
1.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Downloads BIRD financial SQLite into ctrl/seed/data/financial.sqlite and
|
|
# the golden eval set into api/evals/data/. Runs on the HOST (the api image
|
|
# deliberately doesn't include the data). Override BIRD_URL if upstream
|
|
# moves.
|
|
#
|
|
# Layout inside the outer dev.zip (as of the 2024-06-27 dump):
|
|
# dev_<date>/
|
|
# dev.sql, dev.json, dev_tables.json, dev_tied_append.json
|
|
# dev_databases.zip → financial/financial.sqlite (and 10 other DBs)
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")"
|
|
|
|
DEST="data/financial.sqlite"
|
|
EVAL_DIR="../../api/evals/data"
|
|
mkdir -p data "$EVAL_DIR"
|
|
|
|
if [ -f "$DEST" ]; then
|
|
echo "already have $DEST ($(du -h "$DEST" | cut -f1))"
|
|
exit 0
|
|
fi
|
|
|
|
URL="${BIRD_URL:-https://bird-bench.oss-cn-beijing.aliyuncs.com/dev.zip}"
|
|
TMP="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP"' EXIT
|
|
OUTER="$TMP/dev.zip"
|
|
|
|
echo "fetching $URL (~330MB compressed; full dev set is ~3GB extracted)..."
|
|
curl -fL --progress-bar -o "$OUTER" "$URL"
|
|
|
|
echo "extracting outer archive..."
|
|
unzip -q "$OUTER" -d "$TMP"
|
|
ROOT=$(find "$TMP" -maxdepth 1 -type d -name 'dev_*' | head -1)
|
|
test -n "$ROOT" || { echo "could not locate dev_* root inside outer zip" >&2; exit 1; }
|
|
|
|
echo "extracting financial.sqlite from inner archive..."
|
|
unzip -q -j "$ROOT/dev_databases.zip" '*/financial/financial.sqlite' -d data/
|
|
|
|
echo "copying golden eval set to $EVAL_DIR/..."
|
|
cp "$ROOT/dev.json" "$ROOT/dev.sql" "$ROOT/dev_tables.json" "$EVAL_DIR/"
|
|
|
|
echo "ready:"
|
|
echo " $DEST ($(du -h "$DEST" | cut -f1))"
|
|
echo " $(ls -1 "$EVAL_DIR/" | wc -l) files in $EVAL_DIR/"
|