190 lines
9.3 KiB
Python
190 lines
9.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
dataconvert
|
|
Converts CSV, Excel (.xlsx, .xls), OpenDocument (.ods), directories, ZIP archives,
|
|
or wildcard file patterns into schema-agnostic, individual SQL seed files, plus a
|
|
SCHEMA.md describing every table.
|
|
|
|
Usage:
|
|
python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20
|
|
python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6
|
|
python3 dataconvert.py --input data/ --config their-exports.json
|
|
|
|
How a given producer lays out its sheets is not built in: it is read from a
|
|
dataconvert.json in the source folder, or --config. See config.py and
|
|
dataconvert-example.json.
|
|
|
|
--max-rows is for understanding the data rather than loading it: every table
|
|
still gets its file and its SCHEMA.md entry, with only the first N rows, and a
|
|
note of how many rows there are and how big the full file would be.
|
|
|
|
The modules beside this file are imported by name, so the folder works wherever
|
|
it is copied: run this script from anywhere, no install step.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from output import write_tables
|
|
from sqlgen import DEFAULT_BATCH_ROWS, FORMATS
|
|
from progress import say
|
|
import config as cfg
|
|
from readers import expand_inputs, iter_sources
|
|
from schema import SchemaReport
|
|
|
|
|
|
def load_dotenv():
|
|
env_path = Path(".env")
|
|
if env_path.is_file():
|
|
for line in env_path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if line and not line.startswith("#") and "=" in line:
|
|
key, val = line.split("=", 1)
|
|
if key not in os.environ:
|
|
os.environ[key] = val.strip().strip('"\'')
|
|
|
|
|
|
def get_bool_env(key, default=False):
|
|
val = os.getenv(key)
|
|
if val is None:
|
|
return default
|
|
return val.strip().lower() in ('true', '1', 'yes', 'on')
|
|
|
|
|
|
def size(value):
|
|
try:
|
|
return cfg.parse_size(value)
|
|
except ValueError:
|
|
raise argparse.ArgumentTypeError('a size such as 100000000 or "100M"')
|
|
|
|
|
|
def positive_int(value):
|
|
n = int(value)
|
|
if n < 1:
|
|
raise argparse.ArgumentTypeError("must be 1 or more")
|
|
return n
|
|
|
|
|
|
def main():
|
|
load_dotenv()
|
|
|
|
default_input = os.getenv("DATACONVERT_INPUT")
|
|
default_exclude = os.getenv("DATACONVERT_EXCLUDE", "").split(",") if os.getenv("DATACONVERT_EXCLUDE") else []
|
|
|
|
parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
|
|
parser.add_argument("--input", nargs="+", default=[default_input] if default_input else None, required=not bool(default_input),
|
|
help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
|
|
parser.add_argument("--out-dir", default=os.getenv("DATACONVERT_OUT_DIR"),
|
|
help="Directory where individual .sql files will be written (default: the config's out_dir or env)")
|
|
parser.add_argument("--max-rows", type=positive_int, default=os.getenv("DATACONVERT_MAX_ROWS"),
|
|
help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
|
|
parser.add_argument("--no-schema", action="store_true", default=get_bool_env("DATACONVERT_NO_SCHEMA"),
|
|
help="Do not write SCHEMA.md, whatever the config says")
|
|
parser.add_argument("--no-ddl", action="store_true", default=get_bool_env("DATACONVERT_NO_DDL"),
|
|
help="Do not emit CREATE TABLE IF NOT EXISTS statements before seeding")
|
|
parser.add_argument("--sql-format", choices=FORMATS, default=os.getenv("DATACONVERT_SQL_FORMAT"),
|
|
help="insert: a statement per row (default). batch: one INSERT per --batch-rows rows. "
|
|
"copy: COPY FROM stdin, smallest, psql only, load once")
|
|
parser.add_argument("--batch-rows", type=positive_int, default=os.getenv("DATACONVERT_BATCH_ROWS", DEFAULT_BATCH_ROWS),
|
|
help=f"Rows per INSERT with --sql-format batch (default {DEFAULT_BATCH_ROWS})")
|
|
parser.add_argument("--max-file-size", type=size, default=os.getenv("DATACONVERT_MAX_FILE_SIZE"),
|
|
help='Split a table into numbered parts past this size, e.g. "100M"')
|
|
parser.add_argument("--all-text", action="store_true", default=get_bool_env("DATACONVERT_ALL_TEXT", None),
|
|
help="Read every column as text, exactly as in the file; only empty cells become NULL.")
|
|
parser.add_argument("--keep-folders", action="store_true", default=get_bool_env("DATACONVERT_KEEP_FOLDERS", None),
|
|
help="Mirror the source folders under the output directory")
|
|
parser.add_argument("--exclude", action="append", default=default_exclude, metavar="GLOB",
|
|
help='Skip matching files, e.g. "*.xlsx.ods". Repeatable, and added to the config\'s')
|
|
parser.add_argument("--header-row", type=positive_int, default=1,
|
|
help="Spreadsheet row holding the column names, for every file; overrides the config's layouts (default 1)")
|
|
parser.add_argument("--data-row", type=positive_int, default=None,
|
|
help="First row of data, when rows sit between it and the header (default: the row after the header)")
|
|
parser.add_argument("--config", default=os.getenv("DATACONVERT_CONFIG"),
|
|
help="JSON with layouts, naming and run settings, for every input")
|
|
args = parser.parse_args()
|
|
|
|
forced = None
|
|
try:
|
|
if args.header_row != 1 or args.data_row is not None:
|
|
forced = cfg.Layout(args.header_row, args.data_row, "command line")
|
|
given = cfg.load(args.config, forced) if args.config else None
|
|
except cfg.ConfigError as e:
|
|
parser.error(str(e))
|
|
|
|
# Every config is read and checked before anything is written, so a broken
|
|
# one in the third folder does not leave the first two converted.
|
|
inputs = expand_inputs(args.input)
|
|
loaded = {}
|
|
plan = []
|
|
for in_path in inputs:
|
|
if given is not None:
|
|
plan.append((in_path, given))
|
|
continue
|
|
found = cfg.source_config_path(in_path)
|
|
if found is not None and found not in loaded:
|
|
try:
|
|
loaded[found] = cfg.load(found, forced)
|
|
except cfg.ConfigError as e:
|
|
parser.error(str(e))
|
|
plan.append((in_path, loaded[found] if found is not None else cfg.Config(forced=forced)))
|
|
|
|
# No silent default directory. A run whose config was not found would
|
|
# otherwise write, unlaid-out, into ./seed wherever it was started, and look
|
|
# like it worked. Checked for every input before anything is written.
|
|
if not args.out_dir:
|
|
for in_path, config in plan:
|
|
if config.out_dir is not None:
|
|
continue
|
|
where = (f"{config.source} sets no out_dir" if config.source is not None
|
|
else f"no {cfg.FILENAME} found in its folder")
|
|
parser.error(f"no output directory for {in_path} ({where}): pass --out-dir, or set out_dir in the config")
|
|
|
|
# One report per output directory: inputs whose configs name different
|
|
# out_dirs each get their own SCHEMA.md, beside their own .sql files.
|
|
reports = {}
|
|
announced = set()
|
|
for in_path, config in plan:
|
|
if config.source is not None and config.source not in announced:
|
|
announced.add(config.source)
|
|
said = config.settings()
|
|
say(f"config: {config.source}" + (f" ({said})" if said else ""))
|
|
|
|
# The command line wins, then the config.
|
|
out_dir = Path(args.out_dir) if args.out_dir else config.out_dir
|
|
max_rows = int(args.max_rows) if args.max_rows is not None else config.max_rows
|
|
schema = not args.no_schema and config.schema is not False
|
|
include_ddl = not args.no_ddl and config.include_ddl is not False
|
|
fmt = args.sql_format or config.sql_format or "insert"
|
|
batch_rows = int(args.batch_rows) if args.batch_rows else (config.batch_rows or DEFAULT_BATCH_ROWS)
|
|
max_bytes = args.max_file_size or config.max_file_size
|
|
all_text = bool(args.all_text or config.all_text)
|
|
keep_folders = bool(args.keep_folders or config.keep_folders)
|
|
exclude = tuple(config.exclude) + tuple(args.exclude)
|
|
load_order = config.load_order
|
|
|
|
key = out_dir.resolve()
|
|
if key not in reports:
|
|
reports[key] = (out_dir, SchemaReport(), set())
|
|
_, report, caps = reports[key]
|
|
caps.add(max_rows)
|
|
|
|
for source_name, folder, dfs in iter_sources(in_path, config, all_text, exclude):
|
|
write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes,
|
|
fmt, batch_rows, max_bytes, folder if keep_folders else "", include_ddl=include_ddl,
|
|
load_order=load_order)
|
|
|
|
for out_dir, report, caps in reports.values():
|
|
if report.entries and out_dir.is_dir():
|
|
# The sampling note names the cap only when every input here used
|
|
# the same one; otherwise each table's row still says what it kept.
|
|
cap = next(iter(caps)) if len(caps) == 1 else None
|
|
say(f"writing SCHEMA.md for {len(report.tables())} tables")
|
|
say(f"Generated: {report.write(out_dir, cap)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|