129 lines
5.9 KiB
Python
129 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
dataconvert
|
|
Converts CSV, Excel (.xlsx, .xls), OpenDocument (.ods), directories, ZIP archives,
|
|
or wildcard file patterns into schema-agnostic, individual SQL seed files, plus a
|
|
SCHEMA.md describing every table.
|
|
|
|
Usage:
|
|
python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20
|
|
python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6
|
|
python3 dataconvert.py --input data/ --config their-exports.json
|
|
|
|
How a given producer lays out its sheets is not built in: it is read from a
|
|
dataconvert.json in the source folder, or --config. See config.py and
|
|
dataconvert-example.json.
|
|
|
|
--max-rows is for understanding the data rather than loading it: every table
|
|
still gets its file and its SCHEMA.md entry, with only the first N rows, and a
|
|
note of how many rows there are and how big the full file would be.
|
|
|
|
The modules beside this file are imported by name, so the folder works wherever
|
|
it is copied: run this script from anywhere, no install step.
|
|
"""
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
from output import write_tables
|
|
import config as cfg
|
|
from readers import expand_inputs, iter_sources
|
|
from schema import SchemaReport
|
|
|
|
|
|
def positive_int(value):
|
|
n = int(value)
|
|
if n < 1:
|
|
raise argparse.ArgumentTypeError("must be 1 or more")
|
|
return n
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
|
|
parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
|
|
parser.add_argument("--out-dir", default=None,
|
|
help="Directory where individual .sql files will be written (default: the config's out_dir; one of the two is required)")
|
|
parser.add_argument("--max-rows", type=positive_int, default=None,
|
|
help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size (default: the config's max_rows, else all)")
|
|
parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md, whatever the config says")
|
|
parser.add_argument("--header-row", type=positive_int, default=1,
|
|
help="Spreadsheet row holding the column names, for every file; overrides the config's layouts (default 1)")
|
|
parser.add_argument("--data-row", type=positive_int, default=None,
|
|
help="First row of data, when rows sit between it and the header (default: the row after the header)")
|
|
parser.add_argument("--config", default=None,
|
|
help="JSON with layouts, naming and run settings, for every input (default: dataconvert.json in each input's folder, if there is one)")
|
|
args = parser.parse_args()
|
|
|
|
forced = None
|
|
try:
|
|
if args.header_row != 1 or args.data_row is not None:
|
|
forced = cfg.Layout(args.header_row, args.data_row, "command line")
|
|
given = cfg.load(args.config, forced) if args.config else None
|
|
except cfg.ConfigError as e:
|
|
parser.error(str(e))
|
|
|
|
# Every config is read and checked before anything is written, so a broken
|
|
# one in the third folder does not leave the first two converted.
|
|
inputs = expand_inputs(args.input)
|
|
loaded = {}
|
|
plan = []
|
|
for in_path in inputs:
|
|
if given is not None:
|
|
plan.append((in_path, given))
|
|
continue
|
|
found = cfg.source_config_path(in_path)
|
|
if found is not None and found not in loaded:
|
|
try:
|
|
loaded[found] = cfg.load(found, forced)
|
|
except cfg.ConfigError as e:
|
|
parser.error(str(e))
|
|
plan.append((in_path, loaded[found] if found is not None else cfg.Config(forced=forced)))
|
|
|
|
# No silent default directory. A run whose config was not found would
|
|
# otherwise write, unlaid-out, into ./seed wherever it was started, and look
|
|
# like it worked. Checked for every input before anything is written.
|
|
if not args.out_dir:
|
|
for in_path, config in plan:
|
|
if config.out_dir is not None:
|
|
continue
|
|
where = (f"{config.source} sets no out_dir" if config.source is not None
|
|
else f"no {cfg.FILENAME} found in its folder")
|
|
parser.error(f"no output directory for {in_path} ({where}): pass --out-dir, or set out_dir in the config")
|
|
|
|
# One report per output directory: inputs whose configs name different
|
|
# out_dirs each get their own SCHEMA.md, beside their own .sql files.
|
|
reports = {}
|
|
announced = set()
|
|
for in_path, config in plan:
|
|
if config.source is not None and config.source not in announced:
|
|
announced.add(config.source)
|
|
said = config.settings()
|
|
print(f"[dataconvert] config: {config.source}" + (f" ({said})" if said else ""))
|
|
|
|
# The command line wins, then the config.
|
|
out_dir = Path(args.out_dir) if args.out_dir else config.out_dir
|
|
max_rows = args.max_rows if args.max_rows is not None else config.max_rows
|
|
schema = not args.no_schema and config.schema is not False
|
|
|
|
key = out_dir.resolve()
|
|
if key not in reports:
|
|
reports[key] = (out_dir, SchemaReport(), set())
|
|
_, report, caps = reports[key]
|
|
caps.add(max_rows)
|
|
|
|
for source_name, dfs in iter_sources(in_path, config):
|
|
write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes)
|
|
|
|
for out_dir, report, caps in reports.values():
|
|
if report.entries and out_dir.is_dir():
|
|
# The sampling note names the cap only when every input here used
|
|
# the same one; otherwise each table's row still says what it kept.
|
|
cap = next(iter(caps)) if len(caps) == 1 else None
|
|
print(f"[dataconvert] Generated: {report.write(out_dir, cap)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|