buenosairesam
2f3e9c2634
Converts CSV, xlsx/xls/ods, directories, ZIPs and globs into one INSERT file per table. Producer-specific layouts (header/data rows found by a marker cell) and sheet naming live in a gitignored dataconvert.json, with dataconvert-example.json as the template. --max-rows samples each table and SCHEMA.md records columns, types, row counts and full sizes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
81 lines
3.5 KiB
Python
81 lines
3.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
dataconvert
|
|
Converts CSV, Excel (.xlsx, .xls), OpenDocument (.ods), directories, ZIP archives,
|
|
or wildcard file patterns into schema-agnostic, individual SQL seed files, plus a
|
|
SCHEMA.md describing every table.
|
|
|
|
Usage:
|
|
python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir seed/
|
|
python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20
|
|
python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6
|
|
python3 dataconvert.py --input data/ --config their-exports.json
|
|
|
|
How a given producer lays out its sheets is not built in: see config.py and
|
|
dataconvert-example.json.
|
|
|
|
--max-rows is for understanding the data rather than loading it: every table
|
|
still gets its file and its SCHEMA.md entry, with only the first N rows, and a
|
|
note of how many rows there are and how big the full file would be.
|
|
|
|
The modules beside this file are imported by name, so the folder works wherever
|
|
it is copied: run this script from anywhere, no install step.
|
|
"""
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
from output import write_tables
|
|
import config as cfg
|
|
from readers import expand_inputs, iter_sources
|
|
from schema import SchemaReport
|
|
|
|
|
|
def positive_int(value):
|
|
n = int(value)
|
|
if n < 1:
|
|
raise argparse.ArgumentTypeError("must be 1 or more")
|
|
return n
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
|
|
parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
|
|
parser.add_argument("--out-dir", default="seed", help="Directory where individual .sql files will be written")
|
|
parser.add_argument("--max-rows", type=positive_int, default=None,
|
|
help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
|
|
parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md")
|
|
parser.add_argument("--header-row", type=positive_int, default=1,
|
|
help="Spreadsheet row holding the column names, for every file; overrides the config layouts (default 1)")
|
|
parser.add_argument("--data-row", type=positive_int, default=None,
|
|
help="First row of data, when rows sit between it and the header (default: the row after the header)")
|
|
parser.add_argument("--config", default=None,
|
|
help="Layouts and naming for these files (default: dataconvert.json beside this script, if present)")
|
|
args = parser.parse_args()
|
|
|
|
try:
|
|
forced = None
|
|
if args.header_row != 1 or args.data_row is not None:
|
|
forced = cfg.Layout(args.header_row, args.data_row, "command line")
|
|
config = cfg.load(args.config, forced)
|
|
except cfg.ConfigError as e:
|
|
parser.error(str(e))
|
|
if config.source is not None:
|
|
print(f"[dataconvert] config: {config.source}")
|
|
|
|
out_dir = Path(args.out_dir)
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
report = None if args.no_schema else SchemaReport()
|
|
for in_path in expand_inputs(args.input):
|
|
for source_name, dfs in iter_sources(in_path, config):
|
|
write_tables(dfs, out_dir, source_name, args.max_rows, report, config.bare_sheet_prefixes)
|
|
|
|
if report is not None and report.entries:
|
|
print(f"[dataconvert] Generated: {report.write(out_dir, args.max_rows)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|