dataconvert: spreadsheets to SQL seeds, layouts from config, row cap, SCHEMA.md
Converts CSV, xlsx/xls/ods, directories, ZIPs and globs into one INSERT file per table. Producer-specific layouts (header/data rows found by a marker cell) and sheet naming live in a gitignored dataconvert.json, with dataconvert-example.json as the template. --max-rows samples each table and SCHEMA.md records columns, types, row counts and full sizes. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
43
soleprint/station/tools/dataconvert/output.py
Normal file
43
soleprint/station/tools/dataconvert/output.py
Normal file
@@ -0,0 +1,43 @@
|
||||
"""
|
||||
Output: one .sql file per table or sheet, named after it.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlgen import render_table, sanitize_identifier
|
||||
|
||||
|
||||
def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=()) -> str:
|
||||
"""
|
||||
Sheets of a multi-sheet workbook are prefixed with the workbook, so two
|
||||
workbooks cannot collide, unless the config names the sheet as already
|
||||
unique (bare_sheet_prefixes).
|
||||
"""
|
||||
clean = sanitize_identifier(raw_name)
|
||||
if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)):
|
||||
return f"{sanitize_identifier(source_name)}_{clean}.sql"
|
||||
return f"{clean}.sql"
|
||||
|
||||
|
||||
def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None, bare_prefixes=()):
|
||||
"""Write individual .sql files per table/sheet into the output directory."""
|
||||
for raw_name, df in dfs.items():
|
||||
if df.empty:
|
||||
continue
|
||||
|
||||
table = sanitize_identifier(raw_name)
|
||||
filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes)
|
||||
sql, total, full_bytes, exact = render_table(df, table, max_rows)
|
||||
out_file = out_dir / filename
|
||||
|
||||
# Several sources can feed the same table; they accumulate in one file.
|
||||
mode = "a" if out_file.exists() else "w"
|
||||
with open(out_file, mode, encoding="utf-8") as f:
|
||||
f.write(sql)
|
||||
|
||||
shown = total if exact else max_rows
|
||||
if report is not None:
|
||||
report.add(source_name, table, filename, df, total, full_bytes, exact, shown)
|
||||
|
||||
suffix = "" if exact else f" ({shown} of {total} rows)"
|
||||
print(f"[dataconvert] Generated: {out_file}{suffix}")
|
||||
Reference in New Issue
Block a user