dataconvert
This commit is contained in:
45
soleprint/station/tools/dataconvert/README.md
Normal file
45
soleprint/station/tools/dataconvert/README.md
Normal file
@@ -0,0 +1,45 @@
|
||||
# dataconvert
|
||||
|
||||
Spreadsheets and CSV into schema-agnostic SQL seed files, one per table or sheet,
|
||||
plus a `SCHEMA.md` describing every table.
|
||||
|
||||
```bash
|
||||
uv run dataconvert.py --input data/ --out-dir seed/ # full seeds
|
||||
uv run dataconvert.py --input data/ "*.xlsx" --out-dir seed/ # dirs, files, globs, .zip
|
||||
uv run dataconvert.py --input data/ --out-dir sample/ --max-rows 20 # to understand the data
|
||||
```
|
||||
|
||||
Reads `.csv`, `.xlsx`, `.xls` and `.ods`: single files, directories (recursively),
|
||||
ZIP archives and wildcard patterns. Sheets of a multi-sheet workbook are written as
|
||||
`<workbook>_<sheet>.sql` unless the sheet is already `drm_` or `study_`. Several
|
||||
sources feeding the same table accumulate in one file, and so does re-running into
|
||||
the same `--out-dir`, so point a fresh run at an empty directory.
|
||||
|
||||
## Sampling for a web LLM
|
||||
|
||||
Full seed files get large fast, and a model only needs to see the shape of the data.
|
||||
With `--max-rows N`:
|
||||
|
||||
- every table still gets its `.sql` file, holding only its first N rows, with a header
|
||||
line such as `-- SAMPLE: first 20 of 184233 rows. The full file would be ~48.1M`;
|
||||
- `SCHEMA.md` lists every table with its total rows and the size the full seed file
|
||||
would be, then each table's columns: original header, inferred type, null count
|
||||
and one example value.
|
||||
|
||||
`SCHEMA.md` alone is often enough to hand to the model. Full sizes of sampled tables
|
||||
are estimated from the rows written, so they carry a `~`. Types are inferred from
|
||||
what pandas read: a starting point, not DDL. `--no-schema` skips the file.
|
||||
|
||||
## Layout
|
||||
|
||||
| file | does |
|
||||
|---|---|
|
||||
| `dataconvert.py` | command line |
|
||||
| `readers.py` | files, directories, ZIPs and globs into DataFrames |
|
||||
| `sqlgen.py` | DataFrames into INSERT statements, with the row cap |
|
||||
| `output.py` | file naming and writing |
|
||||
| `schema.py` | `SCHEMA.md` |
|
||||
|
||||
The modules import each other by name, so the folder works wherever it is copied:
|
||||
`uv run dataconvert.py` from inside it, or `python3 path/to/dataconvert.py` with
|
||||
pandas, openpyxl and odfpy installed.
|
||||
58
soleprint/station/tools/dataconvert/dataconvert.py
Normal file
58
soleprint/station/tools/dataconvert/dataconvert.py
Normal file
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
dataconvert
|
||||
Converts CSV, Excel (.xlsx, .xls), OpenDocument (.ods), directories, ZIP archives,
|
||||
or wildcard file patterns into schema-agnostic, individual SQL seed files, plus a
|
||||
SCHEMA.md describing every table.
|
||||
|
||||
Usage:
|
||||
python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/
|
||||
python3 dataconvert.py --input path/to/folder/ --out-dir seed/
|
||||
python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20
|
||||
|
||||
--max-rows is for understanding the data rather than loading it: every table
|
||||
still gets its file and its SCHEMA.md entry, with only the first N rows, and a
|
||||
note of how many rows there are and how big the full file would be.
|
||||
|
||||
The modules beside this file are imported by name, so the folder works wherever
|
||||
it is copied: run this script from anywhere, no install step.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from output import write_tables
|
||||
from readers import expand_inputs, iter_sources
|
||||
from schema import SchemaReport
|
||||
|
||||
|
||||
def positive_int(value):
|
||||
n = int(value)
|
||||
if n < 1:
|
||||
raise argparse.ArgumentTypeError("must be 1 or more")
|
||||
return n
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
|
||||
parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
|
||||
parser.add_argument("--out-dir", default="seed", help="Directory where individual .sql files will be written")
|
||||
parser.add_argument("--max-rows", type=positive_int, default=None,
|
||||
help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
|
||||
parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report = None if args.no_schema else SchemaReport()
|
||||
for in_path in expand_inputs(args.input):
|
||||
for source_name, dfs in iter_sources(in_path):
|
||||
write_tables(dfs, out_dir, source_name, args.max_rows, report)
|
||||
|
||||
if report is not None and report.entries:
|
||||
print(f"[dataconvert] Generated: {report.write(out_dir, args.max_rows)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
39
soleprint/station/tools/dataconvert/output.py
Normal file
39
soleprint/station/tools/dataconvert/output.py
Normal file
@@ -0,0 +1,39 @@
|
||||
"""
|
||||
Output: one .sql file per table or sheet, named after it.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from sqlgen import render_table, sanitize_identifier
|
||||
|
||||
|
||||
def table_filename(raw_name: str, source_name: str, sheet_count: int) -> str:
|
||||
"""Sheets of a multi-sheet workbook are prefixed with the workbook, unless already namespaced."""
|
||||
clean = sanitize_identifier(raw_name)
|
||||
if sheet_count > 1 and not clean.startswith("drm_") and not clean.startswith("study_"):
|
||||
return f"{sanitize_identifier(source_name)}_{clean}.sql"
|
||||
return f"{clean}.sql"
|
||||
|
||||
|
||||
def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None):
|
||||
"""Write individual .sql files per table/sheet into the output directory."""
|
||||
for raw_name, df in dfs.items():
|
||||
if df.empty:
|
||||
continue
|
||||
|
||||
table = sanitize_identifier(raw_name)
|
||||
filename = table_filename(raw_name, source_name, len(dfs))
|
||||
sql, total, full_bytes, exact = render_table(df, table, max_rows)
|
||||
out_file = out_dir / filename
|
||||
|
||||
# Several sources can feed the same table; they accumulate in one file.
|
||||
mode = "a" if out_file.exists() else "w"
|
||||
with open(out_file, mode, encoding="utf-8") as f:
|
||||
f.write(sql)
|
||||
|
||||
shown = total if exact else max_rows
|
||||
if report is not None:
|
||||
report.add(source_name, table, filename, df, total, full_bytes, exact, shown)
|
||||
|
||||
suffix = "" if exact else f" ({shown} of {total} rows)"
|
||||
print(f"[dataconvert] Generated: {out_file}{suffix}")
|
||||
@@ -1,7 +1,7 @@
|
||||
[project]
|
||||
name = "seed-converter"
|
||||
name = "dataconvert"
|
||||
version = "0.1.0"
|
||||
description = "SQL Seed Converter Tool"
|
||||
description = "Spreadsheets and CSV into SQL seed files and a schema summary"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
@@ -11,4 +11,4 @@ dependencies = [
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
package = false
|
||||
|
||||
85
soleprint/station/tools/dataconvert/readers.py
Normal file
85
soleprint/station/tools/dataconvert/readers.py
Normal file
@@ -0,0 +1,85 @@
|
||||
"""
|
||||
Readers: files, directories, ZIP archives and wildcards into DataFrames.
|
||||
|
||||
Each input becomes zero or more (source_name, {entity_name: DataFrame}) pairs,
|
||||
one per spreadsheet or CSV. Nothing here knows about SQL or output files.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import tempfile
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
SUPPORTED = {".csv", ".xlsx", ".xls", ".ods"}
|
||||
|
||||
# DRM catalogue sheets put the real column names on row 1 and start the data
|
||||
# on row 5; the rows in between are descriptions. Recognised by the first cell
|
||||
# of the header row.
|
||||
DRM_HEADER_MARKERS = {
|
||||
"CONTROLLED_TERMINOLOGY", "DATA_ELEMENT", "DATA_DOMAIN", "DATA_STATE", "DATA_CATEGORY",
|
||||
}
|
||||
|
||||
|
||||
def expand_inputs(inputs):
|
||||
"""Wildcard patterns become the paths they match; everything else passes through."""
|
||||
paths = []
|
||||
for in_str in inputs:
|
||||
if any(c in in_str for c in ["*", "?", "["]):
|
||||
matched = glob.glob(in_str, recursive=True)
|
||||
if not matched:
|
||||
print(f"[Warning] No files matched wildcard pattern: '{in_str}'")
|
||||
paths.extend(Path(m) for m in sorted(matched))
|
||||
else:
|
||||
paths.append(Path(in_str))
|
||||
return paths
|
||||
|
||||
|
||||
def load_dataframes_from_file(file_path: Path) -> dict:
|
||||
"""Load a file (.csv, .xlsx, .xls, .ods) into {entity_name: DataFrame}."""
|
||||
ext = file_path.suffix.lower()
|
||||
dfs = {}
|
||||
|
||||
try:
|
||||
if ext == ".csv":
|
||||
dfs[file_path.stem] = pd.read_csv(file_path)
|
||||
elif ext in [".xlsx", ".xls"]:
|
||||
xls = pd.ExcelFile(file_path)
|
||||
for sheet in xls.sheet_names:
|
||||
df_raw = pd.read_excel(xls, sheet_name=sheet, header=None)
|
||||
if len(df_raw) > 1 and str(df_raw.iloc[1, 0]).strip() in DRM_HEADER_MARKERS:
|
||||
cols = [str(c).strip() for c in df_raw.iloc[1]]
|
||||
data_df = df_raw.iloc[5:].copy()
|
||||
data_df.columns = cols
|
||||
dfs[sheet] = data_df
|
||||
else:
|
||||
dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
|
||||
elif ext == ".ods":
|
||||
xls = pd.ExcelFile(file_path, engine="odf")
|
||||
for sheet in xls.sheet_names:
|
||||
dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
|
||||
except Exception as e:
|
||||
print(f"[Warning] Could not read '{file_path.name}': {e}")
|
||||
|
||||
return dfs
|
||||
|
||||
|
||||
def iter_sources(path: Path):
|
||||
"""Yield (source_name, dfs) for a file, a directory (recursively) or a ZIP archive."""
|
||||
if path.is_file() and path.suffix.lower() == ".zip":
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
with zipfile.ZipFile(path, "r") as zip_ref:
|
||||
zip_ref.extractall(tmp_dir)
|
||||
yield from iter_sources(Path(tmp_dir))
|
||||
|
||||
elif path.is_dir():
|
||||
for root, _, files in os.walk(path):
|
||||
for f in sorted(files):
|
||||
f_path = Path(root) / f
|
||||
if f_path.suffix.lower() in SUPPORTED:
|
||||
yield f_path.stem, load_dataframes_from_file(f_path)
|
||||
|
||||
elif path.is_file() and path.suffix.lower() in SUPPORTED:
|
||||
yield path.stem, load_dataframes_from_file(path)
|
||||
122
soleprint/station/tools/dataconvert/schema.py
Normal file
122
soleprint/station/tools/dataconvert/schema.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
SCHEMA.md: what each table looks like, without its rows.
|
||||
|
||||
Written for reading, by a person or a web LLM that has to understand the data
|
||||
before anything else: columns, an inferred type, how many are empty, one
|
||||
example value, and how big the table really is. Types are inferred from what
|
||||
pandas read, so they are a starting point, not a DDL.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
from sqlgen import human_bytes, sanitize_identifier
|
||||
|
||||
EXAMPLE_MAX = 40
|
||||
|
||||
|
||||
def infer_type(series: pd.Series) -> str:
|
||||
values = series.dropna()
|
||||
if values.empty:
|
||||
return "unknown (all null)"
|
||||
if pd.api.types.is_bool_dtype(values):
|
||||
return "boolean"
|
||||
if pd.api.types.is_integer_dtype(values):
|
||||
return "integer"
|
||||
if pd.api.types.is_float_dtype(values):
|
||||
return "integer" if (values % 1 == 0).all() else "numeric"
|
||||
if pd.api.types.is_datetime64_any_dtype(values):
|
||||
return "timestamp"
|
||||
# Object columns from spreadsheets mix types; name the one that is there.
|
||||
kinds = {type(v).__name__ for v in values}
|
||||
if kinds <= {"int", "bool"}:
|
||||
return "integer"
|
||||
if kinds <= {"int", "float"}:
|
||||
return "numeric"
|
||||
if kinds <= {"datetime", "Timestamp"}:
|
||||
return "timestamp"
|
||||
longest = values.astype(str).str.len().max()
|
||||
return f"text (max {longest})" if kinds == {"str"} else f"mixed ({', '.join(sorted(kinds))})"
|
||||
|
||||
|
||||
def example(series: pd.Series) -> str:
|
||||
values = series.dropna()
|
||||
if values.empty:
|
||||
return ""
|
||||
text = str(values.iloc[0]).replace("\n", " ").replace("|", "\\|")
|
||||
return text if len(text) <= EXAMPLE_MAX else text[: EXAMPLE_MAX - 1] + "…"
|
||||
|
||||
|
||||
class SchemaReport:
|
||||
"""Collects one entry per written table, then writes them as one document."""
|
||||
|
||||
def __init__(self):
|
||||
self.entries = []
|
||||
|
||||
def add(self, source, table, filename, df, total_rows, full_bytes, exact, shown_rows):
|
||||
self.entries.append(dict(
|
||||
source=source, table=table, filename=filename, df=df,
|
||||
total_rows=total_rows, full_bytes=full_bytes, exact=exact, shown_rows=shown_rows,
|
||||
))
|
||||
|
||||
def tables(self):
|
||||
"""
|
||||
One entry per output file. Several sources can feed the same table, and
|
||||
the file holds all of them, so the report does too: rows and sizes are
|
||||
summed, and the columns come from the first source.
|
||||
"""
|
||||
merged = {}
|
||||
for e in self.entries:
|
||||
m = merged.get(e["filename"])
|
||||
if m is None:
|
||||
merged[e["filename"]] = dict(e, sources=[e["source"]])
|
||||
continue
|
||||
m["sources"].append(e["source"])
|
||||
m["total_rows"] += e["total_rows"]
|
||||
m["full_bytes"] += e["full_bytes"]
|
||||
m["shown_rows"] += e["shown_rows"]
|
||||
m["exact"] = m["exact"] and e["exact"]
|
||||
return list(merged.values())
|
||||
|
||||
def write(self, out_dir: Path, max_rows):
|
||||
tables = self.tables()
|
||||
lines = ["# Data schema", ""]
|
||||
total_rows = sum(t["total_rows"] for t in tables)
|
||||
total_bytes = sum(t["full_bytes"] for t in tables)
|
||||
lines.append(
|
||||
f"{len(tables)} tables · {total_rows} rows · full seed files "
|
||||
f"{'~' if any(not t['exact'] for t in tables) else ''}{human_bytes(total_bytes)}"
|
||||
)
|
||||
lines.append("")
|
||||
if max_rows is not None:
|
||||
lines += [
|
||||
f"The .sql files beside this one hold at most {max_rows} rows per table from each",
|
||||
"source: they are samples for understanding the data, not seeds to load. Sizes",
|
||||
"marked ~ are estimated from the rows that were written.",
|
||||
"",
|
||||
]
|
||||
lines += ["| table | file | rows | full size | in the file |", "|---|---|---:|---:|---|"]
|
||||
for t in tables:
|
||||
size = ("" if t["exact"] else "~") + human_bytes(t["full_bytes"])
|
||||
kept = "all" if t["exact"] else f"{t['shown_rows']} rows"
|
||||
lines.append(f"| `{t['table']}` | `{t['filename']}` | {t['total_rows']} | {size} | {kept} |")
|
||||
lines.append("")
|
||||
|
||||
for t in tables:
|
||||
df = t["df"]
|
||||
sources = ", ".join(f"`{src}`" for src in t["sources"])
|
||||
lines += [f"## {t['table']}", "", f"From {sources} · {t['total_rows']} rows · {len(df.columns)} columns", ""]
|
||||
lines += ["| column | source header | type | nulls | example |", "|---|---|---|---:|---|"]
|
||||
for col in df.columns:
|
||||
series = df[col]
|
||||
header = str(col).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{sanitize_identifier(col)}` | {header} | {infer_type(series)} | "
|
||||
f"{int(series.isna().sum())} | {example(series)} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
path = out_dir / "SCHEMA.md"
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
return path
|
||||
@@ -1,164 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Modular Seed Converter Tool
|
||||
Converts CSV, Excel (.xlsx, .xls), OpenDocument (.ods), directories, ZIP archives,
|
||||
or wildcard file patterns into schema-agnostic, individual SQL seed files.
|
||||
|
||||
Usage:
|
||||
python3 seed_converter.py --input "data/*" "*.xlsx" --out-dir seed/
|
||||
python3 seed_converter.py --input path/to/folder/ --out-dir seed/
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import glob
|
||||
import argparse
|
||||
import zipfile
|
||||
import tempfile
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sanitize_identifier(identifier: str) -> str:
|
||||
"""Sanitize names for SQL tables, columns, and filenames."""
|
||||
clean = re.sub(r"[^\w]", "_", str(identifier).strip().lower())
|
||||
clean = re.sub(r"_+", "_", clean)
|
||||
return clean.strip("_")
|
||||
|
||||
|
||||
def dataframe_to_sql(df: pd.DataFrame, table_name: str) -> str:
|
||||
"""Generate SQL INSERT statements for a given DataFrame without schema qualification."""
|
||||
if df.empty:
|
||||
return ""
|
||||
|
||||
table_ref = f'"{table_name}"'
|
||||
cols = ", ".join([f'"{sanitize_identifier(c)}"' for c in df.columns])
|
||||
|
||||
lines = [
|
||||
f"-- Generated seed data for table: {table_ref}\n",
|
||||
"BEGIN;\n\n"
|
||||
]
|
||||
|
||||
for _, row in df.iterrows():
|
||||
vals = []
|
||||
for v in row:
|
||||
if pd.isna(v):
|
||||
vals.append("NULL")
|
||||
elif isinstance(v, (bool, int)):
|
||||
vals.append(str(v))
|
||||
elif isinstance(v, float):
|
||||
vals.append(str(int(v)) if v.is_integer() else str(v))
|
||||
else:
|
||||
escaped = str(v).replace("'", "''")
|
||||
vals.append(f"'{escaped}'")
|
||||
|
||||
val_str = ", ".join(vals)
|
||||
lines.append(f"INSERT INTO {table_ref} ({cols}) VALUES ({val_str}) ON CONFLICT DO NOTHING;\n")
|
||||
|
||||
lines.append("\nCOMMIT;\n")
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def load_dataframes_from_file(file_path: Path) -> dict:
|
||||
"""Load a file (.csv, .xlsx, .ods) into a dictionary of {entity_name: DataFrame}."""
|
||||
ext = file_path.suffix.lower()
|
||||
dfs = {}
|
||||
|
||||
try:
|
||||
if ext == ".csv":
|
||||
dfs[file_path.stem] = pd.read_csv(file_path)
|
||||
elif ext in [".xlsx", ".xls"]:
|
||||
xls = pd.ExcelFile(file_path)
|
||||
for sheet in xls.sheet_names:
|
||||
df_raw = pd.read_excel(xls, sheet_name=sheet, header=None)
|
||||
if len(df_raw) > 1 and str(df_raw.iloc[1, 0]).strip() in [
|
||||
"CONTROLLED_TERMINOLOGY", "DATA_ELEMENT", "DATA_DOMAIN", "DATA_STATE", "DATA_CATEGORY"
|
||||
]:
|
||||
cols = [str(c).strip() for c in df_raw.iloc[1]]
|
||||
data_df = df_raw.iloc[5:].copy()
|
||||
data_df.columns = cols
|
||||
dfs[sheet] = data_df
|
||||
else:
|
||||
dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
|
||||
elif ext == ".ods":
|
||||
xls = pd.ExcelFile(file_path, engine="odf")
|
||||
for sheet in xls.sheet_names:
|
||||
dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
|
||||
except Exception as e:
|
||||
print(f"[Warning] Could not read '{file_path.name}': {e}")
|
||||
|
||||
return dfs
|
||||
|
||||
|
||||
def process_and_write(dfs: dict, out_dir: Path, source_name: str):
|
||||
"""Write individual .sql files per table/sheet into the output directory."""
|
||||
for raw_name, df in dfs.items():
|
||||
if df.empty:
|
||||
continue
|
||||
|
||||
clean_table_name = sanitize_identifier(raw_name)
|
||||
|
||||
if len(dfs) > 1 and not clean_table_name.startswith("drm_") and not clean_table_name.startswith("study_"):
|
||||
filename = f"{sanitize_identifier(source_name)}_{clean_table_name}.sql"
|
||||
else:
|
||||
filename = f"{clean_table_name}.sql"
|
||||
|
||||
sql_content = dataframe_to_sql(df, clean_table_name)
|
||||
out_file = out_dir / filename
|
||||
|
||||
mode = "a" if out_file.exists() else "w"
|
||||
with open(out_file, mode, encoding="utf-8") as f:
|
||||
f.write(sql_content)
|
||||
|
||||
print(f"[seed_converter] Generated: {out_file}")
|
||||
|
||||
|
||||
def process_path(path: Path, out_dir: Path):
|
||||
"""Process files, directories, or zip archives recursively."""
|
||||
supported = {".csv", ".xlsx", ".xls", ".ods"}
|
||||
|
||||
if path.is_file() and path.suffix.lower() == ".zip":
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
with zipfile.ZipFile(path, "r") as zip_ref:
|
||||
zip_ref.extractall(tmp_dir)
|
||||
process_path(Path(tmp_dir), out_dir)
|
||||
|
||||
elif path.is_dir():
|
||||
for root, _, files in os.walk(path):
|
||||
for f in sorted(files):
|
||||
f_path = Path(root) / f
|
||||
if f_path.suffix.lower() in supported:
|
||||
dfs = load_dataframes_from_file(f_path)
|
||||
process_and_write(dfs, out_dir, f_path.stem)
|
||||
|
||||
elif path.is_file() and path.suffix.lower() in supported:
|
||||
dfs = load_dataframes_from_file(path)
|
||||
process_and_write(dfs, out_dir, path.stem)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.")
|
||||
parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)")
|
||||
parser.add_argument("--out-dir", default="seed", help="Directory where individual .sql files will be written")
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
expanded_paths = []
|
||||
for in_str in args.input:
|
||||
if any(c in in_str for c in ["*", "?", "["]):
|
||||
matched = glob.glob(in_str, recursive=True)
|
||||
if not matched:
|
||||
print(f"[Warning] No files matched wildcard pattern: '{in_str}'")
|
||||
for m in sorted(matched):
|
||||
expanded_paths.append(Path(m))
|
||||
else:
|
||||
expanded_paths.append(Path(in_str))
|
||||
|
||||
for in_path in expanded_paths:
|
||||
process_path(in_path, out_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
soleprint/station/tools/dataconvert/sqlgen.py
Normal file
81
soleprint/station/tools/dataconvert/sqlgen.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""
|
||||
SQL rendering: DataFrames into schema-agnostic INSERT statements.
|
||||
|
||||
With a row cap, only the first rows are rendered, and the size the full output
|
||||
would have had is estimated from them, so a sample still says how big the real
|
||||
thing is.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def sanitize_identifier(identifier: str) -> str:
|
||||
"""Sanitize names for SQL tables, columns, and filenames."""
|
||||
clean = re.sub(r"[^\w]", "_", str(identifier).strip().lower())
|
||||
clean = re.sub(r"_+", "_", clean)
|
||||
return clean.strip("_")
|
||||
|
||||
|
||||
def sql_value(v) -> str:
|
||||
if pd.isna(v):
|
||||
return "NULL"
|
||||
if isinstance(v, (bool, int)):
|
||||
return str(v)
|
||||
if isinstance(v, float):
|
||||
return str(int(v)) if v.is_integer() else str(v)
|
||||
escaped = str(v).replace("'", "''")
|
||||
return f"'{escaped}'"
|
||||
|
||||
|
||||
def render_table(df: pd.DataFrame, table_name: str, max_rows=None):
|
||||
"""
|
||||
Return (sql_text, total_rows, full_bytes, exact).
|
||||
|
||||
full_bytes is what the file would weigh with every row: measured when every
|
||||
row was rendered, extrapolated from the average rendered row otherwise.
|
||||
"""
|
||||
total = len(df)
|
||||
if total == 0:
|
||||
return "", 0, 0, True
|
||||
|
||||
table_ref = f'"{table_name}"'
|
||||
cols = ", ".join(f'"{sanitize_identifier(c)}"' for c in df.columns)
|
||||
shown = df if max_rows is None else df.head(max_rows)
|
||||
|
||||
rows = []
|
||||
# iterrows, not itertuples: it hands values over the way the original tool
|
||||
# did, and seed files people already load depend on exactly that quoting.
|
||||
for _, row in shown.iterrows():
|
||||
vals = ", ".join(sql_value(v) for v in row)
|
||||
rows.append(f"INSERT INTO {table_ref} ({cols}) VALUES ({vals}) ON CONFLICT DO NOTHING;\n")
|
||||
|
||||
head = f"-- Generated seed data for table: {table_ref}\n"
|
||||
begin, commit = "BEGIN;\n\n", "\nCOMMIT;\n"
|
||||
rows_bytes = sum(len(r.encode("utf-8")) for r in rows)
|
||||
fixed = len((head + begin + commit).encode("utf-8"))
|
||||
|
||||
exact = len(rows) == total
|
||||
if exact:
|
||||
full_bytes = fixed + rows_bytes
|
||||
else:
|
||||
full_bytes = fixed + round(rows_bytes / len(rows) * total)
|
||||
|
||||
note = ""
|
||||
if not exact:
|
||||
note = (
|
||||
f"-- SAMPLE: first {len(rows)} of {total} rows. The full file would be "
|
||||
f"~{human_bytes(full_bytes)}; run without --max-rows for all of it.\n"
|
||||
)
|
||||
|
||||
return head + note + begin + "".join(rows) + commit, total, full_bytes, exact
|
||||
|
||||
|
||||
def human_bytes(n: int) -> str:
|
||||
size = float(n)
|
||||
for unit in ("B", "K", "M", "G"):
|
||||
if size < 1000 or unit == "G":
|
||||
return f"{size:.0f}{unit}" if unit == "B" else f"{size:.1f}{unit}"
|
||||
size /= 1000
|
||||
return f"{n}B"
|
||||
2
soleprint/station/tools/dataconvert/uv.lock
generated
2
soleprint/station/tools/dataconvert/uv.lock
generated
@@ -285,7 +285,7 @@ wheels = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "seed-converter"
|
||||
name = "dataconvert"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
|
||||
Reference in New Issue
Block a user