diff --git a/soleprint/station/tools/dataconvert/.gitignore b/soleprint/station/tools/dataconvert/.gitignore new file mode 100644 index 0000000..d9288fd --- /dev/null +++ b/soleprint/station/tools/dataconvert/.gitignore @@ -0,0 +1,4 @@ +# Local layouts: how one producer's spreadsheets are shaped is not a fact about +# the tool, and the marker values name whoever that is. Copy +# dataconvert-example.json to dataconvert.json and edit that; it stays here. +dataconvert.json diff --git a/soleprint/station/tools/dataconvert/README.md b/soleprint/station/tools/dataconvert/README.md index 3589424..6fbe624 100644 --- a/soleprint/station/tools/dataconvert/README.md +++ b/soleprint/station/tools/dataconvert/README.md @@ -7,14 +7,46 @@ plus a `SCHEMA.md` describing every table. uv run dataconvert.py --input data/ --out-dir seed/ # full seeds uv run dataconvert.py --input data/ "*.xlsx" --out-dir seed/ # dirs, files, globs, .zip uv run dataconvert.py --input data/ --out-dir sample/ --max-rows 20 # to understand the data +uv run dataconvert.py --input export.xlsx --header-row 2 --data-row 6 +uv run dataconvert.py --input data/ --config their-exports.json ``` Reads `.csv`, `.xlsx`, `.xls` and `.ods`: single files, directories (recursively), ZIP archives and wildcard patterns. Sheets of a multi-sheet workbook are written as -`_.sql` unless the sheet is already `drm_` or `study_`. Several +`_.sql` (see `bare_sheet_prefixes` below). Several sources feeding the same table accumulate in one file, and so does re-running into the same `--out-dir`, so point a fresh run at an empty directory. +## Layouts: config, not code + +By default row 1 holds the column names and the data starts on row 2. Exports that +put a title above the header, or description rows between it and the data, are +described in `dataconvert.json` beside the script (or `--config FILE`). It is +gitignored because the marker values name whoever produced the files; start from +`dataconvert-example.json`. + +```json +{ + "layouts": [ + { "name": "catalogue", + "match": { "row": 2, "column": 1, "in": ["CODE", "ITEM_CODE"] }, + "header_row": 2, "data_row": 6 } + ], + "bare_sheet_prefixes": ["ref_"] +} +``` + +- **layouts** are checked against every sheet and CSV, first match wins. A layout + matches when the cell at `match.row`/`match.column` (counted from 1, trimmed) is one + of `match.in`; then the names come from `header_row` and the data from `data_row` + on. The run prints which layout each sheet got. A sheet nothing matches is read + normally. +- **bare_sheet_prefixes**: sheets whose name starts with one of these are written as + `.sql` instead of `_.sql`. + +For a one-off, `--header-row 2 --data-row 6` applies one layout to every file in the +run and skips detection. + ## Sampling for a web LLM Full seed files get large fast, and a model only needs to see the shape of the data. @@ -35,6 +67,7 @@ what pandas read: a starting point, not DDL. `--no-schema` skips the file. | file | does | |---|---| | `dataconvert.py` | command line | +| `config.py` | `dataconvert.json`: layouts and sheet naming | | `readers.py` | files, directories, ZIPs and globs into DataFrames | | `sqlgen.py` | DataFrames into INSERT statements, with the row cap | | `output.py` | file naming and writing | diff --git a/soleprint/station/tools/dataconvert/config.py b/soleprint/station/tools/dataconvert/config.py new file mode 100644 index 0000000..f31ba09 --- /dev/null +++ b/soleprint/station/tools/dataconvert/config.py @@ -0,0 +1,117 @@ +""" +Config: what a particular set of spreadsheets looks like. + +The tool knows nothing about any one exporter. Where a header sits, which +sheets are recognised by a marker cell, and which sheet names are already +unique enough to keep as they are, are facts about whoever produced the files, +so they live in dataconvert.json beside this script. That file is gitignored: +copy dataconvert-example.json and edit it. Without one, every file is read with +its header on row 1. + + { + "layouts": [ + { + "name": "catalogue", + "match": {"row": 2, "column": 1, "in": ["CODE", "ITEM"]}, + "header_row": 2, + "data_row": 6 + } + ], + "bare_sheet_prefixes": ["ref_"] + } + +Rows and columns are counted as the spreadsheet shows them, from 1. A layout +applies to a sheet (or CSV) when the cell at match.row/match.column, trimmed, +is one of match.in. The first layout that matches wins. +""" + +import json +from pathlib import Path + +DEFAULT_PATH = Path(__file__).resolve().parent / "dataconvert.json" + + +class ConfigError(Exception): + pass + + +class Layout: + """Where the column names are and where the data starts, from row 1.""" + + def __init__(self, header_row=1, data_row=None, name="default"): + self.name = name + self.header_row = int(header_row) + self.data_row = int(data_row) if data_row is not None else self.header_row + 1 + if self.header_row < 1: + raise ConfigError(f"layout '{name}': header_row must be 1 or more") + if self.data_row <= self.header_row: + raise ConfigError(f"layout '{name}': the data has to start below the header row") + + @property + def is_default(self): + return self.header_row == 1 and self.data_row == 2 + + +class Rule: + def __init__(self, raw, index): + name = raw.get("name") or f"layout {index + 1}" + match = raw.get("match") or {} + values = match.get("in") + if not isinstance(values, list) or not values: + raise ConfigError(f"layout '{name}': match.in must be a non-empty list") + self.row = int(match.get("row", 1)) + self.column = int(match.get("column", 1)) + if self.row < 1 or self.column < 1: + raise ConfigError(f"layout '{name}': match.row and match.column count from 1") + self.values = {str(v).strip() for v in values} + self.layout = Layout(raw.get("header_row", 1), raw.get("data_row"), name) + + def matches(self, raw_df): + r, c = self.row - 1, self.column - 1 + if len(raw_df) <= r or raw_df.shape[1] <= c: + return False + return str(raw_df.iloc[r, c]).strip() in self.values + + +class Config: + def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None): + self.rules = list(rules) + self.bare_sheet_prefixes = tuple(bare_sheet_prefixes) + self.source = source + # --header-row / --data-row on the command line: one layout for every + # file, no detection. + self.forced = forced + + @property + def detects(self): + return self.forced is None and bool(self.rules) + + def layout_for(self, raw_df): + """The layout for a sheet read without a header, or None for the default read.""" + if self.forced is not None: + return None if self.forced.is_default else self.forced + for rule in self.rules: + if rule.matches(raw_df): + return rule.layout + return None + + +def load(path=None, forced=None): + """Read the given config, else dataconvert.json beside this script if there is one.""" + explicit = path is not None + path = Path(path) if explicit else DEFAULT_PATH + if not path.exists(): + if explicit: + raise ConfigError(f"no such config file: {path}") + return Config(forced=forced) + + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + raise ConfigError(f"{path} is not valid JSON: {e}") + + rules = [Rule(r, i) for i, r in enumerate(raw.get("layouts", []))] + prefixes = raw.get("bare_sheet_prefixes", []) + if not isinstance(prefixes, list): + raise ConfigError(f"{path}: bare_sheet_prefixes must be a list") + return Config(rules, [str(p) for p in prefixes], source=path, forced=forced) diff --git a/soleprint/station/tools/dataconvert/dataconvert-example.json b/soleprint/station/tools/dataconvert/dataconvert-example.json new file mode 100644 index 0000000..483be33 --- /dev/null +++ b/soleprint/station/tools/dataconvert/dataconvert-example.json @@ -0,0 +1,16 @@ +{ + "_comment": "Template for dataconvert.json, which dataconvert.py reads from beside itself (or --config FILE). Copy this to dataconvert.json (gitignored) and describe the files you actually convert there. Rows and columns count from 1, as the spreadsheet shows them.", + + "_layouts": "Checked in order against every sheet and CSV; the first match wins, and a sheet nothing matches is read with its header on row 1. A layout matches when the cell at match.row / match.column, trimmed, is one of match.in. header_row holds the column names; data starts at data_row, and anything in between is skipped.", + "layouts": [ + { + "name": "catalogue", + "match": { "row": 2, "column": 1, "in": ["CODE", "ITEM_CODE"] }, + "header_row": 2, + "data_row": 6 + } + ], + + "_bare_sheet_prefixes": "Sheets of a multi-sheet workbook are written as _.sql. A sheet whose name starts with one of these is written as .sql instead, because its name is already unique.", + "bare_sheet_prefixes": ["ref_"] +} diff --git a/soleprint/station/tools/dataconvert/dataconvert.py b/soleprint/station/tools/dataconvert/dataconvert.py index 6f74e3d..42d5104 100644 --- a/soleprint/station/tools/dataconvert/dataconvert.py +++ b/soleprint/station/tools/dataconvert/dataconvert.py @@ -9,6 +9,11 @@ Usage: python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/ python3 dataconvert.py --input path/to/folder/ --out-dir seed/ python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20 + python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6 + python3 dataconvert.py --input data/ --config their-exports.json + +How a given producer lays out its sheets is not built in: see config.py and +dataconvert-example.json. --max-rows is for understanding the data rather than loading it: every table still gets its file and its SCHEMA.md entry, with only the first N rows, and a @@ -22,6 +27,7 @@ import argparse from pathlib import Path from output import write_tables +import config as cfg from readers import expand_inputs, iter_sources from schema import SchemaReport @@ -40,15 +46,31 @@ def main(): parser.add_argument("--max-rows", type=positive_int, default=None, help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size") parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md") + parser.add_argument("--header-row", type=positive_int, default=1, + help="Spreadsheet row holding the column names, for every file; overrides the config layouts (default 1)") + parser.add_argument("--data-row", type=positive_int, default=None, + help="First row of data, when rows sit between it and the header (default: the row after the header)") + parser.add_argument("--config", default=None, + help="Layouts and naming for these files (default: dataconvert.json beside this script, if present)") args = parser.parse_args() + try: + forced = None + if args.header_row != 1 or args.data_row is not None: + forced = cfg.Layout(args.header_row, args.data_row, "command line") + config = cfg.load(args.config, forced) + except cfg.ConfigError as e: + parser.error(str(e)) + if config.source is not None: + print(f"[dataconvert] config: {config.source}") + out_dir = Path(args.out_dir) out_dir.mkdir(parents=True, exist_ok=True) report = None if args.no_schema else SchemaReport() for in_path in expand_inputs(args.input): - for source_name, dfs in iter_sources(in_path): - write_tables(dfs, out_dir, source_name, args.max_rows, report) + for source_name, dfs in iter_sources(in_path, config): + write_tables(dfs, out_dir, source_name, args.max_rows, report, config.bare_sheet_prefixes) if report is not None and report.entries: print(f"[dataconvert] Generated: {report.write(out_dir, args.max_rows)}") diff --git a/soleprint/station/tools/dataconvert/output.py b/soleprint/station/tools/dataconvert/output.py index 0dea6b7..726268b 100644 --- a/soleprint/station/tools/dataconvert/output.py +++ b/soleprint/station/tools/dataconvert/output.py @@ -7,22 +7,26 @@ from pathlib import Path from sqlgen import render_table, sanitize_identifier -def table_filename(raw_name: str, source_name: str, sheet_count: int) -> str: - """Sheets of a multi-sheet workbook are prefixed with the workbook, unless already namespaced.""" +def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=()) -> str: + """ + Sheets of a multi-sheet workbook are prefixed with the workbook, so two + workbooks cannot collide, unless the config names the sheet as already + unique (bare_sheet_prefixes). + """ clean = sanitize_identifier(raw_name) - if sheet_count > 1 and not clean.startswith("drm_") and not clean.startswith("study_"): + if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)): return f"{sanitize_identifier(source_name)}_{clean}.sql" return f"{clean}.sql" -def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None): +def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None, bare_prefixes=()): """Write individual .sql files per table/sheet into the output directory.""" for raw_name, df in dfs.items(): if df.empty: continue table = sanitize_identifier(raw_name) - filename = table_filename(raw_name, source_name, len(dfs)) + filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes) sql, total, full_bytes, exact = render_table(df, table, max_rows) out_file = out_dir / filename diff --git a/soleprint/station/tools/dataconvert/readers.py b/soleprint/station/tools/dataconvert/readers.py index 3a94bf2..df363ba 100644 --- a/soleprint/station/tools/dataconvert/readers.py +++ b/soleprint/station/tools/dataconvert/readers.py @@ -15,13 +15,6 @@ import pandas as pd SUPPORTED = {".csv", ".xlsx", ".xls", ".ods"} -# DRM catalogue sheets put the real column names on row 1 and start the data -# on row 5; the rows in between are descriptions. Recognised by the first cell -# of the header row. -DRM_HEADER_MARKERS = { - "CONTROLLED_TERMINOLOGY", "DATA_ELEMENT", "DATA_DOMAIN", "DATA_STATE", "DATA_CATEGORY", -} - def expand_inputs(inputs): """Wildcard patterns become the paths they match; everything else passes through.""" @@ -37,49 +30,66 @@ def expand_inputs(inputs): return paths -def load_dataframes_from_file(file_path: Path) -> dict: +def read_sheet(read, config): + """ + One sheet or CSV, laid out as the config says. + + With nothing to detect, a plain read, exactly as before. Otherwise the sheet + is read once without a header, checked against the layouts, and when one + matches the header and data rows are sliced out of that same read. + """ + if not config.detects and config.forced is None: + return read(), None + raw = read(header=None) + layout = config.layout_for(raw) + if layout is None: + return read(), None + data = raw.iloc[layout.data_row - 1:].copy() + data.columns = [str(c).strip() for c in raw.iloc[layout.header_row - 1]] + return data, layout + + +def load_dataframes_from_file(file_path: Path, config) -> dict: """Load a file (.csv, .xlsx, .xls, .ods) into {entity_name: DataFrame}.""" ext = file_path.suffix.lower() dfs = {} try: if ext == ".csv": - dfs[file_path.stem] = pd.read_csv(file_path) - elif ext in [".xlsx", ".xls"]: - xls = pd.ExcelFile(file_path) + dfs[file_path.stem], layout = read_sheet(lambda **kw: pd.read_csv(file_path, **kw), config) + note_layout(file_path.name, None, layout) + elif ext in [".xlsx", ".xls", ".ods"]: + xls = pd.ExcelFile(file_path, engine="odf") if ext == ".ods" else pd.ExcelFile(file_path) for sheet in xls.sheet_names: - df_raw = pd.read_excel(xls, sheet_name=sheet, header=None) - if len(df_raw) > 1 and str(df_raw.iloc[1, 0]).strip() in DRM_HEADER_MARKERS: - cols = [str(c).strip() for c in df_raw.iloc[1]] - data_df = df_raw.iloc[5:].copy() - data_df.columns = cols - dfs[sheet] = data_df - else: - dfs[sheet] = pd.read_excel(xls, sheet_name=sheet) - elif ext == ".ods": - xls = pd.ExcelFile(file_path, engine="odf") - for sheet in xls.sheet_names: - dfs[sheet] = pd.read_excel(xls, sheet_name=sheet) + dfs[sheet], layout = read_sheet( + lambda sheet=sheet, **kw: pd.read_excel(xls, sheet_name=sheet, **kw), config) + note_layout(file_path.name, sheet, layout) except Exception as e: print(f"[Warning] Could not read '{file_path.name}': {e}") return dfs -def iter_sources(path: Path): +def note_layout(file_name, sheet, layout): + if layout is not None: + where = f"{file_name} [{sheet}]" if sheet else file_name + print(f"[dataconvert] {where}: layout '{layout.name}', header row {layout.header_row}, data from row {layout.data_row}") + + +def iter_sources(path: Path, config): """Yield (source_name, dfs) for a file, a directory (recursively) or a ZIP archive.""" if path.is_file() and path.suffix.lower() == ".zip": with tempfile.TemporaryDirectory() as tmp_dir: with zipfile.ZipFile(path, "r") as zip_ref: zip_ref.extractall(tmp_dir) - yield from iter_sources(Path(tmp_dir)) + yield from iter_sources(Path(tmp_dir), config) elif path.is_dir(): for root, _, files in os.walk(path): for f in sorted(files): f_path = Path(root) / f if f_path.suffix.lower() in SUPPORTED: - yield f_path.stem, load_dataframes_from_file(f_path) + yield f_path.stem, load_dataframes_from_file(f_path, config) elif path.is_file() and path.suffix.lower() in SUPPORTED: - yield path.stem, load_dataframes_from_file(path) + yield path.stem, load_dataframes_from_file(path, config)