From e05f8f1fca48936f1a2397c9d8229629581e14b0 Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Wed, 16 Sep 2026 08:57:43 -0300 Subject: [PATCH] dataconvert updates --- .../station/tools/dataconvert/.gitignore | 4 +- soleprint/station/tools/dataconvert/README.md | 38 +++++-- soleprint/station/tools/dataconvert/config.py | 102 +++++++++++++++--- .../dataconvert/dataconvert-example.json | 12 ++- .../station/tools/dataconvert/dataconvert.py | 74 +++++++++---- 5 files changed, 182 insertions(+), 48 deletions(-) diff --git a/soleprint/station/tools/dataconvert/.gitignore b/soleprint/station/tools/dataconvert/.gitignore index d9288fd..ee01e42 100644 --- a/soleprint/station/tools/dataconvert/.gitignore +++ b/soleprint/station/tools/dataconvert/.gitignore @@ -1,4 +1,4 @@ # Local layouts: how one producer's spreadsheets are shaped is not a fact about -# the tool, and the marker values name whoever that is. Copy -# dataconvert-example.json to dataconvert.json and edit that; it stays here. +# the tool, and the marker values name whoever that is. The config lives in the +# data folder, not here; this only stops a stray copy being committed. dataconvert.json diff --git a/soleprint/station/tools/dataconvert/README.md b/soleprint/station/tools/dataconvert/README.md index 6fbe624..424d677 100644 --- a/soleprint/station/tools/dataconvert/README.md +++ b/soleprint/station/tools/dataconvert/README.md @@ -17,16 +17,22 @@ ZIP archives and wildcard patterns. Sheets of a multi-sheet workbook are written sources feeding the same table accumulate in one file, and so does re-running into the same `--out-dir`, so point a fresh run at an empty directory. -## Layouts: config, not code +## Config: dataconvert.json in the source folder -By default row 1 holds the column names and the data starts on row 2. Exports that -put a title above the header, or description rows between it and the data, are -described in `dataconvert.json` beside the script (or `--config FILE`). It is -gitignored because the marker values name whoever produced the files; start from -`dataconvert-example.json`. +Everything about one set of spreadsheets, and how you usually convert them, goes in a +`dataconvert.json` kept **in the source folder**, next to the files it describes: the +input itself when it is a directory, or the folder a file, wildcard match or ZIP sits +in. Each input looks in its own folder, and the run prints every config it uses and +what it set. `--config FILE` uses one file for all inputs instead. It is never read +from beside the script, so the repo never holds one: the marker values name whoever +produced the files. Start from `dataconvert-example.json`. ```json { + "out_dir": "sample", + "max_rows": 20, + "schema": true, + "header_row": 1, "layouts": [ { "name": "catalogue", "match": { "row": 2, "column": 1, "in": ["CODE", "ITEM_CODE"] }, @@ -36,16 +42,28 @@ gitignored because the marker values name whoever produced the files; start from } ``` +Every key is optional. + +- **out_dir, max_rows, schema** are the run settings: the same as `--out-dir`, + `--max-rows` and `--no-schema`, and a flag on the command line wins over the file. + `out_dir` is relative to the folder the config is in, so `"sample"` beside the data + means `/sample`. Inputs whose configs name different `out_dir`s each get + their own `SCHEMA.md`. +- **header_row, data_row** at the top level say where the column names and data are + for sheets that no layout matches. By default row 1 holds the names and the data + starts on row 2. - **layouts** are checked against every sheet and CSV, first match wins. A layout matches when the cell at `match.row`/`match.column` (counted from 1, trimmed) is one of `match.in`; then the names come from `header_row` and the data from `data_row` - on. The run prints which layout each sheet got. A sheet nothing matches is read - normally. + on, skipping anything between. The run prints which layout each sheet got. - **bare_sheet_prefixes**: sheets whose name starts with one of these are written as `.sql` instead of `_.sql`. +Keys starting with `_` are comments; any other unknown key stops the run, so a typo +such as `max_row` is not silently ignored. + For a one-off, `--header-row 2 --data-row 6` applies one layout to every file in the -run and skips detection. +run and skips the config's layouts. ## Sampling for a web LLM @@ -67,7 +85,7 @@ what pandas read: a starting point, not DDL. `--no-schema` skips the file. | file | does | |---|---| | `dataconvert.py` | command line | -| `config.py` | `dataconvert.json`: layouts and sheet naming | +| `config.py` | `dataconvert.json`: layouts, sheet naming and run settings | | `readers.py` | files, directories, ZIPs and globs into DataFrames | | `sqlgen.py` | DataFrames into INSERT statements, with the row cap | | `output.py` | file naming and writing | diff --git a/soleprint/station/tools/dataconvert/config.py b/soleprint/station/tools/dataconvert/config.py index f31ba09..3c5b0e3 100644 --- a/soleprint/station/tools/dataconvert/config.py +++ b/soleprint/station/tools/dataconvert/config.py @@ -4,11 +4,18 @@ Config: what a particular set of spreadsheets looks like. The tool knows nothing about any one exporter. Where a header sits, which sheets are recognised by a marker cell, and which sheet names are already unique enough to keep as they are, are facts about whoever produced the files, -so they live in dataconvert.json beside this script. That file is gitignored: -copy dataconvert-example.json and edit it. Without one, every file is read with -its header on row 1. +so they live with those files: a dataconvert.json in the source folder, or a +file named with --config. The source folder is the input itself when it is a +directory, and the directory a file, wildcard match or ZIP sits in. Never beside +this script, so the repo never holds one and the data carries its own shape. +Start from dataconvert-example.json. With neither, every file is read with its +header on row 1. { + "out_dir": "sample", + "max_rows": 20, + "schema": true, + "header_row": 1, "layouts": [ { "name": "catalogue", @@ -22,14 +29,18 @@ its header on row 1. Rows and columns are counted as the spreadsheet shows them, from 1. A layout applies to a sheet (or CSV) when the cell at match.row/match.column, trimmed, -is one of match.in. The first layout that matches wins. +is one of match.in. The first layout that matches wins; a sheet none matches +uses the top-level header_row/data_row, which default to 1 and 2. + +The run settings (out_dir, max_rows, schema) are the command line's flags with +the same names, and a flag given on the command line wins over the file. out_dir +is relative to the folder the config file is in. Keys starting with _ are +comments; any other unknown key is an error, so a typo is not silently ignored. """ import json from pathlib import Path -DEFAULT_PATH = Path(__file__).resolve().parent / "dataconvert.json" - class ConfigError(Exception): pass @@ -74,17 +85,24 @@ class Rule: class Config: - def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None): + def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None, + fallback=None, max_rows=None, schema=None, out_dir=None): self.rules = list(rules) self.bare_sheet_prefixes = tuple(bare_sheet_prefixes) self.source = source # --header-row / --data-row on the command line: one layout for every # file, no detection. self.forced = forced + # The file's own header_row/data_row: for sheets no layout matches. + self.fallback = fallback if fallback is not None and not fallback.is_default else None + # Run settings; None means the file did not say. + self.max_rows = max_rows + self.schema = schema + self.out_dir = out_dir @property def detects(self): - return self.forced is None and bool(self.rules) + return self.forced is None and (bool(self.rules) or self.fallback is not None) def layout_for(self, raw_df): """The layout for a sheet read without a header, or None for the default read.""" @@ -93,25 +111,77 @@ class Config: for rule in self.rules: if rule.matches(raw_df): return rule.layout - return None + return self.fallback + + def settings(self): + """What the file set, for the line the run prints.""" + said = [] + if self.out_dir is not None: + said.append(f"out_dir {self.out_dir}") + if self.max_rows is not None: + said.append(f"max_rows {self.max_rows}") + if self.schema is not None: + said.append(f"schema {'on' if self.schema else 'off'}") + if self.fallback is not None: + said.append(f"header row {self.fallback.header_row}, data from row {self.fallback.data_row}") + if self.rules: + said.append(f"{len(self.rules)} layout{'s' if len(self.rules) != 1 else ''}") + return ", ".join(said) + + +FILENAME = "dataconvert.json" +KNOWN_KEYS = {"layouts", "bare_sheet_prefixes", "max_rows", "schema", "out_dir", "header_row", "data_row"} + + +def source_config_path(in_path: Path): + """The dataconvert.json that belongs to an input, if its folder has one.""" + folder = in_path if in_path.is_dir() else in_path.parent + candidate = folder / FILENAME + return candidate if candidate.is_file() else None def load(path=None, forced=None): - """Read the given config, else dataconvert.json beside this script if there is one.""" - explicit = path is not None - path = Path(path) if explicit else DEFAULT_PATH - if not path.exists(): - if explicit: - raise ConfigError(f"no such config file: {path}") + """Read a config file; with none, no layouts at all.""" + if path is None: return Config(forced=forced) + path = Path(path) + if not path.is_file(): + raise ConfigError(f"no such config file: {path}") try: raw = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError as e: raise ConfigError(f"{path} is not valid JSON: {e}") + if not isinstance(raw, dict): + raise ConfigError(f"{path}: expected a JSON object") + unknown = sorted(k for k in raw if not k.startswith("_") and k not in KNOWN_KEYS) + if unknown: + raise ConfigError(f"{path}: unknown setting {', '.join(unknown)} (known: {', '.join(sorted(KNOWN_KEYS))})") + rules = [Rule(r, i) for i, r in enumerate(raw.get("layouts", []))] prefixes = raw.get("bare_sheet_prefixes", []) if not isinstance(prefixes, list): raise ConfigError(f"{path}: bare_sheet_prefixes must be a list") - return Config(rules, [str(p) for p in prefixes], source=path, forced=forced) + + max_rows = raw.get("max_rows") + if max_rows is not None and (isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1): + raise ConfigError(f"{path}: max_rows must be a whole number, 1 or more, or null") + schema = raw.get("schema") + if schema is not None and not isinstance(schema, bool): + raise ConfigError(f"{path}: schema must be true or false") + out_dir = raw.get("out_dir") + if out_dir is not None: + if not isinstance(out_dir, str) or not out_dir.strip(): + raise ConfigError(f"{path}: out_dir must be a path") + out_dir = path.parent / Path(out_dir).expanduser() + + fallback = None + if "header_row" in raw or "data_row" in raw: + try: + fallback = Layout(raw.get("header_row", 1), raw.get("data_row"), "top level") + except (TypeError, ValueError): + raise ConfigError(f"{path}: header_row and data_row must be row numbers") + + return Config(rules, [str(p) for p in prefixes], source=path, forced=forced, + fallback=fallback, max_rows=max_rows, schema=schema, out_dir=out_dir) diff --git a/soleprint/station/tools/dataconvert/dataconvert-example.json b/soleprint/station/tools/dataconvert/dataconvert-example.json index 483be33..b8e30d5 100644 --- a/soleprint/station/tools/dataconvert/dataconvert-example.json +++ b/soleprint/station/tools/dataconvert/dataconvert-example.json @@ -1,7 +1,15 @@ { - "_comment": "Template for dataconvert.json, which dataconvert.py reads from beside itself (or --config FILE). Copy this to dataconvert.json (gitignored) and describe the files you actually convert there. Rows and columns count from 1, as the spreadsheet shows them.", + "_comment": "Template for one set of spreadsheets. Copy it to dataconvert.json in the folder that holds them, where it is picked up automatically, or pass any file with --config FILE. Every key is optional; keys starting with _ are comments, any other unknown key is an error. Rows and columns count from 1, as the spreadsheet shows them.", - "_layouts": "Checked in order against every sheet and CSV; the first match wins, and a sheet nothing matches is read with its header on row 1. A layout matches when the cell at match.row / match.column, trimmed, is one of match.in. header_row holds the column names; data starts at data_row, and anything in between is skipped.", + "_run": "The same settings as the command-line flags, and a flag given there wins. out_dir is relative to this file's folder. max_rows samples each table (null for all rows). schema false skips SCHEMA.md.", + "out_dir": "sample", + "max_rows": 20, + "schema": true, + + "_rows": "Where the column names and the data are, for sheets no layout below matches. Defaults: 1 and the row after it.", + "header_row": 1, + + "_layouts": "Checked in order against every sheet and CSV; the first match wins. A layout matches when the cell at match.row / match.column, trimmed, is one of match.in. header_row holds the column names; data starts at data_row, and anything in between is skipped.", "layouts": [ { "name": "catalogue", diff --git a/soleprint/station/tools/dataconvert/dataconvert.py b/soleprint/station/tools/dataconvert/dataconvert.py index 42d5104..d76aceb 100644 --- a/soleprint/station/tools/dataconvert/dataconvert.py +++ b/soleprint/station/tools/dataconvert/dataconvert.py @@ -12,7 +12,8 @@ Usage: python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6 python3 dataconvert.py --input data/ --config their-exports.json -How a given producer lays out its sheets is not built in: see config.py and +How a given producer lays out its sheets is not built in: it is read from a +dataconvert.json in the source folder, or --config. See config.py and dataconvert-example.json. --max-rows is for understanding the data rather than loading it: every table @@ -42,38 +43,75 @@ def positive_int(value): def main(): parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.") parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)") - parser.add_argument("--out-dir", default="seed", help="Directory where individual .sql files will be written") + parser.add_argument("--out-dir", default=None, + help="Directory where individual .sql files will be written (default: the config's out_dir, else ./seed)") parser.add_argument("--max-rows", type=positive_int, default=None, - help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size") - parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md") + help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size (default: the config's max_rows, else all)") + parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md, whatever the config says") parser.add_argument("--header-row", type=positive_int, default=1, - help="Spreadsheet row holding the column names, for every file; overrides the config layouts (default 1)") + help="Spreadsheet row holding the column names, for every file; overrides the config's layouts (default 1)") parser.add_argument("--data-row", type=positive_int, default=None, help="First row of data, when rows sit between it and the header (default: the row after the header)") parser.add_argument("--config", default=None, - help="Layouts and naming for these files (default: dataconvert.json beside this script, if present)") + help="JSON with layouts, naming and run settings, for every input (default: dataconvert.json in each input's folder, if there is one)") args = parser.parse_args() + forced = None try: - forced = None if args.header_row != 1 or args.data_row is not None: forced = cfg.Layout(args.header_row, args.data_row, "command line") - config = cfg.load(args.config, forced) + given = cfg.load(args.config, forced) if args.config else None except cfg.ConfigError as e: parser.error(str(e)) - if config.source is not None: - print(f"[dataconvert] config: {config.source}") - out_dir = Path(args.out_dir) - out_dir.mkdir(parents=True, exist_ok=True) + # Every config is read and checked before anything is written, so a broken + # one in the third folder does not leave the first two converted. + inputs = expand_inputs(args.input) + loaded = {} + plan = [] + for in_path in inputs: + if given is not None: + plan.append((in_path, given)) + continue + found = cfg.source_config_path(in_path) + if found is not None and found not in loaded: + try: + loaded[found] = cfg.load(found, forced) + except cfg.ConfigError as e: + parser.error(str(e)) + plan.append((in_path, loaded[found] if found is not None else cfg.Config(forced=forced))) + + # One report per output directory: inputs whose configs name different + # out_dirs each get their own SCHEMA.md, beside their own .sql files. + reports = {} + announced = set() + for in_path, config in plan: + if config.source is not None and config.source not in announced: + announced.add(config.source) + said = config.settings() + print(f"[dataconvert] config: {config.source}" + (f" ({said})" if said else "")) + + # The command line wins, then the config, then the built-in default. + out_dir = Path(args.out_dir) if args.out_dir else (config.out_dir or Path("seed")) + max_rows = args.max_rows if args.max_rows is not None else config.max_rows + schema = not args.no_schema and config.schema is not False + + out_dir.mkdir(parents=True, exist_ok=True) + key = out_dir.resolve() + if key not in reports: + reports[key] = (out_dir, SchemaReport(), set()) + _, report, caps = reports[key] + caps.add(max_rows) - report = None if args.no_schema else SchemaReport() - for in_path in expand_inputs(args.input): for source_name, dfs in iter_sources(in_path, config): - write_tables(dfs, out_dir, source_name, args.max_rows, report, config.bare_sheet_prefixes) - - if report is not None and report.entries: - print(f"[dataconvert] Generated: {report.write(out_dir, args.max_rows)}") + write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes) + + for out_dir, report, caps in reports.values(): + if report.entries: + # The sampling note names the cap only when every input here used + # the same one; otherwise each table's row still says what it kept. + cap = next(iter(caps)) if len(caps) == 1 else None + print(f"[dataconvert] Generated: {report.write(out_dir, cap)}") if __name__ == "__main__":