From 74e246e67a07f4b24540849eb6e105769ced07eb Mon Sep 17 00:00:00 2001 From: buenosairesam Date: Wed, 16 Sep 2026 09:07:07 -0300 Subject: [PATCH] dataconvert updates --- soleprint/station/tools/dataconvert/README.md | 2 ++ .../station/tools/dataconvert/dataconvert.py | 20 ++++++++++++++----- soleprint/station/tools/dataconvert/output.py | 3 +++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/soleprint/station/tools/dataconvert/README.md b/soleprint/station/tools/dataconvert/README.md index 424d677..9cbea8a 100644 --- a/soleprint/station/tools/dataconvert/README.md +++ b/soleprint/station/tools/dataconvert/README.md @@ -46,6 +46,8 @@ Every key is optional. - **out_dir, max_rows, schema** are the run settings: the same as `--out-dir`, `--max-rows` and `--no-schema`, and a flag on the command line wins over the file. + There is no default output directory: without `--out-dir` or an `out_dir` in the + config, the run stops before writing anything. `out_dir` is relative to the folder the config is in, so `"sample"` beside the data means `/sample`. Inputs whose configs name different `out_dir`s each get their own `SCHEMA.md`. diff --git a/soleprint/station/tools/dataconvert/dataconvert.py b/soleprint/station/tools/dataconvert/dataconvert.py index d76aceb..4ff2f05 100644 --- a/soleprint/station/tools/dataconvert/dataconvert.py +++ b/soleprint/station/tools/dataconvert/dataconvert.py @@ -44,7 +44,7 @@ def main(): parser = argparse.ArgumentParser(description="Convert data sources into schema-agnostic SQL seed files.") parser.add_argument("--input", nargs="+", required=True, help="Input file(s), directory, wildcard pattern(s), or ZIP archive(s)") parser.add_argument("--out-dir", default=None, - help="Directory where individual .sql files will be written (default: the config's out_dir, else ./seed)") + help="Directory where individual .sql files will be written (default: the config's out_dir; one of the two is required)") parser.add_argument("--max-rows", type=positive_int, default=None, help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size (default: the config's max_rows, else all)") parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md, whatever the config says") @@ -81,6 +81,17 @@ def main(): parser.error(str(e)) plan.append((in_path, loaded[found] if found is not None else cfg.Config(forced=forced))) + # No silent default directory. A run whose config was not found would + # otherwise write, unlaid-out, into ./seed wherever it was started, and look + # like it worked. Checked for every input before anything is written. + if not args.out_dir: + for in_path, config in plan: + if config.out_dir is not None: + continue + where = (f"{config.source} sets no out_dir" if config.source is not None + else f"no {cfg.FILENAME} found in its folder") + parser.error(f"no output directory for {in_path} ({where}): pass --out-dir, or set out_dir in the config") + # One report per output directory: inputs whose configs name different # out_dirs each get their own SCHEMA.md, beside their own .sql files. reports = {} @@ -91,12 +102,11 @@ def main(): said = config.settings() print(f"[dataconvert] config: {config.source}" + (f" ({said})" if said else "")) - # The command line wins, then the config, then the built-in default. - out_dir = Path(args.out_dir) if args.out_dir else (config.out_dir or Path("seed")) + # The command line wins, then the config. + out_dir = Path(args.out_dir) if args.out_dir else config.out_dir max_rows = args.max_rows if args.max_rows is not None else config.max_rows schema = not args.no_schema and config.schema is not False - out_dir.mkdir(parents=True, exist_ok=True) key = out_dir.resolve() if key not in reports: reports[key] = (out_dir, SchemaReport(), set()) @@ -107,7 +117,7 @@ def main(): write_tables(dfs, out_dir, source_name, max_rows, report if schema else None, config.bare_sheet_prefixes) for out_dir, report, caps in reports.values(): - if report.entries: + if report.entries and out_dir.is_dir(): # The sampling note names the cap only when every input here used # the same one; otherwise each table's row still says what it kept. cap = next(iter(caps)) if len(caps) == 1 else None diff --git a/soleprint/station/tools/dataconvert/output.py b/soleprint/station/tools/dataconvert/output.py index 726268b..4af5c05 100644 --- a/soleprint/station/tools/dataconvert/output.py +++ b/soleprint/station/tools/dataconvert/output.py @@ -29,6 +29,9 @@ def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, repo filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes) sql, total, full_bytes, exact = render_table(df, table, max_rows) out_file = out_dir / filename + # Created on the first file, so a run that finds nothing to convert + # leaves no empty directory behind. + out_dir.mkdir(parents=True, exist_ok=True) # Several sources can feed the same table; they accumulate in one file. mode = "a" if out_file.exists() else "w"