dataconvert updates

soleprint/station/tools/dataconvert/.gitignore

@@ -0,0 +1,4 @@
+# Local layouts: how one producer's spreadsheets are shaped is not a fact about
+# the tool, and the marker values name whoever that is. Copy
+# dataconvert-example.json to dataconvert.json and edit that; it stays here.
+dataconvert.json

soleprint/station/tools/dataconvert/README.md

@@ -7,14 +7,46 @@ plus a `SCHEMA.md` describing every table.
 uv run dataconvert.py --input data/ --out-dir seed/                 # full seeds
 uv run dataconvert.py --input data/ "*.xlsx" --out-dir seed/        # dirs, files, globs, .zip
 uv run dataconvert.py --input data/ --out-dir sample/ --max-rows 20 # to understand the data
+uv run dataconvert.py --input export.xlsx --header-row 2 --data-row 6
+uv run dataconvert.py --input data/ --config their-exports.json
 ```
 
 Reads `.csv`, `.xlsx`, `.xls` and `.ods`: single files, directories (recursively),
 ZIP archives and wildcard patterns. Sheets of a multi-sheet workbook are written as
-`<workbook>_<sheet>.sql` unless the sheet is already `drm_` or `study_`. Several
+`<workbook>_<sheet>.sql` (see `bare_sheet_prefixes` below). Several
 sources feeding the same table accumulate in one file, and so does re-running into
 the same `--out-dir`, so point a fresh run at an empty directory.
 
+## Layouts: config, not code
+
+By default row 1 holds the column names and the data starts on row 2. Exports that
+put a title above the header, or description rows between it and the data, are
+described in `dataconvert.json` beside the script (or `--config FILE`). It is
+gitignored because the marker values name whoever produced the files; start from
+`dataconvert-example.json`.
+
+```json
+{
+  "layouts": [
+    { "name": "catalogue",
+      "match": { "row": 2, "column": 1, "in": ["CODE", "ITEM_CODE"] },
+      "header_row": 2, "data_row": 6 }
+  ],
+  "bare_sheet_prefixes": ["ref_"]
+}
+```
+
+- **layouts** are checked against every sheet and CSV, first match wins. A layout
+  matches when the cell at `match.row`/`match.column` (counted from 1, trimmed) is one
+  of `match.in`; then the names come from `header_row` and the data from `data_row`
+  on. The run prints which layout each sheet got. A sheet nothing matches is read
+  normally.
+- **bare_sheet_prefixes**: sheets whose name starts with one of these are written as
+  `<sheet>.sql` instead of `<workbook>_<sheet>.sql`.
+
+For a one-off, `--header-row 2 --data-row 6` applies one layout to every file in the
+run and skips detection.
+
 ## Sampling for a web LLM
 
 Full seed files get large fast, and a model only needs to see the shape of the data.
@@ -35,6 +67,7 @@ what pandas read: a starting point, not DDL. `--no-schema` skips the file.
 | file | does |
 |---|---|
 | `dataconvert.py` | command line |
+| `config.py` | `dataconvert.json`: layouts and sheet naming |
 | `readers.py` | files, directories, ZIPs and globs into DataFrames |
 | `sqlgen.py` | DataFrames into INSERT statements, with the row cap |
 | `output.py` | file naming and writing |

soleprint/station/tools/dataconvert/config.py

@@ -0,0 +1,117 @@
+"""
+Config: what a particular set of spreadsheets looks like.
+
+The tool knows nothing about any one exporter. Where a header sits, which
+sheets are recognised by a marker cell, and which sheet names are already
+unique enough to keep as they are, are facts about whoever produced the files,
+so they live in dataconvert.json beside this script. That file is gitignored:
+copy dataconvert-example.json and edit it. Without one, every file is read with
+its header on row 1.
+
+    {
+      "layouts": [
+        {
+          "name": "catalogue",
+          "match": {"row": 2, "column": 1, "in": ["CODE", "ITEM"]},
+          "header_row": 2,
+          "data_row": 6
+        }
+      ],
+      "bare_sheet_prefixes": ["ref_"]
+    }
+
+Rows and columns are counted as the spreadsheet shows them, from 1. A layout
+applies to a sheet (or CSV) when the cell at match.row/match.column, trimmed,
+is one of match.in. The first layout that matches wins.
+"""
+
+import json
+from pathlib import Path
+
+DEFAULT_PATH = Path(__file__).resolve().parent / "dataconvert.json"
+
+
+class ConfigError(Exception):
+    pass
+
+
+class Layout:
+    """Where the column names are and where the data starts, from row 1."""
+
+    def __init__(self, header_row=1, data_row=None, name="default"):
+        self.name = name
+        self.header_row = int(header_row)
+        self.data_row = int(data_row) if data_row is not None else self.header_row + 1
+        if self.header_row < 1:
+            raise ConfigError(f"layout '{name}': header_row must be 1 or more")
+        if self.data_row <= self.header_row:
+            raise ConfigError(f"layout '{name}': the data has to start below the header row")
+
+    @property
+    def is_default(self):
+        return self.header_row == 1 and self.data_row == 2
+
+
+class Rule:
+    def __init__(self, raw, index):
+        name = raw.get("name") or f"layout {index + 1}"
+        match = raw.get("match") or {}
+        values = match.get("in")
+        if not isinstance(values, list) or not values:
+            raise ConfigError(f"layout '{name}': match.in must be a non-empty list")
+        self.row = int(match.get("row", 1))
+        self.column = int(match.get("column", 1))
+        if self.row < 1 or self.column < 1:
+            raise ConfigError(f"layout '{name}': match.row and match.column count from 1")
+        self.values = {str(v).strip() for v in values}
+        self.layout = Layout(raw.get("header_row", 1), raw.get("data_row"), name)
+
+    def matches(self, raw_df):
+        r, c = self.row - 1, self.column - 1
+        if len(raw_df) <= r or raw_df.shape[1] <= c:
+            return False
+        return str(raw_df.iloc[r, c]).strip() in self.values
+
+
+class Config:
+    def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None):
+        self.rules = list(rules)
+        self.bare_sheet_prefixes = tuple(bare_sheet_prefixes)
+        self.source = source
+        # --header-row / --data-row on the command line: one layout for every
+        # file, no detection.
+        self.forced = forced
+
+    @property
+    def detects(self):
+        return self.forced is None and bool(self.rules)
+
+    def layout_for(self, raw_df):
+        """The layout for a sheet read without a header, or None for the default read."""
+        if self.forced is not None:
+            return None if self.forced.is_default else self.forced
+        for rule in self.rules:
+            if rule.matches(raw_df):
+                return rule.layout
+        return None
+
+
+def load(path=None, forced=None):
+    """Read the given config, else dataconvert.json beside this script if there is one."""
+    explicit = path is not None
+    path = Path(path) if explicit else DEFAULT_PATH
+    if not path.exists():
+        if explicit:
+            raise ConfigError(f"no such config file: {path}")
+        return Config(forced=forced)
+
+    try:
+        raw = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as e:
+        raise ConfigError(f"{path} is not valid JSON: {e}")
+
+    rules = [Rule(r, i) for i, r in enumerate(raw.get("layouts", []))]
+    prefixes = raw.get("bare_sheet_prefixes", [])
+    if not isinstance(prefixes, list):
+        raise ConfigError(f"{path}: bare_sheet_prefixes must be a list")
+    return Config(rules, [str(p) for p in prefixes], source=path, forced=forced)

soleprint/station/tools/dataconvert/dataconvert-example.json

@@ -0,0 +1,16 @@
+{
+  "_comment": "Template for dataconvert.json, which dataconvert.py reads from beside itself (or --config FILE). Copy this to dataconvert.json (gitignored) and describe the files you actually convert there. Rows and columns count from 1, as the spreadsheet shows them.",
+
+  "_layouts": "Checked in order against every sheet and CSV; the first match wins, and a sheet nothing matches is read with its header on row 1. A layout matches when the cell at match.row / match.column, trimmed, is one of match.in. header_row holds the column names; data starts at data_row, and anything in between is skipped.",
+  "layouts": [
+    {
+      "name": "catalogue",
+      "match": { "row": 2, "column": 1, "in": ["CODE", "ITEM_CODE"] },
+      "header_row": 2,
+      "data_row": 6
+    }
+  ],
+
+  "_bare_sheet_prefixes": "Sheets of a multi-sheet workbook are written as <workbook>_<sheet>.sql. A sheet whose name starts with one of these is written as <sheet>.sql instead, because its name is already unique.",
+  "bare_sheet_prefixes": ["ref_"]
+}

soleprint/station/tools/dataconvert/dataconvert.py

@@ -9,6 +9,11 @@ Usage:
   python3 dataconvert.py --input "data/*" "*.xlsx" --out-dir seed/
   python3 dataconvert.py --input path/to/folder/ --out-dir seed/
   python3 dataconvert.py --input path/to/folder/ --out-dir sample/ --max-rows 20
+  python3 dataconvert.py --input export.xlsx --header-row 2 --data-row 6
+  python3 dataconvert.py --input data/ --config their-exports.json
+
+How a given producer lays out its sheets is not built in: see config.py and
+dataconvert-example.json.
 
 --max-rows is for understanding the data rather than loading it: every table
 still gets its file and its SCHEMA.md entry, with only the first N rows, and a
@@ -22,6 +27,7 @@ import argparse
 from pathlib import Path
 
 from output import write_tables
+import config as cfg
 from readers import expand_inputs, iter_sources
 from schema import SchemaReport
 
@@ -40,15 +46,31 @@ def main():
     parser.add_argument("--max-rows", type=positive_int, default=None,
                         help="Write at most N rows per table; the header and SCHEMA.md note the full row count and size")
     parser.add_argument("--no-schema", action="store_true", help="Do not write SCHEMA.md")
+    parser.add_argument("--header-row", type=positive_int, default=1,
+                        help="Spreadsheet row holding the column names, for every file; overrides the config layouts (default 1)")
+    parser.add_argument("--data-row", type=positive_int, default=None,
+                        help="First row of data, when rows sit between it and the header (default: the row after the header)")
+    parser.add_argument("--config", default=None,
+                        help="Layouts and naming for these files (default: dataconvert.json beside this script, if present)")
     args = parser.parse_args()
 
+    try:
+        forced = None
+        if args.header_row != 1 or args.data_row is not None:
+            forced = cfg.Layout(args.header_row, args.data_row, "command line")
+        config = cfg.load(args.config, forced)
+    except cfg.ConfigError as e:
+        parser.error(str(e))
+    if config.source is not None:
+        print(f"[dataconvert] config: {config.source}")
+
     out_dir = Path(args.out_dir)
     out_dir.mkdir(parents=True, exist_ok=True)
 
     report = None if args.no_schema else SchemaReport()
     for in_path in expand_inputs(args.input):
-        for source_name, dfs in iter_sources(in_path):
-            write_tables(dfs, out_dir, source_name, args.max_rows, report)
+        for source_name, dfs in iter_sources(in_path, config):
+            write_tables(dfs, out_dir, source_name, args.max_rows, report, config.bare_sheet_prefixes)
 
     if report is not None and report.entries:
         print(f"[dataconvert] Generated: {report.write(out_dir, args.max_rows)}")

soleprint/station/tools/dataconvert/output.py

@@ -7,22 +7,26 @@ from pathlib import Path
 from sqlgen import render_table, sanitize_identifier
 
 
-def table_filename(raw_name: str, source_name: str, sheet_count: int) -> str:
-    """Sheets of a multi-sheet workbook are prefixed with the workbook, unless already namespaced."""
+def table_filename(raw_name: str, source_name: str, sheet_count: int, bare_prefixes=()) -> str:
+    """
+    Sheets of a multi-sheet workbook are prefixed with the workbook, so two
+    workbooks cannot collide, unless the config names the sheet as already
+    unique (bare_sheet_prefixes).
+    """
     clean = sanitize_identifier(raw_name)
-    if sheet_count > 1 and not clean.startswith("drm_") and not clean.startswith("study_"):
+    if sheet_count > 1 and not clean.startswith(tuple(bare_prefixes)):
         return f"{sanitize_identifier(source_name)}_{clean}.sql"
     return f"{clean}.sql"
 
 
-def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None):
+def write_tables(dfs: dict, out_dir: Path, source_name: str, max_rows=None, report=None, bare_prefixes=()):
     """Write individual .sql files per table/sheet into the output directory."""
     for raw_name, df in dfs.items():
         if df.empty:
             continue
 
         table = sanitize_identifier(raw_name)
-        filename = table_filename(raw_name, source_name, len(dfs))
+        filename = table_filename(raw_name, source_name, len(dfs), bare_prefixes)
         sql, total, full_bytes, exact = render_table(df, table, max_rows)
         out_file = out_dir / filename
 

soleprint/station/tools/dataconvert/readers.py

@@ -15,13 +15,6 @@ import pandas as pd
 
 SUPPORTED = {".csv", ".xlsx", ".xls", ".ods"}
 
-# DRM catalogue sheets put the real column names on row 1 and start the data
-# on row 5; the rows in between are descriptions. Recognised by the first cell
-# of the header row.
-DRM_HEADER_MARKERS = {
-    "CONTROLLED_TERMINOLOGY", "DATA_ELEMENT", "DATA_DOMAIN", "DATA_STATE", "DATA_CATEGORY",
-}
-
 
 def expand_inputs(inputs):
     """Wildcard patterns become the paths they match; everything else passes through."""
@@ -37,49 +30,66 @@ def expand_inputs(inputs):
     return paths
 
 
-def load_dataframes_from_file(file_path: Path) -> dict:
+def read_sheet(read, config):
+    """
+    One sheet or CSV, laid out as the config says.
+
+    With nothing to detect, a plain read, exactly as before. Otherwise the sheet
+    is read once without a header, checked against the layouts, and when one
+    matches the header and data rows are sliced out of that same read.
+    """
+    if not config.detects and config.forced is None:
+        return read(), None
+    raw = read(header=None)
+    layout = config.layout_for(raw)
+    if layout is None:
+        return read(), None
+    data = raw.iloc[layout.data_row - 1:].copy()
+    data.columns = [str(c).strip() for c in raw.iloc[layout.header_row - 1]]
+    return data, layout
+
+
+def load_dataframes_from_file(file_path: Path, config) -> dict:
     """Load a file (.csv, .xlsx, .xls, .ods) into {entity_name: DataFrame}."""
     ext = file_path.suffix.lower()
     dfs = {}
 
     try:
         if ext == ".csv":
-            dfs[file_path.stem] = pd.read_csv(file_path)
-        elif ext in [".xlsx", ".xls"]:
-            xls = pd.ExcelFile(file_path)
+            dfs[file_path.stem], layout = read_sheet(lambda **kw: pd.read_csv(file_path, **kw), config)
+            note_layout(file_path.name, None, layout)
+        elif ext in [".xlsx", ".xls", ".ods"]:
+            xls = pd.ExcelFile(file_path, engine="odf") if ext == ".ods" else pd.ExcelFile(file_path)
             for sheet in xls.sheet_names:
-                df_raw = pd.read_excel(xls, sheet_name=sheet, header=None)
-                if len(df_raw) > 1 and str(df_raw.iloc[1, 0]).strip() in DRM_HEADER_MARKERS:
-                    cols = [str(c).strip() for c in df_raw.iloc[1]]
-                    data_df = df_raw.iloc[5:].copy()
-                    data_df.columns = cols
-                    dfs[sheet] = data_df
-                else:
-                    dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
-        elif ext == ".ods":
-            xls = pd.ExcelFile(file_path, engine="odf")
-            for sheet in xls.sheet_names:
-                dfs[sheet] = pd.read_excel(xls, sheet_name=sheet)
+                dfs[sheet], layout = read_sheet(
+                    lambda sheet=sheet, **kw: pd.read_excel(xls, sheet_name=sheet, **kw), config)
+                note_layout(file_path.name, sheet, layout)
     except Exception as e:
         print(f"[Warning] Could not read '{file_path.name}': {e}")
 
     return dfs
 
 
-def iter_sources(path: Path):
+def note_layout(file_name, sheet, layout):
+    if layout is not None:
+        where = f"{file_name} [{sheet}]" if sheet else file_name
+        print(f"[dataconvert] {where}: layout '{layout.name}', header row {layout.header_row}, data from row {layout.data_row}")
+
+
+def iter_sources(path: Path, config):
     """Yield (source_name, dfs) for a file, a directory (recursively) or a ZIP archive."""
     if path.is_file() and path.suffix.lower() == ".zip":
         with tempfile.TemporaryDirectory() as tmp_dir:
             with zipfile.ZipFile(path, "r") as zip_ref:
                 zip_ref.extractall(tmp_dir)
-            yield from iter_sources(Path(tmp_dir))
+            yield from iter_sources(Path(tmp_dir), config)
 
     elif path.is_dir():
         for root, _, files in os.walk(path):
             for f in sorted(files):
                 f_path = Path(root) / f
                 if f_path.suffix.lower() in SUPPORTED:
-                    yield f_path.stem, load_dataframes_from_file(f_path)
+                    yield f_path.stem, load_dataframes_from_file(f_path, config)
 
     elif path.is_file() and path.suffix.lower() in SUPPORTED:
-        yield path.stem, load_dataframes_from_file(path)
+        yield path.stem, load_dataframes_from_file(path, config)