118 lines
4.2 KiB
Python
118 lines
4.2 KiB
Python
"""
|
|
Config: what a particular set of spreadsheets looks like.
|
|
|
|
The tool knows nothing about any one exporter. Where a header sits, which
|
|
sheets are recognised by a marker cell, and which sheet names are already
|
|
unique enough to keep as they are, are facts about whoever produced the files,
|
|
so they live in dataconvert.json beside this script. That file is gitignored:
|
|
copy dataconvert-example.json and edit it. Without one, every file is read with
|
|
its header on row 1.
|
|
|
|
{
|
|
"layouts": [
|
|
{
|
|
"name": "catalogue",
|
|
"match": {"row": 2, "column": 1, "in": ["CODE", "ITEM"]},
|
|
"header_row": 2,
|
|
"data_row": 6
|
|
}
|
|
],
|
|
"bare_sheet_prefixes": ["ref_"]
|
|
}
|
|
|
|
Rows and columns are counted as the spreadsheet shows them, from 1. A layout
|
|
applies to a sheet (or CSV) when the cell at match.row/match.column, trimmed,
|
|
is one of match.in. The first layout that matches wins.
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
DEFAULT_PATH = Path(__file__).resolve().parent / "dataconvert.json"
|
|
|
|
|
|
class ConfigError(Exception):
|
|
pass
|
|
|
|
|
|
class Layout:
|
|
"""Where the column names are and where the data starts, from row 1."""
|
|
|
|
def __init__(self, header_row=1, data_row=None, name="default"):
|
|
self.name = name
|
|
self.header_row = int(header_row)
|
|
self.data_row = int(data_row) if data_row is not None else self.header_row + 1
|
|
if self.header_row < 1:
|
|
raise ConfigError(f"layout '{name}': header_row must be 1 or more")
|
|
if self.data_row <= self.header_row:
|
|
raise ConfigError(f"layout '{name}': the data has to start below the header row")
|
|
|
|
@property
|
|
def is_default(self):
|
|
return self.header_row == 1 and self.data_row == 2
|
|
|
|
|
|
class Rule:
|
|
def __init__(self, raw, index):
|
|
name = raw.get("name") or f"layout {index + 1}"
|
|
match = raw.get("match") or {}
|
|
values = match.get("in")
|
|
if not isinstance(values, list) or not values:
|
|
raise ConfigError(f"layout '{name}': match.in must be a non-empty list")
|
|
self.row = int(match.get("row", 1))
|
|
self.column = int(match.get("column", 1))
|
|
if self.row < 1 or self.column < 1:
|
|
raise ConfigError(f"layout '{name}': match.row and match.column count from 1")
|
|
self.values = {str(v).strip() for v in values}
|
|
self.layout = Layout(raw.get("header_row", 1), raw.get("data_row"), name)
|
|
|
|
def matches(self, raw_df):
|
|
r, c = self.row - 1, self.column - 1
|
|
if len(raw_df) <= r or raw_df.shape[1] <= c:
|
|
return False
|
|
return str(raw_df.iloc[r, c]).strip() in self.values
|
|
|
|
|
|
class Config:
|
|
def __init__(self, rules=(), bare_sheet_prefixes=(), source=None, forced=None):
|
|
self.rules = list(rules)
|
|
self.bare_sheet_prefixes = tuple(bare_sheet_prefixes)
|
|
self.source = source
|
|
# --header-row / --data-row on the command line: one layout for every
|
|
# file, no detection.
|
|
self.forced = forced
|
|
|
|
@property
|
|
def detects(self):
|
|
return self.forced is None and bool(self.rules)
|
|
|
|
def layout_for(self, raw_df):
|
|
"""The layout for a sheet read without a header, or None for the default read."""
|
|
if self.forced is not None:
|
|
return None if self.forced.is_default else self.forced
|
|
for rule in self.rules:
|
|
if rule.matches(raw_df):
|
|
return rule.layout
|
|
return None
|
|
|
|
|
|
def load(path=None, forced=None):
|
|
"""Read the given config, else dataconvert.json beside this script if there is one."""
|
|
explicit = path is not None
|
|
path = Path(path) if explicit else DEFAULT_PATH
|
|
if not path.exists():
|
|
if explicit:
|
|
raise ConfigError(f"no such config file: {path}")
|
|
return Config(forced=forced)
|
|
|
|
try:
|
|
raw = json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError as e:
|
|
raise ConfigError(f"{path} is not valid JSON: {e}")
|
|
|
|
rules = [Rule(r, i) for i, r in enumerate(raw.get("layouts", []))]
|
|
prefixes = raw.get("bare_sheet_prefixes", [])
|
|
if not isinstance(prefixes, list):
|
|
raise ConfigError(f"{path}: bare_sheet_prefixes must be a list")
|
|
return Config(rules, [str(p) for p in prefixes], source=path, forced=forced)
|