82 lines
2.6 KiB
Python
82 lines
2.6 KiB
Python
"""Pull items from the items API, rename them into the app's names, upsert into postgres.
|
|
|
|
Three links, kept apart on purpose:
|
|
|
|
- the API client (`fetch_items`) talks to the wire as it is — here the overlay's
|
|
own simulator, `items-api`, whose field names are the API's;
|
|
- the adapter (`to_app_row`) is the one place the wire's names become the app's:
|
|
`id` -> `item_id`, `name` -> `item_name`, `price.amount_cents` -> `price_cents`.
|
|
It belongs to whoever owns the app's model, so it lives in the overlay, not in rig;
|
|
- the load writes through the `app_db` connection (AIRFLOW_CONN_APP_DB, built by
|
|
addons/airflow.sh from the postgres secret) and is idempotent: an upsert keyed
|
|
on `item_id`, so a retry or a rerun never duplicates a row.
|
|
|
|
Operational logic is explicit and minimal: hourly, no backfill, one retry.
|
|
"""
|
|
import json
|
|
import urllib.request
|
|
from datetime import datetime, timedelta
|
|
|
|
from airflow import DAG
|
|
from airflow.operators.python import PythonOperator
|
|
|
|
ITEMS_URL = "http://items-api/v1/items"
|
|
|
|
CREATE = """
|
|
CREATE TABLE IF NOT EXISTS items (
|
|
item_id text PRIMARY KEY,
|
|
item_name text NOT NULL,
|
|
price_cents integer NOT NULL,
|
|
currency text NOT NULL,
|
|
loaded_at timestamptz NOT NULL DEFAULT now()
|
|
)
|
|
"""
|
|
|
|
UPSERT = """
|
|
INSERT INTO items (item_id, item_name, price_cents, currency)
|
|
VALUES (%(item_id)s, %(item_name)s, %(price_cents)s, %(currency)s)
|
|
ON CONFLICT (item_id) DO UPDATE
|
|
SET item_name = EXCLUDED.item_name,
|
|
price_cents = EXCLUDED.price_cents,
|
|
currency = EXCLUDED.currency,
|
|
loaded_at = now()
|
|
"""
|
|
|
|
|
|
def fetch_items():
|
|
"""The API client: the wire, as the API returns it."""
|
|
with urllib.request.urlopen(ITEMS_URL, timeout=10) as response:
|
|
return json.load(response)["items"]
|
|
|
|
|
|
def to_app_row(item):
|
|
"""The adapter: the API's names in, the app's names out."""
|
|
return {
|
|
"item_id": item["id"],
|
|
"item_name": item["name"],
|
|
"price_cents": item["price"]["amount_cents"],
|
|
"currency": item["price"]["currency"],
|
|
}
|
|
|
|
|
|
def load_items():
|
|
from airflow.providers.postgres.hooks.postgres import PostgresHook
|
|
|
|
rows = [to_app_row(item) for item in fetch_items()]
|
|
hook = PostgresHook(postgres_conn_id="app_db")
|
|
hook.run(CREATE)
|
|
for row in rows:
|
|
hook.run(UPSERT, parameters=row)
|
|
print(f"upserted {len(rows)} items")
|
|
|
|
|
|
with DAG(
|
|
dag_id="items_to_postgres",
|
|
schedule="@hourly",
|
|
start_date=datetime(2026, 1, 1),
|
|
catchup=False,
|
|
default_args={"retries": 1, "retry_delay": timedelta(minutes=1)},
|
|
tags=["example"],
|
|
) as dag:
|
|
PythonOperator(task_id="load_items", python_callable=load_items)
|