From 1af02df62cb23fd56bd31a702e59243c2c6e6c24 Mon Sep 17 00:00:00 2001 From: Stephan van Zyl Date: Sun, 4 Oct 2026 19:05:32 +0000 Subject: [PATCH] loadwatch: 1 Hz P1 collector, event detector, labelling API/UI, sub-path routing, healthcheck --- .env.example | 21 ++++ .gitignore | 9 ++ Dockerfile | 15 +++ README.md | 92 +++++++++++++++ app/api.py | 221 +++++++++++++++++++++++++++++++++++ app/collector.py | 93 +++++++++++++++ app/core.py | 105 +++++++++++++++++ app/detector.py | 152 ++++++++++++++++++++++++ app/meter.py | 148 +++++++++++++++++++++++ app/static/index.html | 191 ++++++++++++++++++++++++++++++ bin/healthcheck.sh | 33 ++++++ deploy/traefik-loadwatch.yml | 44 +++++++ docker-compose.yml | 52 +++++++++ requirements.txt | 4 + sql/001_schema.sql | 90 ++++++++++++++ 15 files changed, 1270 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/api.py create mode 100644 app/collector.py create mode 100644 app/core.py create mode 100644 app/detector.py create mode 100644 app/meter.py create mode 100644 app/static/index.html create mode 100755 bin/healthcheck.sh create mode 100644 deploy/traefik-loadwatch.yml create mode 100644 docker-compose.yml create mode 100644 requirements.txt create mode 100644 sql/001_schema.sql diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9644bc4 --- /dev/null +++ b/.env.example @@ -0,0 +1,21 @@ +# Copy to .env and adjust. +# Postgres / Timescale +POSTGRES_PASSWORD=change-me-db + +# API +API_TOKEN=change-me-token +API_PORT=8099 + +# Meter source(s): comma-separated hosts running the ESPHome native API. +# Add the smart plug's host here later, e.g. 192.168.178.213,192.168.178.240 +METER_HOSTS=192.168.178.213 +METER_PORT=6053 +# ESPHome api encryption key (leave empty: this meter has none) +METER_NOISE_PSK= + +# Detection +EVENT_THRESHOLD_W=80 +EVENT_MIN_DURATION_S=3 +EVENT_CLOSE_BAND_W=40 +EVENT_CLOSE_HOLD_S=3 +EVENT_MAX_S=14400 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f6ca136 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.env +*.pyc +__pycache__/ +.venv/ +venv/ +project_state.md +*.csv +*.parquet +data/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a904a00 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +EXPOSE 8000 +CMD ["python", "-m", "app.collector"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..8a3b0a3 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +# loadwatch — Tier-0 NILM dataset builder + +Non-intrusive load monitoring, working from a **single Dutch DSMR-5 P1 stream at 1 Hz**. +Goal: build a *labelled* dataset of large-load switching events, so a model can later +identify which big appliance turned on. + +## What this is + +| Layer | What it does | +|---|---| +| `collector` | Subscribes to the SlimmeLezer+ **ESPHome native API** (1 Hz) and stores every sample | +| `detector` | Step-change detection → **load episodes** (on-step, duration, energy, peak) | +| `api` + UI | Live power, event list, **one-click labelling**, CSV export for training | +| `db` | TimescaleDB (Postgres 16) — 1-min continuous aggregate, compression after 7 days | + +No Home Assistant involvement on the ingest path (HA stays the notification shell). + +## Architecture decision (2026-10-04) + +Separate service on **dev CT141**, *not* inside HAOS: + +- HA here is **Core-in-Docker** — custom Python services installed there die on every + HA upgrade, and the recorder is not a dataset store (purges raw state ~10 days, writes + on-change). +- Ingest goes **straight to the meter** (`192.168.178.213:6053`, ESPHome API, no + encryption) — no HA token, no secrets on this path. +- Public URL via Coolify: **`loadwatch.xm2561.duckdns.org`** (the `xm2561` *apex* belongs + to Headscale — never take it). + +## Data model + +```sql +samples(ts timestamptz, device text, key text, value double precision) -- hypertable, 1 day chunks +events(id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w, duration_s, + energy_wh, appliance, label_source, labeled_at, notes) +appliances(name, notes, is_large) +``` + +`samples` is **narrow** (device, key) so additional sources — a smart plug, later a +second meter — need no schema change. + +## Run it (dev) + +```bash +cd /root/projects/loadwatch +cp .env.example .env # set API_TOKEN +docker compose up -d --build +docker compose logs -f collector +``` + +UI: `http://192.168.178.20:8099/` (token = `API_TOKEN`). + +## Collector entity keys + +The collector matches ESPHome `object_id`s, discovered at connect time and logged: + +| object_id | meaning | +|---|---| +| `power_consumed_phase_1` | instantaneous import W | +| `power_produced_phase_1` | instantaneous export W | +| `voltage_phase_1`, `current_phase_1` | grid V / A | +| `energy_consumed_tariff_1/2` | import kWh counters | +| `energy_produced_tariff_1/2` | export kWh counters | +| `gas_consumed` | gas m³ (M-Bus, ~5 min) | + +If a key is missing the collector logs **every** object_id it can see — that is the +discovery mechanism, not a guess. + +## Adding the smart plug (ground truth) + +Add its host to `METER_HOSTS` (comma-separated) — samples land with `device = `. +A plug on one appliance gives labelled ground truth for exactly that appliance, which is +what a supervised model needs; the P1 event stream covers everything else. + +## Export for training + +`GET /api/export.csv?days=30` → one row per labelled event plus its raw power window +(`GET /api/events/{id}/window` returns the ±60 s raw trace for that event). + +## Detection defaults + +`EVENT_THRESHOLD_W=80` (step size), `EVENT_MIN_DURATION_S=3`, close back within 40 W held +3 s. Night baseload is ~50–200 W here; a kettle/oven/washer step is 1000–2500 W. +Documented limitation: two loads stepping at the same instant inside one open episode +are seen as one event. + +## Roadmap + +1. ✅ capture + detect + label (this repo) +2. plug lane → labelled ground truth +3. HA REST sensor → "big load started" notifications +4. classifier trained on the labelled events (`trainer/`) diff --git a/app/api.py b/app/api.py new file mode 100644 index 0000000..86d55f2 --- /dev/null +++ b/app/api.py @@ -0,0 +1,221 @@ +"""Loadwatch API + labelling UI.""" +from __future__ import annotations + +import csv +import io +import logging +import os +from contextlib import asynccontextmanager +from pathlib import Path + +import asyncpg +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request +from fastapi.responses import HTMLResponse, StreamingResponse + +from . import core + +log = logging.getLogger("loadwatch.api") +STATIC = Path(__file__).parent / "static" + +# When served behind a Traefik sub-path (https://host/loadwatch) with a +# stripprefix middleware, the app still serves "/" - but every URL it EMITS must +# carry the prefix, because the browser requests the prefixed path. +# Traefik's StripPrefix sets X-Forwarded-Prefix, so the same image works both +# standalone (no header) and behind the portal path. +PREFIX = os.environ.get("PORTAL_PREFIX", "").rstrip("/") + +_pool: asyncpg.Pool | None = None + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + global _pool + core.setup_logging() + _pool = await core.make_pool() + log.info("api up") + try: + yield + finally: + if _pool is not None: + await _pool.close() + + +app = FastAPI(title="loadwatch", version="0.1.0", lifespan=lifespan) + + +async def auth(authorization: str | None = Header(default=None)) -> None: + token = core.API_TOKEN + if not token: + return + if authorization != f"Bearer {token}": + raise HTTPException(status_code=401, detail="bad or missing token") + + +# --------------------------------------------------------------------- pages +@app.get("/") +async def index(request: Request) -> HTMLResponse: + prefix = request.headers.get("x-forwarded-prefix", PREFIX).rstrip("/") + html = (STATIC / "index.html").read_text(encoding="utf-8") + return HTMLResponse(html.replace("__PREFIX__", prefix)) + + +# ----------------------------------------------------------------------- api +@app.get("/api/status", dependencies=[Depends(auth)]) +async def status(): + async with _pool.acquire() as con: + latest = await con.fetch( + """ + SELECT DISTINCT ON (key) key, value, ts + FROM samples WHERE device = $1 ORDER BY key, ts DESC + """, + core.METER_HOSTS[0], + ) + today = await con.fetchrow( + """ + SELECT + (SELECT value FROM samples WHERE key='energy_consumed_tariff_1' + ORDER BY ts DESC LIMIT 1) AS e_cons_t1, + (SELECT value FROM samples WHERE key='energy_consumed_tariff_2' + ORDER BY ts DESC LIMIT 1) AS e_cons_t2, + (SELECT value FROM samples WHERE key='energy_produced_tariff_1' + ORDER BY ts DESC LIMIT 1) AS e_prod_t1, + (SELECT value FROM samples WHERE key='energy_produced_tariff_2' + ORDER BY ts DESC LIMIT 1) AS e_prod_t2 + """ + ) + count = await con.fetchval("SELECT count(*) FROM events") + labelled = await con.fetchval("SELECT count(*) FROM events WHERE appliance IS NOT NULL") + return { + "device": core.METER_HOSTS[0], + "latest": {r["key"]: {"value": r["value"], "ts": r["ts"].isoformat()} for r in latest}, + "counters": dict(today) if today else {}, + "events": {"total": count, "labelled": labelled}, + } + + +@app.get("/api/power", dependencies=[Depends(auth)]) +async def power(hours: float = Query(6, ge=0.05, le=24 * 14), + key: str = Query(core.POWER_KEY)): + """Downsampled series from the 1-minute continuous aggregate.""" + if hours <= 3: + src = "samples" + bucket = "30 seconds" + else: + src = "samples_1m" + bucket = "1 minute" + window = min(hours, 6) if src == "samples" else hours + async with _pool.acquire() as con: + rows = await con.fetch( + f""" + SELECT time_bucket($1::interval, ts) AS bucket, + avg(value) AS avg, max(value) AS max, min(value) AS min + FROM {src} + WHERE key = $2 AND device = $3 AND ts > now() - ($4::float * interval '1 hour') + GROUP BY bucket ORDER BY bucket + """, + bucket, key, core.METER_HOSTS[0], window, + ) + return { + "key": key, + "bucket": bucket, + "points": [ + {"t": r["bucket"].isoformat(), "avg": r["avg"], "max": r["max"], "min": r["min"]} + for r in rows + ], + } + + +@app.get("/api/events", dependencies=[Depends(auth)]) +async def events(limit: int = Query(100, ge=1, le=1000), unlabelled: bool = False): + q = "SELECT * FROM events" + if unlabelled: + q += " WHERE appliance IS NULL" + q += " ORDER BY ts_start DESC LIMIT $1" + async with _pool.acquire() as con: + rows = await con.fetch(q, limit) + apps = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name") + return { + "events": [dict(r) for r in rows], + "appliances": [dict(a) for a in apps], + } + + +@app.post("/api/events/{event_id}/label", dependencies=[Depends(auth)]) +async def label(event_id: int, appliance: str = Query(...), notes: str | None = None): + async with _pool.acquire() as con: + ok = await con.fetchval( + "SELECT 1 FROM appliances WHERE name = $1", appliance + ) + if not ok: + raise HTTPException(400, f"unknown appliance {appliance!r}") + row = await con.fetchrow( + """ + UPDATE events + SET appliance = $2, label_source = 'manual', labeled_at = now(), + notes = COALESCE($3, notes) + WHERE id = $1 + RETURNING id, appliance, labeled_at + """, + event_id, appliance, notes, + ) + if row is None: + raise HTTPException(404, "no such event") + return dict(row) + + +@app.get("/api/events/{event_id}/window", dependencies=[Depends(auth)]) +async def event_window(event_id: int, pad_s: float = 60): + """Raw 1 Hz trace around an event - the training input for that label.""" + async with _pool.acquire() as con: + ev = await con.fetchrow("SELECT * FROM events WHERE id = $1", event_id) + if ev is None: + raise HTTPException(404, "no such event") + rows = await con.fetch( + """ + SELECT ts, value FROM samples + WHERE key = $1 AND device = $2 + AND ts BETWEEN $3 - ($4::float * interval '1 second') + AND COALESCE($5, $3) + ($4::float * interval '1 second') + ORDER BY ts + """, + core.POWER_KEY, core.METER_HOSTS[0], ev["ts_start"], pad_s, ev["ts_end"], + ) + return { + "event": dict(ev), + "samples": [{"t": r["ts"].isoformat(), "w": r["value"]} for r in rows], + } + + +@app.get("/api/export.csv", dependencies=[Depends(auth)]) +async def export_csv(days: float = Query(30, ge=0.1, le=3650)): + """Labelled events, one row each - the training set index.""" + async with _pool.acquire() as con: + rows = await con.fetch( + """ + SELECT id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w, + duration_s, energy_wh, appliance, label_source + FROM events + WHERE ts_start > now() - ($1::float * interval '1 day') + ORDER BY ts_start + """, + days, + ) + buf = io.StringIO() + if rows: + w = csv.DictWriter(buf, fieldnames=list(rows[0].keys())) + w.writeheader() + for r in rows: + w.writerow(dict(r)) + buf.seek(0) + return StreamingResponse( + iter([buf.getvalue()]), + media_type="text/csv", + headers={"Content-Disposition": "attachment; filename=loadwatch_events.csv"}, + ) + + +@app.get("/api/appliances", dependencies=[Depends(auth)]) +async def appliances(): + async with _pool.acquire() as con: + rows = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name") + return [dict(r) for r in rows] diff --git a/app/collector.py b/app/collector.py new file mode 100644 index 0000000..1525dab --- /dev/null +++ b/app/collector.py @@ -0,0 +1,93 @@ +"""Collector: ingest the P1 stream, store it, detect load episodes.""" +from __future__ import annotations + +import asyncio +import logging +import signal +import time + +from . import core +from .detector import EventDetector +from .meter import MeterHub + +log = logging.getLogger("loadwatch.collector") + + +async def amain() -> None: + core.setup_logging() + pool = await core.make_pool() + log.info("db pool ready") + + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except NotImplementedError: + pass + + hub = MeterHub(core.METER_HOSTS, core.METER_PORT, core.METER_NOISE_PSK) + detector = EventDetector() + log.info( + "detector: threshold=%.0fW min_duration=%.0fs close_band=%.0fW hold=%.0fs", + detector.threshold_w, detector.min_duration_s, detector.close_band_w, + detector.close_hold_s, + ) + + # everything is stored; the aggregate power also feeds the detector + pending_events: list = [] + + def hook(device: str, key: str, value: float) -> None: + if device == core.METER_HOSTS[0] and key == core.POWER_KEY: + ev = detector.add(time.time(), value) + if ev is not None and ev.duration_s >= detector.min_duration_s: + pending_events.append(ev) + + hub.hook = hook + + async def flusher() -> None: + while not stop.is_set(): + try: + await asyncio.sleep(core.FLUSH_INTERVAL_S) + rows, hub.buffer = hub.buffer, [] + await core.insert_samples(pool, rows) + while pending_events: + ev = pending_events.pop(0) + eid = await core.insert_event(pool, ev) + log.info( + "event #%s %s %.0fW dur=%.0fs energy=%.1fWh", + eid, "ON" if ev.direction > 0 else "OFF", ev.delta_w, + ev.duration_s, ev.energy_wh, + ) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + log.warning("flush failed: %s", exc) + + tasks = [ + asyncio.create_task(hub.run(stop), name="meter"), + asyncio.create_task(flusher(), name="flusher"), + asyncio.create_task(stop.wait(), name="stop"), + ] + done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) + for t in pending: + t.cancel() + await asyncio.gather(*pending, return_exceptions=True) + for t in done: + exc = t.exception() + if exc: + log.error("task %s died: %s", t.get_name(), exc) + + # final flush + rows, hub.buffer = hub.buffer, [] + await core.insert_samples(pool, rows) + await pool.close() + log.info("collector stopped") + + +def main() -> None: + asyncio.run(amain()) + + +if __name__ == "__main__": + main() diff --git a/app/core.py b/app/core.py new file mode 100644 index 0000000..c08d868 --- /dev/null +++ b/app/core.py @@ -0,0 +1,105 @@ +"""Config + database helpers.""" +from __future__ import annotations + +import logging +import os +from datetime import datetime, timezone + +import asyncpg + +log = logging.getLogger("loadwatch") + + +def env_str(name: str, default: str) -> str: + v = os.environ.get(name) + return v if v not in (None, "") else default + + +def env_float(name: str, default: float) -> float: + try: + return float(os.environ[name]) + except (KeyError, TypeError, ValueError): + return default + + +DATABASE_URL = env_str( + "DATABASE_URL", "postgresql://loadwatch:loadwatch@db:5432/loadwatch" +) + +# --- meter / sources ------------------------------------------------------ +METER_HOSTS = [h.strip() for h in env_str("METER_HOSTS", "192.168.178.213").split(",") if h.strip()] +METER_PORT = int(env_float("METER_PORT", 6053)) +METER_NOISE_PSK = env_str("METER_NOISE_PSK", "") + +# ESPHome object_id -> our column/key name. Matched by object_id first, then by +# suffix (the Zuidwijk firmware keeps the plain dsmr names). +WATCH = { + "power_consumed_phase_1": "power_consumed_phase_1", + "power_produced_phase_1": "power_produced_phase_1", + "voltage_phase_1": "voltage_phase_1", + "current_phase_1": "current_phase_1", + "energy_consumed_tariff_1": "energy_consumed_tariff_1", + "energy_consumed_tariff_2": "energy_consumed_tariff_2", + "energy_produced_tariff_1": "energy_produced_tariff_1", + "energy_produced_tariff_2": "energy_produced_tariff_2", + "gas_consumed": "gas_consumed", +} + +POWER_KEY = "power_consumed_phase_1" # the aggregate signal the detector watches + +# --- detection ------------------------------------------------------------ +EVENT_THRESHOLD_W = env_float("EVENT_THRESHOLD_W", 80.0) +EVENT_MIN_DURATION_S = env_float("EVENT_MIN_DURATION_S", 3.0) +EVENT_CLOSE_BAND_W = env_float("EVENT_CLOSE_BAND_W", 40.0) +EVENT_CLOSE_HOLD_S = env_float("EVENT_CLOSE_HOLD_S", 3.0) +EVENT_MAX_S = env_float("EVENT_MAX_S", 4 * 3600.0) + +API_TOKEN = env_str("API_TOKEN", "") + +FLUSH_INTERVAL_S = env_float("FLUSH_INTERVAL_S", 5.0) +BASELINE_WINDOW_S = env_float("BASELINE_WINDOW_S", 60.0) + + +def setup_logging() -> None: + logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", + ) + + +async def make_pool() -> asyncpg.Pool: + return await asyncpg.create_pool(DATABASE_URL, min_size=1, max_size=4) + + +async def insert_samples(pool: asyncpg.Pool, rows: list[tuple]) -> None: + """rows: (ts, device, key, value). ts may be an epoch float or a datetime.""" + if not rows: + return + norm = [ + ( + ts if not isinstance(ts, (int, float)) + else datetime.fromtimestamp(float(ts), tz=timezone.utc), + device, + key, + float(value), + ) + for ts, device, key, value in rows + ] + async with pool.acquire() as con: + await con.executemany( + "INSERT INTO samples (ts, device, key, value) VALUES ($1, $2, $3, $4)", norm + ) + + +async def insert_event(pool: asyncpg.Pool, ev) -> int: + async with pool.acquire() as con: + return await con.fetchval( + """ + INSERT INTO events (ts_start, ts_end, direction, delta_w, peak_w, + baseline_w, duration_s, energy_wh) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING id + """, + ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w, + ev.baseline_w, ev.duration_s, ev.energy_wh, + ) diff --git a/app/detector.py b/app/detector.py new file mode 100644 index 0000000..eee9d0c --- /dev/null +++ b/app/detector.py @@ -0,0 +1,152 @@ +"""Step-change / episode detection on a 1 Hz real-power stream. + +Model: the house has a running baseline (everything always-on). A load *episode* +starts when instantaneous power steps away from that baseline by more than +`threshold_w` and stays there for `min_duration_s`; it ends when power returns +within `close_band_w` of the baseline and holds for `close_hold_s`. + +The baseline is frozen while an episode is open, so a second load switching on +mid-episode does not corrupt the first one's delta. +""" +from __future__ import annotations + +import statistics +from collections import deque +from dataclasses import dataclass + +from . import core + + +@dataclass +class Event: + ts_start: float + ts_end: float | None = None + direction: int = 1 + delta_w: float = 0.0 + peak_w: float = 0.0 + baseline_w: float = 0.0 + duration_s: float = 0.0 + energy_wh: float = 0.0 + + # helpers for the DB layer (epoch -> aware datetime) + @property + def ts_start_dt(self): + from datetime import datetime, timezone + return datetime.fromtimestamp(self.ts_start, tz=timezone.utc) + + @property + def ts_end_dt(self): + from datetime import datetime, timezone + if self.ts_end is None: + return None + return datetime.fromtimestamp(self.ts_end, tz=timezone.utc) + + +class EventDetector: + def __init__(self, threshold_w=None, min_duration_s=None, close_band_w=None, + close_hold_s=None, baseline_window_s=None, max_event_s=None): + self.threshold_w = core.EVENT_THRESHOLD_W if threshold_w is None else threshold_w + self.min_duration_s = core.EVENT_MIN_DURATION_S if min_duration_s is None else min_duration_s + self.close_band_w = core.EVENT_CLOSE_BAND_W if close_band_w is None else close_band_w + self.close_hold_s = core.EVENT_CLOSE_HOLD_S if close_hold_s is None else close_hold_s + self.baseline_window_s = core.BASELINE_WINDOW_S if baseline_window_s is None else baseline_window_s + self.max_event_s = core.EVENT_MAX_S if max_event_s is None else max_event_s + + self._hist: deque[tuple[float, float]] = deque() # (ts, w) since last event + self._cand: tuple[float, float] | None = None # candidate step + self._ev: Event | None = None + self._back_since: float | None = None + self._last_ts: float | None = None + self._energy_acc = 0.0 + + # ------------------------------------------------------------------ util + def _baseline(self) -> float | None: + if not self._hist: + return None + cutoff = (self._last_ts or 0.0) - self.baseline_window_s + window = [w for ts, w in self._hist if ts >= cutoff] + if len(window) < 3: + return None + return statistics.median(window) + + # ------------------------------------------------------------------- fed + def add(self, ts: float, w: float) -> Event | None: + """Feed one sample. Returns a completed Event when one closes.""" + self._last_ts = ts + completed: Event | None = None + + if self._ev is None: + base = self._baseline() + self._hist.append((ts, w)) + # keep memory bounded + if len(self._hist) > 10000: + self._hist.popleft() + + if base is not None: + if abs(w - base) > self.threshold_w: + if self._cand is None: + self._cand = (ts, w) + elif ts - self._cand[0] >= self.min_duration_s: + # episode opens retroactively at the candidate time + delta = w - base + self._ev = Event( + ts_start=self._cand[0], + direction=1 if delta > 0 else -1, + delta_w=delta, + peak_w=w, + baseline_w=base, + ) + self._energy_acc = 0.0 + self._back_since = None + self._cand = None + self._hist.clear() + self._hist.append((ts, w)) + else: + self._cand = None + else: + ev = self._ev + dev = w - ev.baseline_w + if abs(w) > abs(ev.peak_w): + ev.peak_w = w + # energy above baseline, trapezoid-ish (1 Hz => dt of one step) + dt = max(0.0, ts - (self._last_ts_prev or ts)) + self._energy_acc += max(0.0, dev) * dt / 3600.0 + self._hist.append((ts, w)) + + if abs(dev) < self.close_band_w: + if self._back_since is None: + self._back_since = ts + elif ts - self._back_since >= self.close_hold_s: + ev.ts_end = self._back_since + ev.duration_s = ev.ts_end - ev.ts_start + ev.energy_wh = self._energy_acc + completed = ev + self._ev = None + self._back_since = None + self._cand = None + self._hist.clear() + else: + self._back_since = None + + if self._ev is not None and ts - self._ev.ts_start >= self.max_event_s: + self._ev.ts_end = ts + self._ev.duration_s = ts - self._ev.ts_start + self._ev.energy_wh = self._energy_acc + completed = self._ev + self._ev = None + self._back_since = None + self._hist.clear() + + self._last_ts_prev = ts + return completed + + # for the live status endpoint + @property + def active(self) -> Event | None: + return self._ev + + _last_ts_prev: float | None = None + + def to_rows(self, ev: Event) -> tuple: + return (ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w, + ev.baseline_w, ev.duration_s, ev.energy_wh) diff --git a/app/meter.py b/app/meter.py new file mode 100644 index 0000000..4785723 --- /dev/null +++ b/app/meter.py @@ -0,0 +1,148 @@ +"""ESPHome native-API ingestion. + +One ESPHome device = one source. Subscribes to the sensor states we care about and +hands (ts, device, key, value) tuples to a callback. Reconnects with backoff. +""" +from __future__ import annotations + +import asyncio +import inspect +import logging +import time + +from aioesphomeapi import APIClient + +from . import core + +log = logging.getLogger("loadwatch.meter") + + +class MeterSource: + def __init__(self, host: str, port: int, noise_psk: str, on_sample): + self.host = host + self.port = port + self.noise_psk = noise_psk or None + self.on_sample = on_sample + self._keymap: dict[int, str] = {} + self._cli: APIClient | None = None + + # -- state callback ---------------------------------------------------- + def _handle_state(self, state) -> None: + name = self._keymap.get(getattr(state, "key", None)) + if name is None: + return + value = getattr(state, "state", None) + if getattr(state, "missing_state", False) or value is None: + log.debug("%s: %s missing", self.host, name) + return + try: + self.on_sample(self.host, name, float(value)) + except (TypeError, ValueError): + log.warning("%s: non-numeric %s=%r", self.host, name, value) + + # -- discovery --------------------------------------------------------- + async def _resolve_entities(self) -> list[int]: + assert self._cli is not None + entities, _services = await self._cli.list_entities_services() + available: dict[str, int] = {} + for ent in entities: + oid = getattr(ent, "object_id", None) + key = getattr(ent, "key", None) + if oid and key is not None: + available[oid] = key + + wanted: list[int] = [] + for want in core.WATCH: + key = available.get(want) + if key is None: # tolerate firmware variants: match by suffix + for oid, k in available.items(): + if oid.endswith(want): + key = k + break + if key is None: + continue + self._keymap[key] = want + wanted.append(key) + + missing = sorted(set(core.WATCH) - set(self._keymap.values())) + log.info( + "%s: watching %d/%d entities; missing=%s", + self.host, len(wanted), len(core.WATCH), missing or "none", + ) + if missing: + # Self-diagnosing: show the caller every object_id this device exposes. + log.info("%s: available object_ids: %s", self.host, sorted(available)) + return wanted + + # -- run --------------------------------------------------------------- + async def run(self, stop: asyncio.Event) -> None: + backoff = 5 + while not stop.is_set(): + try: + self._cli = APIClient( + self.host, self.port, password="", noise_psk=self.noise_psk + ) + log.info("%s: connecting to ESPHome API %s:%s", self.host, self.host, self.port) + await self._cli.connect(login=True) + log.info("%s: connected", self.host) + backoff = 5 + + wanted = await self._resolve_entities() + if not wanted: + raise RuntimeError("none of the watched entities exist on this device") + # aioesphomeapi has shipped both sync and async variants of this + res = self._cli.subscribe_states(self._handle_state) + if inspect.isawaitable(res): + await res + log.info("%s: subscribed to %d state keys", self.host, len(wanted)) + + while not stop.is_set(): + await asyncio.sleep(1) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - stay alive on any transport error + log.warning("%s: connection problem: %s (retry in %ss)", self.host, exc, backoff) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 120) + finally: + if self._cli is not None: + try: + await self._cli.disconnect() + except Exception: # noqa: BLE001 + pass + self._cli = None + log.info("%s: stopped", self.host) + + +class MeterHub: + """Collects samples from every configured source into one flush buffer.""" + + def __init__(self, hosts: list[str], port: int, noise_psk: str): + self.hosts = hosts + self.port = port + self.noise_psk = noise_psk + self.buffer: list[tuple] = [] + self.latest: dict[str, float] = {} + self._last: dict[str, float] = {} + # optional extra consumer (the detector), set after construction. + # Kept as an attribute and routed through _dispatch so assigning it + # later actually takes effect (sources hold _dispatch, not the hook). + self.hook = None + self.sources = [ + MeterSource(h, port, noise_psk, self._dispatch) for h in hosts + ] + + def _dispatch(self, device: str, key: str, value: float) -> None: + self._on_sample(device, key, value) + if self.hook is not None: + self.hook(device, key, value) + + def _on_sample(self, device: str, key: str, value: float) -> None: + now = time.time() + self.buffer.append((now, device, key, value)) + if device == self.hosts[0]: + self.latest[key] = value + self._last[key] = now + + async def run(self, stop: asyncio.Event) -> None: + await asyncio.gather(*(s.run(stop) for s in self.sources)) diff --git a/app/static/index.html b/app/static/index.html new file mode 100644 index 0000000..71bdb84 --- /dev/null +++ b/app/static/index.html @@ -0,0 +1,191 @@ + + + + + +loadwatch + + + +
+

loadwatch

+ connecting… + token + + + + export csv +
+ +
+
+
now
– W
+
import today
–
+
export today
–
+
gas
–
+
events / labelled
–
+
+ +
+
aggregate power (W) — 6 h
+
+ + + +
+ +
+ +
+
load episodes — label them (this is the dataset)
+ + +
#startΔWpeakdurWhlabel
loading…
+
+
+ + + + diff --git a/bin/healthcheck.sh b/bin/healthcheck.sh new file mode 100755 index 0000000..d1574b7 --- /dev/null +++ b/bin/healthcheck.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# loadwatch healthcheck — always exits 0, prints exactly ONE line. +# OK s -> healthy, newest sample seconds old +# FAIL -> something is wrong +# Intended to be run on the runtime host (dev CT141), e.g. over SSH from a watchdog. +set -u + +cd "$(dirname "$0")/.." || { echo "FAIL cannot cd to project dir"; exit 0; } + +ps_out=$(docker compose ps --format '{{.Service}}={{.State}}' 2>/dev/null | tr '\n' ' ') +if [ -z "$ps_out" ]; then + echo "FAIL docker compose reports no services (stack down?)" + exit 0 +fi +for svc in db collector api; do + case " $ps_out " in + *" $svc=running "*) : ;; + *) echo "FAIL service $svc is not running (states: $ps_out)"; exit 0 ;; + esac +done + +age=$(docker compose exec -T db psql -U loadwatch -d loadwatch -t -A -c \ + "select coalesce(extract(epoch from (now()-max(ts)))::int,-1) from samples;" 2>/dev/null | tr -d '[:space:]') +case "${age:-x}" in + ''|*[!0-9-]*) echo "FAIL sample-age query failed (raw='${age:-empty}')" ;; + -1) echo "FAIL no samples in the database" ;; + *) if [ "$age" -gt 300 ]; then + echo "FAIL newest sample is ${age}s old (threshold 300s)" + else + echo "OK ${age}s" + fi ;; +esac +exit 0 diff --git a/deploy/traefik-loadwatch.yml b/deploy/traefik-loadwatch.yml new file mode 100644 index 0000000..c1b2928 --- /dev/null +++ b/deploy/traefik-loadwatch.yml @@ -0,0 +1,44 @@ +# loadwatch — Tier-0 NILM dataset service (runs on dev CT141, 192.168.178.20:8099) +# +# Public URL: https://xm2561.duckdns.org/loadwatch +# +# IMPORTANT: this host's apex (xm2561.duckdns.org) is Headscale's coordination +# endpoint (see headscale.yml). That router is a bare `Host()` catch-all and is +# deliberately left untouched. This router is strictly longer +# (`Host() && PathPrefix()`), so Traefik's length-based priority gives it the +# /loadwatch prefix only — /ts2021, /derp, /key, /register, /api/v1/* and +# /health keep flowing to Headscale. +# +# StripPrefix: the container serves "/" and stays standalone-clean; Traefik +# strips /loadwatch and (v2.5+) sets X-Forwarded-Prefix, which the app uses to +# emit prefixed asset/API URLs for the browser. +http: + routers: + loadwatch-http: + rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)" + entryPoints: + - http + service: loadwatch + middlewares: + - redirect-to-https + loadwatch: + rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)" + entryPoints: + - https + tls: + certResolver: letsencrypt + service: loadwatch + middlewares: + - loadwatch-strip + + middlewares: + loadwatch-strip: + stripPrefix: + prefixes: + - "/loadwatch" + + services: + loadwatch: + loadBalancer: + servers: + - url: "http://192.168.178.20:8099" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f687b99 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,52 @@ +name: loadwatch + +services: + db: + image: timescale/timescaledb:latest-pg16 + restart: unless-stopped + environment: + POSTGRES_USER: loadwatch + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-loadwatch} + POSTGRES_DB: loadwatch + volumes: + - dbdata:/var/lib/postgresql/data + - ./sql:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U loadwatch -d loadwatch"] + interval: 10s + timeout: 5s + retries: 12 + + collector: + build: . + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch + METER_HOSTS: ${METER_HOSTS:-192.168.178.213} + METER_PORT: ${METER_PORT:-6053} + METER_NOISE_PSK: ${METER_NOISE_PSK:-} + EVENT_THRESHOLD_W: ${EVENT_THRESHOLD_W:-80} + EVENT_MIN_DURATION_S: ${EVENT_MIN_DURATION_S:-3} + EVENT_CLOSE_BAND_W: ${EVENT_CLOSE_BAND_W:-40} + EVENT_CLOSE_HOLD_S: ${EVENT_CLOSE_HOLD_S:-3} + EVENT_MAX_S: ${EVENT_MAX_S:-14400} + command: ["python", "-m", "app.collector"] + + api: + build: . + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch + API_TOKEN: ${API_TOKEN:-loadwatch-dev} + ports: + - "${API_PORT:-8099}:8000" + command: ["uvicorn", "app.api:app", "--host", "0.0.0.0", "--port", "8000"] + +volumes: + dbdata: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b7c3004 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +aioesphomeapi>=24.0 +asyncpg>=0.29 +fastapi>=0.110 +uvicorn[standard]>=0.29 diff --git a/sql/001_schema.sql b/sql/001_schema.sql new file mode 100644 index 0000000..250df4c --- /dev/null +++ b/sql/001_schema.sql @@ -0,0 +1,90 @@ +-- loadwatch schema v1 +CREATE EXTENSION IF NOT EXISTS timescaledb; + +-- Raw 1 Hz samples. Narrow by design: extra sources (smart plug, later a second +-- meter) are just another (device, key) pair - no schema change. +CREATE TABLE IF NOT EXISTS samples ( + ts timestamptz NOT NULL, + device text NOT NULL, + key text NOT NULL, + value double precision NOT NULL +); + +SELECT create_hypertable('samples', 'ts', chunk_time_interval => interval '1 day', + if_not_exists => TRUE); + +CREATE INDEX IF NOT EXISTS samples_device_key_ts_idx ON samples (device, key, ts DESC); + +-- 1-minute rollup for charts and long-term retention. +CREATE MATERIALIZED VIEW IF NOT EXISTS samples_1m +WITH (timescaledb.continuous) AS +SELECT time_bucket('1 minute', ts) AS bucket, + device, + key, + avg(value) AS avg, + min(value) AS min, + max(value) AS max, + last(value, ts) AS last +FROM samples +GROUP BY bucket, device, key; + +SELECT add_continuous_aggregate_policy('samples_1m', + start_offset => interval '3 hours', + end_offset => interval '1 minute', + schedule_interval => interval '1 minute', + if_not_exists => TRUE); + +-- Compress raw samples after 7 days (keeps the full 1 Hz history, cheaply). +ALTER TABLE samples SET (timescaledb.compress, + timescaledb.compress_segmentby = 'device,key', + timescaledb.compress_orderby = 'ts DESC'); +SELECT add_compression_policy('samples', interval '7 days', if_not_exists => TRUE); + +-- Load episodes detected from the aggregate power stream. +CREATE TABLE IF NOT EXISTS events ( + id bigserial PRIMARY KEY, + ts_start timestamptz NOT NULL, + ts_end timestamptz, + direction smallint NOT NULL, -- +1 = load turned on, -1 = off + delta_w double precision, -- the step size + peak_w double precision, -- peak power during the episode + baseline_w double precision, -- baseline the step came off + duration_s double precision, + energy_wh double precision, -- integral of (P - baseline) over the episode + appliance text, -- label: null until labelled + label_source text, -- manual | plug | model + labeled_at timestamptz, + notes text +); + +CREATE INDEX IF NOT EXISTS events_ts_start_idx ON events (ts_start DESC); +CREATE INDEX IF NOT EXISTS events_appliance_idx ON events (appliance); + +CREATE TABLE IF NOT EXISTS appliances ( + name text PRIMARY KEY, + notes text, + is_large boolean NOT NULL DEFAULT TRUE +); + +INSERT INTO appliances (name, notes) VALUES + ('kettle', 'water kettle / boiler, ~2 kW step'), + ('oven', 'electric oven, 2-3 kW, thermostatic cycling'), + ('microwave', '~1.2 kW with ~50% duty cycle'), + ('dishwasher', 'heater cycling ~1-2 kW'), + ('washing_machine','heater ~2 kW, motor ~500 W'), + ('dryer', '~2-3 kW heater cycling'), + ('fridge', 'compressor ~80-150 W cycling'), + ('freezer', 'compressor ~80-150 W cycling'), + ('iron', '~1-2 kW, duty-cycled'), + ('vacuum', '~1-1.5 kW steady'), + ('water_boiler', 'hot water / boiler'), + ('ev_charger', 'EV charge point'), + ('heater', 'space heater'), + ('unknown_large', 'clearly a big load, unidentifiable'), + ('ignore', 'noise / not a real load') +ON CONFLICT (name) DO NOTHING; + +CREATE TABLE IF NOT EXISTS meta ( + k text PRIMARY KEY, + v text +);