loadwatch: 1 Hz P1 collector, event detector, labelling API/UI, sub-path routing, healthcheck

This commit is contained in:
Stephan van Zyl
2026-10-04 19:05:32 +00:00
commit 1af02df62c
15 changed files with 1270 additions and 0 deletions
+21
View File
@@ -0,0 +1,21 @@
# Copy to .env and adjust.
# Postgres / Timescale
POSTGRES_PASSWORD=change-me-db
# API
API_TOKEN=change-me-token
API_PORT=8099
# Meter source(s): comma-separated hosts running the ESPHome native API.
# Add the smart plug's host here later, e.g. 192.168.178.213,192.168.178.240
METER_HOSTS=192.168.178.213
METER_PORT=6053
# ESPHome api encryption key (leave empty: this meter has none)
METER_NOISE_PSK=
# Detection
EVENT_THRESHOLD_W=80
EVENT_MIN_DURATION_S=3
EVENT_CLOSE_BAND_W=40
EVENT_CLOSE_HOLD_S=3
EVENT_MAX_S=14400
+9
View File
@@ -0,0 +1,9 @@
.env
*.pyc
__pycache__/
.venv/
venv/
project_state.md
*.csv
*.parquet
data/
+15
View File
@@ -0,0 +1,15 @@
FROM python:3.12-slim
ENV PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
PIP_NO_CACHE_DIR=1
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app ./app
EXPOSE 8000
CMD ["python", "-m", "app.collector"]
+92
View File
@@ -0,0 +1,92 @@
# loadwatch — Tier-0 NILM dataset builder
Non-intrusive load monitoring, working from a **single Dutch DSMR-5 P1 stream at 1 Hz**.
Goal: build a *labelled* dataset of large-load switching events, so a model can later
identify which big appliance turned on.
## What this is
| Layer | What it does |
|---|---|
| `collector` | Subscribes to the SlimmeLezer+ **ESPHome native API** (1 Hz) and stores every sample |
| `detector` | Step-change detection → **load episodes** (on-step, duration, energy, peak) |
| `api` + UI | Live power, event list, **one-click labelling**, CSV export for training |
| `db` | TimescaleDB (Postgres 16) — 1-min continuous aggregate, compression after 7 days |
No Home Assistant involvement on the ingest path (HA stays the notification shell).
## Architecture decision (2026-10-04)
Separate service on **dev CT141**, *not* inside HAOS:
- HA here is **Core-in-Docker** — custom Python services installed there die on every
HA upgrade, and the recorder is not a dataset store (purges raw state ~10 days, writes
on-change).
- Ingest goes **straight to the meter** (`192.168.178.213:6053`, ESPHome API, no
encryption) — no HA token, no secrets on this path.
- Public URL via Coolify: **`loadwatch.xm2561.duckdns.org`** (the `xm2561` *apex* belongs
to Headscale — never take it).
## Data model
```sql
samples(ts timestamptz, device text, key text, value double precision) -- hypertable, 1 day chunks
events(id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w, duration_s,
energy_wh, appliance, label_source, labeled_at, notes)
appliances(name, notes, is_large)
```
`samples` is **narrow** (device, key) so additional sources — a smart plug, later a
second meter — need no schema change.
## Run it (dev)
```bash
cd /root/projects/loadwatch
cp .env.example .env # set API_TOKEN
docker compose up -d --build
docker compose logs -f collector
```
UI: `http://192.168.178.20:8099/` (token = `API_TOKEN`).
## Collector entity keys
The collector matches ESPHome `object_id`s, discovered at connect time and logged:
| object_id | meaning |
|---|---|
| `power_consumed_phase_1` | instantaneous import W |
| `power_produced_phase_1` | instantaneous export W |
| `voltage_phase_1`, `current_phase_1` | grid V / A |
| `energy_consumed_tariff_1/2` | import kWh counters |
| `energy_produced_tariff_1/2` | export kWh counters |
| `gas_consumed` | gas m³ (M-Bus, ~5 min) |
If a key is missing the collector logs **every** object_id it can see — that is the
discovery mechanism, not a guess.
## Adding the smart plug (ground truth)
Add its host to `METER_HOSTS` (comma-separated) — samples land with `device = <host>`.
A plug on one appliance gives labelled ground truth for exactly that appliance, which is
what a supervised model needs; the P1 event stream covers everything else.
## Export for training
`GET /api/export.csv?days=30` → one row per labelled event plus its raw power window
(`GET /api/events/{id}/window` returns the ±60 s raw trace for that event).
## Detection defaults
`EVENT_THRESHOLD_W=80` (step size), `EVENT_MIN_DURATION_S=3`, close back within 40 W held
3 s. Night baseload is ~50–200 W here; a kettle/oven/washer step is 1000–2500 W.
Documented limitation: two loads stepping at the same instant inside one open episode
are seen as one event.
## Roadmap
1. ✅ capture + detect + label (this repo)
2. plug lane → labelled ground truth
3. HA REST sensor → "big load started" notifications
4. classifier trained on the labelled events (`trainer/`)
+221
View File
@@ -0,0 +1,221 @@
"""Loadwatch API + labelling UI."""
from __future__ import annotations
import csv
import io
import logging
import os
from contextlib import asynccontextmanager
from pathlib import Path
import asyncpg
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
from fastapi.responses import HTMLResponse, StreamingResponse
from . import core
log = logging.getLogger("loadwatch.api")
STATIC = Path(__file__).parent / "static"
# When served behind a Traefik sub-path (https://host/loadwatch) with a
# stripprefix middleware, the app still serves "/" - but every URL it EMITS must
# carry the prefix, because the browser requests the prefixed path.
# Traefik's StripPrefix sets X-Forwarded-Prefix, so the same image works both
# standalone (no header) and behind the portal path.
PREFIX = os.environ.get("PORTAL_PREFIX", "").rstrip("/")
_pool: asyncpg.Pool | None = None
@asynccontextmanager
async def lifespan(_app: FastAPI):
global _pool
core.setup_logging()
_pool = await core.make_pool()
log.info("api up")
try:
yield
finally:
if _pool is not None:
await _pool.close()
app = FastAPI(title="loadwatch", version="0.1.0", lifespan=lifespan)
async def auth(authorization: str | None = Header(default=None)) -> None:
token = core.API_TOKEN
if not token:
return
if authorization != f"Bearer {token}":
raise HTTPException(status_code=401, detail="bad or missing token")
# --------------------------------------------------------------------- pages
@app.get("/")
async def index(request: Request) -> HTMLResponse:
prefix = request.headers.get("x-forwarded-prefix", PREFIX).rstrip("/")
html = (STATIC / "index.html").read_text(encoding="utf-8")
return HTMLResponse(html.replace("__PREFIX__", prefix))
# ----------------------------------------------------------------------- api
@app.get("/api/status", dependencies=[Depends(auth)])
async def status():
async with _pool.acquire() as con:
latest = await con.fetch(
"""
SELECT DISTINCT ON (key) key, value, ts
FROM samples WHERE device = $1 ORDER BY key, ts DESC
""",
core.METER_HOSTS[0],
)
today = await con.fetchrow(
"""
SELECT
(SELECT value FROM samples WHERE key='energy_consumed_tariff_1'
ORDER BY ts DESC LIMIT 1) AS e_cons_t1,
(SELECT value FROM samples WHERE key='energy_consumed_tariff_2'
ORDER BY ts DESC LIMIT 1) AS e_cons_t2,
(SELECT value FROM samples WHERE key='energy_produced_tariff_1'
ORDER BY ts DESC LIMIT 1) AS e_prod_t1,
(SELECT value FROM samples WHERE key='energy_produced_tariff_2'
ORDER BY ts DESC LIMIT 1) AS e_prod_t2
"""
)
count = await con.fetchval("SELECT count(*) FROM events")
labelled = await con.fetchval("SELECT count(*) FROM events WHERE appliance IS NOT NULL")
return {
"device": core.METER_HOSTS[0],
"latest": {r["key"]: {"value": r["value"], "ts": r["ts"].isoformat()} for r in latest},
"counters": dict(today) if today else {},
"events": {"total": count, "labelled": labelled},
}
@app.get("/api/power", dependencies=[Depends(auth)])
async def power(hours: float = Query(6, ge=0.05, le=24 * 14),
key: str = Query(core.POWER_KEY)):
"""Downsampled series from the 1-minute continuous aggregate."""
if hours <= 3:
src = "samples"
bucket = "30 seconds"
else:
src = "samples_1m"
bucket = "1 minute"
window = min(hours, 6) if src == "samples" else hours
async with _pool.acquire() as con:
rows = await con.fetch(
f"""
SELECT time_bucket($1::interval, ts) AS bucket,
avg(value) AS avg, max(value) AS max, min(value) AS min
FROM {src}
WHERE key = $2 AND device = $3 AND ts > now() - ($4::float * interval '1 hour')
GROUP BY bucket ORDER BY bucket
""",
bucket, key, core.METER_HOSTS[0], window,
)
return {
"key": key,
"bucket": bucket,
"points": [
{"t": r["bucket"].isoformat(), "avg": r["avg"], "max": r["max"], "min": r["min"]}
for r in rows
],
}
@app.get("/api/events", dependencies=[Depends(auth)])
async def events(limit: int = Query(100, ge=1, le=1000), unlabelled: bool = False):
q = "SELECT * FROM events"
if unlabelled:
q += " WHERE appliance IS NULL"
q += " ORDER BY ts_start DESC LIMIT $1"
async with _pool.acquire() as con:
rows = await con.fetch(q, limit)
apps = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name")
return {
"events": [dict(r) for r in rows],
"appliances": [dict(a) for a in apps],
}
@app.post("/api/events/{event_id}/label", dependencies=[Depends(auth)])
async def label(event_id: int, appliance: str = Query(...), notes: str | None = None):
async with _pool.acquire() as con:
ok = await con.fetchval(
"SELECT 1 FROM appliances WHERE name = $1", appliance
)
if not ok:
raise HTTPException(400, f"unknown appliance {appliance!r}")
row = await con.fetchrow(
"""
UPDATE events
SET appliance = $2, label_source = 'manual', labeled_at = now(),
notes = COALESCE($3, notes)
WHERE id = $1
RETURNING id, appliance, labeled_at
""",
event_id, appliance, notes,
)
if row is None:
raise HTTPException(404, "no such event")
return dict(row)
@app.get("/api/events/{event_id}/window", dependencies=[Depends(auth)])
async def event_window(event_id: int, pad_s: float = 60):
"""Raw 1 Hz trace around an event - the training input for that label."""
async with _pool.acquire() as con:
ev = await con.fetchrow("SELECT * FROM events WHERE id = $1", event_id)
if ev is None:
raise HTTPException(404, "no such event")
rows = await con.fetch(
"""
SELECT ts, value FROM samples
WHERE key = $1 AND device = $2
AND ts BETWEEN $3 - ($4::float * interval '1 second')
AND COALESCE($5, $3) + ($4::float * interval '1 second')
ORDER BY ts
""",
core.POWER_KEY, core.METER_HOSTS[0], ev["ts_start"], pad_s, ev["ts_end"],
)
return {
"event": dict(ev),
"samples": [{"t": r["ts"].isoformat(), "w": r["value"]} for r in rows],
}
@app.get("/api/export.csv", dependencies=[Depends(auth)])
async def export_csv(days: float = Query(30, ge=0.1, le=3650)):
"""Labelled events, one row each - the training set index."""
async with _pool.acquire() as con:
rows = await con.fetch(
"""
SELECT id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w,
duration_s, energy_wh, appliance, label_source
FROM events
WHERE ts_start > now() - ($1::float * interval '1 day')
ORDER BY ts_start
""",
days,
)
buf = io.StringIO()
if rows:
w = csv.DictWriter(buf, fieldnames=list(rows[0].keys()))
w.writeheader()
for r in rows:
w.writerow(dict(r))
buf.seek(0)
return StreamingResponse(
iter([buf.getvalue()]),
media_type="text/csv",
headers={"Content-Disposition": "attachment; filename=loadwatch_events.csv"},
)
@app.get("/api/appliances", dependencies=[Depends(auth)])
async def appliances():
async with _pool.acquire() as con:
rows = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name")
return [dict(r) for r in rows]
+93
View File
@@ -0,0 +1,93 @@
"""Collector: ingest the P1 stream, store it, detect load episodes."""
from __future__ import annotations
import asyncio
import logging
import signal
import time
from . import core
from .detector import EventDetector
from .meter import MeterHub
log = logging.getLogger("loadwatch.collector")
async def amain() -> None:
core.setup_logging()
pool = await core.make_pool()
log.info("db pool ready")
stop = asyncio.Event()
loop = asyncio.get_running_loop()
for sig in (signal.SIGTERM, signal.SIGINT):
try:
loop.add_signal_handler(sig, stop.set)
except NotImplementedError:
pass
hub = MeterHub(core.METER_HOSTS, core.METER_PORT, core.METER_NOISE_PSK)
detector = EventDetector()
log.info(
"detector: threshold=%.0fW min_duration=%.0fs close_band=%.0fW hold=%.0fs",
detector.threshold_w, detector.min_duration_s, detector.close_band_w,
detector.close_hold_s,
)
# everything is stored; the aggregate power also feeds the detector
pending_events: list = []
def hook(device: str, key: str, value: float) -> None:
if device == core.METER_HOSTS[0] and key == core.POWER_KEY:
ev = detector.add(time.time(), value)
if ev is not None and ev.duration_s >= detector.min_duration_s:
pending_events.append(ev)
hub.hook = hook
async def flusher() -> None:
while not stop.is_set():
try:
await asyncio.sleep(core.FLUSH_INTERVAL_S)
rows, hub.buffer = hub.buffer, []
await core.insert_samples(pool, rows)
while pending_events:
ev = pending_events.pop(0)
eid = await core.insert_event(pool, ev)
log.info(
"event #%s %s %.0fW dur=%.0fs energy=%.1fWh",
eid, "ON" if ev.direction > 0 else "OFF", ev.delta_w,
ev.duration_s, ev.energy_wh,
)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
log.warning("flush failed: %s", exc)
tasks = [
asyncio.create_task(hub.run(stop), name="meter"),
asyncio.create_task(flusher(), name="flusher"),
asyncio.create_task(stop.wait(), name="stop"),
]
done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
for t in pending:
t.cancel()
await asyncio.gather(*pending, return_exceptions=True)
for t in done:
exc = t.exception()
if exc:
log.error("task %s died: %s", t.get_name(), exc)
# final flush
rows, hub.buffer = hub.buffer, []
await core.insert_samples(pool, rows)
await pool.close()
log.info("collector stopped")
def main() -> None:
asyncio.run(amain())
if __name__ == "__main__":
main()
+105
View File
@@ -0,0 +1,105 @@
"""Config + database helpers."""
from __future__ import annotations
import logging
import os
from datetime import datetime, timezone
import asyncpg
log = logging.getLogger("loadwatch")
def env_str(name: str, default: str) -> str:
v = os.environ.get(name)
return v if v not in (None, "") else default
def env_float(name: str, default: float) -> float:
try:
return float(os.environ[name])
except (KeyError, TypeError, ValueError):
return default
DATABASE_URL = env_str(
"DATABASE_URL", "postgresql://loadwatch:loadwatch@db:5432/loadwatch"
)
# --- meter / sources ------------------------------------------------------
METER_HOSTS = [h.strip() for h in env_str("METER_HOSTS", "192.168.178.213").split(",") if h.strip()]
METER_PORT = int(env_float("METER_PORT", 6053))
METER_NOISE_PSK = env_str("METER_NOISE_PSK", "")
# ESPHome object_id -> our column/key name. Matched by object_id first, then by
# suffix (the Zuidwijk firmware keeps the plain dsmr names).
WATCH = {
"power_consumed_phase_1": "power_consumed_phase_1",
"power_produced_phase_1": "power_produced_phase_1",
"voltage_phase_1": "voltage_phase_1",
"current_phase_1": "current_phase_1",
"energy_consumed_tariff_1": "energy_consumed_tariff_1",
"energy_consumed_tariff_2": "energy_consumed_tariff_2",
"energy_produced_tariff_1": "energy_produced_tariff_1",
"energy_produced_tariff_2": "energy_produced_tariff_2",
"gas_consumed": "gas_consumed",
}
POWER_KEY = "power_consumed_phase_1" # the aggregate signal the detector watches
# --- detection ------------------------------------------------------------
EVENT_THRESHOLD_W = env_float("EVENT_THRESHOLD_W", 80.0)
EVENT_MIN_DURATION_S = env_float("EVENT_MIN_DURATION_S", 3.0)
EVENT_CLOSE_BAND_W = env_float("EVENT_CLOSE_BAND_W", 40.0)
EVENT_CLOSE_HOLD_S = env_float("EVENT_CLOSE_HOLD_S", 3.0)
EVENT_MAX_S = env_float("EVENT_MAX_S", 4 * 3600.0)
API_TOKEN = env_str("API_TOKEN", "")
FLUSH_INTERVAL_S = env_float("FLUSH_INTERVAL_S", 5.0)
BASELINE_WINDOW_S = env_float("BASELINE_WINDOW_S", 60.0)
def setup_logging() -> None:
logging.basicConfig(
level=os.environ.get("LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
)
async def make_pool() -> asyncpg.Pool:
return await asyncpg.create_pool(DATABASE_URL, min_size=1, max_size=4)
async def insert_samples(pool: asyncpg.Pool, rows: list[tuple]) -> None:
"""rows: (ts, device, key, value). ts may be an epoch float or a datetime."""
if not rows:
return
norm = [
(
ts if not isinstance(ts, (int, float))
else datetime.fromtimestamp(float(ts), tz=timezone.utc),
device,
key,
float(value),
)
for ts, device, key, value in rows
]
async with pool.acquire() as con:
await con.executemany(
"INSERT INTO samples (ts, device, key, value) VALUES ($1, $2, $3, $4)", norm
)
async def insert_event(pool: asyncpg.Pool, ev) -> int:
async with pool.acquire() as con:
return await con.fetchval(
"""
INSERT INTO events (ts_start, ts_end, direction, delta_w, peak_w,
baseline_w, duration_s, energy_wh)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING id
""",
ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w,
ev.baseline_w, ev.duration_s, ev.energy_wh,
)
+152
View File
@@ -0,0 +1,152 @@
"""Step-change / episode detection on a 1 Hz real-power stream.
Model: the house has a running baseline (everything always-on). A load *episode*
starts when instantaneous power steps away from that baseline by more than
`threshold_w` and stays there for `min_duration_s`; it ends when power returns
within `close_band_w` of the baseline and holds for `close_hold_s`.
The baseline is frozen while an episode is open, so a second load switching on
mid-episode does not corrupt the first one's delta.
"""
from __future__ import annotations
import statistics
from collections import deque
from dataclasses import dataclass
from . import core
@dataclass
class Event:
ts_start: float
ts_end: float | None = None
direction: int = 1
delta_w: float = 0.0
peak_w: float = 0.0
baseline_w: float = 0.0
duration_s: float = 0.0
energy_wh: float = 0.0
# helpers for the DB layer (epoch -> aware datetime)
@property
def ts_start_dt(self):
from datetime import datetime, timezone
return datetime.fromtimestamp(self.ts_start, tz=timezone.utc)
@property
def ts_end_dt(self):
from datetime import datetime, timezone
if self.ts_end is None:
return None
return datetime.fromtimestamp(self.ts_end, tz=timezone.utc)
class EventDetector:
def __init__(self, threshold_w=None, min_duration_s=None, close_band_w=None,
close_hold_s=None, baseline_window_s=None, max_event_s=None):
self.threshold_w = core.EVENT_THRESHOLD_W if threshold_w is None else threshold_w
self.min_duration_s = core.EVENT_MIN_DURATION_S if min_duration_s is None else min_duration_s
self.close_band_w = core.EVENT_CLOSE_BAND_W if close_band_w is None else close_band_w
self.close_hold_s = core.EVENT_CLOSE_HOLD_S if close_hold_s is None else close_hold_s
self.baseline_window_s = core.BASELINE_WINDOW_S if baseline_window_s is None else baseline_window_s
self.max_event_s = core.EVENT_MAX_S if max_event_s is None else max_event_s
self._hist: deque[tuple[float, float]] = deque() # (ts, w) since last event
self._cand: tuple[float, float] | None = None # candidate step
self._ev: Event | None = None
self._back_since: float | None = None
self._last_ts: float | None = None
self._energy_acc = 0.0
# ------------------------------------------------------------------ util
def _baseline(self) -> float | None:
if not self._hist:
return None
cutoff = (self._last_ts or 0.0) - self.baseline_window_s
window = [w for ts, w in self._hist if ts >= cutoff]
if len(window) < 3:
return None
return statistics.median(window)
# ------------------------------------------------------------------- fed
def add(self, ts: float, w: float) -> Event | None:
"""Feed one sample. Returns a completed Event when one closes."""
self._last_ts = ts
completed: Event | None = None
if self._ev is None:
base = self._baseline()
self._hist.append((ts, w))
# keep memory bounded
if len(self._hist) > 10000:
self._hist.popleft()
if base is not None:
if abs(w - base) > self.threshold_w:
if self._cand is None:
self._cand = (ts, w)
elif ts - self._cand[0] >= self.min_duration_s:
# episode opens retroactively at the candidate time
delta = w - base
self._ev = Event(
ts_start=self._cand[0],
direction=1 if delta > 0 else -1,
delta_w=delta,
peak_w=w,
baseline_w=base,
)
self._energy_acc = 0.0
self._back_since = None
self._cand = None
self._hist.clear()
self._hist.append((ts, w))
else:
self._cand = None
else:
ev = self._ev
dev = w - ev.baseline_w
if abs(w) > abs(ev.peak_w):
ev.peak_w = w
# energy above baseline, trapezoid-ish (1 Hz => dt of one step)
dt = max(0.0, ts - (self._last_ts_prev or ts))
self._energy_acc += max(0.0, dev) * dt / 3600.0
self._hist.append((ts, w))
if abs(dev) < self.close_band_w:
if self._back_since is None:
self._back_since = ts
elif ts - self._back_since >= self.close_hold_s:
ev.ts_end = self._back_since
ev.duration_s = ev.ts_end - ev.ts_start
ev.energy_wh = self._energy_acc
completed = ev
self._ev = None
self._back_since = None
self._cand = None
self._hist.clear()
else:
self._back_since = None
if self._ev is not None and ts - self._ev.ts_start >= self.max_event_s:
self._ev.ts_end = ts
self._ev.duration_s = ts - self._ev.ts_start
self._ev.energy_wh = self._energy_acc
completed = self._ev
self._ev = None
self._back_since = None
self._hist.clear()
self._last_ts_prev = ts
return completed
# for the live status endpoint
@property
def active(self) -> Event | None:
return self._ev
_last_ts_prev: float | None = None
def to_rows(self, ev: Event) -> tuple:
return (ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w,
ev.baseline_w, ev.duration_s, ev.energy_wh)
+148
View File
@@ -0,0 +1,148 @@
"""ESPHome native-API ingestion.
One ESPHome device = one source. Subscribes to the sensor states we care about and
hands (ts, device, key, value) tuples to a callback. Reconnects with backoff.
"""
from __future__ import annotations
import asyncio
import inspect
import logging
import time
from aioesphomeapi import APIClient
from . import core
log = logging.getLogger("loadwatch.meter")
class MeterSource:
def __init__(self, host: str, port: int, noise_psk: str, on_sample):
self.host = host
self.port = port
self.noise_psk = noise_psk or None
self.on_sample = on_sample
self._keymap: dict[int, str] = {}
self._cli: APIClient | None = None
# -- state callback ----------------------------------------------------
def _handle_state(self, state) -> None:
name = self._keymap.get(getattr(state, "key", None))
if name is None:
return
value = getattr(state, "state", None)
if getattr(state, "missing_state", False) or value is None:
log.debug("%s: %s missing", self.host, name)
return
try:
self.on_sample(self.host, name, float(value))
except (TypeError, ValueError):
log.warning("%s: non-numeric %s=%r", self.host, name, value)
# -- discovery ---------------------------------------------------------
async def _resolve_entities(self) -> list[int]:
assert self._cli is not None
entities, _services = await self._cli.list_entities_services()
available: dict[str, int] = {}
for ent in entities:
oid = getattr(ent, "object_id", None)
key = getattr(ent, "key", None)
if oid and key is not None:
available[oid] = key
wanted: list[int] = []
for want in core.WATCH:
key = available.get(want)
if key is None: # tolerate firmware variants: match by suffix
for oid, k in available.items():
if oid.endswith(want):
key = k
break
if key is None:
continue
self._keymap[key] = want
wanted.append(key)
missing = sorted(set(core.WATCH) - set(self._keymap.values()))
log.info(
"%s: watching %d/%d entities; missing=%s",
self.host, len(wanted), len(core.WATCH), missing or "none",
)
if missing:
# Self-diagnosing: show the caller every object_id this device exposes.
log.info("%s: available object_ids: %s", self.host, sorted(available))
return wanted
# -- run ---------------------------------------------------------------
async def run(self, stop: asyncio.Event) -> None:
backoff = 5
while not stop.is_set():
try:
self._cli = APIClient(
self.host, self.port, password="", noise_psk=self.noise_psk
)
log.info("%s: connecting to ESPHome API %s:%s", self.host, self.host, self.port)
await self._cli.connect(login=True)
log.info("%s: connected", self.host)
backoff = 5
wanted = await self._resolve_entities()
if not wanted:
raise RuntimeError("none of the watched entities exist on this device")
# aioesphomeapi has shipped both sync and async variants of this
res = self._cli.subscribe_states(self._handle_state)
if inspect.isawaitable(res):
await res
log.info("%s: subscribed to %d state keys", self.host, len(wanted))
while not stop.is_set():
await asyncio.sleep(1)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - stay alive on any transport error
log.warning("%s: connection problem: %s (retry in %ss)", self.host, exc, backoff)
await asyncio.sleep(backoff)
backoff = min(backoff * 2, 120)
finally:
if self._cli is not None:
try:
await self._cli.disconnect()
except Exception: # noqa: BLE001
pass
self._cli = None
log.info("%s: stopped", self.host)
class MeterHub:
"""Collects samples from every configured source into one flush buffer."""
def __init__(self, hosts: list[str], port: int, noise_psk: str):
self.hosts = hosts
self.port = port
self.noise_psk = noise_psk
self.buffer: list[tuple] = []
self.latest: dict[str, float] = {}
self._last: dict[str, float] = {}
# optional extra consumer (the detector), set after construction.
# Kept as an attribute and routed through _dispatch so assigning it
# later actually takes effect (sources hold _dispatch, not the hook).
self.hook = None
self.sources = [
MeterSource(h, port, noise_psk, self._dispatch) for h in hosts
]
def _dispatch(self, device: str, key: str, value: float) -> None:
self._on_sample(device, key, value)
if self.hook is not None:
self.hook(device, key, value)
def _on_sample(self, device: str, key: str, value: float) -> None:
now = time.time()
self.buffer.append((now, device, key, value))
if device == self.hosts[0]:
self.latest[key] = value
self._last[key] = now
async def run(self, stop: asyncio.Event) -> None:
await asyncio.gather(*(s.run(stop) for s in self.sources))
+191
View File
@@ -0,0 +1,191 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>loadwatch</title>
<style>
:root { --bg:#0f1115; --card:#171a21; --fg:#e8eaf0; --mut:#8b93a7; --acc:#4da3ff; --ok:#39d98a; --warn:#ffb454; }
* { box-sizing:border-box; }
body { margin:0; background:var(--bg); color:var(--fg); font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace; }
header { display:flex; gap:12px; align-items:center; padding:12px 16px; border-bottom:1px solid #23262f; flex-wrap:wrap; }
h1 { font-size:16px; margin:0 12px 0 0; letter-spacing:.5px; }
.mut { color:var(--mut); }
input,select,button { background:#0b0d12; color:var(--fg); border:1px solid #2a2f3a; border-radius:6px; padding:6px 8px; font:inherit; }
button { cursor:pointer; border-color:#31415a; }
button:hover { border-color:var(--acc); }
main { display:grid; grid-template-columns:1fr; gap:16px; padding:16px; max-width:1200px; }
.row { display:grid; grid-template-columns:repeat(auto-fit,minmax(150px,1fr)); gap:12px; }
.card { background:var(--card); border:1px solid #23262f; border-radius:10px; padding:14px; }
.k { color:var(--mut); font-size:12px; text-transform:uppercase; letter-spacing:.6px; }
.v { font-size:26px; margin-top:4px; }
canvas { width:100%; height:220px; display:block; }
table { width:100%; border-collapse:collapse; font-size:13px; }
th,td { text-align:left; padding:6px 8px; border-bottom:1px solid #23262f; white-space:nowrap; }
th { color:var(--mut); font-weight:600; }
tr.done td { color:var(--mut); }
.pos { color:var(--warn); } .neg { color:var(--ok); }
.pill { display:inline-block; padding:1px 7px; border-radius:99px; background:#222833; font-size:12px; }
</style>
</head>
<body>
<header>
<h1>loadwatch</h1>
<span class="mut" id="conn">connecting…</span>
<span class="mut">token</span><input id="token" size="18" placeholder="API token" type="password">
<button onclick="saveToken()">save</button>
<span class="mut" id="device"></span>
<span style="margin-left:auto"></span>
<a class="mut" id="export" href="#" download>export csv</a>
</header>
<main>
<div class="row">
<div class="card"><div class="k">now</div><div class="v" id="now">–<span class="mut" style="font-size:14px"> W</span></div></div>
<div class="card"><div class="k">import today</div><div class="v" id="imp">–</div></div>
<div class="card"><div class="k">export today</div><div class="v" id="exp">–</div></div>
<div class="card"><div class="k">gas</div><div class="v" id="gas">–</div></div>
<div class="card"><div class="k">events / labelled</div><div class="v" id="evc">–</div></div>
</div>
<div class="card">
<div class="k" style="margin-bottom:8px">aggregate power (W) — <span id="rangeLbl">6 h</span></div>
<div class="row" style="margin-bottom:10px">
<button onclick="loadPower(1)">1 h</button><button onclick="loadPower(6)">6 h</button>
<button onclick="loadPower(24)">24 h</button><button onclick="loadPower(168)">7 d</button>
<button onclick="showEvents()">show detected events</button>
</div>
<canvas id="chart" width="1100" height="220"></canvas>
</div>
<div class="card">
<div class="k" style="margin-bottom:8px">load episodes — label them (this is the dataset)</div>
<table><thead><tr>
<th>#</th><th>start</th><th>ΔW</th><th>peak</th><th>dur</th><th>Wh</th><th>label</th><th></th>
</tr></thead><tbody id="evbody"><tr><td colspan="8" class="mut">loading…</td></tr></tbody></table>
</div>
</main>
<script>
// injected by the API: "" standalone, "/loadwatch" behind the portal sub-path
const P = '__PREFIX__';
let TOKEN = localStorage.getItem('lw_token') || '';
document.getElementById('token').value = TOKEN;
let APPLIANCES = [];
function saveToken(){ TOKEN = document.getElementById('token').value.trim(); localStorage.setItem('lw_token', TOKEN); refresh(); }
async function api(path, opts){
opts = opts || {}; opts.headers = Object.assign({'Authorization':'Bearer '+TOKEN}, opts.headers||{});
const r = await fetch(P + path, opts);
if(!r.ok) throw new Error(path+' -> '+r.status);
return r.json();
}
const fmt = n => (n===null||n===undefined) ? '–' : n.toLocaleString(undefined,{maximumFractionDigits:1});
async function refresh(){
try {
const s = await api('/api/status');
document.getElementById('conn').textContent = 'live';
document.getElementById('conn').style.color = 'var(--ok)';
document.getElementById('device').textContent = s.device;
const L = s.latest || {};
document.getElementById('now').innerHTML = fmt(L['power_consumed_phase_1']?.value) + '<span class="mut" style="font-size:14px"> W</span>';
const c = s.counters || {};
document.getElementById('imp').textContent = fmt(c.e_cons_t1) + ' kWh';
document.getElementById('exp').textContent = fmt(c.e_prod_t1) + ' kWh';
document.getElementById('gas').textContent = fmt(L['gas_consumed']?.value) + ' m³';
document.getElementById('evc').textContent = s.events.total + ' / ' + s.events.labelled;
} catch(e){
document.getElementById('conn').textContent = 'error: ' + e.message;
document.getElementById('conn').style.color = '#ff6b6b';
}
}
async function loadPower(hours){
document.getElementById('rangeLbl').textContent = hours >= 24 ? (hours/24)+' d' : hours+' h';
const d = await api('/api/power?hours='+hours);
drawSeries(d.points.map(p => p.max), d.points.map(p => p.avg));
}
async function showEvents(){
const d = await api('/api/power?hours=168');
const ev = await api('/api/events?limit=200');
const t0 = new Date(d.points[0].t).getTime(), t1 = new Date(d.points[d.points.length-1].t).getTime();
const markers = ev.events.map(e => ({t:new Date(e.ts_start).getTime(), dw:e.delta_w}));
drawSeries(d.points.map(p=>p.max), d.points.map(p=>p.avg), {t0,t1,markers});
}
function drawSeries(maxs, avgs, opts){
const c = document.getElementById('chart'), ctx = c.getContext('2d');
const W = c.width, H = c.height, pad = 28;
ctx.clearRect(0,0,W,H);
const all = maxs.concat(avgs).filter(v => v!=null);
if(!all.length){ ctx.fillStyle='#8b93a7'; ctx.fillText('no data yet', pad, H/2); return; }
const hi = Math.max.apply(null, all), lo = Math.min.apply(null, all);
const X = i => pad + i*(W-2*pad)/Math.max(1,maxs.length-1);
const Y = v => H-pad - (v-lo)*(H-2*pad)/Math.max(1,(hi-lo));
ctx.strokeStyle = '#23262f'; ctx.beginPath(); ctx.moveTo(pad,H-pad); ctx.lineTo(W-pad,H-pad); ctx.stroke();
ctx.fillStyle = '#8b93a7'; ctx.fillText(Math.round(hi)+' W', 4, pad); ctx.fillText(Math.round(lo)+' W', 4, H-pad);
// max envelope
ctx.strokeStyle = '#2c4a6e'; ctx.beginPath();
maxs.forEach((v,i) => v==null?0:(i?ctx.lineTo(X(i),Y(v)):ctx.moveTo(X(i),Y(v)))); ctx.stroke();
// avg line
ctx.strokeStyle = '#4da3ff'; ctx.lineWidth = 2; ctx.beginPath();
avgs.forEach((v,i) => v==null?0:(i?ctx.lineTo(X(i),Y(v)):ctx.moveTo(X(i),Y(v)))); ctx.stroke(); ctx.lineWidth = 1;
if(opts && opts.markers){
ctx.fillStyle = '#ffb454';
opts.markers.forEach(m => {
if(m.t < opts.t0 || m.t > opts.t1) return;
const i = Math.round((m.t-opts.t0)/Math.max(1,(opts.t1-opts.t0))*(maxs.length-1));
ctx.fillRect(X(i)-1, pad, 2, H-2*pad);
});
}
}
async function loadEvents(){
const d = await api('/api/events?limit=100');
APPLIANCES = d.appliances.filter(a => a.is_large !== false).map(a => a.name);
const b = document.getElementById('evbody');
if(!d.events.length){ b.innerHTML = '<tr><td colspan="8" class="mut">no events yet — they appear as loads switch</td></tr>'; return; }
b.innerHTML = d.events.map(e => {
const opts = ['<option value="">—</option>'].concat(APPLIANCES.map(a =>
'<option'+(a===e.appliance?' selected':'')+'>'+a+'</option>')).join('');
return '<tr class="'+(e.appliance?'done':'')+'">'
+ '<td>'+e.id+'</td>'
+ '<td>'+new Date(e.ts_start).toLocaleTimeString()+'<span class="mut"> '+new Date(e.ts_start).toLocaleDateString()+'</span></td>'
+ '<td class="'+((e.delta_w||0)>0?'pos':'neg')+'">'+Math.round(e.delta_w)+'</td>'
+ '<td>'+Math.round(e.peak_w)+'</td>'
+ '<td>'+(e.duration_s==null?'open':Math.round(e.duration_s)+'s')+'</td>'
+ '<td>'+fmt(e.energy_wh)+'</td>'
+ '<td><select onchange="label('+e.id+', this.value)">'+opts+'</select>'
+ (e.label_source?' <span class="pill">'+e.label_source+'</span>':'')+'</td>'
+ '<td><a class="mut" href="#" onclick="dump('+e.id+');return false">trace</a></td>'
+ '</tr>';
}).join('');
}
async function label(id, appliance){
if(!appliance) return;
await api('/api/events/'+id+'/label?appliance='+encodeURIComponent(appliance), {method:'POST'});
loadEvents(); refresh();
}
async function dump(id){
const d = await api('/api/events/'+id+'/window');
const lines = d.samples.map(s => s.t+' '+s.w);
const blob = new Blob([lines.join('\n')], {type:'text/plain'});
const a = document.createElement('a');
a.href = URL.createObjectURL(blob); a.download = 'event_'+id+'.txt'; a.click();
}
function updateExport(){ document.getElementById('export').href = P + '/api/export.csv?days=30'; }
(async function init(){
updateExport();
await refresh(); await loadEvents(); await loadPower(6);
setInterval(refresh, 5000);
setInterval(() => { loadEvents(); updateExport(); }, 60000);
})();
</script>
</body>
</html>
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# loadwatch healthcheck — always exits 0, prints exactly ONE line.
# OK <age>s -> healthy, newest sample <age> seconds old
# FAIL <reason> -> something is wrong
# Intended to be run on the runtime host (dev CT141), e.g. over SSH from a watchdog.
set -u
cd "$(dirname "$0")/.." || { echo "FAIL cannot cd to project dir"; exit 0; }
ps_out=$(docker compose ps --format '{{.Service}}={{.State}}' 2>/dev/null | tr '\n' ' ')
if [ -z "$ps_out" ]; then
echo "FAIL docker compose reports no services (stack down?)"
exit 0
fi
for svc in db collector api; do
case " $ps_out " in
*" $svc=running "*) : ;;
*) echo "FAIL service $svc is not running (states: $ps_out)"; exit 0 ;;
esac
done
age=$(docker compose exec -T db psql -U loadwatch -d loadwatch -t -A -c \
"select coalesce(extract(epoch from (now()-max(ts)))::int,-1) from samples;" 2>/dev/null | tr -d '[:space:]')
case "${age:-x}" in
''|*[!0-9-]*) echo "FAIL sample-age query failed (raw='${age:-empty}')" ;;
-1) echo "FAIL no samples in the database" ;;
*) if [ "$age" -gt 300 ]; then
echo "FAIL newest sample is ${age}s old (threshold 300s)"
else
echo "OK ${age}s"
fi ;;
esac
exit 0
+44
View File
@@ -0,0 +1,44 @@
# loadwatch — Tier-0 NILM dataset service (runs on dev CT141, 192.168.178.20:8099)
#
# Public URL: https://xm2561.duckdns.org/loadwatch
#
# IMPORTANT: this host's apex (xm2561.duckdns.org) is Headscale's coordination
# endpoint (see headscale.yml). That router is a bare `Host()` catch-all and is
# deliberately left untouched. This router is strictly longer
# (`Host() && PathPrefix()`), so Traefik's length-based priority gives it the
# /loadwatch prefix only — /ts2021, /derp, /key, /register, /api/v1/* and
# /health keep flowing to Headscale.
#
# StripPrefix: the container serves "/" and stays standalone-clean; Traefik
# strips /loadwatch and (v2.5+) sets X-Forwarded-Prefix, which the app uses to
# emit prefixed asset/API URLs for the browser.
http:
routers:
loadwatch-http:
rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)"
entryPoints:
- http
service: loadwatch
middlewares:
- redirect-to-https
loadwatch:
rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)"
entryPoints:
- https
tls:
certResolver: letsencrypt
service: loadwatch
middlewares:
- loadwatch-strip
middlewares:
loadwatch-strip:
stripPrefix:
prefixes:
- "/loadwatch"
services:
loadwatch:
loadBalancer:
servers:
- url: "http://192.168.178.20:8099"
+52
View File
@@ -0,0 +1,52 @@
name: loadwatch
services:
db:
image: timescale/timescaledb:latest-pg16
restart: unless-stopped
environment:
POSTGRES_USER: loadwatch
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-loadwatch}
POSTGRES_DB: loadwatch
volumes:
- dbdata:/var/lib/postgresql/data
- ./sql:/docker-entrypoint-initdb.d:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U loadwatch -d loadwatch"]
interval: 10s
timeout: 5s
retries: 12
collector:
build: .
restart: unless-stopped
depends_on:
db:
condition: service_healthy
environment:
DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch
METER_HOSTS: ${METER_HOSTS:-192.168.178.213}
METER_PORT: ${METER_PORT:-6053}
METER_NOISE_PSK: ${METER_NOISE_PSK:-}
EVENT_THRESHOLD_W: ${EVENT_THRESHOLD_W:-80}
EVENT_MIN_DURATION_S: ${EVENT_MIN_DURATION_S:-3}
EVENT_CLOSE_BAND_W: ${EVENT_CLOSE_BAND_W:-40}
EVENT_CLOSE_HOLD_S: ${EVENT_CLOSE_HOLD_S:-3}
EVENT_MAX_S: ${EVENT_MAX_S:-14400}
command: ["python", "-m", "app.collector"]
api:
build: .
restart: unless-stopped
depends_on:
db:
condition: service_healthy
environment:
DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch
API_TOKEN: ${API_TOKEN:-loadwatch-dev}
ports:
- "${API_PORT:-8099}:8000"
command: ["uvicorn", "app.api:app", "--host", "0.0.0.0", "--port", "8000"]
volumes:
dbdata:
+4
View File
@@ -0,0 +1,4 @@
aioesphomeapi>=24.0
asyncpg>=0.29
fastapi>=0.110
uvicorn[standard]>=0.29
+90
View File
@@ -0,0 +1,90 @@
-- loadwatch schema v1
CREATE EXTENSION IF NOT EXISTS timescaledb;
-- Raw 1 Hz samples. Narrow by design: extra sources (smart plug, later a second
-- meter) are just another (device, key) pair - no schema change.
CREATE TABLE IF NOT EXISTS samples (
ts timestamptz NOT NULL,
device text NOT NULL,
key text NOT NULL,
value double precision NOT NULL
);
SELECT create_hypertable('samples', 'ts', chunk_time_interval => interval '1 day',
if_not_exists => TRUE);
CREATE INDEX IF NOT EXISTS samples_device_key_ts_idx ON samples (device, key, ts DESC);
-- 1-minute rollup for charts and long-term retention.
CREATE MATERIALIZED VIEW IF NOT EXISTS samples_1m
WITH (timescaledb.continuous) AS
SELECT time_bucket('1 minute', ts) AS bucket,
device,
key,
avg(value) AS avg,
min(value) AS min,
max(value) AS max,
last(value, ts) AS last
FROM samples
GROUP BY bucket, device, key;
SELECT add_continuous_aggregate_policy('samples_1m',
start_offset => interval '3 hours',
end_offset => interval '1 minute',
schedule_interval => interval '1 minute',
if_not_exists => TRUE);
-- Compress raw samples after 7 days (keeps the full 1 Hz history, cheaply).
ALTER TABLE samples SET (timescaledb.compress,
timescaledb.compress_segmentby = 'device,key',
timescaledb.compress_orderby = 'ts DESC');
SELECT add_compression_policy('samples', interval '7 days', if_not_exists => TRUE);
-- Load episodes detected from the aggregate power stream.
CREATE TABLE IF NOT EXISTS events (
id bigserial PRIMARY KEY,
ts_start timestamptz NOT NULL,
ts_end timestamptz,
direction smallint NOT NULL, -- +1 = load turned on, -1 = off
delta_w double precision, -- the step size
peak_w double precision, -- peak power during the episode
baseline_w double precision, -- baseline the step came off
duration_s double precision,
energy_wh double precision, -- integral of (P - baseline) over the episode
appliance text, -- label: null until labelled
label_source text, -- manual | plug | model
labeled_at timestamptz,
notes text
);
CREATE INDEX IF NOT EXISTS events_ts_start_idx ON events (ts_start DESC);
CREATE INDEX IF NOT EXISTS events_appliance_idx ON events (appliance);
CREATE TABLE IF NOT EXISTS appliances (
name text PRIMARY KEY,
notes text,
is_large boolean NOT NULL DEFAULT TRUE
);
INSERT INTO appliances (name, notes) VALUES
('kettle', 'water kettle / boiler, ~2 kW step'),
('oven', 'electric oven, 2-3 kW, thermostatic cycling'),
('microwave', '~1.2 kW with ~50% duty cycle'),
('dishwasher', 'heater cycling ~1-2 kW'),
('washing_machine','heater ~2 kW, motor ~500 W'),
('dryer', '~2-3 kW heater cycling'),
('fridge', 'compressor ~80-150 W cycling'),
('freezer', 'compressor ~80-150 W cycling'),
('iron', '~1-2 kW, duty-cycled'),
('vacuum', '~1-1.5 kW steady'),
('water_boiler', 'hot water / boiler'),
('ev_charger', 'EV charge point'),
('heater', 'space heater'),
('unknown_large', 'clearly a big load, unidentifiable'),
('ignore', 'noise / not a real load')
ON CONFLICT (name) DO NOTHING;
CREATE TABLE IF NOT EXISTS meta (
k text PRIMARY KEY,
v text
);