loadwatch: 1 Hz P1 collector, event detector, labelling API/UI, sub-path routing, healthcheck
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
# Copy to .env and adjust.
|
||||
# Postgres / Timescale
|
||||
POSTGRES_PASSWORD=change-me-db
|
||||
|
||||
# API
|
||||
API_TOKEN=change-me-token
|
||||
API_PORT=8099
|
||||
|
||||
# Meter source(s): comma-separated hosts running the ESPHome native API.
|
||||
# Add the smart plug's host here later, e.g. 192.168.178.213,192.168.178.240
|
||||
METER_HOSTS=192.168.178.213
|
||||
METER_PORT=6053
|
||||
# ESPHome api encryption key (leave empty: this meter has none)
|
||||
METER_NOISE_PSK=
|
||||
|
||||
# Detection
|
||||
EVENT_THRESHOLD_W=80
|
||||
EVENT_MIN_DURATION_S=3
|
||||
EVENT_CLOSE_BAND_W=40
|
||||
EVENT_CLOSE_HOLD_S=3
|
||||
EVENT_MAX_S=14400
|
||||
@@ -0,0 +1,9 @@
|
||||
.env
|
||||
*.pyc
|
||||
__pycache__/
|
||||
.venv/
|
||||
venv/
|
||||
project_state.md
|
||||
*.csv
|
||||
*.parquet
|
||||
data/
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["python", "-m", "app.collector"]
|
||||
@@ -0,0 +1,92 @@
|
||||
# loadwatch — Tier-0 NILM dataset builder
|
||||
|
||||
Non-intrusive load monitoring, working from a **single Dutch DSMR-5 P1 stream at 1 Hz**.
|
||||
Goal: build a *labelled* dataset of large-load switching events, so a model can later
|
||||
identify which big appliance turned on.
|
||||
|
||||
## What this is
|
||||
|
||||
| Layer | What it does |
|
||||
|---|---|
|
||||
| `collector` | Subscribes to the SlimmeLezer+ **ESPHome native API** (1 Hz) and stores every sample |
|
||||
| `detector` | Step-change detection → **load episodes** (on-step, duration, energy, peak) |
|
||||
| `api` + UI | Live power, event list, **one-click labelling**, CSV export for training |
|
||||
| `db` | TimescaleDB (Postgres 16) — 1-min continuous aggregate, compression after 7 days |
|
||||
|
||||
No Home Assistant involvement on the ingest path (HA stays the notification shell).
|
||||
|
||||
## Architecture decision (2026-10-04)
|
||||
|
||||
Separate service on **dev CT141**, *not* inside HAOS:
|
||||
|
||||
- HA here is **Core-in-Docker** — custom Python services installed there die on every
|
||||
HA upgrade, and the recorder is not a dataset store (purges raw state ~10 days, writes
|
||||
on-change).
|
||||
- Ingest goes **straight to the meter** (`192.168.178.213:6053`, ESPHome API, no
|
||||
encryption) — no HA token, no secrets on this path.
|
||||
- Public URL via Coolify: **`loadwatch.xm2561.duckdns.org`** (the `xm2561` *apex* belongs
|
||||
to Headscale — never take it).
|
||||
|
||||
## Data model
|
||||
|
||||
```sql
|
||||
samples(ts timestamptz, device text, key text, value double precision) -- hypertable, 1 day chunks
|
||||
events(id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w, duration_s,
|
||||
energy_wh, appliance, label_source, labeled_at, notes)
|
||||
appliances(name, notes, is_large)
|
||||
```
|
||||
|
||||
`samples` is **narrow** (device, key) so additional sources — a smart plug, later a
|
||||
second meter — need no schema change.
|
||||
|
||||
## Run it (dev)
|
||||
|
||||
```bash
|
||||
cd /root/projects/loadwatch
|
||||
cp .env.example .env # set API_TOKEN
|
||||
docker compose up -d --build
|
||||
docker compose logs -f collector
|
||||
```
|
||||
|
||||
UI: `http://192.168.178.20:8099/` (token = `API_TOKEN`).
|
||||
|
||||
## Collector entity keys
|
||||
|
||||
The collector matches ESPHome `object_id`s, discovered at connect time and logged:
|
||||
|
||||
| object_id | meaning |
|
||||
|---|---|
|
||||
| `power_consumed_phase_1` | instantaneous import W |
|
||||
| `power_produced_phase_1` | instantaneous export W |
|
||||
| `voltage_phase_1`, `current_phase_1` | grid V / A |
|
||||
| `energy_consumed_tariff_1/2` | import kWh counters |
|
||||
| `energy_produced_tariff_1/2` | export kWh counters |
|
||||
| `gas_consumed` | gas m³ (M-Bus, ~5 min) |
|
||||
|
||||
If a key is missing the collector logs **every** object_id it can see — that is the
|
||||
discovery mechanism, not a guess.
|
||||
|
||||
## Adding the smart plug (ground truth)
|
||||
|
||||
Add its host to `METER_HOSTS` (comma-separated) — samples land with `device = <host>`.
|
||||
A plug on one appliance gives labelled ground truth for exactly that appliance, which is
|
||||
what a supervised model needs; the P1 event stream covers everything else.
|
||||
|
||||
## Export for training
|
||||
|
||||
`GET /api/export.csv?days=30` → one row per labelled event plus its raw power window
|
||||
(`GET /api/events/{id}/window` returns the ±60 s raw trace for that event).
|
||||
|
||||
## Detection defaults
|
||||
|
||||
`EVENT_THRESHOLD_W=80` (step size), `EVENT_MIN_DURATION_S=3`, close back within 40 W held
|
||||
3 s. Night baseload is ~50–200 W here; a kettle/oven/washer step is 1000–2500 W.
|
||||
Documented limitation: two loads stepping at the same instant inside one open episode
|
||||
are seen as one event.
|
||||
|
||||
## Roadmap
|
||||
|
||||
1. ✅ capture + detect + label (this repo)
|
||||
2. plug lane → labelled ground truth
|
||||
3. HA REST sensor → "big load started" notifications
|
||||
4. classifier trained on the labelled events (`trainer/`)
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
"""Loadwatch API + labelling UI."""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request
|
||||
from fastapi.responses import HTMLResponse, StreamingResponse
|
||||
|
||||
from . import core
|
||||
|
||||
log = logging.getLogger("loadwatch.api")
|
||||
STATIC = Path(__file__).parent / "static"
|
||||
|
||||
# When served behind a Traefik sub-path (https://host/loadwatch) with a
|
||||
# stripprefix middleware, the app still serves "/" - but every URL it EMITS must
|
||||
# carry the prefix, because the browser requests the prefixed path.
|
||||
# Traefik's StripPrefix sets X-Forwarded-Prefix, so the same image works both
|
||||
# standalone (no header) and behind the portal path.
|
||||
PREFIX = os.environ.get("PORTAL_PREFIX", "").rstrip("/")
|
||||
|
||||
_pool: asyncpg.Pool | None = None
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
global _pool
|
||||
core.setup_logging()
|
||||
_pool = await core.make_pool()
|
||||
log.info("api up")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if _pool is not None:
|
||||
await _pool.close()
|
||||
|
||||
|
||||
app = FastAPI(title="loadwatch", version="0.1.0", lifespan=lifespan)
|
||||
|
||||
|
||||
async def auth(authorization: str | None = Header(default=None)) -> None:
|
||||
token = core.API_TOKEN
|
||||
if not token:
|
||||
return
|
||||
if authorization != f"Bearer {token}":
|
||||
raise HTTPException(status_code=401, detail="bad or missing token")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- pages
|
||||
@app.get("/")
|
||||
async def index(request: Request) -> HTMLResponse:
|
||||
prefix = request.headers.get("x-forwarded-prefix", PREFIX).rstrip("/")
|
||||
html = (STATIC / "index.html").read_text(encoding="utf-8")
|
||||
return HTMLResponse(html.replace("__PREFIX__", prefix))
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------- api
|
||||
@app.get("/api/status", dependencies=[Depends(auth)])
|
||||
async def status():
|
||||
async with _pool.acquire() as con:
|
||||
latest = await con.fetch(
|
||||
"""
|
||||
SELECT DISTINCT ON (key) key, value, ts
|
||||
FROM samples WHERE device = $1 ORDER BY key, ts DESC
|
||||
""",
|
||||
core.METER_HOSTS[0],
|
||||
)
|
||||
today = await con.fetchrow(
|
||||
"""
|
||||
SELECT
|
||||
(SELECT value FROM samples WHERE key='energy_consumed_tariff_1'
|
||||
ORDER BY ts DESC LIMIT 1) AS e_cons_t1,
|
||||
(SELECT value FROM samples WHERE key='energy_consumed_tariff_2'
|
||||
ORDER BY ts DESC LIMIT 1) AS e_cons_t2,
|
||||
(SELECT value FROM samples WHERE key='energy_produced_tariff_1'
|
||||
ORDER BY ts DESC LIMIT 1) AS e_prod_t1,
|
||||
(SELECT value FROM samples WHERE key='energy_produced_tariff_2'
|
||||
ORDER BY ts DESC LIMIT 1) AS e_prod_t2
|
||||
"""
|
||||
)
|
||||
count = await con.fetchval("SELECT count(*) FROM events")
|
||||
labelled = await con.fetchval("SELECT count(*) FROM events WHERE appliance IS NOT NULL")
|
||||
return {
|
||||
"device": core.METER_HOSTS[0],
|
||||
"latest": {r["key"]: {"value": r["value"], "ts": r["ts"].isoformat()} for r in latest},
|
||||
"counters": dict(today) if today else {},
|
||||
"events": {"total": count, "labelled": labelled},
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/power", dependencies=[Depends(auth)])
|
||||
async def power(hours: float = Query(6, ge=0.05, le=24 * 14),
|
||||
key: str = Query(core.POWER_KEY)):
|
||||
"""Downsampled series from the 1-minute continuous aggregate."""
|
||||
if hours <= 3:
|
||||
src = "samples"
|
||||
bucket = "30 seconds"
|
||||
else:
|
||||
src = "samples_1m"
|
||||
bucket = "1 minute"
|
||||
window = min(hours, 6) if src == "samples" else hours
|
||||
async with _pool.acquire() as con:
|
||||
rows = await con.fetch(
|
||||
f"""
|
||||
SELECT time_bucket($1::interval, ts) AS bucket,
|
||||
avg(value) AS avg, max(value) AS max, min(value) AS min
|
||||
FROM {src}
|
||||
WHERE key = $2 AND device = $3 AND ts > now() - ($4::float * interval '1 hour')
|
||||
GROUP BY bucket ORDER BY bucket
|
||||
""",
|
||||
bucket, key, core.METER_HOSTS[0], window,
|
||||
)
|
||||
return {
|
||||
"key": key,
|
||||
"bucket": bucket,
|
||||
"points": [
|
||||
{"t": r["bucket"].isoformat(), "avg": r["avg"], "max": r["max"], "min": r["min"]}
|
||||
for r in rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/events", dependencies=[Depends(auth)])
|
||||
async def events(limit: int = Query(100, ge=1, le=1000), unlabelled: bool = False):
|
||||
q = "SELECT * FROM events"
|
||||
if unlabelled:
|
||||
q += " WHERE appliance IS NULL"
|
||||
q += " ORDER BY ts_start DESC LIMIT $1"
|
||||
async with _pool.acquire() as con:
|
||||
rows = await con.fetch(q, limit)
|
||||
apps = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name")
|
||||
return {
|
||||
"events": [dict(r) for r in rows],
|
||||
"appliances": [dict(a) for a in apps],
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/events/{event_id}/label", dependencies=[Depends(auth)])
|
||||
async def label(event_id: int, appliance: str = Query(...), notes: str | None = None):
|
||||
async with _pool.acquire() as con:
|
||||
ok = await con.fetchval(
|
||||
"SELECT 1 FROM appliances WHERE name = $1", appliance
|
||||
)
|
||||
if not ok:
|
||||
raise HTTPException(400, f"unknown appliance {appliance!r}")
|
||||
row = await con.fetchrow(
|
||||
"""
|
||||
UPDATE events
|
||||
SET appliance = $2, label_source = 'manual', labeled_at = now(),
|
||||
notes = COALESCE($3, notes)
|
||||
WHERE id = $1
|
||||
RETURNING id, appliance, labeled_at
|
||||
""",
|
||||
event_id, appliance, notes,
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(404, "no such event")
|
||||
return dict(row)
|
||||
|
||||
|
||||
@app.get("/api/events/{event_id}/window", dependencies=[Depends(auth)])
|
||||
async def event_window(event_id: int, pad_s: float = 60):
|
||||
"""Raw 1 Hz trace around an event - the training input for that label."""
|
||||
async with _pool.acquire() as con:
|
||||
ev = await con.fetchrow("SELECT * FROM events WHERE id = $1", event_id)
|
||||
if ev is None:
|
||||
raise HTTPException(404, "no such event")
|
||||
rows = await con.fetch(
|
||||
"""
|
||||
SELECT ts, value FROM samples
|
||||
WHERE key = $1 AND device = $2
|
||||
AND ts BETWEEN $3 - ($4::float * interval '1 second')
|
||||
AND COALESCE($5, $3) + ($4::float * interval '1 second')
|
||||
ORDER BY ts
|
||||
""",
|
||||
core.POWER_KEY, core.METER_HOSTS[0], ev["ts_start"], pad_s, ev["ts_end"],
|
||||
)
|
||||
return {
|
||||
"event": dict(ev),
|
||||
"samples": [{"t": r["ts"].isoformat(), "w": r["value"]} for r in rows],
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/export.csv", dependencies=[Depends(auth)])
|
||||
async def export_csv(days: float = Query(30, ge=0.1, le=3650)):
|
||||
"""Labelled events, one row each - the training set index."""
|
||||
async with _pool.acquire() as con:
|
||||
rows = await con.fetch(
|
||||
"""
|
||||
SELECT id, ts_start, ts_end, direction, delta_w, peak_w, baseline_w,
|
||||
duration_s, energy_wh, appliance, label_source
|
||||
FROM events
|
||||
WHERE ts_start > now() - ($1::float * interval '1 day')
|
||||
ORDER BY ts_start
|
||||
""",
|
||||
days,
|
||||
)
|
||||
buf = io.StringIO()
|
||||
if rows:
|
||||
w = csv.DictWriter(buf, fieldnames=list(rows[0].keys()))
|
||||
w.writeheader()
|
||||
for r in rows:
|
||||
w.writerow(dict(r))
|
||||
buf.seek(0)
|
||||
return StreamingResponse(
|
||||
iter([buf.getvalue()]),
|
||||
media_type="text/csv",
|
||||
headers={"Content-Disposition": "attachment; filename=loadwatch_events.csv"},
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/appliances", dependencies=[Depends(auth)])
|
||||
async def appliances():
|
||||
async with _pool.acquire() as con:
|
||||
rows = await con.fetch("SELECT name, notes, is_large FROM appliances ORDER BY name")
|
||||
return [dict(r) for r in rows]
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Collector: ingest the P1 stream, store it, detect load episodes."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
|
||||
from . import core
|
||||
from .detector import EventDetector
|
||||
from .meter import MeterHub
|
||||
|
||||
log = logging.getLogger("loadwatch.collector")
|
||||
|
||||
|
||||
async def amain() -> None:
|
||||
core.setup_logging()
|
||||
pool = await core.make_pool()
|
||||
log.info("db pool ready")
|
||||
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
|
||||
hub = MeterHub(core.METER_HOSTS, core.METER_PORT, core.METER_NOISE_PSK)
|
||||
detector = EventDetector()
|
||||
log.info(
|
||||
"detector: threshold=%.0fW min_duration=%.0fs close_band=%.0fW hold=%.0fs",
|
||||
detector.threshold_w, detector.min_duration_s, detector.close_band_w,
|
||||
detector.close_hold_s,
|
||||
)
|
||||
|
||||
# everything is stored; the aggregate power also feeds the detector
|
||||
pending_events: list = []
|
||||
|
||||
def hook(device: str, key: str, value: float) -> None:
|
||||
if device == core.METER_HOSTS[0] and key == core.POWER_KEY:
|
||||
ev = detector.add(time.time(), value)
|
||||
if ev is not None and ev.duration_s >= detector.min_duration_s:
|
||||
pending_events.append(ev)
|
||||
|
||||
hub.hook = hook
|
||||
|
||||
async def flusher() -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
await asyncio.sleep(core.FLUSH_INTERVAL_S)
|
||||
rows, hub.buffer = hub.buffer, []
|
||||
await core.insert_samples(pool, rows)
|
||||
while pending_events:
|
||||
ev = pending_events.pop(0)
|
||||
eid = await core.insert_event(pool, ev)
|
||||
log.info(
|
||||
"event #%s %s %.0fW dur=%.0fs energy=%.1fWh",
|
||||
eid, "ON" if ev.direction > 0 else "OFF", ev.delta_w,
|
||||
ev.duration_s, ev.energy_wh,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("flush failed: %s", exc)
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(hub.run(stop), name="meter"),
|
||||
asyncio.create_task(flusher(), name="flusher"),
|
||||
asyncio.create_task(stop.wait(), name="stop"),
|
||||
]
|
||||
done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
|
||||
for t in pending:
|
||||
t.cancel()
|
||||
await asyncio.gather(*pending, return_exceptions=True)
|
||||
for t in done:
|
||||
exc = t.exception()
|
||||
if exc:
|
||||
log.error("task %s died: %s", t.get_name(), exc)
|
||||
|
||||
# final flush
|
||||
rows, hub.buffer = hub.buffer, []
|
||||
await core.insert_samples(pool, rows)
|
||||
await pool.close()
|
||||
log.info("collector stopped")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
asyncio.run(amain())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
"""Config + database helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import asyncpg
|
||||
|
||||
log = logging.getLogger("loadwatch")
|
||||
|
||||
|
||||
def env_str(name: str, default: str) -> str:
|
||||
v = os.environ.get(name)
|
||||
return v if v not in (None, "") else default
|
||||
|
||||
|
||||
def env_float(name: str, default: float) -> float:
|
||||
try:
|
||||
return float(os.environ[name])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
DATABASE_URL = env_str(
|
||||
"DATABASE_URL", "postgresql://loadwatch:loadwatch@db:5432/loadwatch"
|
||||
)
|
||||
|
||||
# --- meter / sources ------------------------------------------------------
|
||||
METER_HOSTS = [h.strip() for h in env_str("METER_HOSTS", "192.168.178.213").split(",") if h.strip()]
|
||||
METER_PORT = int(env_float("METER_PORT", 6053))
|
||||
METER_NOISE_PSK = env_str("METER_NOISE_PSK", "")
|
||||
|
||||
# ESPHome object_id -> our column/key name. Matched by object_id first, then by
|
||||
# suffix (the Zuidwijk firmware keeps the plain dsmr names).
|
||||
WATCH = {
|
||||
"power_consumed_phase_1": "power_consumed_phase_1",
|
||||
"power_produced_phase_1": "power_produced_phase_1",
|
||||
"voltage_phase_1": "voltage_phase_1",
|
||||
"current_phase_1": "current_phase_1",
|
||||
"energy_consumed_tariff_1": "energy_consumed_tariff_1",
|
||||
"energy_consumed_tariff_2": "energy_consumed_tariff_2",
|
||||
"energy_produced_tariff_1": "energy_produced_tariff_1",
|
||||
"energy_produced_tariff_2": "energy_produced_tariff_2",
|
||||
"gas_consumed": "gas_consumed",
|
||||
}
|
||||
|
||||
POWER_KEY = "power_consumed_phase_1" # the aggregate signal the detector watches
|
||||
|
||||
# --- detection ------------------------------------------------------------
|
||||
EVENT_THRESHOLD_W = env_float("EVENT_THRESHOLD_W", 80.0)
|
||||
EVENT_MIN_DURATION_S = env_float("EVENT_MIN_DURATION_S", 3.0)
|
||||
EVENT_CLOSE_BAND_W = env_float("EVENT_CLOSE_BAND_W", 40.0)
|
||||
EVENT_CLOSE_HOLD_S = env_float("EVENT_CLOSE_HOLD_S", 3.0)
|
||||
EVENT_MAX_S = env_float("EVENT_MAX_S", 4 * 3600.0)
|
||||
|
||||
API_TOKEN = env_str("API_TOKEN", "")
|
||||
|
||||
FLUSH_INTERVAL_S = env_float("FLUSH_INTERVAL_S", 5.0)
|
||||
BASELINE_WINDOW_S = env_float("BASELINE_WINDOW_S", 60.0)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
logging.basicConfig(
|
||||
level=os.environ.get("LOG_LEVEL", "INFO").upper(),
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
|
||||
async def make_pool() -> asyncpg.Pool:
|
||||
return await asyncpg.create_pool(DATABASE_URL, min_size=1, max_size=4)
|
||||
|
||||
|
||||
async def insert_samples(pool: asyncpg.Pool, rows: list[tuple]) -> None:
|
||||
"""rows: (ts, device, key, value). ts may be an epoch float or a datetime."""
|
||||
if not rows:
|
||||
return
|
||||
norm = [
|
||||
(
|
||||
ts if not isinstance(ts, (int, float))
|
||||
else datetime.fromtimestamp(float(ts), tz=timezone.utc),
|
||||
device,
|
||||
key,
|
||||
float(value),
|
||||
)
|
||||
for ts, device, key, value in rows
|
||||
]
|
||||
async with pool.acquire() as con:
|
||||
await con.executemany(
|
||||
"INSERT INTO samples (ts, device, key, value) VALUES ($1, $2, $3, $4)", norm
|
||||
)
|
||||
|
||||
|
||||
async def insert_event(pool: asyncpg.Pool, ev) -> int:
|
||||
async with pool.acquire() as con:
|
||||
return await con.fetchval(
|
||||
"""
|
||||
INSERT INTO events (ts_start, ts_end, direction, delta_w, peak_w,
|
||||
baseline_w, duration_s, energy_wh)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
RETURNING id
|
||||
""",
|
||||
ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w,
|
||||
ev.baseline_w, ev.duration_s, ev.energy_wh,
|
||||
)
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
"""Step-change / episode detection on a 1 Hz real-power stream.
|
||||
|
||||
Model: the house has a running baseline (everything always-on). A load *episode*
|
||||
starts when instantaneous power steps away from that baseline by more than
|
||||
`threshold_w` and stays there for `min_duration_s`; it ends when power returns
|
||||
within `close_band_w` of the baseline and holds for `close_hold_s`.
|
||||
|
||||
The baseline is frozen while an episode is open, so a second load switching on
|
||||
mid-episode does not corrupt the first one's delta.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import statistics
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
from . import core
|
||||
|
||||
|
||||
@dataclass
|
||||
class Event:
|
||||
ts_start: float
|
||||
ts_end: float | None = None
|
||||
direction: int = 1
|
||||
delta_w: float = 0.0
|
||||
peak_w: float = 0.0
|
||||
baseline_w: float = 0.0
|
||||
duration_s: float = 0.0
|
||||
energy_wh: float = 0.0
|
||||
|
||||
# helpers for the DB layer (epoch -> aware datetime)
|
||||
@property
|
||||
def ts_start_dt(self):
|
||||
from datetime import datetime, timezone
|
||||
return datetime.fromtimestamp(self.ts_start, tz=timezone.utc)
|
||||
|
||||
@property
|
||||
def ts_end_dt(self):
|
||||
from datetime import datetime, timezone
|
||||
if self.ts_end is None:
|
||||
return None
|
||||
return datetime.fromtimestamp(self.ts_end, tz=timezone.utc)
|
||||
|
||||
|
||||
class EventDetector:
|
||||
def __init__(self, threshold_w=None, min_duration_s=None, close_band_w=None,
|
||||
close_hold_s=None, baseline_window_s=None, max_event_s=None):
|
||||
self.threshold_w = core.EVENT_THRESHOLD_W if threshold_w is None else threshold_w
|
||||
self.min_duration_s = core.EVENT_MIN_DURATION_S if min_duration_s is None else min_duration_s
|
||||
self.close_band_w = core.EVENT_CLOSE_BAND_W if close_band_w is None else close_band_w
|
||||
self.close_hold_s = core.EVENT_CLOSE_HOLD_S if close_hold_s is None else close_hold_s
|
||||
self.baseline_window_s = core.BASELINE_WINDOW_S if baseline_window_s is None else baseline_window_s
|
||||
self.max_event_s = core.EVENT_MAX_S if max_event_s is None else max_event_s
|
||||
|
||||
self._hist: deque[tuple[float, float]] = deque() # (ts, w) since last event
|
||||
self._cand: tuple[float, float] | None = None # candidate step
|
||||
self._ev: Event | None = None
|
||||
self._back_since: float | None = None
|
||||
self._last_ts: float | None = None
|
||||
self._energy_acc = 0.0
|
||||
|
||||
# ------------------------------------------------------------------ util
|
||||
def _baseline(self) -> float | None:
|
||||
if not self._hist:
|
||||
return None
|
||||
cutoff = (self._last_ts or 0.0) - self.baseline_window_s
|
||||
window = [w for ts, w in self._hist if ts >= cutoff]
|
||||
if len(window) < 3:
|
||||
return None
|
||||
return statistics.median(window)
|
||||
|
||||
# ------------------------------------------------------------------- fed
|
||||
def add(self, ts: float, w: float) -> Event | None:
|
||||
"""Feed one sample. Returns a completed Event when one closes."""
|
||||
self._last_ts = ts
|
||||
completed: Event | None = None
|
||||
|
||||
if self._ev is None:
|
||||
base = self._baseline()
|
||||
self._hist.append((ts, w))
|
||||
# keep memory bounded
|
||||
if len(self._hist) > 10000:
|
||||
self._hist.popleft()
|
||||
|
||||
if base is not None:
|
||||
if abs(w - base) > self.threshold_w:
|
||||
if self._cand is None:
|
||||
self._cand = (ts, w)
|
||||
elif ts - self._cand[0] >= self.min_duration_s:
|
||||
# episode opens retroactively at the candidate time
|
||||
delta = w - base
|
||||
self._ev = Event(
|
||||
ts_start=self._cand[0],
|
||||
direction=1 if delta > 0 else -1,
|
||||
delta_w=delta,
|
||||
peak_w=w,
|
||||
baseline_w=base,
|
||||
)
|
||||
self._energy_acc = 0.0
|
||||
self._back_since = None
|
||||
self._cand = None
|
||||
self._hist.clear()
|
||||
self._hist.append((ts, w))
|
||||
else:
|
||||
self._cand = None
|
||||
else:
|
||||
ev = self._ev
|
||||
dev = w - ev.baseline_w
|
||||
if abs(w) > abs(ev.peak_w):
|
||||
ev.peak_w = w
|
||||
# energy above baseline, trapezoid-ish (1 Hz => dt of one step)
|
||||
dt = max(0.0, ts - (self._last_ts_prev or ts))
|
||||
self._energy_acc += max(0.0, dev) * dt / 3600.0
|
||||
self._hist.append((ts, w))
|
||||
|
||||
if abs(dev) < self.close_band_w:
|
||||
if self._back_since is None:
|
||||
self._back_since = ts
|
||||
elif ts - self._back_since >= self.close_hold_s:
|
||||
ev.ts_end = self._back_since
|
||||
ev.duration_s = ev.ts_end - ev.ts_start
|
||||
ev.energy_wh = self._energy_acc
|
||||
completed = ev
|
||||
self._ev = None
|
||||
self._back_since = None
|
||||
self._cand = None
|
||||
self._hist.clear()
|
||||
else:
|
||||
self._back_since = None
|
||||
|
||||
if self._ev is not None and ts - self._ev.ts_start >= self.max_event_s:
|
||||
self._ev.ts_end = ts
|
||||
self._ev.duration_s = ts - self._ev.ts_start
|
||||
self._ev.energy_wh = self._energy_acc
|
||||
completed = self._ev
|
||||
self._ev = None
|
||||
self._back_since = None
|
||||
self._hist.clear()
|
||||
|
||||
self._last_ts_prev = ts
|
||||
return completed
|
||||
|
||||
# for the live status endpoint
|
||||
@property
|
||||
def active(self) -> Event | None:
|
||||
return self._ev
|
||||
|
||||
_last_ts_prev: float | None = None
|
||||
|
||||
def to_rows(self, ev: Event) -> tuple:
|
||||
return (ev.ts_start_dt, ev.ts_end_dt, ev.direction, ev.delta_w, ev.peak_w,
|
||||
ev.baseline_w, ev.duration_s, ev.energy_wh)
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
"""ESPHome native-API ingestion.
|
||||
|
||||
One ESPHome device = one source. Subscribes to the sensor states we care about and
|
||||
hands (ts, device, key, value) tuples to a callback. Reconnects with backoff.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import logging
|
||||
import time
|
||||
|
||||
from aioesphomeapi import APIClient
|
||||
|
||||
from . import core
|
||||
|
||||
log = logging.getLogger("loadwatch.meter")
|
||||
|
||||
|
||||
class MeterSource:
|
||||
def __init__(self, host: str, port: int, noise_psk: str, on_sample):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.noise_psk = noise_psk or None
|
||||
self.on_sample = on_sample
|
||||
self._keymap: dict[int, str] = {}
|
||||
self._cli: APIClient | None = None
|
||||
|
||||
# -- state callback ----------------------------------------------------
|
||||
def _handle_state(self, state) -> None:
|
||||
name = self._keymap.get(getattr(state, "key", None))
|
||||
if name is None:
|
||||
return
|
||||
value = getattr(state, "state", None)
|
||||
if getattr(state, "missing_state", False) or value is None:
|
||||
log.debug("%s: %s missing", self.host, name)
|
||||
return
|
||||
try:
|
||||
self.on_sample(self.host, name, float(value))
|
||||
except (TypeError, ValueError):
|
||||
log.warning("%s: non-numeric %s=%r", self.host, name, value)
|
||||
|
||||
# -- discovery ---------------------------------------------------------
|
||||
async def _resolve_entities(self) -> list[int]:
|
||||
assert self._cli is not None
|
||||
entities, _services = await self._cli.list_entities_services()
|
||||
available: dict[str, int] = {}
|
||||
for ent in entities:
|
||||
oid = getattr(ent, "object_id", None)
|
||||
key = getattr(ent, "key", None)
|
||||
if oid and key is not None:
|
||||
available[oid] = key
|
||||
|
||||
wanted: list[int] = []
|
||||
for want in core.WATCH:
|
||||
key = available.get(want)
|
||||
if key is None: # tolerate firmware variants: match by suffix
|
||||
for oid, k in available.items():
|
||||
if oid.endswith(want):
|
||||
key = k
|
||||
break
|
||||
if key is None:
|
||||
continue
|
||||
self._keymap[key] = want
|
||||
wanted.append(key)
|
||||
|
||||
missing = sorted(set(core.WATCH) - set(self._keymap.values()))
|
||||
log.info(
|
||||
"%s: watching %d/%d entities; missing=%s",
|
||||
self.host, len(wanted), len(core.WATCH), missing or "none",
|
||||
)
|
||||
if missing:
|
||||
# Self-diagnosing: show the caller every object_id this device exposes.
|
||||
log.info("%s: available object_ids: %s", self.host, sorted(available))
|
||||
return wanted
|
||||
|
||||
# -- run ---------------------------------------------------------------
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
backoff = 5
|
||||
while not stop.is_set():
|
||||
try:
|
||||
self._cli = APIClient(
|
||||
self.host, self.port, password="", noise_psk=self.noise_psk
|
||||
)
|
||||
log.info("%s: connecting to ESPHome API %s:%s", self.host, self.host, self.port)
|
||||
await self._cli.connect(login=True)
|
||||
log.info("%s: connected", self.host)
|
||||
backoff = 5
|
||||
|
||||
wanted = await self._resolve_entities()
|
||||
if not wanted:
|
||||
raise RuntimeError("none of the watched entities exist on this device")
|
||||
# aioesphomeapi has shipped both sync and async variants of this
|
||||
res = self._cli.subscribe_states(self._handle_state)
|
||||
if inspect.isawaitable(res):
|
||||
await res
|
||||
log.info("%s: subscribed to %d state keys", self.host, len(wanted))
|
||||
|
||||
while not stop.is_set():
|
||||
await asyncio.sleep(1)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - stay alive on any transport error
|
||||
log.warning("%s: connection problem: %s (retry in %ss)", self.host, exc, backoff)
|
||||
await asyncio.sleep(backoff)
|
||||
backoff = min(backoff * 2, 120)
|
||||
finally:
|
||||
if self._cli is not None:
|
||||
try:
|
||||
await self._cli.disconnect()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._cli = None
|
||||
log.info("%s: stopped", self.host)
|
||||
|
||||
|
||||
class MeterHub:
|
||||
"""Collects samples from every configured source into one flush buffer."""
|
||||
|
||||
def __init__(self, hosts: list[str], port: int, noise_psk: str):
|
||||
self.hosts = hosts
|
||||
self.port = port
|
||||
self.noise_psk = noise_psk
|
||||
self.buffer: list[tuple] = []
|
||||
self.latest: dict[str, float] = {}
|
||||
self._last: dict[str, float] = {}
|
||||
# optional extra consumer (the detector), set after construction.
|
||||
# Kept as an attribute and routed through _dispatch so assigning it
|
||||
# later actually takes effect (sources hold _dispatch, not the hook).
|
||||
self.hook = None
|
||||
self.sources = [
|
||||
MeterSource(h, port, noise_psk, self._dispatch) for h in hosts
|
||||
]
|
||||
|
||||
def _dispatch(self, device: str, key: str, value: float) -> None:
|
||||
self._on_sample(device, key, value)
|
||||
if self.hook is not None:
|
||||
self.hook(device, key, value)
|
||||
|
||||
def _on_sample(self, device: str, key: str, value: float) -> None:
|
||||
now = time.time()
|
||||
self.buffer.append((now, device, key, value))
|
||||
if device == self.hosts[0]:
|
||||
self.latest[key] = value
|
||||
self._last[key] = now
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
await asyncio.gather(*(s.run(stop) for s in self.sources))
|
||||
@@ -0,0 +1,191 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>loadwatch</title>
|
||||
<style>
|
||||
:root { --bg:#0f1115; --card:#171a21; --fg:#e8eaf0; --mut:#8b93a7; --acc:#4da3ff; --ok:#39d98a; --warn:#ffb454; }
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--fg); font:14px/1.5 ui-monospace,SFMono-Regular,Menlo,monospace; }
|
||||
header { display:flex; gap:12px; align-items:center; padding:12px 16px; border-bottom:1px solid #23262f; flex-wrap:wrap; }
|
||||
h1 { font-size:16px; margin:0 12px 0 0; letter-spacing:.5px; }
|
||||
.mut { color:var(--mut); }
|
||||
input,select,button { background:#0b0d12; color:var(--fg); border:1px solid #2a2f3a; border-radius:6px; padding:6px 8px; font:inherit; }
|
||||
button { cursor:pointer; border-color:#31415a; }
|
||||
button:hover { border-color:var(--acc); }
|
||||
main { display:grid; grid-template-columns:1fr; gap:16px; padding:16px; max-width:1200px; }
|
||||
.row { display:grid; grid-template-columns:repeat(auto-fit,minmax(150px,1fr)); gap:12px; }
|
||||
.card { background:var(--card); border:1px solid #23262f; border-radius:10px; padding:14px; }
|
||||
.k { color:var(--mut); font-size:12px; text-transform:uppercase; letter-spacing:.6px; }
|
||||
.v { font-size:26px; margin-top:4px; }
|
||||
canvas { width:100%; height:220px; display:block; }
|
||||
table { width:100%; border-collapse:collapse; font-size:13px; }
|
||||
th,td { text-align:left; padding:6px 8px; border-bottom:1px solid #23262f; white-space:nowrap; }
|
||||
th { color:var(--mut); font-weight:600; }
|
||||
tr.done td { color:var(--mut); }
|
||||
.pos { color:var(--warn); } .neg { color:var(--ok); }
|
||||
.pill { display:inline-block; padding:1px 7px; border-radius:99px; background:#222833; font-size:12px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>loadwatch</h1>
|
||||
<span class="mut" id="conn">connecting…</span>
|
||||
<span class="mut">token</span><input id="token" size="18" placeholder="API token" type="password">
|
||||
<button onclick="saveToken()">save</button>
|
||||
<span class="mut" id="device"></span>
|
||||
<span style="margin-left:auto"></span>
|
||||
<a class="mut" id="export" href="#" download>export csv</a>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="row">
|
||||
<div class="card"><div class="k">now</div><div class="v" id="now">–<span class="mut" style="font-size:14px"> W</span></div></div>
|
||||
<div class="card"><div class="k">import today</div><div class="v" id="imp">–</div></div>
|
||||
<div class="card"><div class="k">export today</div><div class="v" id="exp">–</div></div>
|
||||
<div class="card"><div class="k">gas</div><div class="v" id="gas">–</div></div>
|
||||
<div class="card"><div class="k">events / labelled</div><div class="v" id="evc">–</div></div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="k" style="margin-bottom:8px">aggregate power (W) — <span id="rangeLbl">6 h</span></div>
|
||||
<div class="row" style="margin-bottom:10px">
|
||||
<button onclick="loadPower(1)">1 h</button><button onclick="loadPower(6)">6 h</button>
|
||||
<button onclick="loadPower(24)">24 h</button><button onclick="loadPower(168)">7 d</button>
|
||||
<button onclick="showEvents()">show detected events</button>
|
||||
</div>
|
||||
<canvas id="chart" width="1100" height="220"></canvas>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="k" style="margin-bottom:8px">load episodes — label them (this is the dataset)</div>
|
||||
<table><thead><tr>
|
||||
<th>#</th><th>start</th><th>ΔW</th><th>peak</th><th>dur</th><th>Wh</th><th>label</th><th></th>
|
||||
</tr></thead><tbody id="evbody"><tr><td colspan="8" class="mut">loading…</td></tr></tbody></table>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<script>
|
||||
// injected by the API: "" standalone, "/loadwatch" behind the portal sub-path
|
||||
const P = '__PREFIX__';
|
||||
let TOKEN = localStorage.getItem('lw_token') || '';
|
||||
document.getElementById('token').value = TOKEN;
|
||||
let APPLIANCES = [];
|
||||
|
||||
function saveToken(){ TOKEN = document.getElementById('token').value.trim(); localStorage.setItem('lw_token', TOKEN); refresh(); }
|
||||
async function api(path, opts){
|
||||
opts = opts || {}; opts.headers = Object.assign({'Authorization':'Bearer '+TOKEN}, opts.headers||{});
|
||||
const r = await fetch(P + path, opts);
|
||||
if(!r.ok) throw new Error(path+' -> '+r.status);
|
||||
return r.json();
|
||||
}
|
||||
const fmt = n => (n===null||n===undefined) ? '–' : n.toLocaleString(undefined,{maximumFractionDigits:1});
|
||||
|
||||
async function refresh(){
|
||||
try {
|
||||
const s = await api('/api/status');
|
||||
document.getElementById('conn').textContent = 'live';
|
||||
document.getElementById('conn').style.color = 'var(--ok)';
|
||||
document.getElementById('device').textContent = s.device;
|
||||
const L = s.latest || {};
|
||||
document.getElementById('now').innerHTML = fmt(L['power_consumed_phase_1']?.value) + '<span class="mut" style="font-size:14px"> W</span>';
|
||||
const c = s.counters || {};
|
||||
document.getElementById('imp').textContent = fmt(c.e_cons_t1) + ' kWh';
|
||||
document.getElementById('exp').textContent = fmt(c.e_prod_t1) + ' kWh';
|
||||
document.getElementById('gas').textContent = fmt(L['gas_consumed']?.value) + ' m³';
|
||||
document.getElementById('evc').textContent = s.events.total + ' / ' + s.events.labelled;
|
||||
} catch(e){
|
||||
document.getElementById('conn').textContent = 'error: ' + e.message;
|
||||
document.getElementById('conn').style.color = '#ff6b6b';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadPower(hours){
|
||||
document.getElementById('rangeLbl').textContent = hours >= 24 ? (hours/24)+' d' : hours+' h';
|
||||
const d = await api('/api/power?hours='+hours);
|
||||
drawSeries(d.points.map(p => p.max), d.points.map(p => p.avg));
|
||||
}
|
||||
|
||||
async function showEvents(){
|
||||
const d = await api('/api/power?hours=168');
|
||||
const ev = await api('/api/events?limit=200');
|
||||
const t0 = new Date(d.points[0].t).getTime(), t1 = new Date(d.points[d.points.length-1].t).getTime();
|
||||
const markers = ev.events.map(e => ({t:new Date(e.ts_start).getTime(), dw:e.delta_w}));
|
||||
drawSeries(d.points.map(p=>p.max), d.points.map(p=>p.avg), {t0,t1,markers});
|
||||
}
|
||||
|
||||
function drawSeries(maxs, avgs, opts){
|
||||
const c = document.getElementById('chart'), ctx = c.getContext('2d');
|
||||
const W = c.width, H = c.height, pad = 28;
|
||||
ctx.clearRect(0,0,W,H);
|
||||
const all = maxs.concat(avgs).filter(v => v!=null);
|
||||
if(!all.length){ ctx.fillStyle='#8b93a7'; ctx.fillText('no data yet', pad, H/2); return; }
|
||||
const hi = Math.max.apply(null, all), lo = Math.min.apply(null, all);
|
||||
const X = i => pad + i*(W-2*pad)/Math.max(1,maxs.length-1);
|
||||
const Y = v => H-pad - (v-lo)*(H-2*pad)/Math.max(1,(hi-lo));
|
||||
ctx.strokeStyle = '#23262f'; ctx.beginPath(); ctx.moveTo(pad,H-pad); ctx.lineTo(W-pad,H-pad); ctx.stroke();
|
||||
ctx.fillStyle = '#8b93a7'; ctx.fillText(Math.round(hi)+' W', 4, pad); ctx.fillText(Math.round(lo)+' W', 4, H-pad);
|
||||
// max envelope
|
||||
ctx.strokeStyle = '#2c4a6e'; ctx.beginPath();
|
||||
maxs.forEach((v,i) => v==null?0:(i?ctx.lineTo(X(i),Y(v)):ctx.moveTo(X(i),Y(v)))); ctx.stroke();
|
||||
// avg line
|
||||
ctx.strokeStyle = '#4da3ff'; ctx.lineWidth = 2; ctx.beginPath();
|
||||
avgs.forEach((v,i) => v==null?0:(i?ctx.lineTo(X(i),Y(v)):ctx.moveTo(X(i),Y(v)))); ctx.stroke(); ctx.lineWidth = 1;
|
||||
if(opts && opts.markers){
|
||||
ctx.fillStyle = '#ffb454';
|
||||
opts.markers.forEach(m => {
|
||||
if(m.t < opts.t0 || m.t > opts.t1) return;
|
||||
const i = Math.round((m.t-opts.t0)/Math.max(1,(opts.t1-opts.t0))*(maxs.length-1));
|
||||
ctx.fillRect(X(i)-1, pad, 2, H-2*pad);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function loadEvents(){
|
||||
const d = await api('/api/events?limit=100');
|
||||
APPLIANCES = d.appliances.filter(a => a.is_large !== false).map(a => a.name);
|
||||
const b = document.getElementById('evbody');
|
||||
if(!d.events.length){ b.innerHTML = '<tr><td colspan="8" class="mut">no events yet — they appear as loads switch</td></tr>'; return; }
|
||||
b.innerHTML = d.events.map(e => {
|
||||
const opts = ['<option value="">—</option>'].concat(APPLIANCES.map(a =>
|
||||
'<option'+(a===e.appliance?' selected':'')+'>'+a+'</option>')).join('');
|
||||
return '<tr class="'+(e.appliance?'done':'')+'">'
|
||||
+ '<td>'+e.id+'</td>'
|
||||
+ '<td>'+new Date(e.ts_start).toLocaleTimeString()+'<span class="mut"> '+new Date(e.ts_start).toLocaleDateString()+'</span></td>'
|
||||
+ '<td class="'+((e.delta_w||0)>0?'pos':'neg')+'">'+Math.round(e.delta_w)+'</td>'
|
||||
+ '<td>'+Math.round(e.peak_w)+'</td>'
|
||||
+ '<td>'+(e.duration_s==null?'open':Math.round(e.duration_s)+'s')+'</td>'
|
||||
+ '<td>'+fmt(e.energy_wh)+'</td>'
|
||||
+ '<td><select onchange="label('+e.id+', this.value)">'+opts+'</select>'
|
||||
+ (e.label_source?' <span class="pill">'+e.label_source+'</span>':'')+'</td>'
|
||||
+ '<td><a class="mut" href="#" onclick="dump('+e.id+');return false">trace</a></td>'
|
||||
+ '</tr>';
|
||||
}).join('');
|
||||
}
|
||||
|
||||
async function label(id, appliance){
|
||||
if(!appliance) return;
|
||||
await api('/api/events/'+id+'/label?appliance='+encodeURIComponent(appliance), {method:'POST'});
|
||||
loadEvents(); refresh();
|
||||
}
|
||||
|
||||
async function dump(id){
|
||||
const d = await api('/api/events/'+id+'/window');
|
||||
const lines = d.samples.map(s => s.t+' '+s.w);
|
||||
const blob = new Blob([lines.join('\n')], {type:'text/plain'});
|
||||
const a = document.createElement('a');
|
||||
a.href = URL.createObjectURL(blob); a.download = 'event_'+id+'.txt'; a.click();
|
||||
}
|
||||
|
||||
function updateExport(){ document.getElementById('export').href = P + '/api/export.csv?days=30'; }
|
||||
|
||||
(async function init(){
|
||||
updateExport();
|
||||
await refresh(); await loadEvents(); await loadPower(6);
|
||||
setInterval(refresh, 5000);
|
||||
setInterval(() => { loadEvents(); updateExport(); }, 60000);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
# loadwatch healthcheck — always exits 0, prints exactly ONE line.
|
||||
# OK <age>s -> healthy, newest sample <age> seconds old
|
||||
# FAIL <reason> -> something is wrong
|
||||
# Intended to be run on the runtime host (dev CT141), e.g. over SSH from a watchdog.
|
||||
set -u
|
||||
|
||||
cd "$(dirname "$0")/.." || { echo "FAIL cannot cd to project dir"; exit 0; }
|
||||
|
||||
ps_out=$(docker compose ps --format '{{.Service}}={{.State}}' 2>/dev/null | tr '\n' ' ')
|
||||
if [ -z "$ps_out" ]; then
|
||||
echo "FAIL docker compose reports no services (stack down?)"
|
||||
exit 0
|
||||
fi
|
||||
for svc in db collector api; do
|
||||
case " $ps_out " in
|
||||
*" $svc=running "*) : ;;
|
||||
*) echo "FAIL service $svc is not running (states: $ps_out)"; exit 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
age=$(docker compose exec -T db psql -U loadwatch -d loadwatch -t -A -c \
|
||||
"select coalesce(extract(epoch from (now()-max(ts)))::int,-1) from samples;" 2>/dev/null | tr -d '[:space:]')
|
||||
case "${age:-x}" in
|
||||
''|*[!0-9-]*) echo "FAIL sample-age query failed (raw='${age:-empty}')" ;;
|
||||
-1) echo "FAIL no samples in the database" ;;
|
||||
*) if [ "$age" -gt 300 ]; then
|
||||
echo "FAIL newest sample is ${age}s old (threshold 300s)"
|
||||
else
|
||||
echo "OK ${age}s"
|
||||
fi ;;
|
||||
esac
|
||||
exit 0
|
||||
@@ -0,0 +1,44 @@
|
||||
# loadwatch — Tier-0 NILM dataset service (runs on dev CT141, 192.168.178.20:8099)
|
||||
#
|
||||
# Public URL: https://xm2561.duckdns.org/loadwatch
|
||||
#
|
||||
# IMPORTANT: this host's apex (xm2561.duckdns.org) is Headscale's coordination
|
||||
# endpoint (see headscale.yml). That router is a bare `Host()` catch-all and is
|
||||
# deliberately left untouched. This router is strictly longer
|
||||
# (`Host() && PathPrefix()`), so Traefik's length-based priority gives it the
|
||||
# /loadwatch prefix only — /ts2021, /derp, /key, /register, /api/v1/* and
|
||||
# /health keep flowing to Headscale.
|
||||
#
|
||||
# StripPrefix: the container serves "/" and stays standalone-clean; Traefik
|
||||
# strips /loadwatch and (v2.5+) sets X-Forwarded-Prefix, which the app uses to
|
||||
# emit prefixed asset/API URLs for the browser.
|
||||
http:
|
||||
routers:
|
||||
loadwatch-http:
|
||||
rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)"
|
||||
entryPoints:
|
||||
- http
|
||||
service: loadwatch
|
||||
middlewares:
|
||||
- redirect-to-https
|
||||
loadwatch:
|
||||
rule: "Host(`xm2561.duckdns.org`) && PathPrefix(`/loadwatch`)"
|
||||
entryPoints:
|
||||
- https
|
||||
tls:
|
||||
certResolver: letsencrypt
|
||||
service: loadwatch
|
||||
middlewares:
|
||||
- loadwatch-strip
|
||||
|
||||
middlewares:
|
||||
loadwatch-strip:
|
||||
stripPrefix:
|
||||
prefixes:
|
||||
- "/loadwatch"
|
||||
|
||||
services:
|
||||
loadwatch:
|
||||
loadBalancer:
|
||||
servers:
|
||||
- url: "http://192.168.178.20:8099"
|
||||
@@ -0,0 +1,52 @@
|
||||
name: loadwatch
|
||||
|
||||
services:
|
||||
db:
|
||||
image: timescale/timescaledb:latest-pg16
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
POSTGRES_USER: loadwatch
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-loadwatch}
|
||||
POSTGRES_DB: loadwatch
|
||||
volumes:
|
||||
- dbdata:/var/lib/postgresql/data
|
||||
- ./sql:/docker-entrypoint-initdb.d:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U loadwatch -d loadwatch"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
|
||||
collector:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch
|
||||
METER_HOSTS: ${METER_HOSTS:-192.168.178.213}
|
||||
METER_PORT: ${METER_PORT:-6053}
|
||||
METER_NOISE_PSK: ${METER_NOISE_PSK:-}
|
||||
EVENT_THRESHOLD_W: ${EVENT_THRESHOLD_W:-80}
|
||||
EVENT_MIN_DURATION_S: ${EVENT_MIN_DURATION_S:-3}
|
||||
EVENT_CLOSE_BAND_W: ${EVENT_CLOSE_BAND_W:-40}
|
||||
EVENT_CLOSE_HOLD_S: ${EVENT_CLOSE_HOLD_S:-3}
|
||||
EVENT_MAX_S: ${EVENT_MAX_S:-14400}
|
||||
command: ["python", "-m", "app.collector"]
|
||||
|
||||
api:
|
||||
build: .
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgresql://loadwatch:${POSTGRES_PASSWORD:-loadwatch}@db:5432/loadwatch
|
||||
API_TOKEN: ${API_TOKEN:-loadwatch-dev}
|
||||
ports:
|
||||
- "${API_PORT:-8099}:8000"
|
||||
command: ["uvicorn", "app.api:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
|
||||
volumes:
|
||||
dbdata:
|
||||
@@ -0,0 +1,4 @@
|
||||
aioesphomeapi>=24.0
|
||||
asyncpg>=0.29
|
||||
fastapi>=0.110
|
||||
uvicorn[standard]>=0.29
|
||||
@@ -0,0 +1,90 @@
|
||||
-- loadwatch schema v1
|
||||
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
||||
|
||||
-- Raw 1 Hz samples. Narrow by design: extra sources (smart plug, later a second
|
||||
-- meter) are just another (device, key) pair - no schema change.
|
||||
CREATE TABLE IF NOT EXISTS samples (
|
||||
ts timestamptz NOT NULL,
|
||||
device text NOT NULL,
|
||||
key text NOT NULL,
|
||||
value double precision NOT NULL
|
||||
);
|
||||
|
||||
SELECT create_hypertable('samples', 'ts', chunk_time_interval => interval '1 day',
|
||||
if_not_exists => TRUE);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS samples_device_key_ts_idx ON samples (device, key, ts DESC);
|
||||
|
||||
-- 1-minute rollup for charts and long-term retention.
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS samples_1m
|
||||
WITH (timescaledb.continuous) AS
|
||||
SELECT time_bucket('1 minute', ts) AS bucket,
|
||||
device,
|
||||
key,
|
||||
avg(value) AS avg,
|
||||
min(value) AS min,
|
||||
max(value) AS max,
|
||||
last(value, ts) AS last
|
||||
FROM samples
|
||||
GROUP BY bucket, device, key;
|
||||
|
||||
SELECT add_continuous_aggregate_policy('samples_1m',
|
||||
start_offset => interval '3 hours',
|
||||
end_offset => interval '1 minute',
|
||||
schedule_interval => interval '1 minute',
|
||||
if_not_exists => TRUE);
|
||||
|
||||
-- Compress raw samples after 7 days (keeps the full 1 Hz history, cheaply).
|
||||
ALTER TABLE samples SET (timescaledb.compress,
|
||||
timescaledb.compress_segmentby = 'device,key',
|
||||
timescaledb.compress_orderby = 'ts DESC');
|
||||
SELECT add_compression_policy('samples', interval '7 days', if_not_exists => TRUE);
|
||||
|
||||
-- Load episodes detected from the aggregate power stream.
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id bigserial PRIMARY KEY,
|
||||
ts_start timestamptz NOT NULL,
|
||||
ts_end timestamptz,
|
||||
direction smallint NOT NULL, -- +1 = load turned on, -1 = off
|
||||
delta_w double precision, -- the step size
|
||||
peak_w double precision, -- peak power during the episode
|
||||
baseline_w double precision, -- baseline the step came off
|
||||
duration_s double precision,
|
||||
energy_wh double precision, -- integral of (P - baseline) over the episode
|
||||
appliance text, -- label: null until labelled
|
||||
label_source text, -- manual | plug | model
|
||||
labeled_at timestamptz,
|
||||
notes text
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS events_ts_start_idx ON events (ts_start DESC);
|
||||
CREATE INDEX IF NOT EXISTS events_appliance_idx ON events (appliance);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS appliances (
|
||||
name text PRIMARY KEY,
|
||||
notes text,
|
||||
is_large boolean NOT NULL DEFAULT TRUE
|
||||
);
|
||||
|
||||
INSERT INTO appliances (name, notes) VALUES
|
||||
('kettle', 'water kettle / boiler, ~2 kW step'),
|
||||
('oven', 'electric oven, 2-3 kW, thermostatic cycling'),
|
||||
('microwave', '~1.2 kW with ~50% duty cycle'),
|
||||
('dishwasher', 'heater cycling ~1-2 kW'),
|
||||
('washing_machine','heater ~2 kW, motor ~500 W'),
|
||||
('dryer', '~2-3 kW heater cycling'),
|
||||
('fridge', 'compressor ~80-150 W cycling'),
|
||||
('freezer', 'compressor ~80-150 W cycling'),
|
||||
('iron', '~1-2 kW, duty-cycled'),
|
||||
('vacuum', '~1-1.5 kW steady'),
|
||||
('water_boiler', 'hot water / boiler'),
|
||||
('ev_charger', 'EV charge point'),
|
||||
('heater', 'space heater'),
|
||||
('unknown_large', 'clearly a big load, unidentifiable'),
|
||||
('ignore', 'noise / not a real load')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
k text PRIMARY KEY,
|
||||
v text
|
||||
);
|
||||
Reference in New Issue
Block a user