91ea434912
- scripts/import_dad_p1.py converts his 15-min cumulative meter dump (~/Downloads/P1e-2025-1-1-2026-1-1.csv) into our hourly format. Net of PV exports — power_w can be negative on surplus hours. Aligned to our 2023-09 → 2024-09 window by (month, day, hour) so prices and irradiance stay seasonally matched. - data/raw/p1_hourly.csv is now dad's data; ours preserved as p1_hourly_michiel.csv for diff/comparison. - LP gE upper bound was 'pv[t]'; now max(0, pv − demand). This was wrong before but invisible because demand was always positive; with dad's net data demand goes negative and the LP went infeasible. - New 'Export rate' field in Advanced. Override saldering toggle with a literal €/kWh — set to −0.106 for terugleveringskosten.
105 lines
4 KiB
Python
105 lines
4 KiB
Python
#!/usr/bin/env python3
|
|
"""Convert dad's 15-min P1 export into our hourly p1 format.
|
|
|
|
Source: ~/Downloads/P1e-2025-1-1-2026-1-1.csv (cumulative T1/T2 import/export
|
|
meter readings, every 15 min, 2025 calendar year, Europe/Amsterdam local time).
|
|
|
|
Output: data/raw/p1_hourly.csv (timestamp UTC, power_w mean over hour). Net
|
|
of PV exports — `power_w` can be negative when his solar exceeded household
|
|
demand. The simulator interprets `demand_kwh = power_w / 1000`, and a plug-in
|
|
battery's discharge cap `max(0, demand - pv)` correctly handles negative
|
|
demand by forbidding discharge during surplus hours.
|
|
|
|
The simulator's window is 2023-09-01 → 2024-09-01 UTC. Dad's data is the
|
|
2025 calendar year. We align by (month, day, hour) so seasonal patterns
|
|
(winter peaks, summer surplus) line up with our prices and irradiance.
|
|
Feb 29 in our 2024 leap year falls back to Feb 28 from dad's 2025.
|
|
|
|
Run from project root:
|
|
uv run python scripts/import_dad_p1.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
import pandas as pd
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--src", type=Path,
|
|
default=Path.home() / "Downloads/P1e-2025-1-1-2026-1-1.csv")
|
|
p.add_argument("--out", type=Path, default=Path("data/raw/p1_hourly.csv"))
|
|
p.add_argument("--window-start", default="2023-09-01T00:00:00Z")
|
|
p.add_argument("--window-end", default="2024-09-01T00:00:00Z")
|
|
args = p.parse_args()
|
|
|
|
print(f"Reading dad's CSV from {args.src}")
|
|
dad = pd.read_csv(args.src, parse_dates=["time"])
|
|
dad["time"] = dad["time"].dt.tz_localize(
|
|
"Europe/Amsterdam", ambiguous="infer", nonexistent="shift_forward"
|
|
)
|
|
dad = dad.set_index("time").sort_index()
|
|
|
|
# Cumulative net energy (signed). Imports add, exports subtract.
|
|
net_cum = (
|
|
dad["Import T1 kWh"] + dad["Import T2 kWh"]
|
|
- dad["Export T1 kWh"] - dad["Export T2 kWh"]
|
|
)
|
|
|
|
# Sample at exact hour boundaries (the meter records every 15 min, so the
|
|
# H:00 reading exists). Diff to get energy consumed in each hour.
|
|
hourly_stock = net_cum.resample("1h").first()
|
|
hourly_kwh = hourly_stock.diff().dropna()
|
|
# Now hourly_kwh.index[i] = end of the hour just consumed; shift to start.
|
|
hourly_kwh.index = hourly_kwh.index - pd.Timedelta(hours=1)
|
|
hourly_kwh = hourly_kwh.tz_convert("UTC")
|
|
|
|
annual = hourly_kwh.sum()
|
|
print(f"Dad's hourly net (kWh): n={len(hourly_kwh)}, "
|
|
f"annual net = {annual:.0f} kWh "
|
|
f"(min hour {hourly_kwh.min():.2f}, max {hourly_kwh.max():.2f}, "
|
|
f"{(hourly_kwh < 0).sum()} export hours)")
|
|
|
|
# Build (month, day, hour) → kWh lookup. dad's full year covers every key
|
|
# except a possible duplicate around DST (averaged).
|
|
lookup = (
|
|
hourly_kwh.groupby([
|
|
hourly_kwh.index.month,
|
|
hourly_kwh.index.day,
|
|
hourly_kwh.index.hour,
|
|
]).mean()
|
|
)
|
|
|
|
# Build target timestamps for our simulator window.
|
|
target = pd.date_range(args.window_start, args.window_end,
|
|
freq="1h", inclusive="left", tz="UTC")
|
|
out = []
|
|
missing = 0
|
|
for ts in target:
|
|
m, d, h = ts.month, ts.day, ts.hour
|
|
if (m, d, h) in lookup.index:
|
|
kwh = lookup.loc[(m, d, h)]
|
|
elif m == 2 and d == 29 and (2, 28, h) in lookup.index:
|
|
kwh = lookup.loc[(2, 28, h)]
|
|
else:
|
|
missing += 1
|
|
continue
|
|
out.append((ts, kwh * 1000.0)) # kWh per hour → mean watts
|
|
|
|
if missing:
|
|
print(f"WARNING: {missing} target hours had no dad-data match")
|
|
|
|
df_out = pd.DataFrame(out, columns=["timestamp", "power_w"])
|
|
df_out["timestamp"] = df_out["timestamp"].dt.strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
args.out.parent.mkdir(parents=True, exist_ok=True)
|
|
df_out.to_csv(args.out, index=False)
|
|
print(f"Wrote {len(df_out)} hours → {args.out}")
|
|
print(f" annualised mean: {df_out['power_w'].mean():.0f} W "
|
|
f"({df_out['power_w'].sum() / 1000:.0f} kWh net per year)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|