mirror of
https://github.com/Hestia-Homes/Model.git
synced 2026-07-12 13:29:04 +00:00
438 lines
15 KiB
Python
438 lines
15 KiB
Python
"""
|
|
This script prepares the data for the financial model
|
|
"""
|
|
|
|
import os
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv(".env.local")
|
|
# The retired `property_details_epc` table is no longer populated under the new
|
|
# backend, so the EPC descriptive fields are sourced live from the EPC service
|
|
# instead (which needs OPEN_EPC_API_TOKEN — also lives in backend/.env).
|
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
load_dotenv(_REPO_ROOT / "backend" / ".env")
|
|
|
|
import pandas as pd
|
|
import numpy as np
|
|
from backend.app.utils import sap_to_epc
|
|
from sqlalchemy.orm import sessionmaker
|
|
from backend.app.db.connection import db_engine, db_read_session
|
|
from backend.app.db.models.recommendations import (
|
|
Recommendation,
|
|
PlanModel,
|
|
RecommendationMaterials,
|
|
)
|
|
from backend.app.db.models.portfolio import (
|
|
PropertyModel,
|
|
PropertyDetailsSpatial,
|
|
)
|
|
from backend.app.db.functions.materials_functions import get_materials
|
|
from infrastructure.epc_client.epc_client_service import EpcClientService
|
|
from collections import defaultdict
|
|
from sqlalchemy import func
|
|
|
|
|
|
def _description_text(item: Any) -> str:
|
|
"""Display text for one raw-cert EPC feature. Handles both schema shapes:
|
|
20.0.0 stores ``description`` as a plain string; 17.1 wraps it as a
|
|
``{"value": ..., "language": ...}`` LanguageString."""
|
|
if not isinstance(item, dict):
|
|
return ""
|
|
desc = item.get("description")
|
|
if isinstance(desc, dict):
|
|
desc = desc.get("value")
|
|
return str(desc or "")
|
|
|
|
|
|
def _join_descriptions(value: Any) -> str:
|
|
"""Flatten a raw-cert EPC feature into a display string. The new EPC API
|
|
returns these as a list of feature dicts (walls/roofs/floors/main_heating),
|
|
a single feature dict (hot_water/window/lighting), or null."""
|
|
if isinstance(value, list):
|
|
return "; ".join(t for t in (_description_text(d) for d in value) if t)
|
|
return _description_text(value)
|
|
|
|
|
|
def _is_expired(registration_date: Optional[str]) -> Optional[bool]:
|
|
"""An EPC is valid for 10 years from its lodgement (registration) date."""
|
|
if not registration_date:
|
|
return None
|
|
try:
|
|
lodged = datetime.fromisoformat(registration_date[:10]).date()
|
|
except ValueError:
|
|
return None
|
|
return (date.today() - lodged).days > 365 * 10
|
|
|
|
|
|
def epc_details_from_service(svc: EpcClientService, uprn: Optional[int]) -> dict[str, Any]:
|
|
"""Mock the retired ``property_details_epc`` row from the live EPC service:
|
|
fetch the UPRN's latest raw certificate and flatten the descriptive fields
|
|
the export needs. Returns ``{}`` when the UPRN has no EPC (the property then
|
|
carries blank EPC columns rather than being dropped)."""
|
|
if uprn is None:
|
|
return {}
|
|
results = svc._search(uprn=uprn) # pyright: ignore[reportPrivateUsage]
|
|
if not results:
|
|
return {}
|
|
latest = max(results, key=lambda r: r.registration_date)
|
|
raw = svc._fetch_certificate(latest.certificate_number) # pyright: ignore[reportPrivateUsage]
|
|
|
|
def _to_int(value: Any) -> Optional[int]:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
current_sap = _to_int(raw.get("energy_rating_current"))
|
|
return {
|
|
"walls": _join_descriptions(raw.get("walls")),
|
|
"roof": _join_descriptions(raw.get("roofs")),
|
|
"floor": _join_descriptions(raw.get("floors")),
|
|
"windows": _join_descriptions(raw.get("window")),
|
|
"heating": _join_descriptions(raw.get("main_heating")),
|
|
"heating_controls": _join_descriptions(raw.get("main_heating_controls")),
|
|
"hot_water": _join_descriptions(raw.get("hot_water")),
|
|
"lighting": _join_descriptions(raw.get("lighting")),
|
|
"total_floor_area": raw.get("total_floor_area"),
|
|
"lodgement_date": raw.get("registration_date"),
|
|
"is_expired": _is_expired(raw.get("registration_date")),
|
|
# Baseline SAP/band/postcode aren't on the new `property` table, so take
|
|
# the lodged figures off the cert (the assessment re-scores from these).
|
|
"postcode": raw.get("postcode"),
|
|
"current_epc_rating": raw.get("current_energy_efficiency_band"),
|
|
"current_sap_points": current_sap,
|
|
"original_sap_points": current_sap,
|
|
}
|
|
|
|
PORTFOLIO_ID = 785
|
|
SCENARIOS = [1266]
|
|
scenario_names = {
|
|
1266: "EPC C",
|
|
}
|
|
|
|
project_name = "Small request for EON"
|
|
|
|
|
|
def get_data(portfolio_id, scenario_ids):
|
|
session = sessionmaker(bind=db_engine)()
|
|
session.begin()
|
|
|
|
# --------------------
|
|
# Properties
|
|
# --------------------
|
|
# `property_details_epc` is dead under the new backend, so read the base
|
|
# Property rows and source the EPC descriptive fields live from the EPC
|
|
# service (one cert fetch per property).
|
|
properties_query = (
|
|
session.query(PropertyModel)
|
|
.filter(PropertyModel.portfolio_id == portfolio_id)
|
|
.all()
|
|
)
|
|
|
|
epc_service = EpcClientService(os.environ["OPEN_EPC_API_TOKEN"])
|
|
properties_data = []
|
|
for p in properties_query:
|
|
base = {col.name: getattr(p, col.name) for col in PropertyModel.__table__.columns}
|
|
# `property_id` is the key the recommendations merge joins on; the
|
|
# Property's own PK is its `id`.
|
|
base["property_id"] = p.id
|
|
# Fill EPC fields from the service; for columns that also exist on the
|
|
# Property row (postcode, SAP points, rating), only fill when the row's
|
|
# value is missing so genuine Property data is never clobbered.
|
|
for key, value in epc_details_from_service(epc_service, p.uprn).items():
|
|
if base.get(key) is None:
|
|
base[key] = value
|
|
properties_data.append(base)
|
|
|
|
# --------------------
|
|
# Plans
|
|
# --------------------
|
|
latest_plans_subq = (
|
|
session.query(
|
|
PlanModel.scenario_id,
|
|
PlanModel.property_id,
|
|
func.max(PlanModel.created_at).label("latest_created_at"),
|
|
)
|
|
.filter(PlanModel.scenario_id.in_(scenario_ids))
|
|
.group_by(PlanModel.scenario_id, PlanModel.property_id)
|
|
.subquery()
|
|
)
|
|
|
|
# plans_query = session.query(Plan).filter(
|
|
# Plan.scenario_id.in_(scenario_ids)
|
|
# ).all()
|
|
|
|
plans_query = (
|
|
session.query(PlanModel)
|
|
.join(
|
|
latest_plans_subq,
|
|
(PlanModel.scenario_id == latest_plans_subq.c.scenario_id)
|
|
& (PlanModel.property_id == latest_plans_subq.c.property_id)
|
|
& (PlanModel.created_at == latest_plans_subq.c.latest_created_at),
|
|
)
|
|
.all()
|
|
)
|
|
|
|
# plans_query = (
|
|
# session.query(Plan)
|
|
# .join(
|
|
# latest_plans_subq,
|
|
# (Plan.scenario_id == latest_plans_subq.c.scenario_id) &
|
|
# (Plan.created_at == latest_plans_subq.c.latest_created_at)
|
|
# )
|
|
# .all()
|
|
# )
|
|
|
|
plans_data = [
|
|
{col.name: getattr(plan, col.name) for col in PlanModel.__table__.columns}
|
|
for plan in plans_query
|
|
]
|
|
|
|
plan_ids = [p["id"] for p in plans_data]
|
|
|
|
# --------------------
|
|
# Recommendations (NO materials yet)
|
|
# --------------------
|
|
# The `plan_recommendations` m2m is retired (ADR-0017): a Recommendation
|
|
# links to its Plan directly via `recommendation.plan_id`.
|
|
recommendations_query = (
|
|
session.query(Recommendation, PlanModel.scenario_id)
|
|
.join(PlanModel, PlanModel.id == Recommendation.plan_id)
|
|
.filter(
|
|
Recommendation.plan_id.in_(plan_ids),
|
|
Recommendation.default.is_(True),
|
|
Recommendation.already_installed.is_(False),
|
|
)
|
|
.all()
|
|
)
|
|
|
|
recommendations_data = [
|
|
{
|
|
**{
|
|
col.name: getattr(r[0], col.name)
|
|
for col in Recommendation.__table__.columns
|
|
},
|
|
"scenario_id": r.scenario_id,
|
|
"materials": [], # placeholder
|
|
}
|
|
for r in recommendations_query
|
|
]
|
|
|
|
recommendation_ids = [r["id"] for r in recommendations_data]
|
|
|
|
# --------------------
|
|
# Recommendation materials (SEPARATE QUERY)
|
|
# --------------------
|
|
materials_query = (
|
|
session.query(RecommendationMaterials)
|
|
.filter(RecommendationMaterials.recommendation_id.in_(recommendation_ids))
|
|
.all()
|
|
)
|
|
|
|
# Group materials by recommendation_id
|
|
materials_by_recommendation = defaultdict(list)
|
|
|
|
for m in materials_query:
|
|
materials_by_recommendation[m.recommendation_id].append(
|
|
{
|
|
"material_id": m.material_id,
|
|
"depth": m.depth,
|
|
"quantity": m.quantity,
|
|
"quantity_unit": m.quantity_unit,
|
|
"estimated_cost": m.estimated_cost,
|
|
}
|
|
)
|
|
|
|
# Attach materials safely (no filtering side effects)
|
|
for r in recommendations_data:
|
|
r["materials"] = materials_by_recommendation.get(r["id"], [])
|
|
|
|
session.close()
|
|
|
|
return properties_data, plans_data, recommendations_data
|
|
|
|
|
|
properties_data, plans_data, recommendations_data = get_data(
|
|
portfolio_id=PORTFOLIO_ID, scenario_ids=SCENARIOS
|
|
)
|
|
|
|
properties_df = pd.DataFrame(properties_data)
|
|
plans_df = pd.DataFrame(plans_data)
|
|
recommendations_df = pd.DataFrame(recommendations_data)
|
|
|
|
with db_read_session() as session:
|
|
materials = get_materials(session)
|
|
|
|
materials = pd.DataFrame(materials)
|
|
|
|
material_lookup = materials.set_index("id")[["type", "includes_battery"]].to_dict(
|
|
"index"
|
|
)
|
|
|
|
|
|
def has_solar_with_battery(materials_list):
|
|
for m in materials_list or []:
|
|
mat = material_lookup.get(m["material_id"])
|
|
if not mat:
|
|
continue
|
|
if mat["type"] == "solar_pv" and mat["includes_battery"]:
|
|
return True
|
|
return False
|
|
|
|
|
|
recommendations_df["has_solar_with_battery"] = recommendations_df["materials"].apply(
|
|
has_solar_with_battery
|
|
)
|
|
|
|
recommendations_df["measure_type"] = np.where(
|
|
recommendations_df["has_solar_with_battery"] == True,
|
|
recommendations_df["measure_type"] + "_with_battery",
|
|
recommendations_df["measure_type"],
|
|
)
|
|
|
|
# Adjust material type to indicate if there is a battery included
|
|
|
|
from utils.s3 import read_csv_from_s3, read_excel_from_s3
|
|
|
|
# asset_list = read_excel_from_s3(
|
|
# bucket_name="retrofit-plan-inputs-dev", file_key="2/404/20251211T163200754Z/asset_list.xlsx",
|
|
# header_row=0, sheet_name="Standardised Asset List"
|
|
# )
|
|
|
|
|
|
for scenario_id in SCENARIOS:
|
|
# Get recs for this scenario
|
|
recommended_measures_df = recommendations_df[
|
|
recommendations_df["scenario_id"] == scenario_id
|
|
][["property_id", "measure_type", "estimated_cost", "default"]]
|
|
recommended_measures_df = recommended_measures_df[
|
|
recommended_measures_df["default"]
|
|
]
|
|
recommended_measures_df = recommended_measures_df.drop(columns=["default"])
|
|
|
|
post_install_sap = recommendations_df[
|
|
recommendations_df["scenario_id"] == scenario_id
|
|
][["property_id", "default", "sap_points"]]
|
|
post_install_sap = post_install_sap[post_install_sap["default"]]
|
|
# Sum up the sap points by property id
|
|
post_install_sap = (
|
|
post_install_sap.groupby(["property_id"])[["sap_points"]].sum().reset_index()
|
|
)
|
|
|
|
# Find dupes by property id and measure type
|
|
dupes = recommended_measures_df.duplicated(
|
|
subset=["property_id", "measure_type"], keep=False
|
|
)
|
|
dupe_df = recommended_measures_df[dupes]
|
|
|
|
if dupe_df.shape:
|
|
# Drop dupes - happened due to a funny bug
|
|
recommended_measures_df = recommended_measures_df.drop_duplicates(
|
|
subset=["property_id", "measure_type"], keep="first"
|
|
)
|
|
|
|
recommendations_measures_pivot = recommended_measures_df.pivot(
|
|
index="property_id", columns="measure_type", values="estimated_cost"
|
|
)
|
|
recommendations_measures_pivot = recommendations_measures_pivot.reset_index()
|
|
|
|
# Total cost is the row sum, excluding the property_id column
|
|
recommendations_measures_pivot["total_retrofit_cost"] = (
|
|
recommendations_measures_pivot.drop(columns=["property_id"]).sum(axis=1)
|
|
)
|
|
|
|
df = (
|
|
properties_df[
|
|
[
|
|
"landlord_property_id",
|
|
"property_id",
|
|
"uprn",
|
|
"address",
|
|
"postcode",
|
|
"property_type",
|
|
"walls",
|
|
"roof",
|
|
"heating",
|
|
"windows",
|
|
"current_epc_rating",
|
|
"current_sap_points",
|
|
"original_sap_points",
|
|
"total_floor_area",
|
|
"number_of_rooms",
|
|
"lodgement_date",
|
|
"is_expired",
|
|
"id",
|
|
]
|
|
]
|
|
.merge(recommendations_measures_pivot, how="left", on="property_id")
|
|
.merge(post_install_sap, how="left", on="property_id")
|
|
)
|
|
|
|
# df = df.drop(columns=["property_id"])
|
|
df["sap_points"] = df["sap_points"].fillna(0)
|
|
|
|
df["predicted_post_works_sap"] = df["current_sap_points"] + df["sap_points"]
|
|
df["predicted_post_works_sap"] = df["predicted_post_works_sap"]
|
|
df["predicted_post_works_epc"] = df["predicted_post_works_sap"].apply(
|
|
lambda x: sap_to_epc(x)
|
|
)
|
|
df["uprn"] = df["uprn"].astype(str)
|
|
|
|
# Expected columns list
|
|
expected_columns = [
|
|
"suspended_floor_insulation",
|
|
"solid_floor_insulation",
|
|
"external_wall_insulation",
|
|
"internal_wall_insulation",
|
|
"cavity_wall_insulation",
|
|
"loft_insulation",
|
|
"flat_roof_insulation",
|
|
"room_roof_insulation",
|
|
"secondary_glazing",
|
|
"double_glazing",
|
|
"solar_pv",
|
|
"high_heat_retention_storage_heaters",
|
|
"air_source_heat_pump",
|
|
"boiler_upgrade",
|
|
"roomstat_programmer_trvs",
|
|
"time_temperature_zone_control",
|
|
]
|
|
# Add missing columns with default values
|
|
for col in expected_columns:
|
|
if col not in df.columns:
|
|
df[col] = ""
|
|
|
|
# A per-recommendation detail sheet (one row per recommended measure) so the
|
|
# measures and their costs are readable directly, not just pivoted into the
|
|
# wide `properties` sheet.
|
|
recs_detail = recommendations_df[
|
|
recommendations_df["scenario_id"] == scenario_id
|
|
].copy()
|
|
recs_detail = recs_detail[recs_detail["default"]]
|
|
detail_cols = [
|
|
c
|
|
for c in [
|
|
"property_id",
|
|
"measure_type",
|
|
"description",
|
|
"estimated_cost",
|
|
"sap_points",
|
|
"co2_equivalent_savings",
|
|
"kwh_savings",
|
|
"energy_cost_savings",
|
|
]
|
|
if c in recs_detail.columns
|
|
]
|
|
recs_detail = recs_detail[detail_cols].sort_values(
|
|
["property_id", "estimated_cost"], ascending=[True, False]
|
|
)
|
|
|
|
# Create excel to store to
|
|
filename = f"{scenario_names[scenario_id]} - {project_name}.xlsx"
|
|
with pd.ExcelWriter(filename) as writer:
|
|
df.to_excel(writer, sheet_name="properties", index=False)
|
|
recs_detail.to_excel(writer, sheet_name="recommendations", index=False)
|