From ef942ef18ab691217f489ac8a4dd0cedc4e0c05f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 10 Jan 2026 11:40:49 +0000 Subject: [PATCH 1/5] allow double glazing if no restrictions but confirmed performance of secondary glazing --- backend/app/db/functions/__init__.py | 1 + .../functions/already_installed_functions.py | 40 + .../db/functions/recommendations_functions.py | 3 +- backend/app/plan/data_classes.py | 1 - backend/app/plan/utils.py | 7 +- backend/engine/engine.py | 15 +- backend/onboarders/mappings/age_band.py | 14 + backend/onboarders/mappings/built_form.py | 15 + backend/onboarders/mappings/property_type.py | 6 + backend/onboarders/mappings/walls.py | 3 + backend/onboarders/parity.py | 95 ++ .../g_rebaselining_installed_measrues.py | 1020 ++++++++++++++--- .../h_reset_estimated_epcs.py | 56 +- .../i_testing_parity_data.py | 21 + .../j_installed_measures.py | 7 + recommendations/WindowsRecommendations.py | 10 +- 16 files changed, 1147 insertions(+), 167 deletions(-) create mode 100644 backend/app/db/functions/already_installed_functions.py create mode 100644 backend/onboarders/mappings/age_band.py create mode 100644 backend/onboarders/mappings/built_form.py create mode 100644 backend/onboarders/mappings/property_type.py create mode 100644 backend/onboarders/mappings/walls.py create mode 100644 backend/onboarders/parity.py create mode 100644 etl/customers/peabody/Nov 2025 Consulting Project/i_testing_parity_data.py create mode 100644 etl/customers/peabody/Nov 2025 Consulting Project/j_installed_measures.py diff --git a/backend/app/db/functions/__init__.py b/backend/app/db/functions/__init__.py index 0f239d6e..8e7495bf 100644 --- a/backend/app/db/functions/__init__.py +++ b/backend/app/db/functions/__init__.py @@ -10,3 +10,4 @@ from .materials_functions import * from .inspections_functions import * from .non_intrusive_surveys import * from .whlg_functions import * +from .already_installed_functions import * diff --git a/backend/app/db/functions/already_installed_functions.py b/backend/app/db/functions/already_installed_functions.py new file mode 100644 index 00000000..351419b0 --- /dev/null +++ b/backend/app/db/functions/already_installed_functions.py @@ -0,0 +1,40 @@ +from backend.app.db.models.recommendations import InstalledMeasure +from typing import Dict, List, Set +from collections import defaultdict + + +def get_installed_measure_types_by_uprns( + session, + uprns: List[int], +) -> Dict[int, Set[str]]: + """ + Returns installed measure types per UPRN. + + { + uprn: {"cavity_wall_insulation", "mechanical_ventilation", ...} + } + """ + + if not uprns: + return {} + + rows = ( + session.query( + InstalledMeasure.uprn, + InstalledMeasure.measure_type, + ) + .filter(InstalledMeasure.is_active.is_(True)) + .filter(InstalledMeasure.uprn.in_(uprns)) + .all() + ) + + out: Dict[int, Set[str]] = defaultdict(set) + + for uprn, measure_type in rows: + out[uprn].add( + measure_type.value + if hasattr(measure_type, "value") + else measure_type + ) + + return out diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index ae178c8a..4fdd9324 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -27,7 +27,8 @@ def prepare_plan_data( """ # Plan carbon savings co2_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations]) - post_co2_emissions = p.data["co2-emissions-current"] - co2_savings + raise Exception("CHECK ME") + post_co2_emissions = p.energy["co2_emissions"] - co2_savings # Plan bill savings energy_bill_savings = sum([r["energy_cost_savings"] for r in default_recommendations]) diff --git a/backend/app/plan/data_classes.py b/backend/app/plan/data_classes.py index cec5ed11..99f6156b 100644 --- a/backend/app/plan/data_classes.py +++ b/backend/app/plan/data_classes.py @@ -5,6 +5,5 @@ from typing import Any, Optional @dataclass class PropertyRequestData: patch: dict - already_installed: list non_invasive_recommendations: dict valuation: Optional[float] diff --git a/backend/app/plan/utils.py b/backend/app/plan/utils.py index 52e2b0c4..33f391d4 100644 --- a/backend/app/plan/utils.py +++ b/backend/app/plan/utils.py @@ -52,7 +52,7 @@ def patch_epc(patch, epc_records): def extract_property_request_data( - address: Address, patches, already_installed, non_invasive_recommendations, valuation_data, uprn + address: Address, patches, non_invasive_recommendations, valuation_data, uprn ): patch_has_uprn = "uprn" in patches[0] if patches else True if patch_has_uprn: @@ -64,10 +64,6 @@ def extract_property_request_data( x for x in patches if (x["address"] == address.address) and (x["postcode"] == address.postcode) ), {}) - property_already_installed = next(( - x for x in already_installed if (x["address"] == address.address) and (x["postcode"] == address.postcode) - ), []) - # Because we have some non-invasive recommendations that match on address and postcode, but not UPRN # we need to check existence of uprn has_uprn = "uprn" in non_invasive_recommendations[0] if non_invasive_recommendations else False @@ -119,7 +115,6 @@ def extract_property_request_data( # Return data class to give a structured format return PropertyRequestData( patch=patch, - already_installed=property_already_installed, non_invasive_recommendations=property_non_invasive_recommendations, valuation=property_valuation ) diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 4f5ee3c1..f4e3ad3f 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -684,6 +684,9 @@ async def model_engine(body: PlanTriggerRequest): energy_assessments_by_uprn = db_funcs.energy_assessment_functions.get_latest_assessments_for_uprns( session, uprns ) + already_installed_by_uprn = db_funcs.already_installed_functions.get_installed_measure_types_by_uprns( + session, uprns + ) # If we have properties that need to be created, we cerate them in bulk logger.info("Determine new properties to be created") @@ -703,7 +706,7 @@ async def model_engine(body: PlanTriggerRequest): property_lookup[("uprn", uprn)] = prop_id if landlord_property_id: property_lookup[("landlord_property_id", landlord_property_id)] = prop_id - + logger.info("Processing each property for model input preparation") input_properties, inspections_map, eco_packages, epc_upserts = [], {}, {}, [] for addr, config in tqdm( @@ -725,6 +728,8 @@ async def model_engine(body: PlanTriggerRequest): energy_assessment = energy_assessments_by_uprn.get(addr.uprn) + property_already_installed = list(already_installed_by_uprn[addr.uprn]) + epc_searcher = SearchEpc( address1=addr.address1, postcode=addr.postcode, @@ -767,7 +772,6 @@ async def model_engine(body: PlanTriggerRequest): req_data = extract_property_request_data( address=addr, patches=patches, - already_installed=already_installed, non_invasive_recommendations=non_invasive_recommendations, valuation_data=valuation_data, uprn=addr.uprn, @@ -813,7 +817,7 @@ async def model_engine(body: PlanTriggerRequest): address=epc_searcher.address_clean, postcode=epc_searcher.postcode_clean, epc_record=prepared_epc, - already_installed=req_data.already_installed + eco_packages.get(property_id)[3], + already_installed=property_already_installed + eco_packages.get(property_id)[3], property_valuation=req_data.valuation, non_invasive_recommendations=property_non_invasive_recommendations, energy_assessment=energy_assessment, @@ -965,6 +969,8 @@ async def model_engine(body: PlanTriggerRequest): # Temp putting this here recommendations_scoring_data["is_post_sap10_ending"] = True + recommendations_scoring_data["sap_starting"] = 77 + recommendations_scoring_data = recommendations_scoring_data.drop( columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", "carbon_ending"] @@ -1189,7 +1195,8 @@ async def model_engine(body: PlanTriggerRequest): property_updates, property_epc_details, property_spatial_updates = [], [], [] plans_to_create, recommendations_to_create = [], [] - + # TODO: Check the update to carbon + print("NEED TO CHECK THE UPDATE TO CARBON") # Prepare the data that will need to be uploaded in bulk for p in input_properties: recommendations_for_property = recommendations.get(p.id, []) diff --git a/backend/onboarders/mappings/age_band.py b/backend/onboarders/mappings/age_band.py new file mode 100644 index 00000000..2487c921 --- /dev/null +++ b/backend/onboarders/mappings/age_band.py @@ -0,0 +1,14 @@ +party_map = { + "Before 1900": 'England and Wales: before 1900', + "1900-1929": 'England and Wales: 1900-1929', + "1930-1949": 'England and Wales: 1930-1949', + "1950-1966": 'England and Wales: 1950-1966', + "1967-1975": 'England and Wales: 1967-1975', + "1976-1982": 'England and Wales: 1976-1982', + "1983-1990": 'England and Wales: 1983-1990', + "1991-1995": 'England and Wales: 1991-1995', + "1996-2002": 'England and Wales: 1996-2002', + "2003-2006": 'England and Wales: 2003-2006', + "2007-2011": 'England and Wales: 2007-2011', + "2012 onwards": 'England and Wales: 2012-2021', +} diff --git a/backend/onboarders/mappings/built_form.py b/backend/onboarders/mappings/built_form.py new file mode 100644 index 00000000..23901fc6 --- /dev/null +++ b/backend/onboarders/mappings/built_form.py @@ -0,0 +1,15 @@ +parity_map = { + "MidTerrace": "Mid-Terrace", + "EndTerrace": "End-Terrace", + "Detached": "Detached", + "SemiDetached": "Semi-Detached", + "EnclosedMidTerrace": "Enclosed Mid-Terrace", + "EnclosedEndTerrace": "Enclosed End-Terrace", +} + +# MidTerrace 41462 +# EndTerrace 20910 +# Detached 16875 +# SemiDetached 14725 +# EnclosedMidTerrace 3176 +# EnclosedEndTerrace 2393 diff --git a/backend/onboarders/mappings/property_type.py b/backend/onboarders/mappings/property_type.py new file mode 100644 index 00000000..75deef04 --- /dev/null +++ b/backend/onboarders/mappings/property_type.py @@ -0,0 +1,6 @@ +parity_map = { + "Flat": "Flat", + "Maisonette": "Maisonette", + "Bungalow": "Bungalow", + "House": "House", +} diff --git a/backend/onboarders/mappings/walls.py b/backend/onboarders/mappings/walls.py new file mode 100644 index 00000000..9b70b49c --- /dev/null +++ b/backend/onboarders/mappings/walls.py @@ -0,0 +1,3 @@ +parity_map = { + +} diff --git a/backend/onboarders/parity.py b/backend/onboarders/parity.py new file mode 100644 index 00000000..f41ebeaf --- /dev/null +++ b/backend/onboarders/parity.py @@ -0,0 +1,95 @@ +import pandas as pd +from etl.epc.DataProcessor import construction_age_bounds_map +from backend.onboarders.mappings.property_type import parity_map as property_map +from backend.onboarders.mappings.age_band import party_map as age_band_map +from backend.onboarders.mappings.built_form import parity_map as built_form_map + + +def check_nulls(data, original_column, mapped_column): + # We only allow nulls if the oroginal value was null + null_vals = data[pd.isnull(data[mapped_column])] + if null_vals.empty: + return True + # We make sure all original values were null + assert pd.isnull(null_vals[original_column]).all(), ( + f"Some values in {mapped_column} were not mapped, but original values were not null" + ) + + +# Sample input data + +data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/2025_11_11 - Peabody " + "- Data Extracts for Domna.xlsx", + sheet_name="Sustainability" +) + +# We want to map the parity fields to standard EPC references. This will allow us to +# 1) Estimate EPCs, more accurately +# 2) Patch incorrect EPCs with ease +# 3) Indicate already installed measures + +# ------------ construction_age_band ------------ +# Map to EPC age bands +# def construction_date_to_band(year): +# if pd.isnull(year): +# return None +# # Get the year from the date which is numpy datetime format +# for label, ranges in construction_age_bounds_map.items(): +# if ranges["l"] <= year <= ranges["u"]: +# return label +# raise NotImplementedError("year out of bounds") +# +# +# data["construction_age_band"] = pd.to_datetime(data["Construction Date"]).dt.year.apply(construction_date_to_band) + +data["construction_age_band"] = data["Construction Years"].map(age_band_map) + +check_nulls(data, "Construction Years", "construction_age_band") + +# ------------ property_type ------------ +data["property_type"] = data["Type"].map(property_map) + +assert pd.isnull(data["property_type"]).sum() == 0, "Some property types were not mapped" + +# ------------ built_form ------------ +data["built_form"] = data["Attachment"].map(built_form_map) + +assert pd.isnull(data["built_form"]).sum() == 0, "Some built forms were not mapped" + +# ------------ Wall Construction ------------ + +data["walls_combined"] = data["Wall Construction"] + "+" + data["Wall Insulation"].fillna("Unknown Insulation") + +data["Wall Insulation"].value_counts() +data["Wall Construction"].value_counts() + +as_built_map = { + "Cavity": {"insulated_age_bands":[], "partial_insulated_age_bands": []}, + "Solid Brick": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, + "System": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, + "Timber Frame": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, + "Sandstone": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, + "Granite": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, + "Cob": {"insulated_age_bands": [], "partial_insulated_age_bands": []}, +} + +def map_wall_construction(wall_constuction, wall_insulation, construction_age_band): + if wall_insulation == "AsBuilt": + # Deduce based on wall construction and age band + bands = as_built_map.get(wall_constuction, None) + if bands is None: + raise NotImplementedError(f"Wall construction {wall_constuction} not in as built map") + + # We check if the age band is in insulated or partial insulated, and if neither, we assume uninsulated + + + + +# Variables we want to map +'Org Ref', 'Address 1', 'Address 2', 'Address 3', 'Postcode', 'Type', + 'Attachment', 'Construction Years', 'Wall Construction', + 'Wall Insulation', 'Roof Construction', 'Roof Insulation', + 'Floor Construction', 'Floor Insulation', 'Glazing', 'Heating', + 'Boiler Efficiency', 'Main Fuel', 'Controls Adequacy', 'UPRN', + 'Total Floor Area (m2)' \ No newline at end of file diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py b/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py index d310ffa4..8d4bc9da 100644 --- a/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py +++ b/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py @@ -1,10 +1,10 @@ import pandas as pd +from tqdm import tqdm from sqlalchemy.orm import sessionmaker from backend.app.db.connection import db_engine, db_read_session, db_session from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, RecommendationMaterials, \ InstalledMeasure from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel -from sqlalchemy import func from backend.app.utils import sap_to_epc from typing import Dict, List, Set from recommendations.Costs import Costs @@ -87,14 +87,9 @@ def get_all_data(portfolio_id, scenario_ids): return properties_data, plans_data, recommendations_data -PORTFOLIO_ID = 419 # Peabody +PORTFOLIO_ID = 431 # Peabody - new portfolio SCENARIOS = [ - # 871, # EPC C - fabric first, no solid floor, ashp 3.0 - # 863, # EPC B, No EWI/IWI, No Solid Floor, ASHP 3.0 COP - # 862, # EPC B - No solid floor, ASHP COP 3.0 - # 861, # EPC C, No EWI/IWI, No Solid Floor, ASHP 3.0 COP - # 859, # EPC C - no solid floor, ashp 3.0 - 885, # EPC B - fabric first, no solid floor, ashp 3.0 + 891, # EPC B - No solid floor, ASHP COP 3.0 ] # properties_data, plans_data, recommendations_data = get_all_data(portfolio_id=PORTFOLIO_ID, scenario_ids=SCENARIOS) @@ -106,31 +101,31 @@ SCENARIOS = [ # Save CSVs # properties_df.to_csv( # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" -# "f_peabody_properties_data_20260108.csv", +# "Final portfolio datasets/v2/peabody_properties_data_20260108.csv", # index=False # ) # plans_df.to_csv( # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" -# "f_peabody_plans_data_20260108.csv", +# "Final portfolio datasets/v2/peabody_plans_data_20260108.csv", # index=False # ) # recommendations_df.to_csv( # "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" -# "f_peabody_recommendations_data_20260108.csv", +# "Final portfolio datasets/v2/peabody_recommendations_data_20260108.csv", # index=False # ) # Read csvs properties_df = pd.read_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" - "f_peabody_properties_data_20260108.csv" + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting " + "Project/Final portfolio datasets/v2/peabody_properties_data_20260108.csv" ) plans_df = pd.read_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" - "f_peabody_plans_data_20260108.csv" + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final portfolio " + "datasets/v2/peabody_plans_data_20260108.csv" ) recommendations_df = pd.read_csv( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/" - "f_peabody_recommendations_data_20260108.csv" + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final portfolio " + "datasets/v2/peabody_recommendations_data_20260108.csv" ) sustainability_data = pd.read_excel( @@ -138,11 +133,29 @@ sustainability_data = pd.read_excel( "- Data Extracts for Domna.xlsx", sheet_name="Sustainability" ) +sustainability_data_with_sap = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/Parity Data " + "08012026.xlsx", +) -# recommendations_df = pd.read_excel( -# "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/EPC B, " -# "No solid floor, ASHP COP 3.0.xlsx" -# ) +properties_df["uprn"] = properties_df["uprn"].astype(str) +property_data_comparison = properties_df.merge( + sustainability_data, how="inner", left_on="uprn", right_on="UPRN", suffixes=("_prop", "_sust") +) + +property_data_comparison["wall_type"] = property_data_comparison["walls"].str.split(",").str[0].str.strip() + +column_pairs = { + "built_form": "Attachment", + "property_type": "Type", + "wall_type": "Wall Construction", + "heating": "Heating", +} +combination_tables = {} + +for v1, v2 in column_pairs.items(): + df = property_data_comparison.groupby([v1, v2]).size().reset_index(name='count') + combination_tables[v1] = df # We just need all of the measure types, per property recommendation_measure_types = recommendations_df[ @@ -177,7 +190,7 @@ sustainability_data["external_wall_insulation"] = sustainability_data["Wall Insu ["External", "FilledCavityPlusExternal"] ) sustainability_data["loft_insulation"] = sustainability_data["Roof Insulation"].isin( - ["mm300", "mm250"] + ["mm300", "mm250", "mm350", "mm400", "mm270"] ) sustainability_data["double_glazing"] = sustainability_data["Glazing"].isin( ["Double 2002 or later", "Double but age unknown", "Triple", "DoubleKnownData", "Secondary", "TripleKnownData"] @@ -223,86 +236,81 @@ comparison = sustainability_data.merge( # Flag entries where we've been told that walls are already insulated, but we have recommendations for wall insulation # ------------ Walls ------------ -comparison["conflict_cavity_wall_insulation"] = ( +cwi_conflicting = comparison[ (comparison["cavity_wall_insulation"]) & (pd.isnull(comparison["cavity_wall_insulation_from_recs"]) == False) -) -comparison["conflict_iwi_wall_insulation"] = ( + ].copy() +cwi_conflicting["conflict_cavity_wall_insulation"] = True +iwi_conflicting = comparison[ (comparison["internal_wall_insulation"]) & (pd.isnull(comparison["internal_wall_insulation_from_recs"]) == False) -) -comparison["conflict_ewi_wall_insulation"] = ( + ].copy() +iwi_conflicting["conflict_iwi_wall_insulation"] = True + +ewi_conflicting = comparison[ (comparison["external_wall_insulation"]) & (pd.isnull(comparison["external_wall_insulation_from_recs"]) == False) -) - -cwi_conflicting = comparison[comparison["conflict_cavity_wall_insulation"] == True] -iwi_conflicting = comparison[comparison["conflict_iwi_wall_insulation"] == True] -ewi_conflicting = comparison[comparison["conflict_ewi_wall_insulation"] == True] + ].copy() +ewi_conflicting["conflict_ewi_wall_insulation"] = True # ------------ Roof ------------ -comparison["conflict_loft_insulation"] = ( +loft_conflicting = comparison[ (comparison["loft_insulation"]) & (pd.isnull(comparison["loft_insulation_from_recs"]) == False) -) - -loft_conflicting = comparison[comparison["conflict_loft_insulation"] == True] + ].copy() +loft_conflicting["conflict_loft_insulation"] = True # ------------ Windows ------------ -comparison["conflict_double_glazing"] = ( +double_glazing_conflicting = comparison[ (comparison["double_glazing"]) & - ( - (pd.isnull(comparison["double_glazing_from_recs"]) == False) - ) -) -comparison["conflict_secondary_glazing"] = ( + (pd.isnull(comparison["double_glazing_from_recs"]) == False) + ].copy() +double_glazing_conflicting["conflict_double_glazing"] = True +secondary_glazing_conflicting = comparison[ (comparison["secondary_glazing"]) & - ( - (pd.isnull(comparison["secondary_glazing_from_recs"]) == False) - ) -) -double_glazing_conflicting = comparison[comparison["conflict_double_glazing"] == True] -secondary_glazing_conflicting = comparison[comparison["conflict_secondary_glazing"] == True] + (pd.isnull(comparison["secondary_glazing_from_recs"]) == False) + ].copy() +secondary_glazing_conflicting["conflict_secondary_glazing"] = True # ------------ Floors ------------ -comparison["conflict_suspended_floor_insulation"] = ( +floors_conflicting = comparison[ (comparison["suspended_floor_insulation"]) & (pd.isnull(comparison["suspended_floor_insulation_from_recs"]) == False) -) -floors_conflicting = comparison[comparison["conflict_suspended_floor_insulation"] == True] + ].copy() +floors_conflicting["conflict_suspended_floor_insulation"] = True # ------------ Boiler Upgrade ------------ -comparison["conflict_boiler_upgrade"] = ( +boiler_conflicting = comparison[ (comparison["boiler_upgrade"]) & (pd.isnull(comparison["boiler_upgrade_from_recs"]) == False) -) -boiler_conflicting = comparison[comparison["conflict_boiler_upgrade"] == True] + ].copy() +boiler_conflicting["conflict_boiler_upgrade"] = True # ------------ ASHP ------------ -comparison["conflict_air_source_heat_pump"] = ( +ashp_conflicting = comparison[ (comparison["air_source_heat_pump"]) & (pd.isnull(comparison["air_source_heat_pump_from_recs"]) == False) -) -ashp_conflicting = comparison[comparison["conflict_air_source_heat_pump"] == True] + ].copy() +ashp_conflicting["conflict_air_source_heat_pump"] = True # ------------ heat controls ------------ -comparison["conflict_time_temperature_zone_control"] = ( +ttzc_conflicting = comparison[ (comparison["time_temperature_zone_control"]) & (pd.isnull(comparison["time_temperature_zone_control_from_recs"]) == False) -) -comparison["conflict_roomstat_programmer_trvs"] = ( + ].copy() +ttzc_conflicting["conflict_time_temperature_zone_control"] = True +rst_conflicting = comparison[ (comparison["roomstat_programmer_trvs"]) & (pd.isnull(comparison["roomstat_programmer_trvs_from_recs"]) == False) -) -ttzc_conflicting = comparison[comparison["conflict_time_temperature_zone_control"] == True] -rst_conflicting = comparison[comparison["conflict_roomstat_programmer_trvs"] == True] + ].copy() +rst_conflicting["conflict_roomstat_programmer_trvs"] = True # ------------ Flat Roof Insulation ----------- -comparison["conflict_flat_roof_insulation"] = ( +flat_roof_conflicting = comparison[ (comparison["flat_roof_insulation"]) & (pd.isnull(comparison["flat_roof_insulation_from_recs"]) == False) -) -flat_roof_conflicting = comparison[comparison["conflict_flat_roof_insulation"] == True] + ].copy() +flat_roof_conflicting["conflict_flat_roof_insulation"] = True # All properties with conflicts all_conflicts = pd.concat( @@ -389,6 +397,61 @@ installed_measures_df = all_conflicts.merge( installed_measures_df = installed_measures_df[installed_measures_df["already_installed"] == True] +## --- Sense checking ---- + +FABRIC_MEASURES = { + "external_wall_insulation", + "internal_wall_insulation", + "cavity_wall_insulation", +} + + +def add_mechanical_ventilation_for_fabric(installed_measures_df, recs_with_uprn): + """ + If a property has fabric insulation installed, also mark + mechanical ventilation as installed using recommendation metrics. + """ + + # Properties with fabric installed + fabric_uprns = installed_measures_df[ + installed_measures_df["measure_type"].isin(FABRIC_MEASURES) + ]["uprn"].unique() + + # Get MV recommendation metrics (pick max SAP per property as you decided) + mv_recs = ( + recs_with_uprn[ + (recs_with_uprn["measure_type"] == "mechanical_ventilation") + & (recs_with_uprn["uprn"].isin(fabric_uprns)) + ] + .sort_values("sap_points", ascending=False) + .drop_duplicates(subset=["uprn"]) + ) + + mv_installed = mv_recs[[ + "uprn", + "measure_type", + "sap_points", + "heat_demand", + "kwh_savings", + "co2_equivalent_savings", + "energy_cost_savings", + ]].copy() + + mv_installed["already_installed"] = True + + return pd.concat( + [installed_measures_df, mv_installed], + ignore_index=True + ) + + +installed_measures_df = add_mechanical_ventilation_for_fabric( + installed_measures_df, + recs_with_uprn +) + +assert installed_measures_df[["uprn", "measure_type"]].duplicated().sum() == 0 + for col in ["sap_points", "heat_demand", "kwh_savings", "co2_equivalent_savings", "energy_cost_savings"]: print(f"n missings for {col}: {pd.isnull(installed_measures_df[col]).sum()}", ) @@ -401,21 +464,12 @@ old_sap_vs_new = properties_sap.merge( sap_impact, how="inner", on="uprn" ) old_sap_vs_new["new_sap_points"] = old_sap_vs_new["current_sap_points"] + old_sap_vs_new["sap_points"] -old_sap_vs_new["new_epc_rating"] = old_sap_vs_new["new_sap_points"].apply( - lambda x: sap_to_epc(x) -) +old_sap_vs_new["new_epc_rating"] = old_sap_vs_new["new_sap_points"].apply(lambda x: sap_to_epc(x)) # How many properties go from below C to above old_sap_vs_new[old_sap_vs_new["current_sap_points"] < 69]["new_epc_rating"].value_counts() changed = old_sap_vs_new[ (old_sap_vs_new["current_sap_points"] < 69) & (old_sap_vs_new["new_sap_points"] >= 69) ] -properties_df[properties_df["current_sap_points"] < 69].shape - -old_sap_vs_new[old_sap_vs_new["current_epc_rating"].isin(["Epc.F", "Epc.G"])] - -25979 - 3891 - -sustainability_data[sustainability_data["UPRN"] == "100021204260"] # What do I need to do: # TODO: - need to get a view of "all" measures for the property, not just recommended. We can do this but just looking @@ -426,32 +480,92 @@ sustainability_data[sustainability_data["UPRN"] == "100021204260"] # 3) For anything already installed, I should mark already installed as True, and set the cost to zero # 4) I need to update the plan cost to remove the cost of the installed measures +# TODO: +# 1) Need to push the already installed measures to the database +from sqlalchemy.orm import sessionmaker +from datetime import datetime + +BATCH_SIZE = 5000 +SOURCE = "peabody_import_2026_01" + +Session = sessionmaker(bind=db_engine) + + +def bulk_insert_installed_measures(installed_measures_df): + session = Session() + + records = [] + now = datetime.utcnow() + + for _, row in installed_measures_df.iterrows(): + records.append({ + "uprn": int(row["uprn"]), + "measure_type": row["measure_type"], + "installed_at": now, + "sap_points": float(row["sap_points"]) if pd.notna(row["sap_points"]) else None, + "carbon_savings": float(row["co2_equivalent_savings"]) if pd.notna(row["co2_equivalent_savings"]) else None, + "kwh_savings": float(row["kwh_savings"]) if pd.notna(row["kwh_savings"]) else None, + "bill_savings": float(row["energy_cost_savings"]) if pd.notna(row["energy_cost_savings"]) else None, + "heat_demand_savings": float(row["heat_demand"]) if pd.notna(row["heat_demand"]) else None, + "source": SOURCE, + "is_active": True, + }) + + try: + for i in range(0, len(records), BATCH_SIZE): + batch = records[i:i + BATCH_SIZE] + session.bulk_insert_mappings(InstalledMeasure, batch) + session.commit() + print(f"✅ Inserted {i + len(batch)} / {len(records)}") + + except Exception: + session.rollback() + raise + finally: + session.close() + + +# bulk_insert_installed_measures(installed_measures_df) ### Rebaselining - - from typing import Dict from sqlalchemy import func +from typing import Dict +from sqlalchemy import func, case + +REBASING_EXCLUDED_MEASURES = { + "mechanical_ventilation", +} + def get_installed_measure_adjustments_by_uprn_for_portfolio( session, portfolio_id: int, ) -> Dict[int, dict]: """ - Returns per-UPRN installed-measure adjustments. + Returns per-UPRN installed-measure adjustments for PROPERTY / EPC rebasing. + IMPORTANT: + - Mechanical ventilation is EXCLUDED from rebasing calculations + (drag-along measure; should not alter baseline EPC/SAP). + - All other installed measures are fully applied. + + Output shape: { uprn: { - sap_points: float, - co2: float, - energy_kwh: float, - energy_bill: float, - heat_demand: float, + "sap_points": float, + "co2": float, + "energy_kwh": float, + "energy_bill": float, + "heat_demand": float, } } """ + # -------------------------------------------- + # Limit to UPRNs that belong to this portfolio + # -------------------------------------------- uprn_subquery = ( session.query(PropertyModel.uprn) .filter(PropertyModel.portfolio_id == portfolio_id) @@ -459,24 +573,55 @@ def get_installed_measure_adjustments_by_uprn_for_portfolio( .subquery() ) + # -------------------------------------------- + # CASE helper: exclude ventilation from rebasing + # -------------------------------------------- + def exclude_ventilation(column): + return case( + ( + InstalledMeasure.measure_type.notin_( + REBASING_EXCLUDED_MEASURES + ), + column, + ), + else_=0.0, + ) + + # -------------------------------------------- + # Aggregate installed-measure impacts per UPRN + # -------------------------------------------- rows = ( session.query( InstalledMeasure.uprn.label("uprn"), - func.coalesce(func.sum(InstalledMeasure.sap_points), 0.0) - .label("sap_points"), + func.coalesce( + func.sum(exclude_ventilation(InstalledMeasure.sap_points)), + 0.0, + ).label("sap_points"), - func.coalesce(func.sum(InstalledMeasure.carbon_savings), 0.0) - .label("co2"), + func.coalesce( + func.sum(exclude_ventilation(InstalledMeasure.carbon_savings)), + 0.0, + ).label("co2"), - func.coalesce(func.sum(InstalledMeasure.kwh_savings), 0.0) - .label("energy_kwh"), + func.coalesce( + func.sum(exclude_ventilation(InstalledMeasure.kwh_savings)), + 0.0, + ).label("energy_kwh"), - func.coalesce(func.sum(InstalledMeasure.bill_savings), 0.0) - .label("energy_bill"), + func.coalesce( + func.sum(exclude_ventilation(InstalledMeasure.bill_savings)), + 0.0, + ).label("energy_bill"), - func.coalesce(func.sum(InstalledMeasure.heat_demand_savings), 0.0) - .label("heat_demand"), + func.coalesce( + func.sum( + exclude_ventilation( + InstalledMeasure.heat_demand_savings + ) + ), + 0.0, + ).label("heat_demand"), ) .filter(InstalledMeasure.is_active.is_(True)) .filter(InstalledMeasure.uprn.in_(uprn_subquery)) @@ -484,6 +629,9 @@ def get_installed_measure_adjustments_by_uprn_for_portfolio( .all() ) + # -------------------------------------------- + # Shape result for downstream consumers + # -------------------------------------------- return { row.uprn: { "sap_points": float(row.sap_points), @@ -520,20 +668,27 @@ def get_installed_measure_types_by_uprn( def compute_property_sap_updates( properties: List[PropertyModel], - sap_adjustments: Dict[int, float], + sap_adjustments: Dict[int, float], # keyed by uprn ) -> List[dict]: """ Returns property SAP rebasing results. - Does NOT mutate DB objects. + ONLY returns rows where installed measures exist. """ updates = [] for prop in properties: - if prop.uprn is None or prop.original_sap_points is None: + if prop.uprn is None: continue - sap_delta = sap_adjustments.get(prop.uprn, 0.0) + # 🚨 gatekeeper + if prop.uprn not in sap_adjustments: + continue + + if prop.original_sap_points is None: + continue + + sap_delta = sap_adjustments[prop.uprn] new_sap = prop.original_sap_points + sap_delta updates.append({ @@ -542,7 +697,7 @@ def compute_property_sap_updates( "original_sap_points": prop.original_sap_points, "installed_sap_delta": sap_delta, "new_sap_points": new_sap, - "is_adjusted": sap_delta != 0, + "is_adjusted": True, }) return updates @@ -553,17 +708,13 @@ def compute_property_sap_updates( # ------------------------------------------------------------ def get_effective_plan_recommendations( - session, - plan_id: int, - excluded_measure_types: Set[str], + session, plan_id: int, excluded_measure_types: Set[str] ) -> List[Recommendation]: q = ( session.query(Recommendation) .join(PlanRecommendations) .filter(PlanRecommendations.plan_id == plan_id) - .filter(Recommendation.default.is_(True)) - ) - + .filter(Recommendation.default.is_(True))) if excluded_measure_types: q = q.filter( ~Recommendation.measure_type.in_(excluded_measure_types) @@ -587,7 +738,7 @@ def aggregate_plan_metrics(recommendations: list[Recommendation]): agg["sap_points"] += r.sap_points or 0.0 agg["co2_savings"] += r.co2_equivalent_savings or 0.0 agg["energy_bill_savings"] += r.energy_cost_savings or 0.0 - agg["energy_consumption_savings"] += r.energy_savings or 0.0 + agg["energy_consumption_savings"] += r.kwh_savings or 0.0 agg["valuation_increase"] += r.property_valuation_increase or 0.0 base_cost = r.estimated_cost or 0.0 @@ -601,49 +752,173 @@ def aggregate_plan_metrics(recommendations: list[Recommendation]): # PLAN REBASING (READ-ONLY) # ------------------------------------------------------------ +# session,plans,properties_by_id,epcs_by_property_id, property_sap_updates = session, plans, properties_by_id, epcs, +# property_updates_by_id + +from collections import defaultdict + + +def get_installed_measure_types_by_property_id_for_portfolio( + session, + portfolio_id: int, +) -> dict[int, set[str]]: + """ + Returns: + { property_id: {measure_type, ...} } + + Includes drag-along measures (e.g. ventilation). + """ + + rows = ( + session.query( + PropertyModel.id.label("property_id"), + InstalledMeasure.measure_type, + ) + .join( + InstalledMeasure, + InstalledMeasure.uprn == PropertyModel.uprn, + ) + .filter(PropertyModel.portfolio_id == portfolio_id) + .filter(InstalledMeasure.is_active.is_(True)) + .all() + ) + + installed_by_property: dict[int, set[str]] = defaultdict(set) + + for property_id, measure_type in rows: + mt = measure_type.value if hasattr(measure_type, "value") else measure_type + installed_by_property[property_id].add(mt) + + # drag-along rules + if mt in {"cavity_wall_insulation", "internal_wall_insulation", "external_wall_insulation"}: + installed_by_property[property_id].add("mechanical_ventilation") + + return installed_by_property + + +def get_all_default_plan_recommendations( + session, + plan_ids: list[int], +) -> dict[int, list[Recommendation]]: + """ + Returns {plan_id: [Recommendation, ...]} for ALL plans in one query. + """ + + rows = ( + session.query( + PlanRecommendations.plan_id, + Recommendation, + ) + .join(Recommendation, Recommendation.id == PlanRecommendations.recommendation_id) + .filter(PlanRecommendations.plan_id.in_(plan_ids)) + .filter(Recommendation.default.is_(True)) + .all() + ) + + by_plan: dict[int, list[Recommendation]] = {} + + for plan_id, rec in rows: + by_plan.setdefault(plan_id, []).append(rec) + + return by_plan + + +def filter_remaining_recommendations( + recommendations: list[Recommendation], + installed_types: set[str], +) -> list[Recommendation]: + """ + Removes recommendations whose measure_type is already installed. + """ + if not installed_types: + return recommendations + + return [ + r for r in recommendations + if ( + (r.measure_type.value if hasattr(r.measure_type, "value") else r.measure_type) + not in installed_types + ) + ] + + def compute_plan_updates( session, plans: List[Plan], properties_by_id: Dict[int, PropertyModel], epcs_by_property_id: Dict[int, PropertyDetailsEpcModel], - property_sap_updates: Dict[int, dict], + installed_types_by_property_id, + all_ventilation_measures ) -> List[dict]: """ - Computes plan metrics assuming properties are already rebased. + Computes plan metrics after marking some recommendations as already installed. + + Rules: + - Baseline post metrics remain unchanged + - Savings + costs are recomputed excluding installed measures + - ONLY mechanical ventilation alters post metrics """ - updates = [] + all_plan_recs = get_all_default_plan_recommendations( + session, + [p.id for p in plans], + ) - for plan in plans: + updates = [] + property_to_installed_types = {} + for plan in tqdm(plans, total=len(plans)): prop = properties_by_id.get(plan.property_id) epc = epcs_by_property_id.get(plan.property_id) - prop_update = property_sap_updates.get(plan.property_id) - if not prop or not epc or not prop_update: + if not prop or not epc: continue - installed_types = get_installed_measure_types_by_uprn( - session, prop.uprn - ) + installed_types = installed_types_by_property_id.get(prop.id, set()) - future_recs = get_effective_plan_recommendations( - session, - plan.id, + if not installed_types: + continue + + plan_recs = all_plan_recs.get(plan.id, []) + + remaining_recs = filter_remaining_recommendations( + plan_recs, installed_types, ) - metrics = aggregate_plan_metrics(future_recs) + remaining = aggregate_plan_metrics(remaining_recs) - baseline_bill = ( - epc.heating_cost_current - + epc.hot_water_cost_current - + epc.lighting_cost_current - + epc.appliances_cost_current - + epc.gas_standing_charge - + epc.electricity_standing_charge - ) + # Detect ventilation removal + ventilation_removed = "mechanical_ventilation" in installed_types - post_sap = prop_update["new_sap_points"] + metrics["sap_points"] + # ------------------------------- + # Start from the previous plan + # ------------------------------- + post_sap = plan.post_sap_points + post_co2 = plan.post_co2_emissions + post_bill = plan.post_energy_bill + post_kwh = plan.post_energy_consumption + + # ------------------------------- + # Undo ventilation ONLY + # ------------------------------- + ventilation_impact = all_ventilation_measures.get(prop.uprn, None) + + if ventilation_removed and ventilation_impact is not None: + # ventilation impact = baseline - remaining + + post_sap -= ventilation_impact["sap_points"] + + post_co2 += ventilation_impact["co2"] # We save more with ventilation + + post_bill += ventilation_impact["energy_bill"] + post_kwh += ventilation_impact["energy_kwh"] + + # # Skip if nothing changes at all + # if ( + # remaining["cost_of_works"] == baseline["cost_of_works"] + # and not ventilation_removed + # ): + # continue updates.append({ "plan_id": plan.id, @@ -654,40 +929,52 @@ def compute_plan_updates( "post_epc_rating": sap_to_epc(post_sap), # Carbon - "co2_savings": metrics["co2_savings"], - "post_co2_emissions": ( - epc.co2_emissions - metrics["co2_savings"] - if epc.co2_emissions is not None - else None - ), + "co2_savings": remaining["co2_savings"], + "post_co2_emissions": post_co2, # Energy bills - "energy_bill_savings": metrics["energy_bill_savings"], - "post_energy_bill": baseline_bill - metrics["energy_bill_savings"], + "energy_bill_savings": remaining["energy_bill_savings"], + "post_energy_bill": post_bill, # Energy consumption - "energy_consumption_savings": metrics["energy_consumption_savings"], - "post_energy_consumption": ( - epc.primary_energy_consumption - - metrics["energy_consumption_savings"] - ), + "energy_consumption_savings": remaining["energy_consumption_savings"], + "post_energy_consumption": post_kwh, - # Valuation - "valuation_increase": metrics["valuation_increase"], + # Valuation (safe) + "valuation_increase": remaining["valuation_increase"], "valuation_post_retrofit": ( - prop.current_valuation + metrics["valuation_increase"] + prop.current_valuation + + remaining["valuation_increase"] if prop.current_valuation is not None else None ), # Costs - "cost_of_works": metrics["cost_of_works"], - "contingency_cost": metrics["contingency_cost"], + "cost_of_works": remaining["cost_of_works"], + "contingency_cost": remaining["contingency_cost"], }) + property_to_installed_types[prop.id] = installed_types + return updates +def build_installed_recommendation_pairs( + installed_types_by_property_id: dict[int, set[str]], +) -> list[tuple[int, str]]: + """ + Returns: + [(property_id, measure_type), ...] + """ + pairs = [] + + for property_id, measure_types in installed_types_by_property_id.items(): + for mt in measure_types: + pairs.append((property_id, mt)) + + return pairs + + def calculate_contingency_for_recommendation( recommendation, ) -> float: @@ -766,6 +1053,15 @@ def compute_epc_rebasing_updates( if not adj: continue + # if ( + # adj["sap_points"] == 0 + # and adj["co2"] == 0 + # and adj["energy_kwh"] == 0 + # and adj["energy_bill"] == 0 + # and adj["heat_demand"] == 0 + # ): + # continue + updates[property_id] = { "property_id": property_id, @@ -915,13 +1211,187 @@ def persist_epc_rebasing_updates( print(f"✅ Updated {len(epcs)} EPC records") +# For setting the original SAP, carbon, etc to the current values +def initialise_original_property_and_epc_values(portfolio_id: int): + """ + Initialise original_* columns for SAP + EPC. + Safe to re-run. Only fills NULL originals. + """ + + with db_session() as session: + # ------------------------- + # PROPERTY (SAP) + # ------------------------- + properties = ( + session.query(PropertyModel) + .filter(PropertyModel.portfolio_id == portfolio_id) + .filter(PropertyModel.original_sap_points.is_(None)) + .all() + ) + + for prop in properties: + prop.original_sap_points = prop.current_sap_points + + print(f"✅ Initialised original_sap_points for {len(properties)} properties") + + # ------------------------- + # EPC (energy / carbon) + # ------------------------- + epcs = ( + session.query(PropertyDetailsEpcModel) + .filter(PropertyDetailsEpcModel.portfolio_id == portfolio_id) + .all() + ) + + epc_updates = 0 + + for epc in epcs: + updated = False + + if epc.original_co2_emissions is None: + epc.original_co2_emissions = epc.co2_emissions + updated = True + + if epc.original_primary_energy_consumption is None: + epc.original_primary_energy_consumption = ( + epc.primary_energy_consumption + ) + updated = True + + if epc.original_current_energy_demand is None: + epc.original_current_energy_demand = epc.current_energy_demand + updated = True + + if epc.original_current_energy_demand_heating_hotwater is None: + epc.original_current_energy_demand_heating_hotwater = ( + epc.current_energy_demand_heating_hotwater + ) + updated = True + + if updated: + epc_updates += 1 + + print(f"✅ Initialised EPC originals for {epc_updates} EPC records") + + session.commit() + + +from typing import Set, Dict +from sqlalchemy import distinct + +from typing import Dict +from sqlalchemy import func + + +def get_installed_ventilation_adjustments_by_uprn_for_portfolio( + session, + portfolio_id: int, +) -> Dict[int, dict]: + """ + Returns per-UPRN aggregated impact metrics for + already-installed MECHANICAL VENTILATION. + + { + uprn: { + sap_points: float, + co2: float, + energy_kwh: float, + energy_bill: float, + heat_demand: float, + } + } + """ + + # Only consider UPRNs that belong to this portfolio + uprn_subquery = ( + session.query(PropertyModel.uprn) + .filter(PropertyModel.portfolio_id == portfolio_id) + .filter(PropertyModel.uprn.isnot(None)) + .subquery() + ) + + rows = ( + session.query( + InstalledMeasure.uprn.label("uprn"), + + func.coalesce(func.sum(InstalledMeasure.sap_points), 0.0) + .label("sap_points"), + + func.coalesce(func.sum(InstalledMeasure.carbon_savings), 0.0) + .label("co2"), + + func.coalesce(func.sum(InstalledMeasure.kwh_savings), 0.0) + .label("energy_kwh"), + + func.coalesce(func.sum(InstalledMeasure.bill_savings), 0.0) + .label("energy_bill"), + + func.coalesce(func.sum(InstalledMeasure.heat_demand_savings), 0.0) + .label("heat_demand"), + ) + .filter(InstalledMeasure.is_active.is_(True)) + .filter(InstalledMeasure.measure_type == "mechanical_ventilation") + .filter(InstalledMeasure.uprn.in_(uprn_subquery)) + .group_by(InstalledMeasure.uprn) + .all() + ) + + return { + row.uprn: { + "sap_points": float(row.sap_points), + "co2": float(row.co2), + "energy_kwh": float(row.energy_kwh), + "energy_bill": float(row.energy_bill), + "heat_demand": float(row.heat_demand), + } + for row in rows + } + + +from sqlalchemy import update, tuple_ + + +def mark_recommendations_as_installed( + session, + property_measure_pairs: list[tuple[int, str]], + dry_run: bool = True, +): + if not property_measure_pairs: + print("No recommendations to update") + return + + print(f"{len(property_measure_pairs)} recommendation matches found") + + if dry_run: + print("DRY RUN — no database changes") + return + + stmt = ( + update(Recommendation) + .where( + tuple_(Recommendation.property_id, Recommendation.measure_type) + .in_(property_measure_pairs) + ) + .values(already_installed=True) + ) + + result = session.execute(stmt) + session.commit() + + print(f"✅ Updated {result.rowcount} recommendations") + + # ------------------------------------------------------------ # EXECUTION (DRY RUN) # ------------------------------------------------------------ -PORTFOLIO_ID = 430 +PORTFOLIO_ID = 431 # TODO - run the original sap points update on the peabody portfolio +# Initialising +# initialise_original_property_and_epc_values(PORTFOLIO_ID) + + with db_read_session() as session: properties = ( session.query(PropertyModel) @@ -929,6 +1399,9 @@ with db_read_session() as session: .all() ) + all_ventilation_measures = get_installed_ventilation_adjustments_by_uprn_for_portfolio(session, PORTFOLIO_ID) + installed_types_by_property_id = get_installed_measure_types_by_property_id_for_portfolio(session, PORTFOLIO_ID) + plans = ( session.query(Plan) .filter(Plan.portfolio_id == PORTFOLIO_ID) @@ -974,10 +1447,255 @@ with db_read_session() as session: plans, properties_by_id, epcs, - property_updates_by_id, + installed_types_by_property_id, + all_ventilation_measures, ) + # Used to mark recommendations + pairs = build_installed_recommendation_pairs( + installed_types_by_property_id + ) + +from copy import deepcopy + +plan_updates_comparison = deepcopy(plan_updates) +plans_by_planid = {p.id: p for p in plans} +for u in plan_updates_comparison: + before = plans_by_planid.get(u["plan_id"]) + if not before: + continue + + u.update({ + # SAP + "before_sap_points": before.post_sap_points, + "after_sap_points": u["post_sap_points"], + + # Carbon + "before_post_co2_emissions": before.post_co2_emissions, + "after_post_co2_emissions": u["post_co2_emissions"], + + # Costs + "before_cost_of_works": before.cost_of_works, + "after_cost_of_works": u["cost_of_works"], + + "before_contingency_cost": before.contingency_cost, + "after_contingency_cost": u["contingency_cost"], + }) + +plan_updates_df = pd.DataFrame(plan_updates_comparison) + +plan_updates_df["delta_sap_points"] = ( + plan_updates_df["after_sap_points"] + - plan_updates_df["before_sap_points"] +) +plan_updates_df["delta_carbon"] = ( + plan_updates_df["after_post_co2_emissions"] + - plan_updates_df["before_post_co2_emissions"] +) +plan_updates_df["delta_cost_of_works"] = ( + plan_updates_df["after_cost_of_works"] + - plan_updates_df["before_cost_of_works"] +) +plan_updates_df["delta_contingency_cost"] = ( + plan_updates_df["after_contingency_cost"] + - plan_updates_df["before_contingency_cost"] +) + +# High-level sanity checks +summary = plan_updates_df[[ + "delta_sap_points", + "delta_carbon", + "delta_cost_of_works", + "delta_contingency_cost", +]].sum() + +print(summary) + +# Grab some random samples +example = plan_updates_df[plan_updates_df["delta_cost_of_works"] < -1000].sample(1) +# example = plan_updates_df[plan_updates_df["delta_sap_points"] == 0].sample(1) +example = plan_updates_df[plan_updates_df["property_id"] == 434936].squeeze() + +print(example["property_id"]) +# Go the the db and get the UPRN +uprn_example = 202149883 +installed_adjustments[uprn_example] + +[x for x in plan_updates if x["property_id"] == example["property_id"].values[0]] + +installed_measures_example = {} + +example.squeeze() # When ready to run! -persist_property_sap_updates(property_updates_by_id) -persist_plan_updates(plan_updates) -persist_epc_rebasing_updates(epc_updates) +# persist_property_sap_updates(property_updates_by_id) +# persist_plan_updates(plan_updates) +# persist_epc_rebasing_updates(epc_updates) +# BATCH_SIZE = 1000 +# +# with db_session() as session: +# for i in range(0, len(pairs), BATCH_SIZE): +# batch = pairs[i:i + BATCH_SIZE] +# +# mark_recommendations_as_installed( +# session, +# batch, +# dry_run=False, +# ) +# +# session.commit() + +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/435084/ +# Current EPC rating should go to 68.6 - no it shouldn't! less + +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/434930/ +# Should now be a C72, +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/434930 +# Carbon should be 2.02, energy_kwh should be, 12311.5 + +# We need a follow-up query which switches off ventilation if ewi, iwi or cwi are already installed +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/435154/plans/1024673 +# Should go to C73 +# This is a good one to test also, marking the recommendation as non-default + +# Good example to check: +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/434936/plans/1024455 +# Should go down by these: +# {'sap_points': 11.299999, 'co2': 1.85, 'energy_kwh': 7882.1997, 'energy_bill': 549.89935, 'heat_demand': 77.7} +# Before SAP: 55 +# Carbon 7.56 +# kwh: 28207 + +# Good example to check: +# https://assessment-model-git-main-hestiahomes.vercel.app/portfolio/430/building-passport/434444/plans/1024063 +# SHould change by these +# {'sap_points': 10.3, 'co2': 2.54, 'energy_kwh': 3713.5, 'energy_bill': 1028.2682, 'heat_demand': 151.61} +# Current: SAP 54 +# Carbon: 4.45 +# kwh: 10307 + +# There's one final thing to do - we had an error in post carbon so we need to increase it by the appliances +# amount for all units +from backend.ml_models.AnnualBillSavings import AnnualBillSavings + + +# Need to add this on to the plan for each property +def calculate_appliance_carbon_tonnes(total_floor_area: float) -> float: + """ + Returns appliance carbon emissions in tonnes CO2. + """ + appliance_energy_kwh = AnnualBillSavings.estimate_appliances_energy_use( + total_floor_area=total_floor_area + ) + + # kgCO2 → tonnes CO2 + appliance_carbon_tonnes = (appliance_energy_kwh * 0.232) / 1000 + return appliance_carbon_tonnes + + +from sqlalchemy.orm import joinedload +from tqdm import tqdm + +from tqdm import tqdm + + +def apply_appliance_carbon_to_plans( + session, + portfolio_id: int, + dry_run: bool = True, +): + """ + Adds appliance-related carbon emissions to plan.post_co2_emissions + using EPC total_floor_area. + """ + + # -------------------------------------------- + # Load EPCs (floor area source of truth) + # -------------------------------------------- + epcs = ( + session.query(PropertyDetailsEpcModel) + .filter(PropertyDetailsEpcModel.portfolio_id == portfolio_id) + .filter(PropertyDetailsEpcModel.total_floor_area.isnot(None)) + .all() + ) + + epc_by_property_id = { + e.property_id: e for e in epcs + } + + # -------------------------------------------- + # Load plans with post carbon + # -------------------------------------------- + plans = ( + session.query(Plan) + .filter(Plan.portfolio_id == portfolio_id) + .filter(Plan.post_co2_emissions.isnot(None)) + .all() + ) + + updates = [] + total_delta = 0.0 + + for plan in tqdm(plans, total=len(plans)): + epc = epc_by_property_id.get(plan.property_id) + if not epc: + continue + + floor_area = epc.total_floor_area + if not floor_area or floor_area <= 0: + continue + + delta = float(calculate_appliance_carbon_tonnes(floor_area)) + + if delta == 0: + continue + + updates.append((plan, delta)) + total_delta += delta + + # -------------------------------------------- + # Reporting + # -------------------------------------------- + print(f"Plans affected: {len(updates)}") + print(f"Total appliance carbon added (tCO2): {total_delta:.4f}") + + if dry_run: + print("🟡 DRY RUN — no updates applied") + return + + # -------------------------------------------- + # Apply updates + # -------------------------------------------- + for plan, delta in updates: + plan.post_co2_emissions += delta + + session.commit() + print("✅ Appliance carbon successfully applied") + + +# with db_session() as session: +# apply_appliance_carbon_to_plans( +# session, +# portfolio_id=PORTFOLIO_ID, +# dry_run=False, +# ) + +# Get all uprns for entries in already installed, from the database +with db_read_session() as session: + db_uprns = { + str(r[0]) + for r in ( + session.query(InstalledMeasure.uprn) + .all() + ) + } + +# What is the overlap of these properties and the properties in portfolo 430 +sal_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260107 " + "corrected batch 6 sal.xlsx", + sheet_name="batch 1", +) + +len(sal_data[sal_data["epc_os_uprn"].astype(str).isin(db_uprns)]["epc_os_uprn"]) + +# len([uprn for uprn, v in installed_adjustments.items() if str(uprn) in sal_data["epc_os_uprn"].astype(str).tolist()]) diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/h_reset_estimated_epcs.py b/etl/customers/peabody/Nov 2025 Consulting Project/h_reset_estimated_epcs.py index 83799eff..9c7b3d2f 100644 --- a/etl/customers/peabody/Nov 2025 Consulting Project/h_reset_estimated_epcs.py +++ b/etl/customers/peabody/Nov 2025 Consulting Project/h_reset_estimated_epcs.py @@ -75,6 +75,10 @@ df = df.sort_values("property_id", ascending=True) agg = df.groupby("property_id").size().reset_index(name="n_plans") agg = agg.sort_values("n_plans", ascending=True) + +agg[agg["n_plans"] == 3] +agg[agg["n_plans"] == 2].shape + agg[agg["n_plans"] != 3] assert all(agg["n_plans"] == 3) @@ -153,4 +157,54 @@ with pd.ExcelWriter(filename) as writer: sal.iloc[41000:61000, :].to_excel(writer, sheet_name="batch 4", index=False) sal.iloc[61000:81000, :].to_excel(writer, sheet_name="batch 5", index=False) - sal.iloc[81000:, :].to_excel(writer, sheet_name="batch 5", index=False) + sal.iloc[81000:, :].to_excel(writer, sheet_name="batch 6", index=False) + +# TODO - mistake was made when creating the final SAL +b1 = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260101 " + "sal.xlsx", + sheet_name="batch 1" +) +b2 = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260101 " + "sal.xlsx", + sheet_name="batch 2" +) +b3 = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260101 " + "sal.xlsx", + sheet_name="batch 3" +) +b4 = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260101 " + "sal.xlsx", + sheet_name="batch 4" +) +b5 = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/20260101 " + "sal.xlsx", + sheet_name="batch 5" +) +# Batch 6 should be the remaining +total = pd.concat([b1, b2, b3, b4, b5]) +remaining = sal[~sal["epc_os_uprn"].isin(total["epc_os_uprn"].values)] +# Create new output +filename = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/Final SAL/" + "20260107 corrected batch 6 sal.xlsx") + +with pd.ExcelWriter(filename) as writer: + sal.to_excel(writer, sheet_name="Standardised Asset List", index=False) + # Top 1000 for testing + b1.to_excel(writer, sheet_name="batch 1", index=False) + # Batch 2 is the next 20,000 + b2.to_excel(writer, sheet_name="batch 2", index=False) + # Batch 3 is the next 20,000 + b3.to_excel(writer, sheet_name="batch 3", index=False) + + b4.to_excel(writer, sheet_name="batch 4", index=False) + b5.to_excel(writer, sheet_name="batch 5", index=False) + remaining.to_excel(writer, sheet_name="batch 6", index=False) + +all_together = pd.concat( + [b1, b2, b3, b4, b5, remaining] +) diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/i_testing_parity_data.py b/etl/customers/peabody/Nov 2025 Consulting Project/i_testing_parity_data.py new file mode 100644 index 00000000..c6fb86ea --- /dev/null +++ b/etl/customers/peabody/Nov 2025 Consulting Project/i_testing_parity_data.py @@ -0,0 +1,21 @@ +import pandas as pd + +df = pd.read_excel( + "/Users/khalimconn-kowlessar/Downloads/Parity Data 08012026.xlsx" +) + +df['SAP Score'].mean() + +df[~pd.isnull(df["Lodged EPC Score"])]["Lodged EPC Score"].mean() +df[~pd.isnull(df["Lodged EPC Score"])]["SAP Score"].mean() + +df['Difference'] = abs(df['SAP Score'] - df['Lodged EPC Score']) +df[~pd.isnull(df["Lodged EPC Score"])]["Difference"].mean() + +df["Lodged EPC Band"].value_counts(normalize=True) +df["SAP Band"].value_counts(normalize=True) + +z = df[df["SAP Band"] != df["Lodged EPC Band"]] +agg = z.groupby(["Lodged EPC Band", "SAP Band"]).size().reset_index(name="count") + +zz = z[z["Lodged EPC Band"] == "A"] diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/j_installed_measures.py b/etl/customers/peabody/Nov 2025 Consulting Project/j_installed_measures.py new file mode 100644 index 00000000..370473a1 --- /dev/null +++ b/etl/customers/peabody/Nov 2025 Consulting Project/j_installed_measures.py @@ -0,0 +1,7 @@ +import pandas as pd + +sustainability_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Peabody/Nov 2025 Consulting Project/2025_11_11 - Peabody " + "- Data Extracts for Domna.xlsx", + sheet_name="Sustainability" +) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index 7b85ac49..bac20af4 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -72,9 +72,12 @@ class WindowsRecommendations: elif "secondary_glazing" in measures and "double_glazing" not in measures: is_secondary_glazing = True else: - is_secondary_glazing = self.property.restricted_measures or ( - self.property.windows["glazing_type"] == "secondary" + # If the property currently has some secondary glazing but isn't in a conservation area + # + is_secondary_glazing = self.property.restricted_measures and ( + self.property.data["windows-energy-eff"] in ["Poor", "Very Poor"] ) + windows_area = self.property.windows_area if not number_of_windows: @@ -200,6 +203,8 @@ class WindowsRecommendations: else: glazed_type_ending = "secondary glazing" new_windows_description = "Multiple glazing throughout" + # Windows only end up with an average efficiency + windows_energy_eff = "Average" else: raise ValueError("Invalid glazing type - implement me") @@ -208,7 +213,6 @@ class WindowsRecommendations: windows_energy_eff = "Very Good" # For post 2002 windows, the energy efficiency is "Good" and so for the simulation, we simulate with "Good" - windows_ending_config = WindowAttributes(new_windows_description).process() windows_simulation_config = check_simulation_difference( From b156513524b3883825a3b568c0d99204216fe47f Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 10 Jan 2026 11:48:56 +0000 Subject: [PATCH 2/5] updating windows description for secondary glazing replacement to double --- recommendations/WindowsRecommendations.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py index bac20af4..917a1667 100644 --- a/recommendations/WindowsRecommendations.py +++ b/recommendations/WindowsRecommendations.py @@ -78,15 +78,17 @@ class WindowsRecommendations: self.property.data["windows-energy-eff"] in ["Poor", "Very Poor"] ) - windows_area = self.property.windows_area + # We check if the windows are partially insulated but we're recommending double glazing as a complete + # replacement + double_glazing_replacement = ( + not is_secondary_glazing and + # As defined in coverage_map in windows attributes + self.property.windows["glazing_coverage"] in ["partial", "most"] + ) if not number_of_windows: raise ValueError("Number of windows not specified") - if windows_area is not None: - # TODO - we don't have a price for this so we can't recommend it - print("We have windows area, we should use this data for our recommendations!!!") - # We scale the number of windows based on the proportion of existing glazing if self.property.data["multi-glaze-proportion"] != "": @@ -118,7 +120,10 @@ class WindowsRecommendations: is_secondary_glazing=is_secondary_glazing, ) - already_installed = "windows_glazing" in self.property.already_installed + measure_type = "double_glazing" if not is_secondary_glazing else "secondary_glazing" + + already_installed = measure_type in self.property.already_installed + if already_installed: cost_result = override_costs(cost_result) description = "The property already has double glazing installed. No further action is required." @@ -126,7 +131,7 @@ class WindowsRecommendations: glazing_type = ( "secondary glazing" if is_secondary_glazing else "double glazing" ) - if self.property.windows["glazing_coverage"] in ["partial", "most"]: + if self.property.windows["glazing_coverage"] in ["partial", "most"] and not double_glazing_replacement: description = f"Install {glazing_type} to the remaining windows" else: description = f"Install {glazing_type} to all windows" @@ -234,8 +239,6 @@ class WindowsRecommendations: "glazed-type": glazed_type_ending, } - measure_type = "double_glazing" if not is_secondary_glazing else "secondary_glazing" - non_invasive_recommendation = next( (r for r in self.property.non_invasive_recommendations if r["type"] in ["windows_glazing", measure_type]), {} From 808a5122ee8b1b73677eeb4ce3d37421e115c36c Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 10 Jan 2026 14:59:46 +0000 Subject: [PATCH 3/5] added re-baselining to the property model --- backend/Property.py | 15 ++++- .../db/functions/recommendations_functions.py | 1 - backend/engine/engine.py | 51 +++++++++------ backend/ml_models/api.py | 3 +- recommendations/HeatingRecommender.py | 16 +++-- recommendations/Recommendations.py | 64 ++++++++++++++++++- 6 files changed, 118 insertions(+), 32 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 70a70307..7df947ce 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -800,13 +800,19 @@ class Property: to_update[k] = None return to_update - def get_full_property_data(self, current_valuation=None): + def get_full_property_data(self, current_valuation=None, needs_rebaselining=False, rebaselining_sap=0): """ This method extracts the data which is pushed to the database, containing core information, from the EPC about a property :return: """ + current_sap_rating = self.data["current-energy-efficiency"] + if needs_rebaselining: + current_sap_rating += rebaselining_sap + + current_epc_rating = sap_to_epc(current_sap_rating) + property_data = { "creation_status": "READY", "uprn": int(self.data["uprn"]), @@ -823,9 +829,12 @@ class Property: "number_of_rooms": self.number_of_rooms, "year_built": self.year_built, "tenure": self.data["tenure"], - "current_epc_rating": self.data["current-energy-rating"], - "current_sap_points": self.data["current-energy-efficiency"], + "current_epc_rating": current_epc_rating, + "current_sap_points": current_sap_rating, "current_valuation": current_valuation, + "original_sap_points": self.data["current-energy-efficiency"], + "is_sap_points_adjusted_for_installed_measures": needs_rebaselining, + "installed_measures_sap_point_adjustment": rebaselining_sap, } property_data = self._clean_upload_data(property_data) diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 4fdd9324..7d448aa0 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -27,7 +27,6 @@ def prepare_plan_data( """ # Plan carbon savings co2_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations]) - raise Exception("CHECK ME") post_co2_emissions = p.energy["co2_emissions"] - co2_savings # Plan bill savings diff --git a/backend/engine/engine.py b/backend/engine/engine.py index f4e3ad3f..e0c5fdb7 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -929,9 +929,7 @@ async def model_engine(body: PlanTriggerRequest): # any panel performance, we ensure that we have a 3kWp and 4kWp option for the property logger.info("Identifying property recommendations") - recommendations = {} - recommendations_scoring_data = [] - representative_recommendations = {} + recommendations, recommendations_scoring_data, representative_recommendations = {}, [], {} for p in tqdm(input_properties): # We set the ECO package data, if we have it property_eco_package = eco_packages.get(p.id, (None, None, None)) @@ -965,17 +963,15 @@ async def model_engine(body: PlanTriggerRequest): recommendations_scoring_data.extend(p.recommendations_scoring_data) logger.info("Preparing data for scoring in sap change api") - recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data) + recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data).drop( + columns=[ + "rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", + "carbon_ending" + ] + ) # Temp putting this here recommendations_scoring_data["is_post_sap10_ending"] = True - recommendations_scoring_data["sap_starting"] = 77 - - recommendations_scoring_data = recommendations_scoring_data.drop( - columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending", - "carbon_ending"] - ) - all_predictions = await model_api.async_paginated_predictions( data=recommendations_scoring_data, bucket=get_settings().DATA_BUCKET, @@ -1015,19 +1011,19 @@ async def model_engine(body: PlanTriggerRequest): # We now insert kwh estimates and costs into the recommendations logger.info("Calculating tenant savings - kwh and bills") - for property_id in tqdm([p.id for p in input_properties]): + for p in tqdm(input_properties): + property_id = p.id property_recommendations = recommendations.get(property_id, []) - property_instance = [p for p in input_properties if p.id == property_id][0] property_current_energy_bill = ( Recommendations.calculate_recommendation_tenant_savings( - property_instance=property_instance, + property_instance=p, kwh_simulation_predictions=kwh_simulation_predictions, property_recommendations=property_recommendations, ashp_cop=body.ashp_cop ) ) - property_instance.current_energy_bill = property_current_energy_bill + p.current_energy_bill = property_current_energy_bill # Insert the predictions into the recommendations and run the optimiser logger.info("Optimising measures") @@ -1195,23 +1191,40 @@ async def model_engine(body: PlanTriggerRequest): property_updates, property_epc_details, property_spatial_updates = [], [], [] plans_to_create, recommendations_to_create = [], [] - # TODO: Check the update to carbon - print("NEED TO CHECK THE UPDATE TO CARBON") # Prepare the data that will need to be uploaded in bulk for p in input_properties: recommendations_for_property = recommendations.get(p.id, []) default_recommendations = [r for r in recommendations_for_property if r["default"]] + + # We need to: + # Get already installed measures + already_installed_default = [r for r in default_recommendations if r["already_installed"]] + # Property should be have increased SAP + needs_rebaselining = bool(len(already_installed_default)) + rebaselining_sap = float(sum([r["sap_points"] for r in already_installed_default])) + rebaselining_carbon = float(sum([r["co2_equivalent_savings"] for r in already_installed_default])) + rebaselining_heat_demand = float(sum([r["heat_demand"] for r in already_installed_default])) + rebaselining_kwh = float(sum([r["kwh_savings"] for r in already_installed_default])) + rebaselining_bills = float(sum([r["energy_cost_savings"] for r in already_installed_default])) + # TODO - gotta apply the adjustments to the property table, and the property_details_epc table + + # This will include everything, including already installed total_sap_points = sum([r["sap_points"] for r in default_recommendations]) new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points new_epc = sap_to_epc(new_sap_points) - total_cost = sum([r["total"] for r in default_recommendations]) + # Already installed measures do not have a cost but we remove anyway + total_cost = sum([r["total"] for r in default_recommendations if not r["already_installed"]]) valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc, total_cost=total_cost) # --- property-level updates (always) --- property_updates.append({ "property_id": p.id, "portfolio_id": body.portfolio_id, - "data": p.get_full_property_data(current_valuation=valuations["current_value"]) + "data": p.get_full_property_data( + current_valuation=valuations["current_value"], + needs_rebaselining=needs_rebaselining, + rebaselining_sap=rebaselining_sap, + ) }) property_epc_details.append(p.get_property_details_epc(portfolio_id=body.portfolio_id)) diff --git a/backend/ml_models/api.py b/backend/ml_models/api.py index 7f3e5873..daf4b715 100644 --- a/backend/ml_models/api.py +++ b/backend/ml_models/api.py @@ -142,7 +142,8 @@ class ModelApi: @staticmethod def extract_phase(recommendation_id): if 'phase=' in recommendation_id: - return int(recommendation_id.split('phase=')[1][0]) + extracted = recommendation_id.split('phase=')[1] + return int(extracted.strip()) else: return None diff --git a/recommendations/HeatingRecommender.py b/recommendations/HeatingRecommender.py index fdc25bf9..15a7b0b0 100644 --- a/recommendations/HeatingRecommender.py +++ b/recommendations/HeatingRecommender.py @@ -1,5 +1,6 @@ import re import backend.app.assumptions as assumptions +from etl.customers.immo.pilot.asset_list import already_installed from recommendations.recommendation_utils import ( check_simulation_difference, override_costs, combine_recommendation_configs ) @@ -320,12 +321,6 @@ class HeatingRecommender: measures = MEASURE_MAP["heating"] if measures is None else measures - # TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace - # the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this - # in the Costs class, stored as SYSTEM_FLUSH_COST - - # TODO: Right now, we don't have recommendations for electric boilers - we should probably have one - # if we have a non-invasive ashp recommendation, we get the configuration directly from the property instance non_invasive_ashp_recommendation = next( (r for r in self.property.non_invasive_recommendations if r["type"] == "air_source_heat_pump"), @@ -1115,6 +1110,7 @@ class HeatingRecommender: "hot-water-energy-eff": heating_simulation_config["hot_water_energy_eff_ending"] } + # TODO: Probably don't need to use this for HHRSH - simplify recommendations = self.combine_heating_and_controls( controls_recommendations=controls_recommender.recommendation, heating_simulation_config=heating_simulation_config, @@ -1128,6 +1124,12 @@ class HeatingRecommender: non_intrusive_recommendation=non_intrusive_recommendation, heating_product=hhrsh_product ) + + # Check if HHRSH are already installed + already_installed = "high_heat_retention_storage_heaters" in self.property.already_installed + for rec in recommendations: + rec["already_installed"] = already_installed + if _return: return recommendations @@ -1347,7 +1349,7 @@ class HeatingRecommender: n_rooms=self.property.number_of_rooms ) - already_installed = "heating" in self.property.already_installed + already_installed = "boiler_upgrade" in self.property.already_installed if already_installed: boiler_costs = override_costs(boiler_costs) description = "Heating system has already been upgraded, no further action needed." diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index 29ba267a..e1d63592 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -272,6 +272,36 @@ class Recommendations: property_recommendations.append(self.solar_recommender.recommendation) phase += 1 + if self.property_instance.already_installed: + # We need to re-shuffle our measures + property_recommendations_removed_installed = [] + already_installed_recs = [] + for recs in property_recommendations: + phase_recs = [] + phase_already_installed_recs = [] + for rec in recs: + if rec["already_installed"]: + phase_already_installed_recs.append(rec) + else: + phase_recs.append(rec) + if phase_recs: + property_recommendations_removed_installed.append(phase_recs) + if phase_already_installed_recs: + already_installed_recs.append(phase_already_installed_recs) + + # We re-set the phases + for i, recs in enumerate(property_recommendations_removed_installed): + for rec in recs: + rec["phase"] = i + # already installed recs get negative phasing + already_installed_phase = -len(already_installed_recs) + for recs in already_installed_recs: + for rec in recs: + rec["phase"] = already_installed_phase + already_installed_phase += 1 + + property_recommendations = already_installed_recs + property_recommendations_removed_installed + # We insert temporary ids into the recommendations which is important for the optimiser later property_recommendations = self.insert_temp_recommendation_id(property_recommendations) @@ -486,6 +516,11 @@ class Recommendations: mv_increasing_variables = ["carbon", "heat_demand"] mv_decreasing_variables = ["sap"] + # We allow for negative phase + starting_phase = min( + rec["phase"] for recs in property_recommendations for rec in recs + ) + impact_summary = [] for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: @@ -526,7 +561,7 @@ class Recommendations: # We structure this so that depending on the phase, we capture the previous phase impacts and # then just have one piece of code to calculate the difference - if rec["phase"] == 0: + if rec["phase"] == starting_phase: # These are just the starting values, from the EPC. When we score the ML models, # heating_cost_starting and heating_cost_ending are just the values in the EPC. However, with # heating_cost_ending, we expect that the EPC will predict a heating cost based on what would happen @@ -954,6 +989,33 @@ class Recommendations: pd.isnull(kwh_impact_table["hotwater_fuel_type"]).sum()): raise Exception("Fuel type is missing") + # As one final adjustment, if we + # 1) have a boiler upgrade recommendation + # 2) Have an average efficiency boiler, we adjust the COP of the existing boiler down to 75% + heating_upgrades = [x for x in property_recommendations if x[0]["type"] == "heating"] + boiler_upgrade = [r for recs in heating_upgrades for r in recs if r["measure_type"] == "boiler_upgrade"] + existing_heating_efficiency = property_instance.data["mainheat-energy-eff"] + + if len(boiler_upgrade) and existing_heating_efficiency in ["Very Poor", "Poor", "Average"]: + efficiency_map = {"Very Poor": 0.6, "Poor": 0.65, "Average": 0.7} + adjusted_cop = efficiency_map[existing_heating_efficiency] + boiler_phase = boiler_upgrade[0]["phase"] + heating_measure_types_to_id = [ + {"recommendation_id": r["recommendation_id"], "measure_type": r["measure_type"]} + for r in heating_upgrades[0] + ] + kwh_impact_table = kwh_impact_table.merge( + pd.DataFrame(heating_measure_types_to_id), how="left", on="recommendation_id" + ) + for col in ["heating_cop", "hotwater_cop"]: + kwh_impact_table[col] = np.where( + (kwh_impact_table["phase"] <= boiler_phase) & + (kwh_impact_table["heating_fuel_type"] == "Natural Gas") & + (kwh_impact_table["measure_type"] != "boiler_upgrade"), + adjusted_cop, kwh_impact_table[col] + ) + kwh_impact_table = kwh_impact_table.drop(columns=["measure_type"]) + # We now calculate the fuel cost for k in ["heating", "hotwater"]: kwh_impact_table[f"{k}_cost"] = kwh_impact_table.apply( From 3fe102c385ceb543d3c4361d9f7ba4d5e18e51f3 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 10 Jan 2026 18:52:26 +0000 Subject: [PATCH 4/5] added rebaselining for installed measures --- backend/Property.py | 37 +++++++-- .../db/functions/recommendations_functions.py | 31 +++++--- backend/engine/engine.py | 76 +++++++++++++++++-- recommendations/Recommendations.py | 2 +- .../optimiser/funding_optimiser.py | 20 +++-- 5 files changed, 136 insertions(+), 30 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 7df947ce..e0bc2199 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -852,7 +852,10 @@ class Property: else None ) - def get_property_details_epc(self, portfolio_id: int): + def get_property_details_epc( + self, portfolio_id: int, needs_rebaselining: bool = False, rebaselining_carbon: float = 0, + rebaselining_heat_demand: float = 0, rebaselining_kwh: float = 0, rebaselining_bills: float = 0 + ): if self.current_energy_bill is None: raise ValueError("Current energy bill has not been set") @@ -875,6 +878,19 @@ class Property: # We check if the lodgement date is more than 10 years old is_expired = (datetime.now() - pd.to_datetime(lodgement_date)) > timedelta(days=3650) + # Handle re-baselining + co2_emissions = self.energy["co2_emissions"] + primary_energy_consumption = self.energy["primary_energy_consumption"] + current_kwh_demand = self.current_energy_consumption + current_kwh_heating_hotwater = self.current_energy_consumption_heating_hotwater + if needs_rebaselining: + # Carbon will be reduced + co2_emissions -= rebaselining_carbon + # Heat demand will be reduced + primary_energy_consumption -= rebaselining_heat_demand + current_kwh_demand -= rebaselining_kwh + current_kwh_heating_hotwater -= rebaselining_kwh + property_details_epc = { "property_id": self.id, "portfolio_id": portfolio_id, @@ -911,16 +927,25 @@ class Property: "number_of_storeys": self.number_of_storeys["number_of_storeys"], "mains_gas": self.mains_gas, "energy_tariff": self.data["energy-tariff"], - "primary_energy_consumption": self.energy["primary_energy_consumption"], - "co2_emissions": self.energy["co2_emissions"], - "current_energy_demand": self.current_energy_consumption, - "current_energy_demand_heating_hotwater": self.current_energy_consumption_heating_hotwater, + "primary_energy_consumption": primary_energy_consumption, + "co2_emissions": co2_emissions, + "current_energy_demand": current_kwh_demand, # This is kwh - naming is confusing + "current_energy_demand_heating_hotwater": current_kwh_heating_hotwater, # This is kwh "estimated": self.data.get("estimated", False), # We indicate if we've overwritten a SAP 05 EPC "sap_05_overwritten": sap_05_overwritten, "sap_05_score": sap_05_score, "sap_05_epc_rating": sap_05_epc_rating, - **self.current_energy_bill + **self.current_energy_bill, + "original_co2_emissions": self.energy["co2_emissions"], + "original_primary_energy_consumption": self.energy["primary_energy_consumption"], + "original_current_energy_demand": self.current_energy_consumption, # Bad naming, this is kwh + "original_current_energy_demand_heating_hotwater": self.current_energy_consumption_heating_hotwater, # kwh + "installed_measures_co2_adjustment": rebaselining_carbon, + "installed_measures_energy_demand_adjustment": rebaselining_kwh, # kwh + "installed_measures_total_energy_bill_adjustment": rebaselining_bills, + "installed_measures_heat_demand_adjustment": rebaselining_heat_demand, + "is_epc_adjusted_for_installed_measures": needs_rebaselining, } return property_details_epc diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py index 7d448aa0..51562f55 100644 --- a/backend/app/db/functions/recommendations_functions.py +++ b/backend/app/db/functions/recommendations_functions.py @@ -10,7 +10,8 @@ from backend.app.db.connection import db_session, db_read_session def prepare_plan_data( - p, body, scenario_id, eco_packages, valuations, new_sap_points, new_epc, default_recommendations + p, body, scenario_id, eco_packages, valuations, new_sap_points, new_epc, default_recommendations, + rebaselining_carbon=0, rebaselining_heat_demand=0, rebaselining_kwh=0, rebaselining_bills=0, ): """ Utility function to prepare the data that goes into the production of a plan. Is a fairly rough and unstructured @@ -23,19 +24,29 @@ def prepare_plan_data( :param new_sap_points: sap points, post default recommendations :param new_epc: new epc rating, post default recommendations :param default_recommendations: list of default recommendations for a property + :param rebaselining_carbon: carbon emissions adjustment for rebaselining + :param rebaselining_heat_demand: heat demand adjustment for rebaselining + :param rebaselining_kwh: kwh consumption adjustment for rebaselining + :param rebaselining_bills: energy bill adjustment for rebaselining :return: """ # Plan carbon savings - co2_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations]) - post_co2_emissions = p.energy["co2_emissions"] - co2_savings + co2_savings = sum( + [r["co2_equivalent_savings"] for r in default_recommendations if not r.get("already_installed", False)] + ) + post_co2_emissions = p.energy["co2_emissions"] - rebaselining_carbon - co2_savings # Plan bill savings - energy_bill_savings = sum([r["energy_cost_savings"] for r in default_recommendations]) - post_energy_bill = sum(p.current_energy_bill.values()) - energy_bill_savings + energy_bill_savings = sum( + [r["energy_cost_savings"] for r in default_recommendations if not r.get("already_installed", False)] + ) + post_energy_bill = sum(p.current_energy_bill.values()) - rebaselining_bills - energy_bill_savings # energy consumption - energy_consumption_savings = sum([r["kwh_savings"] for r in default_recommendations]) - post_energy_consumption = p.current_energy_consumption - energy_consumption_savings + energy_consumption_savings = sum( + [r["kwh_savings"] for r in default_recommendations if not r.get("already_installed", False)] + ) + post_energy_consumption = p.current_energy_consumption - rebaselining_kwh - energy_consumption_savings valuation_post_retrofit, valuation_increase = None, None if valuations["current_value"]: @@ -43,8 +54,10 @@ def prepare_plan_data( valuation_post_retrofit = valuations["average_increased_value"] # plan costing data - cost_of_works = sum([r["total"] for r in default_recommendations]) - contingency_cost = sum([r.get("contingency", 0) for r in default_recommendations]) + cost_of_works = sum([r["total"] for r in default_recommendations if not r.get("already_installed", False)]) + contingency_cost = sum( + [r.get("contingency", 0) for r in default_recommendations if not r.get("already_installed", False)] + ) return { "portfolio_id": body.portfolio_id, diff --git a/backend/engine/engine.py b/backend/engine/engine.py index e0c5fdb7..740b9581 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -525,6 +525,22 @@ def extract_address_data(config, body): return uprn, address1, full_address +def keep_max_sap_per_measure_type(items): + # First pass: find max sap_points per measure_type + max_by_type = {} + for item in items: + t = item["measure_type"] + max_by_type[t] = max(max_by_type.get(t, float("-inf")), item["sap_points"]) + + # Second pass: keep only items matching the max for their type + output = [] + for measure_type, points in max_by_type.items(): + to_consider = [x for x in items if x["measure_type"] == measure_type and x["sap_points"] == points] + output.append(to_consider[0]) # pick the first one in case of ties + + return output + + async def model_engine(body: PlanTriggerRequest): logger.info("Model Engine triggered with body: %s", json.loads(body.model_dump_json())) @@ -1063,8 +1079,33 @@ async def model_engine(body: PlanTriggerRequest): (r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], r["uplift_project_score"]) = (0, 0, 0, 0) + already_installed_measures = [] + for measures in measures_to_optimise_with_uplift: + for m in measures: + # A) We're going to make the already installed measures default + # B) We need to SAP points for all already installed measures to avoid double counting + if m["already_installed"]: + already_installed_measures.append( + { + "id": m["recommendation_id"], + "measure_type": m["measure_type"], + "sap_points": m["sap_points"], + } + ) + + # We get the ones with the highest SAP + default_already_installed = keep_max_sap_per_measure_type(already_installed_measures) + already_installed_sap = float(sum(d["sap_points"] for d in default_already_installed)) + + # Remove them from the optimisation pool + finalised_measures_to_optimise = [] + for m in measures_to_optimise_with_uplift: + filtered = [x for x in m if not x["already_installed"]] + if filtered: + finalised_measures_to_optimise.append(filtered) + input_measures = optimiser_functions.prepare_input_measures( - measures_to_optimise_with_uplift, body.goal, needs_ventilation, funding=True, + finalised_measures_to_optimise, body.goal, needs_ventilation, funding=True, property_eco_packages=eco_packages.get(p.id) ) @@ -1075,9 +1116,10 @@ async def model_engine(body: PlanTriggerRequest): p=p, input_measures=input_measures, budget=body.budget, - target_gain=gain, + target_gain=gain - already_installed_sap, enforce_heat_pump_insulation=True, - enforce_fabric_first=body.enforce_fabric_first + enforce_fabric_first=body.enforce_fabric_first, + already_installed_sap=already_installed_sap, # To be passed to output ) # if handle the empty case @@ -1120,7 +1162,8 @@ async def model_engine(body: PlanTriggerRequest): ) battery_sap_score = BatterySAPScorer.score(starting_sap=post_sap, pv_size=pv_size) - selected = {r["id"] for r in solution} + # We add the defauly already installed measures to the solution + selected = {r["id"] for r in solution + default_already_installed} if property_required_measures: solution = optimiser_functions.add_required_measures( @@ -1206,7 +1249,6 @@ async def model_engine(body: PlanTriggerRequest): rebaselining_heat_demand = float(sum([r["heat_demand"] for r in already_installed_default])) rebaselining_kwh = float(sum([r["kwh_savings"] for r in already_installed_default])) rebaselining_bills = float(sum([r["energy_cost_savings"] for r in already_installed_default])) - # TODO - gotta apply the adjustments to the property table, and the property_details_epc table # This will include everything, including already installed total_sap_points = sum([r["sap_points"] for r in default_recommendations]) @@ -1227,7 +1269,16 @@ async def model_engine(body: PlanTriggerRequest): ) }) - property_epc_details.append(p.get_property_details_epc(portfolio_id=body.portfolio_id)) + property_epc_details.append( + p.get_property_details_epc( + portfolio_id=body.portfolio_id, + needs_rebaselining=needs_rebaselining, + rebaselining_carbon=rebaselining_carbon, + rebaselining_heat_demand=rebaselining_heat_demand, + rebaselining_kwh=rebaselining_kwh, + rebaselining_bills=rebaselining_bills, + ) + ) property_spatial_updates.append({"uprn": p.uprn, "data": p.spatial}) @@ -1236,7 +1287,18 @@ async def model_engine(body: PlanTriggerRequest): continue plan_data = db_funcs.recommendations_functions.prepare_plan_data( - p, body, scenario_id, eco_packages, valuations, new_sap_points, new_epc, default_recommendations + p=p, + body=body, + scenario_id=scenario_id, + eco_packages=eco_packages, + valuations=valuations, + new_sap_points=new_sap_points, + new_epc=new_epc, + default_recommendations=default_recommendations, + rebaselining_carbon=rebaselining_carbon, + rebaselining_heat_demand=rebaselining_heat_demand, + rebaselining_kwh=rebaselining_kwh, + rebaselining_bills=rebaselining_bills, ) plans_to_create.append({"property_id": p.id, "plan_data": plan_data}) diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py index e1d63592..2466ea4e 100644 --- a/recommendations/Recommendations.py +++ b/recommendations/Recommendations.py @@ -298,7 +298,7 @@ class Recommendations: for recs in already_installed_recs: for rec in recs: rec["phase"] = already_installed_phase - already_installed_phase += 1 + already_installed_phase += 1 property_recommendations = already_installed_recs + property_recommendations_removed_installed diff --git a/recommendations/optimiser/funding_optimiser.py b/recommendations/optimiser/funding_optimiser.py index 1d4fc682..f9e471ce 100644 --- a/recommendations/optimiser/funding_optimiser.py +++ b/recommendations/optimiser/funding_optimiser.py @@ -643,7 +643,8 @@ def optimise_with_scenarios( budget=None, target_gain=None, enforce_heat_pump_insulation=True, - enforce_fabric_first=False + enforce_fabric_first=False, + already_installed_sap=0 ): """ Scenario-based optimiser (funding-agnostic). @@ -754,7 +755,11 @@ def optimise_with_scenarios( heat_pump_paths = build_heat_pump_paths(remaining_wall_measures, remaining_roof_measures) paths.extend(heat_pump_paths) - fixed_selections = expand_funding_path(optimisation_measures, paths) + fixed_selections = [] + for path in paths: + result = expand_funding_path(input_measures, [path]) + if result: + fixed_selections.extend(result) for fixed in fixed_selections: @@ -825,7 +830,7 @@ def optimise_with_scenarios( "already_installed_gain": sum([x["gain"] for x in picked if x["already_installed"]]) }) - solutions_df = append_solution_metrics(solutions, target_gain, p) + solutions_df = append_solution_metrics(solutions, target_gain, p, already_installed_sap) return solutions_df @@ -835,12 +840,14 @@ def _get_ending_sap_without_battery(x): return float(sum(gain)) -def append_solution_metrics(solutions, target_gain, p): +def append_solution_metrics(solutions, target_gain, p, already_installed_sap=0): """ Given a set of solutions, this function will return a dataframe, with cost metrics appended, to allow the end user to select the optimal solution. :param solutions: :param target_gain: + :param p: + :param already_installed_sap: :return: """ @@ -852,7 +859,7 @@ def append_solution_metrics(solutions, target_gain, p): # Given the scheme, we now check if the packages are eligible. If they *are* eligible, but they don't meet the # final upgrade target, we then look to perform a final optimisation pass to meet the target gain. - solutions_df["meets_upgrade_target"] = solutions_df["total_gain"] >= target_gain - 0.1 + solutions_df["meets_upgrade_target"] = solutions_df["total_gain"] >= target_gain # We now can calculate the project ABS, which subtracts from the cost, but this is only relevant for ECO4 # We flag projects that are including batteries solutions_df["has_battery"] = solutions_df["items"].apply(has_battery) @@ -863,7 +870,7 @@ def append_solution_metrics(solutions, target_gain, p): # We need the ending SAP, but we'll need to remove the battery SAP uplift first solutions_df["ending_sap_without_battery"] = solutions_df.apply( - lambda x: int(p.data["current-energy-efficiency"]) + _get_ending_sap_without_battery(x), + lambda x: int(p.data["current-energy-efficiency"]) + already_installed_sap + _get_ending_sap_without_battery(x), axis=1 ) @@ -1015,7 +1022,6 @@ def expand_funding_path(input_measures, path_spec): cands = iter_and_candidates(input_measures, elem["AND"]) else: raise ValueError("unknown path element; expected 'OR' or 'AND'") - if not cands: return [] From 1b3aa926715f23506ff15965b5723fea62791cee Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sat, 10 Jan 2026 21:21:18 +0000 Subject: [PATCH 5/5] already installed working --- backend/Property.py | 4 +--- backend/engine/engine.py | 1 - .../g_rebaselining_installed_measrues.py | 15 ++++++++------- recommendations/WallRecommendations.py | 15 +++++++++++++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index e0bc2199..49dc15d0 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -469,10 +469,8 @@ class Property: # It means we've recommended HHR with electric immersion, and shouldn't overwrite # the hot water description continue + # Set the new value otherwise as it's due to already installed measures - do nothing - raise NotImplementedError( - "Already have this key in the phase_epc_transformation - implement me" - ) phase_epc_transformation[k] = v simulation_epc.update(phase_epc_transformation) self.simulation_epcs[rec["recommendation_id"]] = simulation_epc diff --git a/backend/engine/engine.py b/backend/engine/engine.py index 740b9581..4a503a08 100644 --- a/backend/engine/engine.py +++ b/backend/engine/engine.py @@ -1073,7 +1073,6 @@ async def model_engine(body: PlanTriggerRequest): # We insert the innovation uplift measures_to_optimise_with_uplift = deepcopy(measures_to_optimise) - # TODO: Turn this into a function and store the innovaiton uplift for group in measures_to_optimise_with_uplift: for r in group: (r["partial_project_score"], r["partial_project_funding"], r["innovation_uplift"], diff --git a/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py b/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py index 8d4bc9da..4405d113 100644 --- a/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py +++ b/etl/customers/peabody/Nov 2025 Consulting Project/g_rebaselining_installed_measrues.py @@ -196,7 +196,7 @@ sustainability_data["double_glazing"] = sustainability_data["Glazing"].isin( ["Double 2002 or later", "Double but age unknown", "Triple", "DoubleKnownData", "Secondary", "TripleKnownData"] ) sustainability_data["secondary_glazing"] = sustainability_data["Glazing"].isin( - ["Double 2002 or later", "Double but age unknown", "Triple", "DoubleKnownData", "Secondary", "TripleKnownData"] + ["Secondary"] ) sustainability_data["suspended_floor_insulation"] = sustainability_data["Floor Insulation"].isin( @@ -262,8 +262,9 @@ loft_conflicting["conflict_loft_insulation"] = True # ------------ Windows ------------ double_glazing_conflicting = comparison[ - (comparison["double_glazing"]) & - (pd.isnull(comparison["double_glazing_from_recs"]) == False) + (comparison["double_glazing"] | comparison["secondary_glazing"]) & + (pd.isnull(comparison["double_glazing_from_recs"]) == False) & + (pd.isnull(comparison["secondary_glazing_from_recs"]) == True) ].copy() double_glazing_conflicting["conflict_double_glazing"] = True secondary_glazing_conflicting = comparison[ @@ -445,10 +446,10 @@ def add_mechanical_ventilation_for_fabric(installed_measures_df, recs_with_uprn) ) -installed_measures_df = add_mechanical_ventilation_for_fabric( - installed_measures_df, - recs_with_uprn -) +# installed_measures_df = add_mechanical_ventilation_for_fabric( +# installed_measures_df, +# recs_with_uprn +# ) assert installed_measures_df[["uprn", "measure_type"]].duplicated().sum() == 0 diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py index 49483d2f..e1770838 100644 --- a/recommendations/WallRecommendations.py +++ b/recommendations/WallRecommendations.py @@ -8,6 +8,7 @@ from datatypes.enums import QuantityUnits from backend.Property import Property from backend.app.plan.schemas import MEASURE_MAP from BaseUtility import Definitions +from etl.customers.vander_elliot.non_intrusives import already_installed from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes from recommendations.recommendation_utils import ( r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value, @@ -641,8 +642,18 @@ class WallRecommendations(Definitions): # we separate the logic for for recommending them, therefore we don't # consider diminishing returns between the two as they are considered to be separate measures + prop_already_installed = self.property.already_installed + # So, we'll end up with problems if e.g. an external wall insulation is already installed and we try and + # recommend internal wall insulation. To avoid this, we check if either measure is already installed + # and: + # 1) If EWI is installed, we don't recommend IWI + # 2) If IWI is installed, we don't recommend EWI + # We only produce the recommendation for the moment, for the purpose of re-baselining + ewi_recommendations = [] - if self.ewi_valid() and "external_wall_insulation" in measures: + if self.ewi_valid() and "external_wall_insulation" in measures and ( + "internal_wall_insulation" not in prop_already_installed + ): ewi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame( @@ -653,7 +664,7 @@ class WallRecommendations(Definitions): ) iwi_recommendations = [] - if "internal_wall_insulation" in measures: + if "internal_wall_insulation" in measures and "external_wall_insulation" not in prop_already_installed: iwi_recommendations = self._find_insulation( u_value=u_value, insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials),