junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

updating database pushes for rebaselined properties

Browse source
This commit is contained in:
Khalim Conn-Kowlessar 2026-03-25 22:16:30 +00:00
parent 28b39407d0
commit e946b7254a
13 changed files with 73 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.idea/Model.iml generated
View file

@ -6,6 +6,7 @@
<sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/infrastructure/terraform/.terraform" />
</content>
<orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />

6
backend/Property.py
View file

@ -772,7 +772,7 @@ class Property:
"current_epc_rating": current_epc_rating,
"current_sap_points": current_sap_rating,
"current_valuation": current_valuation,
"original_sap_points": self.epc_record.current_energy_efficiency,
"original_sap_points": self.epc_record.original_epc["current-energy-efficiency"],
"is_sap_points_adjusted_for_installed_measures": needs_rebaselining,
"installed_measures_sap_point_adjustment": rebaselining_sap,
}
@ -886,6 +886,10 @@ class Property:
"installed_measures_total_energy_bill_adjustment": rebaselining_bills,
"installed_measures_heat_demand_adjustment": rebaselining_heat_demand,
"is_epc_adjusted_for_installed_measures": needs_rebaselining,
# Re-baselining variables - to replace already installed variables entirely
"lodged_co2_emissions": float(self.epc_record.original_epc["co2-emissions-current"]),
"lodged_heat_demand": float(self.epc_record.original_epc["energy-consumption-current"]),
"has_been_remodelled": self.epc_record.has_been_remodelled,
}
return property_details_epc

2
backend/app/db/functions/energy_assessment_functions.py
View file

@ -101,7 +101,7 @@ def get_latest_assessments_for_uprns(
found_set = set(result.keys())
missing_uprns = uprn_set - found_set
for uprn in missing_uprns:
result[uprn] = EnergyAssessment.empty_response()

77
backend/engine/engine.py
View file

@ -719,8 +719,10 @@ async def model_engine(body: PlanTriggerRequest):
# Otherwise, we use the newest EPC
# energy_assessment_is_newer will tell us if the energy assessment is newer than the newest EPC that
# has been publically lodged
epc_records, energy_assessment_is_newer = create_epc_records(
epc_searcher, energy_assessment if energy_assessment is not None else {"epc": None}
if energy_assessment is None:
energy_assessment = {}
epc_records, energy_assessment["energy_assessment_is_newer"] = create_epc_records(
epc_searcher, energy_assessment
)
req_data = extract_property_request_data(
@ -845,61 +847,7 @@ async def model_engine(body: PlanTriggerRequest):
extract_uprn=True
)
# TODO: TEMP: Compare values - and summarise the differences
compare_scores = []
for x in rebaselining_scoring_data["uprn"].unique():
record = [p for p in input_properties if p.uprn == x][0].epc_record
original_sap = record.current_energy_efficiency
new_sap = rebaselining_response["retrofit_sap_baseline_predictions"][
rebaselining_response["retrofit_sap_baseline_predictions"]["uprn"] == x
]["predictions"].values[0]
lodgement_date = record.lodgement_date
ll_differences = record.landlord_differences
# 🔑 Normalise original keys to match LL format
original = {
k.replace("-", "_"): v
for k, v in record.original_epc.items()
if k.replace("-", "_") in ll_differences
}
row = {
"uprn": x,
"original_sap": original_sap,
"new_sap": new_sap,
"differences": ll_differences,
"lodgement_date": lodgement_date,
}
# 🔑 Add paired columns in order
for key in ll_differences.keys():
row[f"{key}_ori"] = original.get(key)
row[f"{key}_ll"] = ll_differences.get(key)
compare_scores.append(row)
compare_scores = pd.DataFrame(compare_scores)
df = compare_scores.copy()
ori_cols = [c for c in df.columns if c.endswith("_ori")]
for ori_col in ori_cols:
ll_col = ori_col.replace("_ori", "_ll")
if ll_col in df.columns:
# Handle NaNs properly
same = (
df[ori_col].fillna("NULL")
== df[ll_col].fillna("NULL")
)
df.loc[same, [ori_col, ll_col]] = None
# --- Refactored: Efficiently update EPC records with new model predictions ---
# Pre-index input_properties by UPRN for fast lookup
# Update EPC records with new model predictions
input_properties_by_uprn = {int(p.uprn): p for p in input_properties if p.uprn is not None}
# Pre-index predictions for each model by UPRN
@ -913,10 +861,9 @@ async def model_engine(body: PlanTriggerRequest):
df = rebaselining_response[model]
predictions_by_model_and_uprn[model] = dict(zip(df["uprn"].astype(int), df["predictions"]))
for uprn in rebaselining_scoring_data["uprn"].unique():
for uprn_int in rebaselining_scoring_data["uprn"].unique().astype(int):
try:
uprn_int = int(uprn)
property_instance = input_properties_by_uprn.get(uprn_int)
property_instance = input_properties_by_uprn[uprn_int]
if property_instance is None:
logger.warning(f"No property found for UPRN {uprn_int} during rebaselining update.")
continue
@ -935,10 +882,8 @@ async def model_engine(body: PlanTriggerRequest):
new_carbon=new_carbon,
new_heat_demand=new_heat_demand,
)
logger.info(f"Updated EPC record for UPRN {uprn_int} with new model predictions.")
except Exception as e:
logger.error(f"Error updating EPC record for UPRN {uprn}: {e}")
# --- End refactor ---
logger.error(f"Error updating EPC record for UPRN {uprn_int}: {e}")
kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True)
@ -1015,6 +960,12 @@ async def model_engine(body: PlanTriggerRequest):
if not property_recommendations:
continue
# Perform a check for properties (temp) where we've remodelled
if p.epc_record.has_been_remodelled:
for x in property_recommendations:
if any(y.get("survey") for y in x):
raise ValueError("Should not have survey true for remodelled properties")
recommendations[p.id] = property_recommendations
representative_recommendations[p.id] = property_representative_recommendations

1
recommendations/FireplaceRecommendations.py
View file

@ -1,4 +1,3 @@
import pandas as pd
from BaseUtility import Definitions
from backend.Property import Property

7
recommendations/FloorRecommendations.py
View file

@ -9,7 +9,7 @@ from backend.app.plan.schemas import MEASURE_MAP
from backend.Property import Property
from recommendations.recommendation_utils import (
r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
get_recommended_part, get_floor_u_value, override_costs, check_simulation_difference
get_recommended_part, get_floor_u_value, override_costs, check_simulation_difference, check_use_survey
)
from recommendations.Costs import Costs
from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes
@ -226,7 +226,6 @@ class FloorRecommendations(Definitions):
raise NotImplementedError("Implement me!")
sap_points = non_invasive_recs.get("sap_points", None)
survey = non_invasive_recs.get("survey", False)
floor_ending_config = FloorAttributes(new_description).process()
floor_simulation_config = check_simulation_difference(
@ -257,7 +256,9 @@ class FloorRecommendations(Definitions):
"starting_u_value": u_value,
"new_u_value": new_u_value,
"sap_points": sap_points,
"survey": survey,
"survey": check_use_survey(
non_invasive_recs, self.property.epc_record.has_been_remodelled
),
"already_installed": already_installed,
"simulation_config": simulation_config,
"description_simulation": {

8
recommendations/HeatingRecommender.py
View file

@ -1,7 +1,7 @@
import re
import backend.app.assumptions as assumptions
from recommendations.recommendation_utils import (
check_simulation_difference, override_costs, combine_recommendation_configs
check_simulation_difference, override_costs, combine_recommendation_configs, check_use_survey
)
from backend.Property import Property
from backend.app.plan.schemas import MEASURE_MAP
@ -865,7 +865,9 @@ class HeatingRecommender:
"description_simulation": recommendation_description_simulation,
# We insert the heating system type here
"system_type": system_type,
"survey": non_intrusive_recommendation.get("survey", False),
"survey": check_use_survey(
non_intrusive_recommendation, self.property.epc_record.has_been_remodelled
),
# In this instance, we are recommending an entire heating system so the innovation rate is becased
# on the heating system as whole
"innovation_rate": heating_product["innovation_rate"],
@ -1367,7 +1369,7 @@ class HeatingRecommender:
"description_simulation": description_simulation,
**boiler_costs,
"system_type": "boiler_upgrade",
"survey": non_invasive_recommendation.get("survey", None),
"survey": check_use_survey(non_invasive_recommendation, self.property.epc_record.has_been_remodelled),
"innovation_rate": 0,
}

6
recommendations/HotwaterRecommendations.py
View file

@ -1,6 +1,6 @@
from backend.Property import Property
from recommendations.Costs import Costs
from recommendations.recommendation_utils import override_costs, check_simulation_difference
from recommendations.recommendation_utils import override_costs, check_simulation_difference, check_use_survey
from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes
@ -39,7 +39,7 @@ class HotwaterRecommendations:
self.recommend_tank_insulation(
phase=recommendations_phase,
sap_points=non_invasive_rec["sap_points"],
survey=non_invasive_rec["survey"],
survey=check_use_survey(non_invasive_rec, self.property.epc_record.has_been_remodelled),
)
recommendations_phase += 1
@ -47,7 +47,7 @@ class HotwaterRecommendations:
self.recommend_cylinder_thermostat(
phase=recommendations_phase,
sap_points=non_invasive_rec["sap_points"],
survey=non_invasive_rec["survey"],
survey=check_use_survey(non_invasive_rec, self.property.epc_record.has_been_remodelled),
)
recommendations_phase += 1

6
recommendations/LightingRecommendations.py
View file

@ -3,7 +3,7 @@ import pandas as pd
from backend.Property import Property
from typing import List
from recommendations.Costs import Costs
from recommendations.recommendation_utils import override_costs
from recommendations.recommendation_utils import override_costs, check_use_survey
from backend.ml_models.AnnualBillSavings import AnnualBillSavings
@ -169,7 +169,9 @@ class LightingRecommendations:
"low-energy-lighting": 100,
},
**cost_result,
"survey": leds_recommendation_config.get("survey", False),
"survey": check_use_survey(
leds_recommendation_config, self.property.epc_record.has_been_remodelled
),
"innovation_rate": self.material["innovation_rate"],
}
]

14
recommendations/RoofRecommendations.py
View file

@ -7,7 +7,7 @@ from datatypes.enums import QuantityUnits
from recommendations.recommendation_utils import (
get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns,
update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric, override_costs,
check_simulation_difference
check_simulation_difference, check_use_survey
)
from recommendations.Costs import Costs
from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
@ -874,7 +874,9 @@ class RoofRecommendations:
"roof-energy-eff": new_efficiency
},
**cost_result,
"survey": non_invasive_recommendations.get("survey", False),
"survey": check_use_survey(
non_invasive_recommendations, self.property.epc_record.has_been_remodelled
),
"innovation_rate": material.to_dict()["innovation_rate"]
}
)
@ -1009,7 +1011,9 @@ class RoofRecommendations:
},
**cost_result,
"already_installed": already_installed,
"survey": rir_non_invasive_recommendation.get("survey", None),
"survey": check_use_survey(
rir_non_invasive_recommendation, self.property.epc_record.has_been_remodelled
),
"innovation_rate": material.innovation_rate
}
)
@ -1079,7 +1083,9 @@ class RoofRecommendations:
},
**cost_result,
"already_installed": "sloping_ceiling_insulation" in self.property.already_installed,
"survey": sloping_ceiling_recommendation.get("survey", None),
"survey": check_use_survey(
sloping_ceiling_recommendation, self.property.epc_record.has_been_remodelled
),
"innovation_rate": 0
}
]

12
recommendations/WallRecommendations.py
View file

@ -11,7 +11,8 @@ from BaseUtility import Definitions
from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes
from recommendations.recommendation_utils import (
r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
get_recommended_part, get_wall_u_value, override_costs, check_simulation_difference
get_recommended_part, get_wall_u_value, override_costs, check_simulation_difference,
check_use_survey
)
from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
from recommendations.Costs import Costs
@ -443,7 +444,9 @@ class WallRecommendations(Definitions):
"walls-energy-eff": "Good"
},
**cost_result,
"survey": non_invasive_recommendations.get("survey", False),
"survey": check_use_survey(
non_invasive_recommendations, self.property.epc_record.has_been_remodelled
),
"innovation_rate": material.to_dict()["innovation_rate"]
}
)
@ -573,7 +576,6 @@ class WallRecommendations(Definitions):
raise ValueError("Invalid material type")
sap_points = non_invasive_recommendations.get("sap_points", None)
survey = non_invasive_recommendations.get("survey", False)
wall_ending_config = WallAttributes(new_description).process()
@ -624,7 +626,9 @@ class WallRecommendations(Definitions):
"walls-energy-eff": simulation_config["walls_energy_eff_ending"]
},
**cost_result,
"survey": survey,
"survey": check_use_survey(
non_invasive_recommendations, self.property.epc_record.has_been_remodelled
),
"innovation_rate": material.to_dict()["innovation_rate"]
}
)

6
recommendations/WindowsRecommendations.py
View file

@ -6,7 +6,7 @@ from backend.Property import Property
from backend.app.plan.schemas import MEASURE_MAP
from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes
from recommendations.Costs import Costs
from recommendations.recommendation_utils import override_costs, check_simulation_difference
from recommendations.recommendation_utils import override_costs, check_simulation_difference, check_use_survey
class WindowsRecommendations:
@ -259,7 +259,9 @@ class WindowsRecommendations:
"is_secondary_glazing": is_secondary_glazing,
"description_simulation": description_simulation,
"simulation_config": simulation_config,
"survey": non_invasive_recommendation.get("survey", None),
"survey": check_use_survey(
non_invasive_recommendation, self.property.epc_record.has_been_remodelled
),
"innovation_rate": self.glazing_material["innovation_rate"],
}
]

15
recommendations/recommendation_utils.py
View file

@ -1,7 +1,7 @@
import math
from datetime import datetime
from copy import deepcopy
from typing import Union
from typing import Union, Dict
import numpy as np
import pandas as pd
@ -975,3 +975,16 @@ def combine_recommendation_configs(recommendation_config1, recommendation_config
combined[key] = eff_2[key]
return combined
def check_use_survey(non_invasive_recommendations: Dict[str, bool], has_been_remodelled: bool):
"""
Determines if we should use a survey SAP points or not
:return:
"""
use_survey = (
non_invasive_recommendations.get("survey", False) if not
has_been_remodelled else False
)
return use_survey