updating database pushes for rebaselined properties

.idea/Model.iml

@@ -6,6 +6,7 @@
       <sourceFolder url="file://$MODULE_DIR$/model_data" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
       <excludeFolder url="file://$MODULE_DIR$/infrastructure/terraform/.terraform" />
     </content>
     <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />

backend/Property.py

@@ -772,7 +772,7 @@ class Property:
             "current_epc_rating": current_epc_rating,
             "current_sap_points": current_sap_rating,
             "current_valuation": current_valuation,
-            "original_sap_points": self.epc_record.current_energy_efficiency,
+            "original_sap_points": self.epc_record.original_epc["current-energy-efficiency"],
             "is_sap_points_adjusted_for_installed_measures": needs_rebaselining,
             "installed_measures_sap_point_adjustment": rebaselining_sap,
         }
@@ -886,6 +886,10 @@ class Property:
             "installed_measures_total_energy_bill_adjustment": rebaselining_bills,
             "installed_measures_heat_demand_adjustment": rebaselining_heat_demand,
             "is_epc_adjusted_for_installed_measures": needs_rebaselining,
+            # Re-baselining variables - to replace already installed variables entirely
+            "lodged_co2_emissions": float(self.epc_record.original_epc["co2-emissions-current"]),
+            "lodged_heat_demand": float(self.epc_record.original_epc["energy-consumption-current"]),
+            "has_been_remodelled": self.epc_record.has_been_remodelled,
         }
 
         return property_details_epc

backend/app/db/functions/energy_assessment_functions.py

@@ -101,7 +101,7 @@ def get_latest_assessments_for_uprns(
     found_set = set(result.keys())
 
     missing_uprns = uprn_set - found_set
-    
+
     for uprn in missing_uprns:
         result[uprn] = EnergyAssessment.empty_response()
 

backend/engine/engine.py

@@ -719,8 +719,10 @@ async def model_engine(body: PlanTriggerRequest):
             # Otherwise, we use the newest EPC
             # energy_assessment_is_newer will tell us if the energy assessment is newer than the newest EPC that
             # has been publically lodged
-            epc_records, energy_assessment_is_newer = create_epc_records(
+            if energy_assessment is None:
-                epc_searcher, energy_assessment if energy_assessment is not None else {"epc": None}
+                energy_assessment = {}
+            epc_records, energy_assessment["energy_assessment_is_newer"] = create_epc_records(
+                epc_searcher, energy_assessment
             )
 
             req_data = extract_property_request_data(
@@ -845,61 +847,7 @@ async def model_engine(body: PlanTriggerRequest):
             extract_uprn=True
         )
 
-        # TODO: TEMP: Compare values - and summarise the differences
+        # Update EPC records with new model predictions
-        compare_scores = []
-
-        for x in rebaselining_scoring_data["uprn"].unique():
-            record = [p for p in input_properties if p.uprn == x][0].epc_record
-
-            original_sap = record.current_energy_efficiency
-            new_sap = rebaselining_response["retrofit_sap_baseline_predictions"][
-                rebaselining_response["retrofit_sap_baseline_predictions"]["uprn"] == x
-                ]["predictions"].values[0]
-
-            lodgement_date = record.lodgement_date
-            ll_differences = record.landlord_differences
-
-            # 🔑 Normalise original keys to match LL format
-            original = {
-                k.replace("-", "_"): v
-                for k, v in record.original_epc.items()
-                if k.replace("-", "_") in ll_differences
-            }
-
-            row = {
-                "uprn": x,
-                "original_sap": original_sap,
-                "new_sap": new_sap,
-                "differences": ll_differences,
-                "lodgement_date": lodgement_date,
-            }
-
-            # 🔑 Add paired columns in order
-            for key in ll_differences.keys():
-                row[f"{key}_ori"] = original.get(key)
-                row[f"{key}_ll"] = ll_differences.get(key)
-
-            compare_scores.append(row)
-
-        compare_scores = pd.DataFrame(compare_scores)
-        df = compare_scores.copy()
-
-        ori_cols = [c for c in df.columns if c.endswith("_ori")]
-
-        for ori_col in ori_cols:
-            ll_col = ori_col.replace("_ori", "_ll")
-
-            if ll_col in df.columns:
-                # Handle NaNs properly
-                same = (
-                    df[ori_col].fillna("NULL")
-                    == df[ll_col].fillna("NULL")
-                )
-
-                df.loc[same, [ori_col, ll_col]] = None
-
-        # --- Refactored: Efficiently update EPC records with new model predictions ---
-        # Pre-index input_properties by UPRN for fast lookup
         input_properties_by_uprn = {int(p.uprn): p for p in input_properties if p.uprn is not None}
 
         # Pre-index predictions for each model by UPRN
@@ -913,10 +861,9 @@ async def model_engine(body: PlanTriggerRequest):
             df = rebaselining_response[model]
             predictions_by_model_and_uprn[model] = dict(zip(df["uprn"].astype(int), df["predictions"]))
 
-        for uprn in rebaselining_scoring_data["uprn"].unique():
+        for uprn_int in rebaselining_scoring_data["uprn"].unique().astype(int):
             try:
-                uprn_int = int(uprn)
+                property_instance = input_properties_by_uprn[uprn_int]
-                property_instance = input_properties_by_uprn.get(uprn_int)
                 if property_instance is None:
                     logger.warning(f"No property found for UPRN {uprn_int} during rebaselining update.")
                     continue
@@ -935,10 +882,8 @@ async def model_engine(body: PlanTriggerRequest):
                     new_carbon=new_carbon,
                     new_heat_demand=new_heat_demand,
                 )
-                logger.info(f"Updated EPC record for UPRN {uprn_int} with new model predictions.")
             except Exception as e:
-                logger.error(f"Error updating EPC record for UPRN {uprn}: {e}")
+                logger.error(f"Error updating EPC record for UPRN {uprn_int}: {e}")
-        # --- End refactor ---
 
         kwh_client = KwhData(bucket=get_settings().DATA_BUCKET, read_consumption_data=True)
 
@@ -1015,6 +960,12 @@ async def model_engine(body: PlanTriggerRequest):
             if not property_recommendations:
                 continue
 
+            # Perform a check for properties (temp) where we've remodelled
+            if p.epc_record.has_been_remodelled:
+                for x in property_recommendations:
+                    if any(y.get("survey") for y in x):
+                        raise ValueError("Should not have survey true for remodelled properties")
+
             recommendations[p.id] = property_recommendations
             representative_recommendations[p.id] = property_representative_recommendations
 

recommendations/FireplaceRecommendations.py

@@ -1,4 +1,3 @@
-import pandas as pd
 from BaseUtility import Definitions
 from backend.Property import Property
 

recommendations/FloorRecommendations.py

@@ -9,7 +9,7 @@ from backend.app.plan.schemas import MEASURE_MAP
 from backend.Property import Property
 from recommendations.recommendation_utils import (
     r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
-    get_recommended_part, get_floor_u_value, override_costs, check_simulation_difference
+    get_recommended_part, get_floor_u_value, override_costs, check_simulation_difference, check_use_survey
 )
 from recommendations.Costs import Costs
 from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes
@@ -226,7 +226,6 @@ class FloorRecommendations(Definitions):
                         raise NotImplementedError("Implement me!")
 
                     sap_points = non_invasive_recs.get("sap_points", None)
-                    survey = non_invasive_recs.get("survey", False)
 
                     floor_ending_config = FloorAttributes(new_description).process()
                     floor_simulation_config = check_simulation_difference(
@@ -257,7 +256,9 @@ class FloorRecommendations(Definitions):
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": sap_points,
-                            "survey": survey,
+                            "survey": check_use_survey(
+                                non_invasive_recs, self.property.epc_record.has_been_remodelled
+                            ),
                             "already_installed": already_installed,
                             "simulation_config": simulation_config,
                             "description_simulation": {

recommendations/HeatingRecommender.py

@@ -1,7 +1,7 @@
 import re
 import backend.app.assumptions as assumptions
 from recommendations.recommendation_utils import (
-    check_simulation_difference, override_costs, combine_recommendation_configs
+    check_simulation_difference, override_costs, combine_recommendation_configs, check_use_survey
 )
 from backend.Property import Property
 from backend.app.plan.schemas import MEASURE_MAP
@@ -865,7 +865,9 @@ class HeatingRecommender:
                 "description_simulation": recommendation_description_simulation,
                 # We insert the heating system type here
                 "system_type": system_type,
-                "survey": non_intrusive_recommendation.get("survey", False),
+                "survey": check_use_survey(
+                    non_intrusive_recommendation, self.property.epc_record.has_been_remodelled
+                ),
                 # In this instance, we are recommending an entire heating system so the innovation rate is becased
                 # on the heating system as whole
                 "innovation_rate": heating_product["innovation_rate"],
@@ -1367,7 +1369,7 @@ class HeatingRecommender:
                 "description_simulation": description_simulation,
                 **boiler_costs,
                 "system_type": "boiler_upgrade",
-                "survey": non_invasive_recommendation.get("survey", None),
+                "survey": check_use_survey(non_invasive_recommendation, self.property.epc_record.has_been_remodelled),
                 "innovation_rate": 0,
             }
 

recommendations/HotwaterRecommendations.py

@@ -1,6 +1,6 @@
 from backend.Property import Property
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import override_costs, check_simulation_difference
+from recommendations.recommendation_utils import override_costs, check_simulation_difference, check_use_survey
 from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes
 
 
@@ -39,7 +39,7 @@ class HotwaterRecommendations:
                     self.recommend_tank_insulation(
                         phase=recommendations_phase,
                         sap_points=non_invasive_rec["sap_points"],
-                        survey=non_invasive_rec["survey"],
+                        survey=check_use_survey(non_invasive_rec, self.property.epc_record.has_been_remodelled),
                     )
 
                     recommendations_phase += 1
@@ -47,7 +47,7 @@ class HotwaterRecommendations:
                     self.recommend_cylinder_thermostat(
                         phase=recommendations_phase,
                         sap_points=non_invasive_rec["sap_points"],
-                        survey=non_invasive_rec["survey"],
+                        survey=check_use_survey(non_invasive_rec, self.property.epc_record.has_been_remodelled),
                     )
                     recommendations_phase += 1
 

recommendations/LightingRecommendations.py

@@ -3,7 +3,7 @@ import pandas as pd
 from backend.Property import Property
 from typing import List
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import override_costs
+from recommendations.recommendation_utils import override_costs, check_use_survey
 from backend.ml_models.AnnualBillSavings import AnnualBillSavings
 
 
@@ -169,7 +169,9 @@ class LightingRecommendations:
                     "low-energy-lighting": 100,
                 },
                 **cost_result,
-                "survey": leds_recommendation_config.get("survey", False),
+                "survey": check_use_survey(
+                    leds_recommendation_config, self.property.epc_record.has_been_remodelled
+                ),
                 "innovation_rate": self.material["innovation_rate"],
             }
         ]

recommendations/RoofRecommendations.py

@@ -7,7 +7,7 @@ from datatypes.enums import QuantityUnits
 from recommendations.recommendation_utils import (
     get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns,
     update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric, override_costs,
-    check_simulation_difference
+    check_simulation_difference, check_use_survey
 )
 from recommendations.Costs import Costs
 from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
@@ -874,7 +874,9 @@ class RoofRecommendations:
                                 "roof-energy-eff": new_efficiency
                             },
                             **cost_result,
-                            "survey": non_invasive_recommendations.get("survey", False),
+                            "survey": check_use_survey(
+                                non_invasive_recommendations, self.property.epc_record.has_been_remodelled
+                            ),
                             "innovation_rate": material.to_dict()["innovation_rate"]
                         }
                     )
@@ -1009,7 +1011,9 @@ class RoofRecommendations:
                         },
                         **cost_result,
                         "already_installed": already_installed,
-                        "survey": rir_non_invasive_recommendation.get("survey", None),
+                        "survey": check_use_survey(
+                            rir_non_invasive_recommendation, self.property.epc_record.has_been_remodelled
+                        ),
                         "innovation_rate": material.innovation_rate
                     }
                 )
@@ -1079,7 +1083,9 @@ class RoofRecommendations:
                 },
                 **cost_result,
                 "already_installed": "sloping_ceiling_insulation" in self.property.already_installed,
-                "survey": sloping_ceiling_recommendation.get("survey", None),
+                "survey": check_use_survey(
+                    sloping_ceiling_recommendation, self.property.epc_record.has_been_remodelled
+                ),
                 "innovation_rate": 0
             }
         ]

recommendations/WallRecommendations.py

@@ -11,7 +11,8 @@ from BaseUtility import Definitions
 from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes
 from recommendations.recommendation_utils import (
     r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
-    get_recommended_part, get_wall_u_value, override_costs, check_simulation_difference
+    get_recommended_part, get_wall_u_value, override_costs, check_simulation_difference,
+    check_use_survey
 )
 from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
 from recommendations.Costs import Costs
@@ -443,7 +444,9 @@ class WallRecommendations(Definitions):
                             "walls-energy-eff": "Good"
                         },
                         **cost_result,
-                        "survey": non_invasive_recommendations.get("survey", False),
+                        "survey": check_use_survey(
+                            non_invasive_recommendations, self.property.epc_record.has_been_remodelled
+                        ),
                         "innovation_rate": material.to_dict()["innovation_rate"]
                     }
                 )
@@ -573,7 +576,6 @@ class WallRecommendations(Definitions):
                         raise ValueError("Invalid material type")
 
                     sap_points = non_invasive_recommendations.get("sap_points", None)
-                    survey = non_invasive_recommendations.get("survey", False)
 
                     wall_ending_config = WallAttributes(new_description).process()
 
@@ -624,7 +626,9 @@ class WallRecommendations(Definitions):
                                 "walls-energy-eff": simulation_config["walls_energy_eff_ending"]
                             },
                             **cost_result,
-                            "survey": survey,
+                            "survey": check_use_survey(
+                                non_invasive_recommendations, self.property.epc_record.has_been_remodelled
+                            ),
                             "innovation_rate": material.to_dict()["innovation_rate"]
                         }
                     )

recommendations/WindowsRecommendations.py

@@ -6,7 +6,7 @@ from backend.Property import Property
 from backend.app.plan.schemas import MEASURE_MAP
 from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import override_costs, check_simulation_difference
+from recommendations.recommendation_utils import override_costs, check_simulation_difference, check_use_survey
 
 
 class WindowsRecommendations:
@@ -259,7 +259,9 @@ class WindowsRecommendations:
                 "is_secondary_glazing": is_secondary_glazing,
                 "description_simulation": description_simulation,
                 "simulation_config": simulation_config,
-                "survey": non_invasive_recommendation.get("survey", None),
+                "survey": check_use_survey(
+                    non_invasive_recommendation, self.property.epc_record.has_been_remodelled
+                ),
                 "innovation_rate": self.glazing_material["innovation_rate"],
             }
         ]

recommendations/recommendation_utils.py

@@ -1,7 +1,7 @@
 import math
 from datetime import datetime
 from copy import deepcopy
-from typing import Union
+from typing import Union, Dict
 
 import numpy as np
 import pandas as pd
@@ -975,3 +975,16 @@ def combine_recommendation_configs(recommendation_config1, recommendation_config
                 combined[key] = eff_2[key]
 
     return combined
+
+
+def check_use_survey(non_invasive_recommendations: Dict[str, bool], has_been_remodelled: bool):
+    """
+    Determines if we should use a survey SAP points or not
+    :return: 
+    """
+
+    use_survey = (
+        non_invasive_recommendations.get("survey", False) if not
+        has_been_remodelled else False
+    )
+    return use_survey