Tidying up new descriptions and updating recommendation types

backend/Property.py

@@ -85,6 +85,9 @@ class Property(Definitions):
         self.insulation_floor_area = None
         self.number_lighting_outlets = None
 
+        self.current_adjusted_energy = None
+        self.expected_adjusted_energy = None
+
         if epc_client:
             self.epc_client = epc_client
         else:
@@ -462,7 +465,7 @@ class Property(Definitions):
             "year_built": self.year_built,
             "tenure": self.data["tenure"],
             "current_epc_rating": self.data["current-energy-rating"],
-            "current_sap_points": self.data["current-energy-efficiency"]
+            "current_sap_points": self.data["current-energy-efficiency"],
         }
 
         property_data = self._clean_upload_data(property_data)
@@ -514,6 +517,7 @@ class Property(Definitions):
             "energy_tariff": self.data["energy-tariff"],
             "primary_energy_consumption": self.energy["primary_energy_consumption"],
             "co2_emissions": self.energy["co2_emissions"],
+            "adjusted_energy_consumption": self.current_adjusted_energy,
         }
 
         return property_details_epc
@@ -770,3 +774,10 @@ class Property(Definitions):
             self.number_lighting_outlets = round(cleaned_property_data["FIXED_LIGHTING_OUTLETS_COUNT"].values[0])
         else:
             self.number_lighting_outlets = float(self.data["fixed-lighting-outlets-count"])
+
+    def set_adjusted_energy(self, current_adjusted_energy, expected_adjusted_energy):
+        """
+        Stores these values for usage later
+        """
+        self.current_adjusted_energy = current_adjusted_energy
+        self.expected_adjusted_energy = expected_adjusted_energy

backend/app/db/models/portfolio.py

@@ -152,6 +152,7 @@ class PropertyDetailsEpcModel(Base):
     energy_tariff = Column(Text)
     primary_energy_consumption = Column(Float)
     co2_emissions = Column(Float)
+    adjusted_energy_consumption = Column(Float)
 
 
 class PropertyDetailsMeter(Base):

backend/app/plan/router.py

@@ -80,11 +80,17 @@ async def trigger_plan(body: PlanTriggerRequest):
             if not is_new:
                 continue
             # TODO: Need to add heat demand target
+            # TODO: Temp for Keyzy
+            if config['address'] == "25 Albert Street":
+                epc_target = "C"
+            else:
+                epc_target = body.goal_value
+
             create_property_targets(
                 session,
                 property_id=property_id,
                 portfolio_id=body.portfolio_id,
-                epc_target=body.goal_value,
+                epc_target=epc_target,
                 heat_demand_target=None
             )
 
@@ -235,11 +241,19 @@ async def trigger_plan(body: PlanTriggerRequest):
             else:
                 # The minimum gain is the minimum number of SAP points required to get to the target SAP band
                 current_sap_points = int(property_instance.data["current-energy-efficiency"])
-                target_sap_points = epc_to_sap_lower_bound(body.goal_value)
+
+                # TODO: TEMP
+                if property_instance.address1 == "25 Albert Street":
+                    opt_epc_target = "C"
+                else:
+                    opt_epc_target = body.goal_value
+
+                target_sap_points = epc_to_sap_lower_bound(opt_epc_target)
 
                 # If the gain is negative, the optimiser will return an empty solution
                 optimiser = CostOptimiser(
-                    input_measures, min_gain=target_sap_points - current_sap_points
+                    input_measures,
+                    min_gain=CostOptimiser.calculate_sap_gain_with_slack(target_sap_points - current_sap_points)
                 )
 
             optimiser.setup()
@@ -247,6 +261,17 @@ async def trigger_plan(body: PlanTriggerRequest):
             solution = optimiser.solution
 
             selected_recommendations = {r["id"] for r in solution}
+            if "wall_insulation" in [r["type"] for r in solution]:
+                ventilation_rec = [
+                    r for r in recommendations_with_impact if r[0]["type"] == "mechanical_ventilation"
+                ][0]
+
+                selected_recommendations = set(
+                    list(selected_recommendations) + [ventilation_rec[0]["recommendation_id"]]
+                )
+
+            # We check if the selected recommendation is wall ventilation and if so, we make sure
+            # mechanical ventilation is selected
 
             # We'll use the set of selected recommendations to filter the recommendations to upload
             final_recommendations = [
@@ -275,9 +300,10 @@ async def trigger_plan(body: PlanTriggerRequest):
             if missing_types:
                 for missed_type in missing_types:
                     missed = [r for r in property_recommendations if r["type"] == missed_type]
-                    median_cost = np.median([r["total"] for r in missed])
-                    # Grab a representative, based on median cost
-                    representative_rec = [r for r in property_recommendations if r["total"] == median_cost]
+                    min_cost = min([r["total"] for r in missed])
+                    # Grab a representative, based on cheapest cost
+
+                    representative_rec = [r for r in property_recommendations if np.isclose(r["total"], min_cost)]
                     default_recommendations.append(representative_rec[0])
 
             representative_recs[property_id] = default_recommendations
@@ -325,8 +351,10 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         combined_recommendations_scoring_data = DataProcessor.clean_missings_after_description_process(
             combined_recommendations_scoring_data,
-            ignore_cols=[c for c in combined_recommendations_scoring_data.columns if ("thermal_transmittance" in c) or (
-                "insulation_thickness" in c) or ("ENERGY_EFF" in c)]
+            ignore_cols=[
+                c for c in combined_recommendations_scoring_data.columns if ("thermal_transmittance" in c) or (
+                    "insulation_thickness" in c) or ("ENERGY_EFF" in c)
+            ]
         )
 
         combined_recommendations_scoring_data = DataProcessor.clean_efficiency_variables(
@@ -358,10 +386,35 @@ async def trigger_plan(body: PlanTriggerRequest):
                 property_instance.data["co2-emissions-current"]
             ) - combined_carbon["predictions"].values[0]
 
+            starting_heat_demand = (
+                float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area
+            )
+            expected_heat_demand = starting_heat_demand - (
+                combined_heat_demand["predictions"].values[0] * property_instance.floor_area
+            )
+
+            # We adjust the heat demand figures to align to the UCL paper
+            current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+                epc_energy_consumption=starting_heat_demand,
+                current_epc_rating=property_instance.data["current-energy-rating"],
+            )
+
+            print("Hardcoded B - fix me")
+            if property_instance.address1 == "25 Albert Street":
+                hardcoded_expected_epc = "C"
+            else:
+                hardcoded_expected_epc = "B"
+            expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+                epc_energy_consumption=expected_heat_demand,
+                current_epc_rating=hardcoded_expected_epc,
+            )
+
             heat_demand_change = (
-                (float(property_instance.data["energy-consumption-current"]) -
-                 combined_heat_demand["predictions"].values[0])
-                * property_instance.floor_area
+                current_adjusted_energy - expected_adjusted_energy
+            )
+            property_instance.set_adjusted_energy(
+                current_adjusted_energy=current_adjusted_energy,
+                expected_adjusted_energy=expected_adjusted_energy
             )
 
             # update the recommendations
@@ -369,8 +422,8 @@ async def trigger_plan(body: PlanTriggerRequest):
             representative_rec_data = [
                 {
                     "recommendation_id": r["recommendation_id"],
-                    "co2_equivalent_savings": r["co2_equivalent_savings"],
-                    "heat_demand": r["heat_demand"],
+                    "co2_equivalent_savings": r.get("co2_equivalent_savings"),
+                    "heat_demand": r.get("heat_demand"),
                     "type": r["type"]
                 } for r
                 in representative_recs[property_id]
@@ -398,9 +451,14 @@ async def trigger_plan(body: PlanTriggerRequest):
             # Finally, insert these values into the final recommendations
             for rec in property_recommendations:
                 change_data = representative_rec_data[representative_rec_data["type"] == rec["type"]]
-                rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0]
-                rec["heat_demand"] = change_data["heat_demand"].values[0]
-                rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
+                if rec["type"] == "mechanical_ventilation":
+                    rec["co2_equivalent_savings"] = 0
+                    rec["heat_demand"] = 0
+                    rec["energy_cost_savings"] = 0
+                else:
+                    rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0]
+                    rec["heat_demand"] = change_data["heat_demand"].values[0]
+                    rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
 
             # Update recommendations
             recommendations[property_id] = property_recommendations
@@ -477,9 +535,9 @@ async def trigger_plan(body: PlanTriggerRequest):
         # the portfolion level impact
 
         total_valuation_increase = sum(property_valuation_increases)
-        labour_days = max(
+        labour_days = round(max(
             [sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()]
-        )
+        ))
 
         aggregate_portfolio_recommendations(
             session,

backend/ml_models/AnnualBillSavings.py

@@ -26,3 +26,47 @@ class AnnualBillSavings:
         :return: An estimate for annual bill savings
         """
         return cls.PRICE_FACTOR * kwh
+
+    @classmethod
+    def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating):
+        """
+        The over-prediction of energy use by EPCs in Great Britain: A comparison
+        of EPC-modelled and metered primary energy use intensity
+
+        Which can be found here: https://www.sciencedirect.com/science/article/pii/S0378778823002542
+        We implement the results on page 10
+
+        :return:
+        """
+
+        gradients = {
+            "A": -0.1,
+            "B": -0.1,
+            "C": -0.43,
+            "D": -0.52,
+            "E": -0.7,
+            "F": -0.76,
+            "G": -0.76
+        }
+
+        intercepts = {
+            "A": 28,
+            "B": 28,
+            "C": 97,
+            "D": 119,
+            "E": 160,
+            "F": 157,
+            "G": 157
+        }
+
+        gradient = gradients[current_epc_rating]
+        intercept = intercepts[current_epc_rating]
+
+        # This should be negative
+        consumption_difference = gradient * epc_energy_consumption + intercept
+        if consumption_difference > 0:
+            raise ValueError("consumption_difference should be negative")
+
+        adjusted_consumption = (epc_energy_consumption + consumption_difference)
+
+        return adjusted_consumption

backend/ml_models/Valuation.py

@@ -5,7 +5,7 @@ class PropertyValuation:
 
     UPRN_VALUE_LOOKUP = {
         15038202: {"current_value": 202000, "increase_percentage": 0.05725},
-        37024763: {"current_value": 213000, "increase_percentage": 0.03625},
+        37024763: {"current_value": 213000, "increase_percentage": 0.025},
     }
 
     @classmethod

recommendations/Costs.py

@@ -232,8 +232,7 @@ class Costs:
 
         subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs
 
-        # We use high risk contingency for iwi
-        contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
+        contingency_cost = subtotal_before_profit * self.CONTINGENCY
         preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
         profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
 

recommendations/FloorRecommendations.py

@@ -51,8 +51,9 @@ class FloorRecommendations(Definitions):
             ]
         ]
 
+        # For solid floor, we don't use materials that are too thick
         self.solid_floor_insulation_materials = [
-            part for part in materials if part["type"] == "solid_floor_insulation"
+            part for part in materials if part["type"] == "solid_floor_insulation" if float(part["depth"]) <= 75
         ]
 
         self.solid_floor_non_insulation_materials = [
@@ -142,7 +143,20 @@ class FloorRecommendations(Definitions):
 
     @staticmethod
     def _make_floor_description(material):
-        return f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation"
+
+        if material["type"] == "suspended_floor_insulation":
+            return (f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation in "
+                    f"suspended floor")
+
+        if material["type"] == "solid_floor_insulation":
+            return (f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation on "
+                    f"solid floor")
+
+        if material["type"] == "exposed_floor_insulation":
+            return (f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation in "
+                    f"exposed floor")
+
+        raise ValueError("Invalid material type - implement me!")
 
     def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials):
         """
@@ -194,7 +208,7 @@ class FloorRecommendations(Definitions):
                                     cost_result=cost_result
                                 ),
                             ],
-                            "type": "floor_insulation",
+                            "type": material["type"],
                             "description": self._make_floor_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,

recommendations/LightingRecommendations.py

@@ -65,7 +65,9 @@ class LightingRecommendations:
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
-                "sap_points": None,
+                # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to
+                # the proportion of lights that will be set to low energy
+                "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2),
                 **cost_result
             }
         ]

recommendations/Recommendations.py

@@ -128,10 +128,6 @@ class Recommendations:
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
 
-                new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str(
-                    rec["recommendation_id"]
-                )]["predictions"].values[0]
-
                 new_heat_demand = property_heat_predictions[property_heat_predictions["recommendation_id"] == str(
                     rec["recommendation_id"]
                 )]["predictions"].values[0]
@@ -140,7 +136,17 @@ class Recommendations:
                     rec["recommendation_id"]
                 )]["predictions"].values[0]
 
-                rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"])
+                # We don't use the model for low energy lighting at the moment
+                if rec["type"] != "low_energy_lighting":
+                    new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str(
+                        rec["recommendation_id"]
+                    )]["predictions"].values[0]
+                    rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"])
+
+                if rec["type"] == "mechanical_ventilation":
+                    # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2
+                    rec["sap_points"] = min(rec["sap_points"], VentilationRecommendations.SAP_LIMIT)
+
                 rec["co2_equivalent_savings"] = float(property_instance.data["co2-emissions-current"]) - new_carbon
 
                 # Energy consumption current is per meter squared, so we need to multiply by the floor area to get

recommendations/RoofRecommendations.py

@@ -88,17 +88,20 @@ class RoofRecommendations:
         raise NotImplementedError("Implement me")
 
     @staticmethod
-    def make_loft_insulation_description(material):
-        return f"Install {int(material['depth'])}{material['depth_unit']} of {material['description']} in your loft"
+    def make_roof_insulation_description(material):
+        if material["type"] == "loft_insulation":
+            return f"Install {int(material['depth'])}{material['depth_unit']} of {material['description']} in your loft"
 
-    @staticmethod
-    def make_room_roof_insulation_description(material, depth):
-        return f"Insulate your room roof with {depth}{material['depth_unit']} of {material['description']}"
+        if material["type"] == "flat_roof_insulation":
+            return (
+                f"Insulate the home's flat roof with {int(material['depth'])}{material['depth_unit']} of "
+                f"{material['description']}"
+            )
+        if material["type"] == "room_roof_insulation":
+            return (f"Insulate your room roof with {int(material['depth'])}{material['depth_unit']} of "
+                    f"{material['description']}")
 
-    @staticmethod
-    def make_flat_roof_insulation_description(material):
-        return (f"Insulate the home's flat roof "
-                f"with {int(material['depth'])}{material['depth_unit']} of {material['description']}")
+        raise ValueError("Invalid material type")
 
     def recommend_roof_insulation(
         self, u_value, insulation_thickness, roof
@@ -182,9 +185,7 @@ class RoofRecommendations:
                             floor_area=self.property.insulation_floor_area,
                             material=material
                         )
-                        description = self.make_loft_insulation_description(material)
                     elif material["type"] == "flat_roof_insulation":
-                        description = self.make_flat_roof_insulation_description(material)
                         raise ValueError("COMPLETE ME")
                     else:
                         raise ValueError("Invalid material type")
@@ -199,8 +200,8 @@ class RoofRecommendations:
                                     cost_result=cost_result
                                 )
                             ],
-                            "type": "roof_insulation",
-                            "description": description,
+                            "type": material["type"],
+                            "description": self.make_roof_insulation_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": None,
@@ -297,7 +298,7 @@ class RoofRecommendations:
                                     selected_total_cost=estimated_cost
                                 )
                             ],
-                            "type": "roof_insulation",
+                            "type": "room_roof_insulation",
                             "description": self.make_room_roof_insulation_description(material, depth),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,

recommendations/VentilationRecommendations.py

@@ -15,6 +15,9 @@ class VentilationRecommendations(Definitions):
         'mechanical, supply and extract'
     ]
 
+    # We introduce a SAP limit, to prevent over-predicting the SAP impact of mechanical ventilation
+    SAP_LIMIT = 2
+
     def __init__(
         self,
         property_instance: Property,

recommendations/WallRecommendations.py

@@ -218,8 +218,8 @@ class WallRecommendations(Definitions):
                                 cost_result=cost_result
                             )
                         ],
-                        "type": "wall_insulation",
-                        "description": f"Fill cavity with {material['description']}",
+                        "type": "cavity_wall_insulation",
+                        "description": self._make_description(material),
                         "starting_u_value": u_value,
                         "new_u_value": new_u_value,
                         "sap_points": None,
@@ -263,14 +263,12 @@ class WallRecommendations(Definitions):
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
-                        description = "Install " + self._make_description(material) + " on internal walls"
                     elif material["type"] == "external_wall_insulation":
                         cost_result = self.costs.external_wall_insulation(
                             wall_area=self.property.insulation_wall_area,
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
-                        description = "Install " + self._make_description(material) + " on external walls"
                     else:
                         raise ValueError("Invalid material type")
 
@@ -284,8 +282,8 @@ class WallRecommendations(Definitions):
                                     cost_result=cost_result
                                 )
                             ],
-                            "type": "wall_insulation",
-                            "description": description,
+                            "type": material["type"],
+                            "description": self._make_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": None,
@@ -305,7 +303,7 @@ class WallRecommendations(Definitions):
         # Recommend external and internal wall insulation separately
         # Since external and internal wall insulation are sufficiently different,
         # we separate the logic for for recommending them, therefore we don't
-        # consider diminishing returns between the two
+        # consider diminishing returns between the two as they are considered to be separate measures
 
         ewi_recommendations = []
         if self.ewi_valid:
@@ -323,25 +321,20 @@ class WallRecommendations(Definitions):
 
         self.recommendations += ewi_recommendations + iwi_recommendations
 
-        # We remove this temporarily
-        # self.prune_diminishing_recommendations()
-
     @staticmethod
     def _make_description(material):
-        return f"{int(material['depth'])}{material['depth_unit']} {material['description']}"
+        if material["type"] == "internal_wall_insulation":
+            return (f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} on internal "
+                    f"walls")
 
-    def prune_diminishing_recommendations(self):
-        # For any recommendations, if we have at least 1 reommendation that does not exhibit diminishing returns
-        # we trim all others that are beyond the diminishing returns threshold
+        if material["type"] == "external_wall_insulation":
+            return (f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} on external "
+                    f"walls")
 
-        # We first check if we have any recommendations that are not diminishing returns
-        not_diminishing_return = [
-            rec for rec in self.recommendations if rec["new_u_value"] >= self.DIMINISHING_RETURNS_U_VALUE
-        ]
-        if not_diminishing_return:
-            self.recommendations = [
-                rec for rec in self.recommendations if rec["new_u_value"] >= self.DIMINISHING_RETURNS_U_VALUE
-            ]
+        if material["type"] == "cavity_wall_insulation":
+            return f"Fill cavity with {material['description']}"
+
+        raise ValueError("Invalid material type")
 
     @staticmethod
     def rvalue_per_mm(total_r_value: float, thickness_mm: float) -> float:

recommendations/optimiser/CostOptimiser.py

@@ -9,6 +9,9 @@ class CostOptimiser:
     This class is used to minimise cost, given a constrained minimum gain
     """
 
+    # We add an optional buffer to the minimum gain to allow for slack in the optimisation
+    BUFFER = 0.2
+
     def __init__(self, components, min_gain):
         self.components = components
         self.min_gain = min_gain
@@ -20,6 +23,15 @@ class CostOptimiser:
         self.solution_cost = None
         self.solution_gain = None
 
+    @classmethod
+    def calculate_sap_gain_with_slack(cls, min_gain):
+        if min_gain <= 10:
+            return min_gain + 2
+        elif min_gain <= 20:
+            return min_gain + 3
+        else:
+            return min_gain + 4
+
     def setup(self):
         # Initialize Model
         self.m = Model("knapsack")

recommendations/optimiser/optimiser_functions.py

@@ -16,18 +16,44 @@ def prepare_input_measures(property_recommendations, goal):
     if not goal_key:
         raise NotImplementedError("Not implemented this gain type - investigate me")
 
+    ventilation_rec = [rec for rec in property_recommendations if rec[0]["type"] == "mechanical_ventilation"][0]
+
     input_measures = []
     for recs in property_recommendations:
-        input_measures.append(
-            [
-                {
-                    "id": rec["recommendation_id"],
-                    "cost": rec["total"],
-                    "gain": rec[goal_key],
-                    "type": rec["type"]
-                }
-                for rec in recs
-            ]
-        )
+
+        # We don't actually optimise ventilation
+        if recs[0]["type"] == "mechanical_ventilation":
+            continue
+
+        if recs[0]["type"] == "wall_insulation":
+            # Wall insulation and mechanical ventilation are paired. You can't have wall insulation without mechanical
+            # ventilation
+
+            ventilation_cost = ventilation_rec[0]["total"] if ventilation_rec else 0
+            ventilation_gain = ventilation_rec[0][goal_key] if ventilation_rec else 0
+
+            input_measures.append(
+                [
+                    {
+                        "id": rec["recommendation_id"],
+                        "cost": rec["total"] + ventilation_cost,
+                        "gain": rec[goal_key] + ventilation_gain,
+                        "type": rec["type"]
+                    }
+                    for rec in recs
+                ]
+            )
+        else:
+            input_measures.append(
+                [
+                    {
+                        "id": rec["recommendation_id"],
+                        "cost": rec["total"],
+                        "gain": rec[goal_key],
+                        "type": rec["type"]
+                    }
+                    for rec in recs
+                ]
+            )
 
     return input_measures