Merge pull request #746 from Hestia-Homes/main

khalims changes

backend/Property.py

@@ -490,7 +490,7 @@ class Property:
         for rec_id in rec_ids:
             sim_epc = self.simulation_epcs[rec_id].copy()
             rec_impact = [x for x in impact_summary if x["recommendation_id"] == rec_id][0]
-            # We update all of the features that should have an impact on the kwh model
+            # We update all features that should have an impact on the kwh model
 
             sim_epc.update(
                 {

backend/engine/engine.py

@@ -1053,14 +1053,18 @@ async def model_engine(body: PlanTriggerRequest):
             property_required_measures = [m for m in recommendations[p.id] if m[0]["type"] in body.required_measures]
             measures_to_optimise = [m for m in recommendations[p.id] if m[0]["type"] not in body.required_measures]
 
-            ventilation_included = "ventilation" in property_measure_types
+            # TODO - formalise property measure types into an enum
+            ventilation_included = (
+                "ventilation" in property_measure_types or "mechanical_ventilation" in property_measure_types
+            )
 
             # If a measure requiring ventilation is selected, and the property does not have ventilation, we enfore
             # its inclusion
 
-            needs_ventilation = any(
-                x in property_measure_types for x in assumptions.measures_needing_ventilation
-            ) and not p.has_ventilation and ventilation_included
+            needs_ventilation = optimiser_functions.check_needs_ventilation(
+                property_measure_types, assumptions.measures_needing_ventilation, p.has_ventilation,
+                ventilation_included
+            )
 
             if not measures_to_optimise:
                 # Nothing to do, we just reshape the recommendations
@@ -1177,8 +1181,10 @@ async def model_engine(body: PlanTriggerRequest):
                     recommendations=recommendations, selected=selected,
                 )
 
-            # Add best practice measures (ventilation/trickle vents)
-            selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected)
+            # Add best practice measures (ventilation/trickle vents) - pass needs_ventilation flag
+            selected = optimiser_functions.add_best_practice_measures(
+                p.id, solution, recommendations, selected, needs_ventilation
+            )
             # Final flattening - we pass what the battery SAP score would be, regardless if the battery was selected
             recommendations[p.id] = optimiser_functions.flatten_recommendations_with_defaults(
                 p.id, recommendations, selected, battery_sap_score

recommendations/Recommendations.py

@@ -499,8 +499,16 @@ class Recommendations:
         return predicted_appliances_cost_reduction, predicted_appliances_kwh_reduction
 
     @staticmethod
-    def _check_ventilation_out_of_bounds(sap_impact, ventilation_sap_limit):
-        return (sap_impact < ventilation_sap_limit) or (sap_impact >= 0)
+    def _check_ventilation_out_of_bounds(sap_impact: float, ventilation_sap_limit: float) -> bool:
+        """
+        Checks if the SAP impact of a ventilation recommendation is out of bounds, which would indicate that the
+        recommendation is not appropriate.
+        :param sap_impact: The SAP impact of the ventilation recommendation, which is typically negative or zero
+        :param ventilation_sap_limit: The SAP limit for ventilation recommendations, which is typically a negative
+        number. E.g. -4
+        :return:
+        """
+        return (sap_impact < ventilation_sap_limit) or (sap_impact > 0)
 
     @staticmethod
     def _adjust_ventilation_sap(sap_impact, ventilation_sap_limit):
@@ -691,7 +699,8 @@ class Recommendations:
         previous_phase_values: dict,
         current_phase_values: dict,
         adjustments: list,
-        property_instance,
+        property_instance: Property,
+        model_predicted_sap: float,
     ):
         # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2
         if rec["type"] == "low_energy_lighting":
@@ -785,7 +794,6 @@ class Recommendations:
 
                 # Update the current phase values
                 current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
-
         elif rec["type"] == "loft_insulation":
             # When we have a loft insulation recommendation, where there is an extension and the existing
             # amount of loft insulation is already good, we limit the SAP points
@@ -831,6 +839,27 @@ class Recommendations:
 
             # Update the current phase values
             current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
+        elif rec["measure_type"] in ["roomstat_programmer_trvs", "time_temperature_zone_control"]:
+            # We trim the SAP point recommendations based on the minimum of the predicted and the survey SAP
+            # points
+            predicted_difference = model_predicted_sap - previous_phase_values["sap_prediction"]
+            proposed_impact = property_phase_impact["sap"]
+            numerically_the_same = np.isclose(proposed_impact, predicted_difference)
+
+            if predicted_difference > 0 and (predicted_difference < proposed_impact) and not numerically_the_same:
+                # We constrain the impact based on what the model predicts.
+                # We update the proposed impact to be the predicted difference
+                adjustments.append(
+                    {
+                        "recommendation_id": rec["recommendation_id"],
+                        "phase": rec["phase"],
+                        # If we've made an adjustment, it will be negative
+                        "sap_adjustment": property_phase_impact["sap"] - predicted_difference,
+                    }
+                )
+                property_phase_impact["sap"] = predicted_difference
+                # Update the current phase values
+                current_phase_values["sap"] = previous_phase_values["sap"] + property_phase_impact["sap"]
 
         return property_phase_impact, current_phase_values, adjustments
 
@@ -963,7 +992,8 @@ class Recommendations:
                     previous_phase_values=previous_phase_values,
                     current_phase_values=current_phase_values,
                     adjustments=adjustments,
-                    property_instance=property_instance
+                    property_instance=property_instance,
+                    model_predicted_sap=phase_energy_efficiency_metrics["sap_change"],
                 )
 
                 # Insert this information into the recommendation.

recommendations/SecondaryHeating.py

@@ -18,6 +18,9 @@ class SecondaryHeating:
     def recommend(self, phase: int):
         # Reset
         self.recommendation = []
+        if self.property.epc_record.secondheat_description in ["None", None]:
+            # No secondary heating system, so no recommendation to remove it
+            return
 
         if self.property.data['number-habitable-rooms'] > self.property.data['number-heated-rooms']:
             n_rooms = self.property.data['number-habitable-rooms'] - self.property.data['number-heated-rooms']

recommendations/optimiser/optimiser_functions.py

@@ -1,4 +1,5 @@
 import pandas as pd
+from typing import List, Dict, Any, Set
 import backend.app.assumptions as assumptions
 from backend.Property import Property
 from backend.app.plan.schemas import PlanTriggerRequest
@@ -78,14 +79,14 @@ def prepare_input_measures(
         # if recs[0]["type"] == "solar_pv":
         #     recs = [r for r in recs if ~r["has_battery"]]
 
-        # Only include measures with non-negative cost savings
+        # Only include measures with non-negative cost savings - we allow for a minor negative impact
         if eco_measures:
             recs_to_append = [
-                rec for rec in recs if (rec["energy_cost_savings"] >= 0) or (rec["measure_type"] in eco_measures)
+                rec for rec in recs if (rec["energy_cost_savings"] >= -10) or (rec["measure_type"] in eco_measures)
             ]
         else:
             recs_to_append = [
-                rec for rec in recs if (rec["energy_cost_savings"] >= 0)
+                rec for rec in recs if (rec["energy_cost_savings"] >= -10)
             ]
         if not recs_to_append:
             continue
@@ -300,7 +301,13 @@ def add_required_measures(property_id, property_required_measures, recommendatio
     ]
 
 
-def add_best_practice_measures(property_id, solution, recommendations, selected):
+def add_best_practice_measures(
+    property_id: int,
+    solution: List[Dict[str, Any]],
+    recommendations: Dict[int, List[List[Dict[str, Any]]]],
+    selected: Set[str],
+    needs_ventilation: bool
+):
     """
     Ensures best-practice measures like ventilation and trickle vents are included
     in the selected recommendations when appropriate.
@@ -320,6 +327,8 @@ def add_best_practice_measures(property_id, solution, recommendations, selected)
         All recommendations for all properties, keyed by property id.
     selected : set
         Set of already selected recommendation IDs.
+    needs_ventilation : bool
+        Whether the property requires mechanical ventilation to accompany certain measures.
 
     Returns
     -------
@@ -329,12 +338,6 @@ def add_best_practice_measures(property_id, solution, recommendations, selected)
     # Check if any selected measure requires ventilation
     ventilation_selected = [r for r in solution if "+mechanical_ventilation" in r["type"]]
 
-    # If ventilation has been selected, or one of the measures needs ventilation, we need to ensure ventilation is
-    # included
-    needs_ventilation = any(
-        x in [r["type"] for r in solution] for x in assumptions.measures_needing_ventilation
-    ) or len(ventilation_selected) > 0
-
     if needs_ventilation:
         ventilation_rec = next(
             (r[0] for r in recommendations[property_id] if r[0]["type"] == "mechanical_ventilation"),
@@ -395,3 +398,30 @@ def flatten_recommendations_with_defaults(property_id, recommendations, selected
 
     # Flatten the nested list of lists into a single list
     return [rec for recommendations_by_type in final_recommendations for rec in recommendations_by_type]
+
+
+def check_needs_ventilation(
+    property_measure_types: Set[str],
+    measures_needing_ventilation: List[str],
+    property_already_has_ventilation: bool,
+    ventilation_in_included_measures: bool
+) -> bool:
+    """
+    Function to check if we need to include ventilation based on the measures selected and the property
+    features
+    :param property_measure_types: The set of measure types recommended for the property
+    :param measures_needing_ventilation: The set of measure types that require ventilation
+    :param property_already_has_ventilation: Whether the property currently has ventilation
+    :param ventilation_in_included_measures: Whether ventilation is already included in the recommended
+    measures
+    :return: Boolean indicating whether ventilation needs to be included in the recommendations
+
+    # TODO - none of the inputs of this function are well structured and so this is quite brittle - we should
+             consider refactoring to make this more robust
+    """
+
+    needs_ventilation = any(
+        x in property_measure_types for x in measures_needing_ventilation
+    )
+
+    return needs_ventilation and not property_already_has_ventilation and ventilation_in_included_measures

recommendations/tests/test_optimiser_functions.py

@@ -143,7 +143,9 @@ class TestAddBestPracticeMeasures:
             ]
         }
         selected = set()
-        updated = optimiser_functions.add_best_practice_measures(property_id, solution, recommendations, selected)
+        updated = optimiser_functions.add_best_practice_measures(
+            property_id, solution, recommendations, selected, True
+        )
         assert "vent1" in updated
         assert "trickle1" in updated
 
@@ -273,7 +275,7 @@ class TestIncreasingEpcE2e:
         total_optimised_gain = sum(m["gain"] for m in solution)
         assert total_optimised_gain == 17.6, "Total gain of optimised measures should meet or exceed target gain"
 
-        selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected)
+        selected = optimiser_functions.add_best_practice_measures(p.id, solution, recommendations, selected, False)
 
         # Flatten recommendations for output
         flattened = optimiser_functions.flatten_recommendations_with_defaults(p.id, recommendations, selected)
@@ -510,3 +512,63 @@ class TestStrategicOptimiser:
         assert opt.strategy_used.value == "case_2_solve_max_gain_under_budget"
         assert opt.solution_cost == 7787.068
         assert opt.solution_gain == 28.8
+
+
+class TestCheckNeedsVentilation:
+
+    def measure_types_includes_ventilation_no_existing_ventilation(self):
+        property_measure_types = {'mechanical_ventilation', 'cavity_wall_insulation', 'suspended_floor_insulation',
+                                  'secondary_heating', 'loft_insulation', 'heating', 'low_energy_lighting'}
+
+        measures_needing_ventilation = ['internal_wall_insulation', 'external_wall_insulation',
+                                        'cavity_wall_insulation']
+
+        has_ventilation = False
+
+        ventilation_included = True
+
+        result = optimiser_functions.check_needs_ventilation(
+            property_measure_types, measures_needing_ventilation, has_ventilation,
+            ventilation_included
+        )
+
+        assert result == True
+
+    def measure_types_includes_ventilation_existing_ventilation(self):
+        property_measure_types = {'mechanical_ventilation', 'cavity_wall_insulation', 'suspended_floor_insulation',
+                                  'secondary_heating', 'loft_insulation', 'heating', 'low_energy_lighting'}
+
+        measures_needing_ventilation = ['internal_wall_insulation', 'external_wall_insulation',
+                                        'cavity_wall_insulation']
+
+        has_ventilation = True
+
+        ventilation_included = True
+
+        result = optimiser_functions.check_needs_ventilation(
+            property_measure_types, measures_needing_ventilation, has_ventilation,
+            ventilation_included
+        )
+
+        assert result == False
+
+    def measure_types_includes_ventilation_existing_ventilation(self):
+        property_measure_types_without_ventilation = {
+            'cavity_wall_insulation', 'suspended_floor_insulation',
+            'secondary_heating', 'loft_insulation', 'heating',
+            'low_energy_lighting'
+        }
+
+        measures_needing_ventilation = ['internal_wall_insulation', 'external_wall_insulation',
+                                        'cavity_wall_insulation']
+
+        has_ventilation = False
+
+        ventilation_included = True
+
+        result = optimiser_functions.check_needs_ventilation(
+            property_measure_types_without_ventilation, measures_needing_ventilation, has_ventilation,
+            ventilation_included
+        )
+
+        assert result == False

recommendations/tests/test_recommendations.py

@@ -373,7 +373,7 @@ def test_filter_phase_adjustment(input_data, expected):
     "sap_impact, limit, expected",
     [
         (1.0, -4, True),  # positive SAP not allowed
-        (0.0, -4, True),  # zero not allowed
+        (0.0, -4, False),  # zero is allowed
         (-1.0, -4, False),  # valid range
         (-3.9, -4, False),  # valid range
         (-4.0, -4, False),  # exact lower bound allowed
@@ -1476,7 +1476,9 @@ def test_lighting_and_loft_adjustment_combined(property_instance, heat_demand_pr
 
     assert adjustments2 == [
         {'recommendation_id': '0_phase=0', 'phase': 0, 'sap_adjustment': np.float64(1.7)},
-        {'recommendation_id': '4_phase=2', 'phase': 2, 'sap_adjustment': np.float64(4.0)}
+        {'recommendation_id': '4_phase=2', 'phase': 2, 'sap_adjustment': np.float64(4.0)},
+        {'recommendation_id': '5_phase=3', 'phase': 3, 'sap_adjustment': np.float64(1.0)},
+        {'recommendation_id': '6_phase=3', 'phase': 3, 'sap_adjustment': np.float64(1.0000000000000027)}
     ]
 
 
@@ -1499,7 +1501,8 @@ def test_mechanical_ventilation_sap_floor(property_instance):
             previous_phase_values=previous_phase_values,
             current_phase_values=current_phase_values,
             adjustments=adjustments,
-            property_instance=property_instance
+            property_instance=property_instance,
+            model_predicted_sap=0
         )
     )
 
@@ -1538,7 +1541,8 @@ def test_mechanical_ventilation_no_floor_adjustment(property_instance):
             previous_phase_values=previous_phase_values,
             current_phase_values=current_phase_values,
             adjustments=adjustments,
-            property_instance=property_instance
+            property_instance=property_instance,
+            model_predicted_sap=0
         )
     )
 
@@ -1570,7 +1574,8 @@ def test_mechanical_ventilation_exactly_one_no_adjustment(property_instance):
             previous_phase_values=previous_phase_values,
             current_phase_values=current_phase_values,
             adjustments=adjustments,
-            property_instance=property_instance
+            property_instance=property_instance,
+            model_predicted_sap=0
         )
     )
 
@@ -1578,3 +1583,182 @@ def test_mechanical_ventilation_exactly_one_no_adjustment(property_instance):
     assert updated_adjustments == []
     assert updated_current["sap"] == 1.0
     assert updated_impact["sap"] == -1.0
+
+
+def test_mechanical_ventilation_sap_zero_no_adjustment(property_instance):
+    # Test when SAP = 0
+    rec = {
+        "type": "mechanical_ventilation",
+        "recommendation_id": "mv_test",
+        "phase": 1,
+    }
+
+    previous_phase_values = {'phase': 0, 'representative': True, 'recommendation_id': '0_phase=0',
+                             'measure_type': 'flat_roof_insulation', 'sap': 68.0, 'carbon': np.float64(0.5),
+                             'heat_demand': np.float64(300.1), 'sap_prediction': np.float64(71.7)}
+    current_phase_values = {'sap': 68.0, 'carbon': np.float64(0.5), 'heat_demand': np.float64(307.0)}
+    property_phase_impact = {'sap': 0, 'carbon': 0, 'heat_demand': np.float64(-6.899999999999977)}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec=rec,
+            property_phase_impact=property_phase_impact,
+            previous_phase_values=previous_phase_values,
+            current_phase_values=current_phase_values,
+            adjustments=adjustments,
+            property_instance=property_instance,
+            model_predicted_sap=0
+        )
+    )
+
+    # SAP is already at 0 → no adjustment expected
+    assert updated_adjustments == []
+    assert updated_current["sap"] == 68.0
+    assert updated_impact["sap"] == 0
+
+
+def test_mv_valid_negative_no_adjustment(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 70.0}
+    current = {"sap": 67.0}
+    impact = {"sap": -3.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert updated_adjustments == []
+    assert updated_current["sap"] == 67.0
+    assert updated_impact["sap"] == -3.0
+
+
+def test_mv_zero_impact_allowed(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 68.0, "sap_prediction": 71.7}
+    current = {"sap": 68.0}
+    impact = {"sap": 0.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert updated_adjustments == []
+    assert updated_current["sap"] == 68.0
+    assert updated_impact["sap"] == 0.0
+
+
+def test_mv_positive_impact_corrected(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 60.0}
+    current = {"sap": 61.0}
+    impact = {"sap": 1.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert len(updated_adjustments) == 1
+    assert updated_current["sap"] == previous["sap"] + updated_impact["sap"]
+    assert updated_impact["sap"] <= 0
+
+
+def test_mv_below_lower_bound_corrected(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 70.0}
+    current = {"sap": 64.0}
+    impact = {"sap": -6.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert len(updated_adjustments) == 1
+    assert updated_impact["sap"] >= -4
+
+
+def test_mv_floor_triggered(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 2.0}
+    current = {"sap": 0.5}
+    impact = {"sap": -1.5, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert updated_current["sap"] == 1.0
+    assert updated_adjustments[0]["sap_adjustment"] > 0
+
+
+def test_mv_exactly_one_no_floor(property_instance):
+    rec = {"type": "mechanical_ventilation", "recommendation_id": "mv", "phase": 1}
+
+    previous = {"sap": 2.0}
+    current = {"sap": 1.0}
+    impact = {"sap": -1.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert updated_adjustments == []
+    assert updated_current["sap"] == 1.0
+
+
+def test_lighting_no_cap(property_instance):
+    rec = {"type": "low_energy_lighting", "recommendation_id": "led", "phase": 1,
+           "co2_equivalent_savings": 0}
+
+    previous = {"sap": 60.0, "carbon": 2.0}
+    current = {"sap": 61.0, "carbon": 2.0}
+    impact = {"sap": 1.0, "carbon": 0, "heat_demand": 0}
+    adjustments = []
+
+    updated_impact, updated_current, updated_adjustments = (
+        Recommendations._apply_measure_specific_rules(
+            rec, impact, previous, current, adjustments, property_instance, 0
+        )
+    )
+
+    assert updated_adjustments == []
+
+
+def test_filter_phase_adjustments():
+    example_adjustments = [
+        {'recommendation_id': '0_phase=0', 'phase': 0, 'sap_adjustment': np.float64(1.7)},
+        {'recommendation_id': '4_phase=2', 'phase': 2, 'sap_adjustment': np.float64(4.0)},
+        {'recommendation_id': '5_phase=3', 'phase': 3, 'sap_adjustment': np.float64(1.0)},
+        {'recommendation_id': '6_phase=3', 'phase': 3, 'sap_adjustment': np.float64(1.0000000000000027)}
+    ]
+
+    res = Recommendations._filter_phase_adjustment(example_adjustments)
+
+    assert res == [
+        {'recommendation_id': '0_phase=0', 'phase': 0, 'sap_adjustment': np.float64(1.7)},
+        {'recommendation_id': '4_phase=2', 'phase': 2, 'sap_adjustment': np.float64(4.0)},
+        {'recommendation_id': '6_phase=3', 'phase': 3, 'sap_adjustment': np.float64(1.0000000000000027)}
+    ]