wip integrating additional recs with new code

backend/Property.py

@@ -357,6 +357,27 @@ class Property:
             for config in epc_transformations:
                 for k, v in config.items():
                     if k in phase_epc_transformation:
+                        if "-energy-eff" in k:
+                            # We take the highest value
+                            if phase_epc_transformation[k] == "Very Good":
+                                continue
+                            elif phase_epc_transformation[k] == "Good":
+                                if v == "Very Good":
+                                    phase_epc_transformation[k] = v
+                            elif phase_epc_transformation[k] == "Average":
+                                if v in ["Good", "Very Good"]:
+                                    phase_epc_transformation[k] = v
+                            elif phase_epc_transformation[k] == "Poor":
+                                if v in ["Average", "Good", "Very Good"]:
+                                    phase_epc_transformation[k] = v
+                            else:
+                                phase_epc_transformation[k] = v
+
+                            continue
+
+                        if phase_epc_transformation[k] == v:
+                            continue
+
                         raise NotImplementedError(
                             "Already have this key in the phase_epc_transformation - implement me")
                     phase_epc_transformation[k] = v

backend/apis/GoogleSolarApi.py

@@ -7,6 +7,7 @@ from functools import lru_cache
 import time
 from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data
 from utils.logger import setup_logger
+from sklearn.preprocessing import MinMaxScaler
 
 logger = setup_logger()
 
@@ -198,6 +199,7 @@ class GoogleSolarApi:
         scenarios_data["scenario_type"] = scenario_type
         scenarios_data = scenarios_data.to_dict(orient="records")
 
+        # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it exists
         store_batch_data(
             session=session,
             api_data=self.insights_data,
@@ -244,7 +246,7 @@ class GoogleSolarApi:
                 wattage = segment["panelsCount"] * self.insights_data["solarPotential"]["panelCapacityWatts"]
                 generated_dc_energy = segment["yearlyEnergyDcKwh"]
                 ratio = generated_dc_energy / wattage
-                cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (generated_dc_energy / 1000)
+                cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000)
                 roi_summary.append(
                     {
                         "segmentIndex": segment["segmentIndex"],
@@ -309,17 +311,19 @@ class GoogleSolarApi:
         )
 
         # Now that we know the lifetime cnsumption of ac kwh, we can estimate the roi
+        lifetime_energy_consumption = energy_consumption * self.installation_life_span
         roi_results = []
         for _, panel_config in panel_performance.iterrows():
             lifetime_ac_kwh = panel_config["lifetime_ac_kwh"]
-            lifetime_energy_consumption = energy_consumption * self.installation_life_span
 
+            surplus = 0
             if lifetime_ac_kwh < lifetime_energy_consumption:
                 # We estimate the amount of electricity generated, based on the price cap
                 generation_value = lifetime_ac_kwh * AnnualBillSavings.ELECTRICITY_PRICE_CAP
                 roi = generation_value / panel_config["total_cost"]
                 generation_deficit = lifetime_energy_consumption - lifetime_ac_kwh
             else:
+
                 # We now have a surplus of energy, which we can sell back to the grid
                 surplus = lifetime_ac_kwh - lifetime_energy_consumption
                 surplus_value = surplus * AnnualBillSavings.ELECTRICITY_EXPORT_PAYMENT
@@ -341,7 +345,8 @@ class GoogleSolarApi:
                     "roi": roi,
                     "generation_value": generation_value,
                     "generation_deficit": generation_deficit,
-                    "expected_payback_years": expected_payback_years
+                    "expected_payback_years": expected_payback_years,
+                    "surplus": surplus
                 }
             )
 
@@ -351,12 +356,28 @@ class GoogleSolarApi:
             roi_results, how="left", on="n_panels"
         )
 
-        # We prioritise maximal roi, then minimal geneartion deficit, then maximal generation value (if there is still
+        # We want max roi, minimal generation deficit, and max generation value - we create a ranking score
-        # a tie). Ideally, we want the best roi over the lifetime of the solar panels, but we also want to ensure that
+        # Assign equal weights to each metric
-        # we can meet the energy demands of the building.
+        weights = {'roi': 0.6, 'generation_value': 0.2, 'generation_deficit': 0.2}
-        panel_performance = panel_performance.sort_values(
+        metrics = panel_performance[['roi', 'generation_value', 'generation_deficit']]
-            ["roi", "generation_deficit", "generation_value"], ascending=[False, True, False]
+
+        # Normalize the columns (0 to 1 scale)
+        scaler = MinMaxScaler()
+        normalized_metrics = scaler.fit_transform(metrics)
+
+        # Convert normalized metrics back to a dataframe
+        normalized_metrics_df = pd.DataFrame(
+            normalized_metrics, columns=['roi', 'generation_value', 'generation_deficit']
         )
+        normalized_metrics_df['combined_score'] = (
+            normalized_metrics_df['roi'] * weights['roi'] +
+            normalized_metrics_df['generation_value'] * weights['generation_value'] +
+            (1 - normalized_metrics_df['generation_deficit']) * weights['generation_deficit']
+        )
+
+        panel_performance['combined_score'] = normalized_metrics_df['combined_score'].values
+        panel_performance['rank'] = panel_performance['combined_score'].rank(ascending=False)
+        panel_performance = panel_performance.sort_values(by='rank')
 
         panel_performance["expected_payback_years"] = np.ceil(panel_performance["expected_payback_years"]).astype(int)
 

backend/app/plan/router.py

@@ -431,6 +431,7 @@ async def trigger_plan(body: PlanTriggerRequest):
                 }
 
                 # Store the data in the database
+                # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it exists
                 solar_api_client.save_to_db(
                     session=session, uprns_to_location=building_uprns[building_id], scenario_type="building"
                 )

etl/customers/places_for_people/demo_portfolio.py

@@ -3,6 +3,7 @@ import pandas as pd
 from utils.s3 import save_csv_to_s3
 
 PORTFOLIO_ID = 83
+SECOND_PORTFOLIO_ID = 84
 USER_ID = 8
 
 
@@ -67,6 +68,181 @@ def app():
         "patches_file_path": "",
         "non_invasive_recommendations_file_path": "",
         "budget": None,
+        "exclusions": ["floor_insulation"]
+    }
+    print(body)
+
+    # Get an example of flats with solar panels from epc data
+
+    # import inspect
+    # import pandas as pd
+    # from tqdm import tqdm
+    # from pathlib import Path
+    #
+    # src_file_path = inspect.getfile(lambda: None)
+    #
+    # EPC_DIRECTORY = Path(src_file_path).parent / "local_data" / "all-domestic-certificates"
+    #
+    # epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
+    #
+    # directory = epc_directories[1]
+    # data = pd.read_csv(directory / "certificates.csv", low_memory=False)
+    # # Get flats
+    # data = data[data["PROPERTY_TYPE"].str.lower().str.contains("flat")]
+    # data = data[~pd.isnull(data["UPRN"])]
+    # data["UPRN"] = data["UPRN"].astype(int).astype(str)
+    # data = data[pd.to_datetime(data["LODGEMENT_DATE"]) > "2020-01-01"]
+    # flats_with_solar = data[data['PHOTO_SUPPLY'] > 0]
+    #
+    # print(flats_with_solar["UPRN"])
+    #
+    # flats_with_solar[["ADDRESS", "UPRN"]]
+    #
+    # # Good example:
+    # # UPRN: 10013160824, Flat 39, The Meadow, 30 Busk Meadow S5 7JH (care home with 39 flats, have solar panels)
+    # #
+    # # Mostly, For a mid-floor flat, the property doesn't show as having solar panels through the photo_supply variable
+    # # But actually for UPRN: 10013245713, Apartment 4, Orchard House, Gill Lane PR4 5QN, this has a dwelling above
+    # # but the photo_supply variable is 20
+    #
+    # # Small flat consisting of 2 units
+    # # UPRN: 42172953, FLAT 2, 276 CLAUGHTON ROAD, BIRKENHEAD CH41 4DX
+    #
+    # # Flat containing 5 units
+    # # UPRN: 10013247127 Flat 1, Old Church House PR4 5GE
+    # # UPRN: 10013247130 Flat 4, Old Church House PR4 5GE
+    #
+    # # Flat containing multiple units:
+    # # UPRNS: 10013245710, 10013245716, 10013245711, 10013245717, 10013245714, 10013245715, 10013245712, 10013245713
+    #
+    # # Look for flats with air source heat pumps!
+    # flats_with_asps = data[data["MAINHEAT_DESCRIPTION"].str.lower().str.contains("air source heat pump")]
+    # print(flats_with_asps[["UPRN", "ADDRESS"]])
+
+
+def app_epc_b():
+    # TODO: We can insert a variable, indicating the they own all of the units in the building
+    asset_list = [
+        {
+            "address": "Flat 1, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140644,
+            "building_id": 1,
+        },
+        {
+            "address": "Flat 2, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140645,
+            "building_id": 1,
+        },
+        {
+            "address": "Flat 3, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140646,
+            "building_id": 1,
+        },
+        {
+            "address": "Flat 4, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140647,
+            "building_id": 1,
+        },
+        {
+            "address": "Flat 5, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140648,
+            "building_id": 1,
+        },
+        {
+            "address": "Flat 6, Fenton Court",
+            "postcode": "N2 8DS",
+            "uprn": 200140649,
+            "building_id": 1,
+        }
+    ]
+
+    non_invasive_recommendations = [
+        {
+            "address": "Flat 1, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                # 'air_source_heat_pump'
+            ]
+        },
+        {
+            "address": "Flat 2, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                # 'air_source_heat_pump'
+            ]
+        },
+        {
+            "address": "Flat 3, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                # 'air_source_heat_pump'
+            ]
+        },
+        {
+            "address": "Flat 4, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                # 'air_source_heat_pump'
+            ]
+        },
+        {
+            "address": "Flat 5, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                'loft_insulation',
+                # 'air_source_heat_pump'
+            ]
+        },
+        {
+            "address": "Flat 6, Fenton Court",
+            "postcode": "N2 8DS",
+            'recommendations': [
+                'cavity_extract_and_refill',
+                'loft_insulation',
+                # 'air_source_heat_pump'
+            ]
+        },
+    ]
+
+    asset_list = pd.DataFrame(asset_list)
+
+    # Store the asset list in s3
+    filename = f"{USER_ID}/{SECOND_PORTFOLIO_ID}/non_intrusives.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{SECOND_PORTFOLIO_ID}/non_invasive_recommendations.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    body = {
+        "portfolio_id": str(SECOND_PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "B",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+        "exclusions": ["floor_insulation"]
     }
     print(body)
 

recommendations/Costs.py

@@ -18,23 +18,23 @@ regional_labour_variations = [
     {"Region": "Northern Ireland", "Adjustment_Factor": 0.76}
 ]
 
-# This data is based on the MCS database
+# This data is based on the MCS database - taken the figures for June 2024
 MCS_SOLAR_PV_COST_DATA = {
-    "last_updated": "2024-06-10",
+    "last_updated": "2024-07-10",
-    "average_cost_per_kwh": 1750,
+    "average_cost_per_kwh": 1825,
-    "average_cost_per_kwh-Outer London": 1776,
+    "average_cost_per_kwh-Outer London": 1950,
-    "average_cost_per_kwh-Inner London": 1776,
+    "average_cost_per_kwh-Inner London": 1950,
-    "average_cost_per_kwh-South East England": 1672,
+    "average_cost_per_kwh-South East England": 1966,
-    "average_cost_per_kwh-South West England": 1732,
+    "average_cost_per_kwh-South West England": 1864,
-    "average_cost_per_kwh-East of England": 1721,
+    "average_cost_per_kwh-East of England": 1719,
     "average_cost_per_kwh-East Midlands": 1730,
-    "average_cost_per_kwh-West Midlands": 1761,
+    "average_cost_per_kwh-West Midlands": 1789,
-    "average_cost_per_kwh-North East England": 1669,
+    "average_cost_per_kwh-North East England": 1872,
-    "average_cost_per_kwh-North West England": 1764,
+    "average_cost_per_kwh-North West England": 1860,
-    "average_cost_per_kwh-Yorkshire and the Humber": 1705,
+    "average_cost_per_kwh-Yorkshire and the Humber": 1789,
-    "average_cost_per_kwh-Wales": 1896,
+    "average_cost_per_kwh-Wales": 1676,
-    "average_cost_per_kwh-Scotland": 1767,
+    "average_cost_per_kwh-Scotland": 1781,
-    "average_cost_per_kwh-Northern Ireland": 1767,
+    "average_cost_per_kwh-Northern Ireland": 1347,
 }
 
 # This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average,

recommendations/HeatingRecommender.py

@@ -183,6 +183,10 @@ class HeatingRecommender:
                            "boiler")
 
     def is_ashp_valid(self):
+
+        if "air_source_heat_pump" in self.property.non_invasive_recommendations:
+            return True
+
         suitable_property_type = self.property.data["property-type"] in ["House", "Bungalow"]
         has_air_source_heat_pump = self.property.main_heating["has_air_source_heat_pump"]
 
@@ -232,6 +236,12 @@ class HeatingRecommender:
             "mainheat_energy_eff_ending": "Good",
             "hot_water_energy_eff_ending": "Good"
         }
+        description_simulation = {
+            "mainheat-description": "Air source heat pump, radiators, electric",
+            "mainheat-energy-eff": "Good",
+            "hot-water-energy-eff": "Good",
+            "hotwater-description": "From main system",
+        }
         # Installation of a boiler improves the hot water system so we need to reflect this in
         # the outcome of the recommendation
         heating_ending_config = MainHeatAttributes("Air source heat pump, radiators, electric").process()
@@ -241,6 +251,10 @@ class HeatingRecommender:
         fuel_ending_config = {}
         if self.property.main_fuel["fuel_type"] != "electricity":
             fuel_ending_config = MainFuelAttributes("electricity (not community)").process()
+            description_simulation = {
+                **description_simulation,
+                "main-fuel": "electricity (not community)"
+            }
 
         # Check the simulation differences
         heating_simulation_config = check_simulation_difference(
@@ -270,6 +284,12 @@ class HeatingRecommender:
                 **controls_recommender.recommendation[0]["simulation_config"]
             }
 
+            description_simulation = {
+                **description_simulation,
+                "mainheatcont-description": "time and temperature zone control",
+                "mainheatc-energy-eff": "Very Good"
+            }
+
         ashp_recommendation = {
             "phase": phase,
             "parts": [
@@ -282,6 +302,7 @@ class HeatingRecommender:
             "sap_points": None,
             "already_installed": already_installed,
             "simulation_config": simulation_config,
+            "description_simulation": description_simulation,
             **ashp_costs
         }
 

recommendations/Recommendations.py

@@ -752,6 +752,23 @@ class Recommendations:
                         predicted_appliances_kwh_reduction
                     )
 
+                    # We store this value for later
+                    phase_lighting_costs[rec["phase"]] = {
+                        "adjusted": new_lighting_cost,
+                        "unadjusted": scoring_lighting_cost
+                    }
+
+                    phase_kwh_figures[rec["phase"]] = {
+                        "adjusted": {
+                            "heating": new_heating_kwh_adjusted,
+                            "hot_water": new_hot_water_kwh_adjusted
+                        },
+                        "unadjusted": {
+                            "heating": new_heating_kwh,
+                            "hot_water": new_hot_water_kwh
+                        }
+                    }
+
                 if rec["type"] == "low_energy_lighting":
                     # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2
                     rec["sap_points"] = min(predicted_sap_points, LightingRecommendations.SAP_LIMIT)

recommendations/RoofRecommendations.py

@@ -23,6 +23,7 @@ class RoofRecommendations:
     # It is recommended that lofts should have at least 270mm of insulation. If the property has more than 200mm of
     # loft insulation in place already, we do not recommend anything for the moment
     MINIMUM_LOFT_ISULATION_MM = 200
+    MINIMUM_RECOMMENDED_LOFT_INSULATION = 280
     # Flat roof should have at least 100mm of insulation
     MINIMUM_FLAT_ROOF_ISULATION_MM = 100
 
@@ -79,6 +80,11 @@ class RoofRecommendations:
         """
         Check if the loft is already insulated
         """
+
+        # If we have a non-invasive recommendation for the loft insulation, we can assume that the loft is not insulated
+        if "loft_insulation" in self.property.non_invasive_recommendations:
+            return False
+
         return (self.insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]
 
     def recommend(self, phase):
@@ -115,12 +121,17 @@ class RoofRecommendations:
         u_value = get_roof_u_value(**{**self.property.roof, "age_band": self.property.age_band})
 
         self.estimated_u_value = u_value
-        if u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
+        if (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) and (
+            "loft_insulation" not in self.property.non_invasive_recommendations
+        ):
             # The Roof is already compliant
             return
 
         if self.property.roof["is_pitched"] or self.property.roof["is_flat"]:
-            self.recommend_roof_insulation(u_value, self.insulation_thickness, self.property.roof, phase)
+            insulation_thickness = (
+                0 if "loft_insulation" not in self.property.non_invasive_recommendations else self.insulation_thickness
+            )
+            self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
             return
 
         if self.property.roof["is_roof_room"]:
@@ -200,7 +211,9 @@ class RoofRecommendations:
                 # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the
                 # loft is already partially insulated.
                 # Note: This requirement is only for loft insulation
-                if ((material["depth"] + insulation_thickness) < self.MINIMUM_LOFT_ISULATION_MM) and roof["is_pitched"]:
+                if (
+                    (material["depth"] + insulation_thickness) < self.MINIMUM_RECOMMENDED_LOFT_INSULATION
+                ) and roof["is_pitched"]:
                     continue
 
                 part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
@@ -245,6 +258,35 @@ class RoofRecommendations:
                     else:
                         raise ValueError("Invalid material type")
 
+                    # This is based on the values we have in the training data
+                    valid_numeric_values = [
+                        12,
+                        25,
+                        50,
+                        75,
+                        100,
+                        150,
+                        200,
+                        250,
+                        270,
+                        300,
+                        350,
+                        400,
+                    ]
+
+                    proposed_depth = new_thickness
+                    if new_thickness not in valid_numeric_values:
+                        # Take the nearest value for scoring
+                        proposed_depth = min(
+                            valid_numeric_values, key=lambda x: abs(x - proposed_depth)
+                        )
+
+                    if proposed_depth >= 270:
+                        new_efficiency = "Very Good"
+                    else:
+                        if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
+                            new_efficiency = "Good"
+
                     recommendations.append(
                         {
                             "phase": phase,
@@ -263,6 +305,10 @@ class RoofRecommendations:
                             "sap_points": None,
                             "already_installed": already_installed,
                             "new_thickness": new_thickness,
+                            "description_simulation": {
+                                "roof-description": f"Pitched, {int(proposed_depth)}mm loft insulation",
+                                "roof-energy-eff": new_efficiency
+                            },
                             **cost_result
                         }
                     )

recommendations/VentilationRecommendations.py

@@ -72,7 +72,7 @@ class VentilationRecommendations(Definitions):
                 "already_installed": already_installed,
                 "sap_points": 0,
                 "heat_demand": 0,
-                "adjusted_heat_demand": 0,
+                "kwh_savings": 0,
                 "co2_equivalent_savings": 0,
                 "energy_cost_savings": 0,
                 "total": estimated_cost,

recommendations/WallRecommendations.py

@@ -252,7 +252,7 @@ class WallRecommendations(Definitions):
 
         self.estimated_u_value = u_value
 
-        if is_cavity_wall:
+        if is_cavity_wall or "cavity_extract_and_refill" in self.property.non_invasive_recommendations:
             if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
                 # Test filling cavity
                 self.find_cavity_insulation(u_value, insulation_thickness, phase)
@@ -357,7 +357,7 @@ class WallRecommendations(Definitions):
                 simulation_config = {
                     **simulation_config,
                     **walls_simulation_config,
-                    "walls_thermal_transmittance_ending": new_u_value
+                    "walls_thermal_transmittance_ending": new_u_value,
                 }
 
                 recommendations.append(
@@ -378,6 +378,10 @@ class WallRecommendations(Definitions):
                         "sap_points": None,
                         "already_installed": already_installed,
                         "simulation_config": simulation_config,
+                        "description_simulation": {
+                            "walls-description": "Cavity wall, filled cavity",
+                            "walls-energy-eff": "Good"
+                        },
                         **cost_result
                     }
                 )