Merge pull request #361 from Hestia-Homes/main

Backend deployment with new image

.dockerignore

@@ -1,6 +1,9 @@
+# Ignore all test directories
 model_data/local_data/*
 backend/tests/*
 backend/node_modules/*
+backend/.idea/*
+backend/.env
 recommendations/tests/*
 model_data/tests/*
 infrastructure/*
@@ -12,3 +15,8 @@ land_registry/*
 pytest.ini
 */README.md
 utils/tests/*
+etl/epc/tests/*
+etl/epc_clean/tests/*
+etl/spatial/tests/*
+
+

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Engine" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Engine" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

backend/Outputs.py

@@ -0,0 +1,343 @@
+import msgpack
+import pandas as pd
+import numpy as np
+from sqlalchemy.orm import sessionmaker
+from datetime import datetime
+
+from utils.s3 import read_from_s3, save_excel_to_s3
+from backend.app.utils import sap_to_epc
+from backend.app.db.connection import db_engine
+from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel
+from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations
+
+
+class Outputs:
+    FORMATS = ["mds"]
+
+    MDS_MEASURE_MAPPING = {
+        "external_wall_insulation": "EWI (Trad Const)",
+        "cavity_wall_insulation": "CWI",
+        "loft_insulation": "LI",
+        "party_wall_insulation": "Party Wall Insu",
+        "internal_wall_insulation": "IWI (POA - Prov Sum Only)",
+        "suspended_floor_insulation": "U/F Insu (Manual install)",
+        "solid_floor_insulation": "Solid floor insl (Out of scope - Prov sum only)",
+        "air_source_heat_pump": "ASHP Htg",
+        "ground_source_heat_pump": "GSHP Htg",
+        "shared_ground_loops": "Shared ground loops",
+        "communal_heat_networks": "Communal heat networks",
+        "district_heating_networks": "District heating networks",
+        "high_heat_retention_storage_heaters": "Elec Storage Htrs (Out of scope -Prov sum only)",
+        "low_energy_lighting": "Low Energy Bulbs",
+        "cylinder_insulation": "Cyl Insulation",
+        "smart_controls": "Smart controls",
+        "zone_controls": "Zone controls",
+        "trvs": "Upgrade TRV's",
+        "solar_pv": "Solar PV",
+        "solar_thermal": "Solar Thermal",
+        "double_glazing": "Double Glazing (POA - Prov sum only)",
+        "draught_proofing": "Draught Proofing",
+        "mechanical_ventilation": "Ventilation upgrade",
+        "gas_boiler": "Gas Boiler Replacement",
+        "flat_roof_insulation": "Flat roof (Out of scope - prov sum only)",
+        "room_in_roof_insulation": "RIR (POA - Prov sum only)",
+        "ev_charging": "EV Charging",
+        "battery": "Battery"
+    }
+
+    def __init__(self, format, portfolio_id):
+        """
+        This class handles the creation of standard outputs for the backend. For example, creation of
+        an excel output, to be used for the MDS data sheet, required by E.ON
+
+        :param format: The format of the output, e.g. mds
+        :param portfolio_id: The id of the portfolio for which the output is being created
+        """
+
+        if format not in self.FORMATS:
+            raise ValueError("Invalid format, should be one of {}".format(self.FORMATS))
+
+        self.format = format
+        self.portfolio_id = portfolio_id
+        self.today = datetime.now().strftime("%Y-%m-%d")
+
+        # Connect to the database
+        self.session = sessionmaker(bind=db_engine)()
+
+        # Download cleaned data
+        self.cleaned_epc_lookup = read_from_s3(
+            s3_file_name="cleaned_epc_data/cleaned.bson",
+            bucket_name="retrofit-data-dev"
+        )
+
+        self.cleaned_epc_lookup = msgpack.unpackb(self.cleaned_epc_lookup, raw=False)
+
+    def get_properties_from_db(self):
+        # Get properties and their details for a specific portfolio
+        properties_query = self.session.query(
+            PropertyModel,
+            PropertyDetailsEpcModel
+        ).join(
+            PropertyDetailsEpcModel,
+            PropertyModel.id == PropertyDetailsEpcModel.property_id
+        ).filter(
+            PropertyModel.portfolio_id == self.portfolio_id  # Filter by portfolio ID
+        ).all()
+
+        # Transform properties data to include all fields dynamically
+        properties_data = [
+            {**{col.name: getattr(prop.PropertyModel, col.name) for col in PropertyModel.__table__.columns},
+             **{col.name: getattr(prop.PropertyDetailsEpcModel, col.name) for col in
+                PropertyDetailsEpcModel.__table__.columns}}
+            for prop in properties_query
+        ]
+
+        return properties_data
+
+    def get_plans_from_db(self):
+
+        plans_query = self.session.query(Plan).filter(Plan.portfolio_id == self.portfolio_id).all()
+        # Transform plans data to include all fields dynamically
+        plans_data = [
+            {col.name: getattr(plan, col.name) for col in Plan.__table__.columns}
+            for plan in plans_query
+        ]
+
+        return plans_data
+
+    def get_recommendations_from_db(self, plan_ids):
+        # Get recommendations through PlanRecommendations for those plans and that are default
+        recommendations_query = self.session.query(
+            Recommendation,
+            Plan.scenario_id
+        ).join(
+            PlanRecommendations, Recommendation.id == PlanRecommendations.recommendation_id
+        ).join(
+            Plan, Plan.id == PlanRecommendations.plan_id  # Join with Plan to access scenario_id
+        ).filter(
+            PlanRecommendations.plan_id.in_(plan_ids),
+            Recommendation.default == True  # Filtering for default recommendations
+        ).all()
+
+        # Transform recommendations data to include all fields dynamically and include scenario_id
+        recommendations_data = [
+            {
+                **{
+                    col.name: getattr(rec.Recommendation, col.name) if
+                    hasattr(rec, 'Recommendation') else getattr(rec, col.name)
+                    for col in Recommendation.__table__.columns
+                },
+                "Scenario ID": rec.scenario_id
+            } for rec in recommendations_query
+        ]
+
+        return recommendations_data
+
+    def make_mds_measure_matrix(self, scenario_recommendations):
+        all_measures = list(self.MDS_MEASURE_MAPPING.values())
+
+        # Collect rows in a list
+        rows = []
+
+        # Populate the rows list
+        for idx, row in scenario_recommendations.iterrows():
+            property_id = row["property_id"]
+            measure_type = row["measure_type"]
+
+            # Get the label for the current type
+            measure_label = self.MDS_MEASURE_MAPPING.get(measure_type, None)
+
+            # If the property_id already exists in the collected rows, update it
+            existing_row = next((item for item in rows if item["property_id"] == property_id), None)
+            if existing_row is None:
+                # Create a new row if the property_id doesn't exist
+                new_row = {measure: None for measure in all_measures}
+                new_row["property_id"] = property_id
+                rows.append(new_row)
+            else:
+                new_row = existing_row
+
+            # Set the corresponding measure label in the row
+            new_row[measure_label] = measure_label
+
+        # Convert the list of dictionaries to a DataFrame
+        matrix = pd.DataFrame(rows)
+
+        # Reset the index for cleanliness
+        matrix.reset_index(drop=True, inplace=True)
+
+        return matrix
+
+    def export_mds(self):
+        """
+        This function will export the data in the MDS format
+        Core data required:
+        - Property address
+        - Property postcode
+        - uprn
+        - recommended measures
+        - pre-EPC
+        - pre-SAP
+        - pre Heat Demand
+        - Property Type
+        - Built form
+        - Wall type
+        - Tenure
+        - Fuel type
+        - Estimated bill
+        - Recommended measures
+        - Post EPC
+        - Post heat demand
+        - Bill savings
+        - Kwh savings
+        """
+
+        self.session.begin()
+        properties_data = self.get_properties_from_db()
+
+        plans_data = self.get_plans_from_db()
+        plan_ids = [plan['id'] for plan in plans_data]
+
+        recommendations_data = self.get_recommendations_from_db(plan_ids)
+        self.session.close()
+
+        # Convert these tables to dataframes
+        properties_df = pd.DataFrame(properties_data)
+        plans_df = pd.DataFrame(plans_data)
+        recommendations_df = pd.DataFrame(recommendations_data)
+
+        scenario_ids = plans_df["scenario_id"].unique()
+
+        # We start to create the MDS sheet
+        mds = properties_df[
+            [
+                "property_id",
+                "address",
+                "postcode",
+                "uprn",
+                "current_epc_rating",
+                "current_sap_points",
+                "primary_energy_consumption",
+                "property_type",
+                "built_form",
+                "total_floor_area",
+                "walls",
+                "tenure",
+                "mainfuel",
+                # The bills columns are split out - we include them and aggregate, without appliances
+                "heating_cost_current",
+                "hot_water_cost_current",
+                "lighting_cost_current",
+                "gas_standing_charge",
+                "electricity_standing_charge"
+            ]
+        ].copy().rename(
+            columns={
+                "address": "Address",
+                "postcode": "Postcode",
+                "uprn": "UPRN",
+                "current_epc_rating": "Pre EPC",
+                "current_sap_points": "EPC Source",
+                "primary_energy_consumption": "Existing Heating Demand Kwh/m2/y",
+                "property_type": "Property Type",
+                "built_form": "Built Form",
+                "total_floor_area": "Floor area m2 (If known)",
+                "walls": "Wall Type (Mandatory field)",
+                "tenure": "Tenure",
+            }
+        )
+
+        mds["Estimated bill (£ per year)"] = (
+            mds["heating_cost_current"] +
+            mds["hot_water_cost_current"] +
+            mds["lighting_cost_current"] +
+            mds["gas_standing_charge"] +
+            mds["electricity_standing_charge"]
+        )
+
+        mds = mds.drop(
+            columns=[
+                "heating_cost_current",
+                "hot_water_cost_current",
+                "lighting_cost_current",
+                "gas_standing_charge",
+                "electricity_standing_charge"
+            ]
+        )
+
+        # Formatting - Pre EPC is an enum
+        mds["Pre EPC"] = [x.value for x in mds["Pre EPC"].values]
+        mds["Wall Type (Mandatory field)"] = mds["Wall Type (Mandatory field)"].str.split(",").str[0]
+        # Remove average thermal transmittance field
+        mds["Wall Type (Mandatory field)"] = np.where(
+            mds["Wall Type (Mandatory field)"].str.contains("Average thermal transmittance"),
+            "",
+            mds["Wall Type (Mandatory field)"]
+        )
+
+        mds = mds.merge(
+            pd.DataFrame(self.cleaned_epc_lookup["main-fuel"])[["clean_description", "fuel_type"]],
+            left_on="mainfuel",
+            right_on="clean_description",
+            how="left"
+        )
+        mds = mds.rename(columns={"fuel_type": "Existing Fuel Type"}).drop(columns=["clean_description", "mainfuel"])
+
+        mds["Existing Fuel Type"].value_counts()
+
+        mds_output_by_scenario = {}
+        for scenario_id in scenario_ids:
+            scenario_recommendations = recommendations_df[recommendations_df["Scenario ID"] == scenario_id]
+
+            # For each measure, we create the measure matrix
+            scenario_measure_matrix = self.make_mds_measure_matrix(scenario_recommendations)
+
+            # Calculate the predicted impact on: SAP, heat demand, bills, kwh
+            recommendation_impacts = scenario_recommendations.groupby("property_id")[
+                ["sap_points", "heat_demand", "kwh_savings", "energy_cost_savings"]
+            ].sum().reset_index()
+
+            scenario_mds = mds.merge(
+                scenario_measure_matrix, how="left", on="property_id"
+            ).merge(
+                recommendation_impacts, how="left", on="property_id"
+            )
+            # If we have no recommendations, sap_points, kwh_savings, head_demand will be NaN
+            to_clean = [c for c in recommendation_impacts.columns if c != "property_id"]
+            for col in to_clean:
+                scenario_mds[col].fillna(0, inplace=True)
+            scenario_mds.fillna(0, inplace=True)
+            scenario_mds["Post SAP"] = scenario_mds["EPC Source"] + scenario_mds["sap_points"]
+            # Round Post SAP down to the nearest integer
+            scenario_mds["Post SAP"] = scenario_mds["Post SAP"].apply(lambda x: int(x))
+            scenario_mds["Post EPC"] = scenario_mds["Post SAP"].apply(lambda x: sap_to_epc(x))
+            scenario_mds["Heating Demand Kwh/m2/y"] = (
+                scenario_mds["Existing Heating Demand Kwh/m2/y"] - scenario_mds["heat_demand"]
+            )
+
+            scenario_mds = scenario_mds.rename(
+                columns={
+                    "sap_points": "Predicted SAP Points",
+                    "kwh_savings": "Energy Saving (Kwh)",
+                    "energy_cost_savings": "Bill Reduction (£ per yr)"
+                }
+            )
+
+            mds_output_by_scenario[scenario_id] = scenario_mds
+
+        # We now save them to s3 as excels
+        for scenario_id, scenario_mds in mds_output_by_scenario.items():
+            save_excel_to_s3(
+                df=scenario_mds,
+                file_key=f"engine_outputs/{self.format}/{self.today}_scenario_id={scenario_id}.xlsx",
+                bucket_name="retrofit-data-dev"
+            )
+
+    def export(self):
+        """
+        This function will export the data in the required format
+        """
+        if self.format == "mds":
+            self.export_mds()
+
+        raise NotImplementedError("Export format not implemented")

backend/Property.py

@@ -18,6 +18,7 @@ from recommendations.recommendation_utils import (
     get_wall_type,
     estimate_external_wall_area,
     estimate_windows,
+    estimate_pitched_roof_area
 )
 from backend.ml_models.AnnualBillSavings import AnnualBillSavings
 from backend.app.utils import sap_to_epc
@@ -74,6 +75,7 @@ class Property:
         postcode,
         address,
         epc_record,
+        property_valuation=None,
         already_installed=None,
         non_invasive_recommendations=None,
         measures=None,
@@ -110,7 +112,11 @@ class Property:
         else:
             self.measures = ast.literal_eval(measures) if measures else None
 
+        self.valuation = property_valuation
+
         self.uprn = epc_record.get("uprn")
+        self.uprn_source = self.data.get("uprn-source")
+
         self.full_sap_epc = epc_record.get("full_sap_epc")
         self.in_conservation_area, self.is_listed, self.is_heritage = None, None, None
         self.restricted_measures = False
@@ -187,6 +193,9 @@ class Property:
 
         # This additional condition data should change how we pass kwargs to this. We should no longer need to pass
         # kwargs to this class, but instead, we should pass the energy assessment condition data
+        energy_assessment = (
+            {"condition": {}, "energy_assessment_is_newer": False} if energy_assessment is None else energy_assessment
+        )
         self.energy_assessment_condition_data = energy_assessment["condition"]
         self.energy_assessment_is_newer = energy_assessment["energy_assessment_is_newer"]
 
@@ -499,44 +508,12 @@ class Property:
                 output["low_energy_lighting_ending"] = 100
                 output["lighting_energy_eff_ending"] = "Very Good"
 
-            if recommendation["type"] == "windows_glazing":
-                output["multi_glaze_proportion_ending"] = 100
-                if output["windows_energy_eff_ending"] not in ["Average", "Good", "Very Good"]:
-                    output["windows_energy_eff_ending"] = "Average"
-
-                is_secondary_glazing = recommendation["is_secondary_glazing"]
-
-                if output["glazing_type_ending"] == "multiple":
-                    pass
-                elif output["glazing_type_ending"] == "single":
-                    output["glazing_type_ending"] = (
-                        "secondary" if is_secondary_glazing else "double"
-                    )
-                elif output["glazing_type_ending"] == "double":
-                    output["glazing_type_ending"] = (
-                        "multiple" if is_secondary_glazing else "double"
-                    )
-                elif output["glazing_type_ending"] == "secondary":
-                    output["glazing_type_ending"] = (
-                        "secondary" if is_secondary_glazing else "multiple"
-                    )
-                elif output["glazing_type_ending"] in ["triple", "high performance"]:
-                    output["glazing_type_ending"] = "multiple"
-                else:
-                    raise ValueError("Invalid glazing type - implement me")
-
-                if is_secondary_glazing:
-                    output["glazed_type_ending"] = "secondary glazing"
-                else:
-                    output["glazed_type_ending"] = (
-                        "double glazing installed during or after 2002"
-                    )
-
             if recommendation["type"] in [
                 "heating", "hot_water_tank_insulation", "heating_control", "secondary_heating",
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
                 "cylinder_thermostat", "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
-                "solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing"
+                "solid_floor_insulation", "suspended_floor_insulation", "mixed_glazing",
+                "windows_glazing"
             ]:
                 # We update the data, as defined in the recommendaton
                 for prefix in ["walls", "roof", "floor"]:
@@ -561,7 +538,8 @@ class Property:
                 "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
                 "solid_floor_insulation", "suspended_floor_insulation",
                 "windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation",
-                "heating_control", "secondary_heating", "cylinder_thermostat", "mixed_glazing"
+                "heating_control", "secondary_heating", "cylinder_thermostat", "mixed_glazing",
+                "extension_cavity_wall_insulation",
             ]:
                 raise NotImplementedError(
                     "Implement me, given type %s" % recommendation["type"]
@@ -592,8 +570,6 @@ class Property:
         if not self.data:
             raise ValueError("Property does not contain data")
 
-        self.set_basic_property_dimensions()
-
         for description, attribute in cleaned.items():
 
             if self.data[description] in self.DATA_ANOMALY_MATCHES:
@@ -641,26 +617,22 @@ class Property:
 
             setattr(self, self.ATTRIBUTE_MAP[description], attributes[0])
 
+        self.set_basic_property_dimensions()
         self.set_wall_type()
         self.set_floor_type()
         self.set_floor_level()
         self.set_windows_count()
         self.set_energy_source()
         self.find_energy_sources()
-        self.set_current_energy_bill(kwh_client, kwh_predictions)
+        self.set_current_energy(kwh_client, kwh_predictions)
 
-    def set_solar_panel_configuration(
-        self, solar_panel_configuration, roof_area
-    ):
+    def set_solar_panel_configuration(self, solar_panel_configuration):
         """
         This funtion inserts the solar panel configuration into the property object
         """
         self.solar_panel_configuration = solar_panel_configuration
 
-        # We also set the roof area
-        self.roof_area = roof_area
-
-    def set_current_energy_bill(self, kwh_client, kwh_predictions):
+    def set_current_energy(self, kwh_client, kwh_predictions):
         """
         Given what we know about the property now, estimates the current energy consumption using the UCL paper
         https://www.sciencedirect.com/science/article/pii/S0378778823002542
@@ -817,6 +789,9 @@ class Property:
 
     def get_property_details_epc(self, portfolio_id: int, rating_lookup):
 
+        if self.current_energy_bill is None:
+            raise ValueError("Current energy bill has not been set")
+
         property_details_epc = {
             "property_id": self.id,
             "portfolio_id": portfolio_id,
@@ -874,6 +849,7 @@ class Property:
             "current_energy_demand": self.current_energy_consumption,
             "current_energy_demand_heating_hotwater": self.current_energy_consumption_heating_hotwater,
             "estimated": self.data.get("estimated", False),
+            **self.current_energy_bill
         }
 
         return property_details_epc
@@ -990,6 +966,13 @@ class Property:
                 self.floor_area / self.number_of_floors
             )
 
+        if not self.roof["is_flat"]:
+            self.roof_area = estimate_pitched_roof_area(
+                floor_area=self.insulation_floor_area,
+            )
+        else:
+            self.roof_area = self.insulation_floor_area
+
     def set_floor_level(self):
         self.floor_level = (
             FLOOR_LEVEL_MAP[self.data["floor-level"]]
@@ -1224,7 +1207,15 @@ class Property:
         if "air_source_heat_pump" not in measures:
             return False
 
-        suitable_property_type = self.data["property-type"] in ["House", "Bungalow"]
+        suitable_house = self.data["property-type"] == "House" and self.data["built-form"] in [
+            "Detached", "Semi-Detached", "End-Terrace",
+        ]
+
+        suitable_bungalow = self.data["property-type"] == "Bungalow" and self.data["built-form"] in [
+            "Detached", "Semi-Detached"
+        ]
+
+        suitable_property_type = suitable_house or suitable_bungalow
         has_air_source_heat_pump = self.main_heating["has_air_source_heat_pump"]
 
         return suitable_property_type and not has_air_source_heat_pump

backend/SearchEpc.py

@@ -7,6 +7,9 @@ import pandas as pd
 import numpy as np
 from epc_api.client import EpcClient
 from backend.OrdnanceSurvey import OrdnanceSuveyClient
+from etl.epc_clean.epc_attributes.WallAttributes import WallAttributes
+from etl.epc_clean.epc_attributes.FloorAttributes import FloorAttributes
+from etl.epc_clean.epc_attributes.RoofAttributes import RoofAttributes
 from BaseUtility import Definitions
 from utils.logger import setup_logger
 from typing import List
@@ -123,6 +126,9 @@ class SearchEpc:
     combinations about the home to find the property
     """
 
+    # If we create the uprn based on a hash, we mark it as simulated
+    UPRN_SOURCE_SIMULATED = "SIMULATED"
+
     MAX_RETRIES = 5
 
     SUCCESS = {
@@ -181,6 +187,7 @@ class SearchEpc:
         self.newest_epc = None
         self.older_epcs = None
         self.full_sap_epc = None
+        self.metadata = None
 
         # These are the address and postcode values, which we store in the database
         self.address_clean = None
@@ -306,7 +313,10 @@ class SearchEpc:
         if (property_type is None) and (address is None):
             return rows
 
-        if len(uprns) == 1:
+        unique_property_types = {r["property-type"] for r in rows}
+
+        # We allow for variation in property type across flats/maisonettes
+        if (len(uprns) == 1) and ((len(unique_property_types) == 1) or unique_property_types == {"Flat", "Maisonette"}):
             return rows
 
         if property_type is not None:
@@ -398,7 +408,11 @@ class SearchEpc:
             else:
                 raise ValueError("Multiple UPRNs found - investigate me")
 
-        uprn = uprns.pop() if uprns else None
+        if uprns:
+            uprn = uprns.pop()
+        else:
+            newest_epc["uprn-source"] = self.UPRN_SOURCE_SIMULATED
+            uprn = hash(self.address1 + self.postcode)
 
         if self.fast:
             return newest_epc, [], {}, "", "", None
@@ -784,3 +798,86 @@ class SearchEpc:
         self.address_clean = self.ordnance_survey_client.address_os
         self.postcode_clean = self.ordnance_survey_client.postcode_os
         return
+
+    def check_attribute_variations(self):
+        attribute_map = {
+            "walls-description": {
+                "cleaner": WallAttributes,
+                "attribute": [
+                    "is_cavity_wall", "is_solid_brick", "is_system_built", "is_timber_frame",
+                    "is_granite_or_whinstone", "is_cob", "is_sandstone_or_limestone", "is_park_home"
+                ],
+                "name": "has_wall_type_ever_varied"
+            },
+            "roof-description": {
+                "cleaner": RoofAttributes,
+                "attribute": [
+                    "is_flat", "is_pitched", "is_roof_room", "is_thatched", "has_dwelling_above"
+                ],
+                "name": "has_roof_type_ever_varied"
+            },
+            "floor-description": {
+                "cleaner": FloorAttributes,
+                "attribute": [
+                    "is_to_unheated_space", "is_to_external_air", "is_suspended", "is_solid", "is_to_external_air",
+                ],
+                "name": "has_floor_type_ever_varied"
+            }
+        }
+
+        attribute_variations = {}
+        for attribute, attribute_objs in attribute_map.items():
+            attribute_variations[attribute_objs["name"]] = False
+            cleaner = attribute_objs["cleaner"]
+            type_timeline = pd.DataFrame([cleaner(epc[attribute]).process() for epc in self.older_epcs] + [
+                cleaner(self.newest_epc[attribute]).process()
+            ])
+            # For eac col in attribute_objs["attribute"] we check if the timeline has ever varied, i.e has gone
+            # from true to false
+            for col in attribute_objs["attribute"]:
+                if type_timeline[col].nunique() > 1:
+                    attribute_variations[attribute_objs["name"]] = True
+                    break
+
+            return attribute_variations
+
+    def identify_flat_floor(self):
+        # If there is no dwelling above, it is a top floor flat
+        processed_roof = RoofAttributes(self.newest_epc["roof-description"]).process()
+        if not processed_roof["has_dwelling_above"]:
+            return "top"
+
+        # We know that there is a dwelling above. If there's also a drwelling below, it is a mid floor flat
+        processed_floor = FloorAttributes(self.newest_epc["floor-description"]).process()
+        if processed_floor["another_property_below"]:
+            return "mid"
+
+        # Otherwise ground floor
+        return "ground"
+
+    def get_metadata(self):
+        if self.newest_epc is None:
+            raise ValueError("No EPC data available")
+
+        # We check if the property has ever been downgraded on SAP
+        has_sap_ever_downgraded = False
+        sap_timeline = [int(epc["current-energy-efficiency"]) for epc in self.older_epcs] + [
+            int(self.newest_epc["current-energy-efficiency"])
+        ]
+        # We check if there has ever been a decrease by differencing
+        has_sap_ever_downgraded = any(np.diff(sap_timeline) < 0)
+
+        # We check if the wall type has ever varied over time
+        attribute_varations = self.check_attribute_variations()
+
+        # If the property is a flat, we distinguish between top, mid, ground floor
+        floor = None
+        if self.newest_epc["property-type"] == "Flat":
+            floor = self.identify_flat_floor()
+
+        self.metadata = {
+            "days_since_last_epc": (pd.Timestamp.now() - pd.Timestamp(self.newest_epc["lodgement-date"])).days,
+            "has_sap_ever_downgraded": has_sap_ever_downgraded,
+            "floor": floor,
+            **attribute_varations
+        }

backend/apis/GoogleSolarApi.py

@@ -1,16 +1,22 @@
+import time
+import requests
 import pandas as pd
 import numpy as np
-from recommendations.Costs import MCS_SOLAR_PV_COST_DATA
-from backend.ml_models.AnnualBillSavings import AnnualBillSavings
-import requests
+from typing import List
 from functools import lru_cache
-import time
-from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data
-from utils.logger import setup_logger
 from sklearn.preprocessing import MinMaxScaler
-from recommendations.Costs import Costs
+from tqdm import tqdm
 from math import sin, cos, sqrt, atan2, radians
 
+from utils.logger import setup_logger
+from recommendations.Costs import Costs, MCS_SOLAR_PV_COST_DATA
+from etl.bill_savings.EnergyConsumptionModel import EnergyConsumptionModel
+from backend.ml_models.AnnualBillSavings import AnnualBillSavings
+from backend.Property import Property
+from backend.app.db.functions.solar_functions import get_solar_data, store_batch_data
+import backend.app.assumptions as assumptions
+from backend.app.plan.schemas import PlanTriggerRequest
+
 logger = setup_logger()
 
 
@@ -42,6 +48,9 @@ class GoogleSolarApi:
     # your area
     installation_life_span = 20
 
+    MIN_UNIT_PANELS = 4  # Minimum number of panels we allow for a domestic building
+    MIN_BUILDING_PANELS = 10  # Minimum number of panels we allow for a block of flats
+
     def __init__(self, api_key, max_retries=5):
         """
         Initialize the GoogleSolarApi class with the provided API key and maximum retries.
@@ -60,7 +69,7 @@ class GoogleSolarApi:
         self.floor_area = None
         self.roof_area = None
         self.roof_segment_indexes = None
-        self.panel_area = None
+        self.panel_area = assumptions.RDSAP_AREA_PER_PANEL
         self.panel_wattage = None
         self.panel_performance = None
 
@@ -157,10 +166,6 @@ class GoogleSolarApi:
 
         self.roof_area = self.insights_data["solarPotential"]["wholeRoofStats"]['areaMeters2']
         self.floor_area = self.insights_data["solarPotential"]["wholeRoofStats"]['groundAreaMeters2']
-        self.panel_area = (
-            self.insights_data["solarPotential"]["panelHeightMeters"] *
-            self.insights_data["solarPotential"]["panelWidthMeters"]
-        )
         self.panel_wattage = self.insights_data["solarPotential"]["panelCapacityWatts"]
         if self.panel_wattage != 400:
             # In the API documentation, it claims that the default output is 250W, however we've only seen 400W, so if
@@ -192,8 +197,6 @@ class GoogleSolarApi:
         if not self.need_to_store:
             return
 
-        logger.info("Storing to database")
-
         scenarios_data = self.panel_performance.head(1)[
             [
                 "n_panels",
@@ -221,7 +224,6 @@ class GoogleSolarApi:
         scenarios_data["scenario_type"] = scenario_type
         scenarios_data = scenarios_data.to_dict(orient="records")
 
-        # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it exists
         store_batch_data(
             session=session,
             api_data=self.insights_data,
@@ -253,6 +255,9 @@ class GoogleSolarApi:
         Optimise the solar panel configuration for the building.
         :return:
         """
+        # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the
+        # minimum is 4
+        min_panels = self.MIN_BUILDING_PANELS if is_building else self.MIN_UNIT_PANELS
 
         cost_instance = Costs(property_instance=property_instance) if property_instance is not None else None
 
@@ -271,13 +276,6 @@ class GoogleSolarApi:
                 generated_dc_energy = segment["yearlyEnergyDcKwh"]
                 ratio = generated_dc_energy / wattage
 
-                if cost_instance is None:
-                    cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000)
-                else:
-                    cost = cost_instance.solar_pv(
-                        wattage=wattage, has_battery=False
-                    )["total"]
-
                 roi_summary.append(
                     {
                         "segmentIndex": segment["segmentIndex"],
@@ -285,7 +283,6 @@ class GoogleSolarApi:
                         "generated_dc_energy": generated_dc_energy,
                         "ratio": ratio,
                         "n_panels": segment["panelsCount"],
-                        "cost": cost,
                         "panneled_roof_area": self.panel_area * int(segment["panelsCount"])
                     }
                 )
@@ -294,10 +291,21 @@ class GoogleSolarApi:
             if roi_summary.empty:
                 continue
 
+            if roi_summary["n_panels"].sum() < min_panels:
+                continue
+
+            if cost_instance is None:
+                total_cost = MCS_SOLAR_PV_COST_DATA["average_cost_per_kwh"] * (wattage / 1000)
+            else:
+                total_cost = cost_instance.solar_pv(
+                    n_panels=roi_summary["n_panels"].sum(),
+                    has_battery=False,
+                    n_floors=property_instance.number_of_floors,
+                )["total"]
+
             weighted_ratio = np.average(
                 roi_summary["ratio"].values, weights=roi_summary["generated_dc_energy"].values
             )
-            total_cost = roi_summary["cost"].sum()
             yearly_dc_energy = roi_summary["generated_dc_energy"].sum()
 
             panel_performance.append(
@@ -333,10 +341,6 @@ class GoogleSolarApi:
             # We can have duplicate configurations
 
         panel_performance = panel_performance.drop_duplicates()
-        # If we look at the building level, we don't include any projects fewer than 10 panels, otherwise the
-        # minimum is 4
-        min_panels = 10 if is_building else 4
-        panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels]
 
         if panel_performance.empty:
             self.panel_performance = pd.DataFrame(
@@ -439,8 +443,8 @@ class GoogleSolarApi:
 
         # We want max roi, minimal generation deficit, and max generation value - we create a ranking score
         # Assign equal weights to each metric
-        weights = {'roi': 0.6, 'generation_value': 0.2, 'generation_deficit': 0.2}
-        metrics = panel_performance[['roi', 'generation_value', 'generation_deficit']]
+        weights = {'roi': 0.8, 'generation_value': 0.2}
+        metrics = panel_performance[['roi', 'generation_value']].copy()
 
         # Normalize the columns (0 to 1 scale)
         scaler = MinMaxScaler()
@@ -448,12 +452,11 @@ class GoogleSolarApi:
 
         # Convert normalized metrics back to a dataframe
         normalized_metrics_df = pd.DataFrame(
-            normalized_metrics, columns=['roi', 'generation_value', 'generation_deficit']
+            normalized_metrics, columns=['roi', 'generation_value']
         )
         normalized_metrics_df['combined_score'] = (
             normalized_metrics_df['roi'] * weights['roi'] +
-            normalized_metrics_df['generation_value'] * weights['generation_value'] +
-            (1 - normalized_metrics_df['generation_deficit']) * weights['generation_deficit']
+            normalized_metrics_df['generation_value'] * weights['generation_value']
         )
 
         panel_performance['combined_score'] = normalized_metrics_df['combined_score'].values
@@ -486,6 +489,7 @@ class GoogleSolarApi:
 
             panel_performance["n_panels"] = panel_performance["n_panels_halved"]
             panel_performance = panel_performance.drop(columns=["n_panels_halved"])
+            panel_performance = panel_performance[panel_performance["n_panels"] >= min_panels]
 
         self.panel_performance = panel_performance
 
@@ -583,3 +587,317 @@ class GoogleSolarApi:
             # we need to do is perform the solar analysis and then half the results. We set an indicator which
             # implies we should do this
             self.double_property = True
+
+    @staticmethod
+    def calculate_percentage_decrease(start_efficiency, end_efficiency, consumption_averages):
+        """
+        Calculate the percentage decrease in consumption between two energy efficiency ratings.
+        :param start_efficiency: The starting energy efficiency rating.
+        :param end_efficiency: The ending energy efficiency rating.
+        :param consumption_averages: The DataFrame containing the consumption averages.
+        :return:
+        """
+
+        start_consumption = consumption_averages.loc[
+            consumption_averages["current-energy-efficiency"].astype(str) == str(start_efficiency), "total_consumption"
+        ].values[0]
+
+        end_consumption = consumption_averages.loc[
+            consumption_averages["current-energy-efficiency"].astype(str) == str(end_efficiency), "total_consumption"
+        ].values[0]
+
+        percentage_decrease = ((start_consumption - end_consumption) / start_consumption) * 100
+        # percentage_decrease cannot be nehative
+        if percentage_decrease < 0:
+            percentage_decrease = 0
+        return percentage_decrease
+
+    @classmethod
+    def estimate_new_consumption(
+        cls, current_energy_efficiency, target_efficiency, current_consumption, ofgem_consumption_averages
+    ):
+        """
+        Given then consumption_averages dataset, which is produced as a result of the training_data.py script,
+        for the energy kwh models, this function will estimate the new consumption based on the current consumption,
+        based on the expected reduction in consumption from the current rating to the target rating.
+        :param current_energy_efficiency: The current energy efficiency rating
+        :param target_efficiency: The target energy efficiency rating
+        :param current_consumption: The current consumption of the property
+        :param ofgem_consumption_averages: DataFrame of the Ofgem consumption averages
+        :return:
+        """
+        percentage_decrease = cls.calculate_percentage_decrease(
+            start_efficiency=current_energy_efficiency,
+            end_efficiency=target_efficiency,
+            consumption_averages=ofgem_consumption_averages
+        )
+        new_consumption = current_consumption * (1 - percentage_decrease / 100)
+        return new_consumption
+
+    @classmethod
+    def prepare_input_data(
+        cls,
+        input_properties: List[Property],
+        ofgem_consumption_averages: pd.DataFrame,
+        body: PlanTriggerRequest
+    ):
+        """
+        :param input_properties: List of properties
+        :param ofgem_consumption_averages: DataFrame of the Ofgem consumption averages
+        :param body: PlanTriggerRequest instance
+        This sets up the data required to make the solar api request
+        :return:
+        """
+
+        building_solar_config = [
+            {
+                "building_id": p.building_id,
+                "longitude": p.spatial["longitude"],
+                "latitude": p.spatial["latitude"],
+                # Energy consumption is adjusted for the property's expected post retrofit state
+                # We set the target rating to EPC C, which is the typical EPC rating we would expect the
+                # property to achieve post retrofit of just the fabric
+                "energy_consumption": cls.estimate_new_consumption(
+                    current_energy_efficiency=p.data["current-energy-efficiency"],
+                    target_efficiency="69",
+                    current_consumption=p.estimate_electrical_consumption(
+                        assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions
+                    ),
+                    ofgem_consumption_averages=ofgem_consumption_averages
+                ),
+                "property_id": p.id,
+                "uprn": p.uprn
+            } for p in input_properties if p.building_id is not None
+        ]
+        unit_solar_config = [
+            {
+                "longitude": p.spatial["longitude"],
+                "latitude": p.spatial["latitude"],
+                # Energy consumption is adjusted for the property's expected post retrofit state
+                # We set the target rating to EPC C, which is the typical EPC rating we would expect the
+                # property to achieve post retrofit of just the fabric
+                "energy_consumption": cls.estimate_new_consumption(
+                    current_energy_efficiency=p.data["current-energy-efficiency"],
+                    target_efficiency="69",
+                    current_consumption=p.estimate_electrical_consumption(
+                        assumed_ashp_efficiency=assumptions.AVERAGE_ASHP_EFFICIENCY, exclusions=body.exclusions
+                    ),
+                    ofgem_consumption_averages=ofgem_consumption_averages
+                ),
+                "property_id": p.id,
+                "uprn": p.uprn
+            } for p in input_properties if p.building_id is None
+        ]
+
+        return building_solar_config, unit_solar_config
+
+    @classmethod
+    def building_solar_analysis(
+        cls, building_solar_config: List, input_properties: List[Property], session, google_solar_api_key: str
+    ):
+        """
+        Perform the solar analysis for the building level
+        :param building_solar_config: List of building solar configurations
+        :param input_properties: List of properties
+        :param session: Database session
+        :param google_solar_api_key: Google Solar API key
+        :return:
+        """
+
+        if not building_solar_config:
+            return input_properties
+
+        # Find the unique longitude and latitude pairs for each building id
+        unique_coordinates = {}
+        building_uprns = {}
+        for entry in building_solar_config:
+            building_id = entry['building_id']
+            coordinate_pair = {'longitude': entry['longitude'], 'latitude': entry['latitude']}
+
+            if building_id not in unique_coordinates:
+                unique_coordinates[building_id] = []
+
+            if coordinate_pair not in unique_coordinates[building_id]:
+                unique_coordinates[building_id].append(coordinate_pair)
+
+            if building_id not in building_uprns:
+                building_uprns[building_id] = []
+
+            if entry['uprn'] not in building_uprns[building_id]:
+                building_uprns[building_id].append(
+                    {
+                        "uprn": entry['uprn'], "longitude": entry['longitude'], "latitude": entry['latitude']
+                    }
+                )
+
+        solar_panel_configuration = {}
+        for building_id, coordinates in unique_coordinates.items():
+            if len(coordinates) > 1:
+                raise NotImplementedError("more than one coordinate for a building - handle me")
+
+            coordinates = coordinates[0]
+            energy_consumption = sum(
+                [entry['energy_consumption'] for entry in building_solar_config if entry['building_id'] == building_id]
+            )
+            solar_api_client = cls(api_key=google_solar_api_key)
+            solar_api_client.get(
+                longitude=coordinates["longitude"],
+                latitude=coordinates["latitude"],
+                energy_consumption=energy_consumption,
+                is_building=True,
+                session=session
+            )
+            solar_panel_configuration[building_id] = {
+                "insights_data": solar_api_client.insights_data,
+                "panel_performance": solar_api_client.panel_performance,
+                "n_units": len([entry for entry in building_solar_config if entry['building_id'] == building_id])
+            }
+
+            # Store the data in the database
+            # TODO: Rather than just doing a straight insert, we should overwrite what's already there if it
+            #  exists
+            solar_api_client.save_to_db(
+                session=session, uprns_to_location=building_uprns[building_id], scenario_type="building"
+            )
+
+            # Insert this into the properties that have this building id
+            for p in input_properties:
+                if p.building_id == building_id:
+                    unit_solar_panel_configuration = solar_panel_configuration[building_id].copy()
+
+                    unit_solar_panel_configuration["unit_share_of_energy"] = (
+                        [x for x in building_solar_config if x["property_id"] == p.id][0]["energy_consumption"] /
+                        energy_consumption
+                    )
+                    p.set_solar_panel_configuration(unit_solar_panel_configuration)
+
+        return input_properties
+
+    @classmethod
+    def unit_solar_analysis(
+        cls, unit_solar_config: List, input_properties: List[Property], session, body, google_solar_api_key: str
+    ):
+
+        if not unit_solar_config:
+            return input_properties
+
+        # Model the solar potential at the property level
+        for unit in tqdm(unit_solar_config):
+
+            # We don't need to do this if we have global inclusions that don't include solar
+            if body.inclusions:
+                if "solar_pv" not in body.inclusions:
+                    continue
+
+            property_instance = [p for p in input_properties if p.id == unit["property_id"]][0]
+            # At this level, we check if the property is suitable for solar and if now, skip
+            # Or if we have a solar non-invasive recommendation
+            if (
+                (not property_instance.is_solar_pv_valid()) or
+                [r for r in property_instance.non_invasive_recommendations if r["type"] == "solar_pv"]
+            ):
+                continue
+
+            if unit["longitude"] is None or unit["latitude"] is None:
+                # At this point, we've checked that solar PV is valid, and so we provide some defaults
+
+                property_instance.set_solar_panel_configuration(
+                    solar_panel_configuration={
+                        "insights_data": None,
+                        "panel_performance": cls.default_panel_performance(property_instance=property_instance),
+                        "unit_share_of_energy": 1
+                    },
+                )
+                continue
+
+            solar_api_client = cls(api_key=google_solar_api_key)
+            solar_api_client.get(
+                longitude=unit["longitude"],
+                latitude=unit["latitude"],
+                energy_consumption=unit["energy_consumption"],
+                is_building=False,
+                session=session,
+                uprn=unit["uprn"],
+                property_instance=property_instance
+            )
+
+            # Store the data in the database
+            solar_api_client.save_to_db(
+                session=session,
+                uprns_to_location=[
+                    {
+                        "uprn": property_instance.uprn,
+                        "longitude": property_instance.spatial["longitude"],
+                        "latitude": property_instance.spatial["latitude"]
+                    }
+                ],
+                scenario_type="unit"
+            )
+
+            property_instance.set_solar_panel_configuration(
+                solar_panel_configuration={
+                    "insights_data": solar_api_client.insights_data,
+                    "panel_performance": solar_api_client.panel_performance,
+                    "unit_share_of_energy": 1
+                },
+            )
+
+        return input_properties
+
+    @classmethod
+    def default_panel_performance(cls, property_instance):
+        """
+        In a small number of cases, where properties have simulated uprns, we do not have a longitude and latitude
+        value and therefore we just return a default panel performance
+        :param property_instance:
+        :return:
+        """
+
+        cost_instance = Costs(property_instance=property_instance)
+
+        # We return a 2.4 and 4 kwp system
+        panel_performance = pd.DataFrame(
+            [
+                {
+                    'n_panels': 10,
+                    'yearly_dc_energy': 4000 * 0.99,  # Assumed 99% efficient wattage -> dc
+                    'total_cost': cost_instance.solar_pv(
+                        n_panels=10, has_battery=False, n_floors=property_instance.number_of_floors
+                    )["total"],
+                    'weighted_ratio': None,
+                    'panneled_roof_area': 10 * assumptions.RDSAP_AREA_PER_PANEL,
+                    'array_wattage': 4000,
+                    'initial_ac_kwh_per_year': 4000 * 0.95,  # Assumed 95% efficient wattage -> ac
+                    'lifetime_ac_kwh': None,
+                    'lifetime_dc_kwh': None,
+                    'roi': None,
+                    'generation_value': None,
+                    'generation_deficit': None,
+                    'expected_payback_years': None,
+                    'surplus': None,
+                    'combined_score': None,
+                    'rank': None
+                },
+                {
+                    'n_panels': 6,
+                    'yearly_dc_energy': 2400 * 0.99,  # Assumed 99% efficient wattage -> dc
+                    'total_cost': cost_instance.solar_pv(
+                        n_panels=6, has_battery=False, n_floors=property_instance.number_of_floors
+                    )["total"],
+                    'weighted_ratio': None,
+                    'panneled_roof_area': 6 * assumptions.RDSAP_AREA_PER_PANEL,
+                    'array_wattage': 2400,
+                    'initial_ac_kwh_per_year': 2400 * 0.95,  # Assumed 95% efficient wattage -> ac
+                    'lifetime_ac_kwh': None,
+                    'lifetime_dc_kwh': None,
+                    'roi': None,
+                    'generation_value': None,
+                    'generation_deficit': None,
+                    'expected_payback_years': None,
+                    'surplus': None,
+                    'combined_score': None,
+                    'rank': None
+                },
+            ]
+        )
+        return panel_performance

backend/app/assumptions.py

@@ -1,44 +1,53 @@
 # Assumes that the average efficiency of an air source heat pump is 250%, taking the median of the 200-400% range,
 # which is often quoted as a sensible efficiency range for air source heat pumps.
 PESSIMISTIC_ASHP_EFFICIENCY = 200
-AVERAGE_ASHP_EFFICIENCY = 300
+AVERAGE_ASHP_EFFICIENCY = 250
 
 # Conservative estimate of the proportion of electricity that will be consumed, whereas the rest will
-# be exported
+# be exported. These are averages based on Google research. E.g
+# https://www.nea.org.uk/who-we-are/innovation-technical-evaluation/solarpv/solarpv-batteries
 SOLAR_CONSUMPTION_PROPORTION = 0.5
+SOLAR_CONSUMPTION_WITH_BATTERY_PROPORTION = 0.7
+
+# Typically, each solar panel takes up around 3.4 m2 of roof space under RdSAP. This was been verified in Elmhurst
+RDSAP_AREA_PER_PANEL = 3.4
+
+SOCIAL_TENURES = ["Rented (social)", "rental (social)"]
 
 DESCRIPTIONS_TO_FUEL_TYPES = {
     "Air source heat pump, radiators, electric": {
         "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100
     },
-    "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.9},
+    "Boiler and radiators, mains gas": {"fuel": 'Natural Gas', "cop": 0.85},
     'Electric storage heaters': {"fuel": 'Electricity', "cop": 1},
     "Electric immersion, off-peak": {"fuel": 'Electricity', "cop": 1},
     "Electric storage heaters, radiators": {"fuel": 'Electricity', "cop": 1},
     "Room heaters, electric": {"fuel": 'Electricity', "cop": 1},
     "Electric immersion, standard tariff": {"fuel": 'Electricity', "cop": 1},
     "Portable electric heaters assumed for most rooms": {"fuel": 'Electricity', "cop": 1},
-    "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.9},
+    "Boiler and radiators, LPG": {"fuel": 'LPG', "cop": 0.85},
     "Room heaters, dual fuel (mineral and wood)": {"fuel": 'Wood Logs', "cop": 1},
-    "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.9},
-    "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.9},
-    "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.9},
-    "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.9},
+    "Room heaters, mains gas": {"fuel": 'Natural Gas', "cop": 0.85},
+    "Warm air, mains gas": {"fuel": 'Natural Gas', "cop": 0.85},
+    "Boiler, mains gas": {"fuel": 'Natural Gas', "cop": 0.85},
+    "Gas multipoint": {"fuel": "Natural Gas", "cop": 0.85},
     "Warm air, Electricaire": {"fuel": "Electricity", "cop": 1},
-    "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.9},
-    "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.9},
+    "Gas boiler/circulator": {"fuel": "Natural Gas", "cop": 0.85},
+    "Boiler and underfloor heating, mains gas": {"fuel": "Natural Gas", "cop": 0.85},
     "No system present: electric heaters assumed": {"fuel": "Electricity", "cop": 1},
     "Electric instantaneous at point of use": {"fuel": "Electricity", "cop": 1},
-    "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.9},
+    "Boiler and radiators, oil": {"fuel": "Oil", "cop": 0.85},
     "Electric storage heaters, Electric storage heaters": {"fuel": "Electricity", "cop": 1},
-    "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.9},
-    "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.9},
-    "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.9},
+    "Boiler and radiators, electric": {"fuel": "Electricity", "cop": 0.85},
+    "Gas boiler/circulator, no cylinder thermostat": {"fuel": "Natural Gas", "cop": 0.85},
+    "Boiler and radiators, dual fuel (mineral and wood)": {"fuel": "Wood Logs", "cop": 0.85},
     "Electric immersion, standard tariff, plus solar": {"fuel": "Electricity + Solar Thermal", "cop": 1},
-    "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.9},
+    "From main system, flue gas heat recovery": {"fuel": "Natural Gas", "cop": 0.85},
     "Electric underfloor heating": {"fuel": "Electricity", "cop": 1},
     "No system present: electric immersion assumed": {"fuel": "Electricity", "cop": 1},
     "Air source heat pump, underfloor, electric": {
         "fuel": "Electricity", "cop": AVERAGE_ASHP_EFFICIENCY / 100
     },
+    "Gas instantaneous at point of use": {"fuel": "Natural Gas", "cop": 0.85},
+    "Room heaters, wood logs": {"fuel": "Wood Logs", "cop": 1},
 }

backend/app/config.py

@@ -1,5 +1,5 @@
 from functools import lru_cache
-from pydantic import BaseSettings
+from pydantic_settings import BaseSettings
 
 
 class Settings(BaseSettings):

backend/app/db/functions/energy_assessment_functions.py

@@ -71,6 +71,10 @@ def get_latest_assessment_by_uprn(session: Session, uprn: int) -> Optional[Energ
     :param uprn: The unique property reference number
     :return: The latest EnergyAssessment object or None if not found
     """
+
+    if not uprn:
+        return EnergyAssessment.empty_response()
+
     try:
         # Query the EnergyAssessment model, filter by uprn, order by inspection_date in descending order
         latest_assessment = session.query(EnergyAssessment).filter_by(uprn=uprn).order_by(

backend/app/db/functions/property_functions.py

@@ -11,7 +11,8 @@ from backend.app.db.models.portfolio import (
 from sqlalchemy.orm.exc import NoResultFound
 
 
-def create_property(session: Session, portfolio_id: int, address: str, postcode: str, uprn: str) -> (int, bool):
+def create_property(session: Session, portfolio_id: int, address: str, postcode: str, uprn: str,
+                    energy_assessment: dict) -> (int, bool):
     """
     This function will create a record for the property in the database if it does not exist.
     If it does exist, it will just update the updated_at field.
@@ -39,13 +40,17 @@ def create_property(session: Session, portfolio_id: int, address: str, postcode:
 
     except NoResultFound:
         # Property doesn't exist, create a new one
+
+        status = PortfolioStatus.ASSESSMENT.value if len(energy_assessment["epc"]) == 0 \
+            else PortfolioStatus.SURVEY.value
+
         new_property = PropertyModel(
             address=address,
             postcode=postcode,
             portfolio_id=portfolio_id,
             uprn=uprn,
             creation_status=PropertyCreationStatus.LOADING,
-            status=PortfolioStatus.ASSESSMENT.value,
+            status=status,
             has_pre_condition_report=False,
             has_recommendations=False
         )

backend/app/db/functions/recommendations_functions.py

@@ -108,19 +108,21 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
             {
                 "property_id": property_id,
                 "type": rec["type"],
+                "measure_type": rec["measure_type"],
                 "description": rec["description"],
-                "estimated_cost": rec["total"],
+                "estimated_cost": float(rec["total"]),
                 "default": rec["default"],
-                "starting_u_value": rec.get("starting_u_value"),
-                "new_u_value": rec.get("new_u_value"),
-                "sap_points": rec["sap_points"],
-                "energy_savings": rec["heat_demand"],
-                "kwh_savings": rec["kwh_savings"],
-                "co2_equivalent_savings": rec["co2_equivalent_savings"],
-                "total_work_hours": rec["labour_hours"],
-                "energy_cost_savings": rec["energy_cost_savings"],
-                "labour_days": rec["labour_days"],
+                "starting_u_value": float(rec.get("starting_u_value")) if rec.get("starting_u_value") else None,
+                "new_u_value": float(rec.get("new_u_value")) if rec.get("new_u_value") else None,
+                "sap_points": float(rec["sap_points"]),
+                "energy_savings": float(rec["heat_demand"]),
+                "kwh_savings": float(rec["kwh_savings"]),
+                "co2_equivalent_savings": float(rec["co2_equivalent_savings"]),
+                "total_work_hours": float(rec["labour_hours"]),
+                "energy_cost_savings": float(rec["energy_cost_savings"]),
+                "labour_days": float(rec["labour_days"]),
                 "already_installed": rec["already_installed"],
+                "heat_demand": float(rec["heat_demand"])
             }
             for rec in recommendations_to_upload
         ]

backend/app/db/functions/solar_functions.py

@@ -1,5 +1,6 @@
 import datetime
 import pytz
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound
 from backend.app.db.models.solar import Solar, SolarScenario
@@ -38,57 +39,80 @@ def get_solar_data(session: Session, longitude: float = None, latitude: float =
 def store_batch_data(session: Session, api_data: dict, uprns_to_location: list, scenarios_data: list):
     """
     This function will store the API data to the solar table against all of the UPRNs with longitude and latitude.
+    If a record already exists in the Solar table by UPRN, it will be updated instead of creating a new one.
+    Similarly, if a scenario exists in SolarScenario by number_panels, it will also be updated.
+
     :param session: The database session
     :param api_data: The API data to store
-    :param uprns_to_location: A list of dictionaries containing uprn, longitude, and latitude
+    :param uprns_to_location: A list of dictionaries containing UPRN, longitude, and latitude
     :param scenarios_data: A list of dictionaries containing scenario data for each UPRN
     """
     try:
-
-        # Insert data into the Solar table and get the IDs
-        solar_records = []
+        # Insert or update data into the Solar table
         for data in uprns_to_location:
-            solar_record = Solar(
-                uprn=data['uprn'],
-                longitude=data['longitude'],
-                latitude=data['latitude'],
-                google_api_response=api_data,
-                updated_at=datetime.datetime.now(pytz.utc)
-            )
-            solar_records.append(solar_record)
-            session.add(solar_record)
+            existing_solar = session.execute(select(Solar).where(Solar.uprn == data['uprn'])).scalar_one_or_none()
 
-        session.flush()  # Flush to get the IDs generated
-
-        for record in solar_records:
-            session.refresh(record)  # Refresh to populate the ID fields
-
-        # Retrieve the IDs of the inserted records
-        inserted_ids = {record.uprn: record.id for record in solar_records}
-
-        # Prepare the data for SolarScenario
-        scenario_records = []
-        for data in uprns_to_location:
-            solar_id = inserted_ids.get(data['uprn'])
-            for scenario in scenarios_data:
-                scenario_record = SolarScenario(
-                    solar_id=solar_id,
-                    scenario_type=scenario['scenario_type'],
-                    number_panels=scenario['number_panels'],
-                    array_kwhp=scenario['array_kwhp'],
-                    lifetime_dc_kwh=scenario['lifetime_dc_kwh'],
-                    yearly_dc_kwh=scenario['yearly_dc_kwh'],
-                    lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'),  # Optional field
-                    yearly_ac_kwh=scenario.get('yearly_ac_kwh'),  # Optional field
-                    cost=scenario['cost'],
-                    expected_payback_years=scenario.get('expected_payback_years'),  # Optional field
-                    panelled_roof_area=scenario['panelled_roof_area'],
-                    is_default=scenario['is_default']
+            if existing_solar:
+                # Update the existing record
+                existing_solar.longitude = data['longitude']
+                existing_solar.latitude = data['latitude']
+                existing_solar.google_api_response = api_data
+                existing_solar.updated_at = datetime.datetime.now(pytz.utc)
+                solar_id = existing_solar.id
+            else:
+                # Insert a new record
+                solar_record = Solar(
+                    uprn=data['uprn'],
+                    longitude=data['longitude'],
+                    latitude=data['latitude'],
+                    google_api_response=api_data,
+                    updated_at=datetime.datetime.now(pytz.utc)
                 )
-                scenario_records.append(scenario_record)
+                session.add(solar_record)
+                session.flush()  # Flush to get the IDs generated
+                session.refresh(solar_record)  # Refresh to populate the ID field
+                solar_id = solar_record.id
 
-        # Insert data into the SolarScenario table
-        session.bulk_save_objects(scenario_records)
+            # Insert or update data in the SolarScenario table
+            for scenario in scenarios_data:
+                existing_scenario = session.execute(
+                    select(SolarScenario).where(
+                        SolarScenario.solar_id == solar_id,
+                        SolarScenario.number_panels == scenario['number_panels']
+                    )
+                ).scalar_one_or_none()
+
+                if existing_scenario:
+                    # Update the existing scenario record
+                    existing_scenario.scenario_type = scenario['scenario_type']
+                    existing_scenario.array_kwhp = scenario['array_kwhp']
+                    existing_scenario.lifetime_dc_kwh = scenario['lifetime_dc_kwh']
+                    existing_scenario.yearly_dc_kwh = scenario['yearly_dc_kwh']
+                    existing_scenario.lifetime_ac_kwh = scenario.get('lifetime_ac_kwh')  # Optional field
+                    existing_scenario.yearly_ac_kwh = scenario.get('yearly_ac_kwh')  # Optional field
+                    existing_scenario.cost = scenario['cost']
+                    existing_scenario.expected_payback_years = scenario.get('expected_payback_years')  # Optional field
+                    existing_scenario.panelled_roof_area = scenario['panelled_roof_area']
+                    existing_scenario.is_default = scenario['is_default']
+                else:
+                    # Insert a new scenario record
+                    scenario_record = SolarScenario(
+                        solar_id=solar_id,
+                        scenario_type=scenario['scenario_type'],
+                        number_panels=scenario['number_panels'],
+                        array_kwhp=scenario['array_kwhp'],
+                        lifetime_dc_kwh=scenario['lifetime_dc_kwh'],
+                        yearly_dc_kwh=scenario['yearly_dc_kwh'],
+                        lifetime_ac_kwh=scenario.get('lifetime_ac_kwh'),  # Optional field
+                        yearly_ac_kwh=scenario.get('yearly_ac_kwh'),  # Optional field
+                        cost=scenario['cost'],
+                        expected_payback_years=scenario.get('expected_payback_years'),  # Optional field
+                        panelled_roof_area=scenario['panelled_roof_area'],
+                        is_default=scenario['is_default']
+                    )
+                    session.add(scenario_record)
+
+        # Commit the changes after all operations
         session.commit()
 
     except Exception as e:

backend/app/db/models/portfolio.py

@@ -11,6 +11,7 @@ Base = declarative_base()
 class PortfolioStatus(enum.Enum):
     SCOPING = "scoping"
     ASSESSMENT = "assessment"
+    SURVEY = "survey"
     TENDERING = "tendering"
     PROJECT_UNDERWAY = "project underway"
     COMPLETION_ON_TRACK = "completion; status: on track"
@@ -172,6 +173,13 @@ class PropertyDetailsEpcModel(Base):
     current_energy_demand = Column(Float)
     current_energy_demand_heating_hotwater = Column(Float)
     estimated = Column(Boolean, default=False)
+    # Include estimates for energy bills, across the different types of energy
+    heating_cost_current = Column(Float)
+    hot_water_cost_current = Column(Float)
+    lighting_cost_current = Column(Float)
+    appliances_cost_current = Column(Float)
+    gas_standing_charge = Column(Float)
+    electricity_standing_charge = Column(Float)
 
 
 class PropertyDetailsSpatial(Base):

backend/app/db/models/recommendations.py

@@ -15,6 +15,7 @@ class Recommendation(Base):
     property_id = Column(BigInteger, ForeignKey(PropertyModel.id), nullable=False)
     created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())
     type = Column(String, nullable=False)
+    measure_type = Column(String)
     description = Column(String, nullable=False)
     estimated_cost = Column(Float)
     default = Column(Boolean, nullable=False)

backend/app/plan/schemas.py

@@ -1,108 +1,82 @@
-from pydantic import BaseModel, conlist, validator
-from typing import Optional
+from pydantic import BaseModel, Field, BeforeValidator
+from typing import Annotated, List, Optional
 
+# Example constants for validation
 TYPICAL_MEASURE_TYPES = [
-    "wall_insulation",
-    "roof_insulation",
-    "ventilation",
-    "floor_insulation",
-    "windows",
-    "fireplace",
-    "heating",
-    "hot_water",
-    "low_energy_lighting",
-    "secondary_heating",
-    "solar_pv"
+    "wall_insulation", "roof_insulation", "ventilation", "floor_insulation",
+    "windows", "fireplace", "heating", "hot_water", "low_energy_lighting",
+    "secondary_heating", "solar_pv"
 ]
 
 SPECIFIC_MEASURES = [
-    # Specific measures
-    # Walls
-    "internal_wall_insulation",
-    "external_wall_insulation",
-    "cavity_wall_insulation"
-    # Roof
-    "loft_insulation",
-    "flat_roof_insulation",
-    "room_roof_insulation",
-    # Floor
-    "suspended_floor_insulation",
-    "solid_floor_insulation",
-    # Heating
-    "boiler_upgrade",
-    "high_heat_retention_storage_heater",
-    "air_source_heat_pump",
+    "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
+    "loft_insulation", "flat_roof_insulation", "room_roof_insulation",
+    "suspended_floor_insulation", "solid_floor_insulation",
+    "boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump",
+    "secondary_heating", "solar_pv", "double_glazing", "secondary_glazing",
+    "ventilation", "low_energy_lighting", "fireplace", "hot_water"
+]
 
-    # Specific measures that will typically come from an energy assessment
-    "trickle_vents",
-    "draught_proofing",
-    "mixed_glazing",  # This covers partial double glazing and secondary glazing
-    "cavity_extract_and_refill",
+NON_INVASIVE_SPECIFIC_MEASURES = [
+    "trickle_vents", "draught_proofing", "mixed_glazing", "cavity_extract_and_refill",
+    "extension_cavity_wall_insulation"
 ]
 
 # This allows us to extend high level categories for measures such as "wall_insulation" to the specific measures
 # such as "external_wall_insulation", "internal_wall_insulation", "cavity_wall_insulation"
 MEASURE_MAP = {
     "wall_insulation": [
-        "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation", "cavity_extract_and_refill"
+        "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
     ],
     "roof_insulation": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"],
     "floor_insulation": ["suspended_floor_insulation", "solid_floor_insulation"],
     "heating": ["boiler_upgrade", "high_heat_retention_storage_heater", "air_source_heat_pump"],
+    "windows": ["double_glazing", "secondary_glazing"],
+    "heating_controls": ["roomstat_programmer_trvs", "time_temperature_zone_control"]
 }
 
+VALID_GOALS = ["Increasing EPC"]
+VALID_HOUSING_TYPES = ["Social", "Private"]
+
+
+# Define the validation function for inclusions/exclusions
+def check_inclusion_or_exclusion(value: str) -> str:
+    if value not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES + NON_INVASIVE_SPECIFIC_MEASURES:
+        raise ValueError(f"{value} is not an allowed inclusion")
+    return value
+
+
+def check_goals(value: str) -> str:
+    assert value in VALID_GOALS, f"{value} is not a valid goal"
+    return value
+
+
+def check_housing_type(value: str) -> str:
+    assert value in VALID_HOUSING_TYPES, f"{value} is not a valid housing type"
+    return value
+
+
+# Use Annotated with BeforeValidator for each list item validation
+InclusionOrExclusionItem = Annotated[str, BeforeValidator(check_inclusion_or_exclusion)]
+Goal = Annotated[str, BeforeValidator(check_goals)]
+HousingType = Annotated[str, BeforeValidator(check_housing_type)]
+
 
 class PlanTriggerRequest(BaseModel):
     budget: Optional[float] = None
-    goal: str
-    housing_type: str
+    goal: Goal
+    housing_type: HousingType
     goal_value: str
     portfolio_id: int
     trigger_file_path: str
     already_installed_file_path: Optional[str] = None
     patches_file_path: Optional[str] = None
     non_invasive_recommendations_file_path: Optional[str] = None
-    exclusions: Optional[conlist(str, min_items=1)] = None
-    inclusions: Optional[conlist(str, min_items=1)] = None
+    valuation_file_path: Optional[str] = None
+    exclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1)
+    inclusions: Optional[List[InclusionOrExclusionItem]] = Field(default=None, min_length=1)
 
     scenario_name: Optional[str] = ""
-    # If true, will allow us to create multiple plans for the same portfolio, whereas if this is false, if this property
-    # exists in the portfolio, it will be ignored
     multi_plan: Optional[bool] = False
-
-    _allowed_goals = {"Increasing EPC"}
-
-    _allowed_housing_types = {"Social", "Private"}
-
-    # Validator to ensure exclusions are within the pre-defined possibilities
-    @validator('exclusions', each_item=True)
-    def check_exclusions(cls, v):
-        if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES:
-            raise ValueError(f"{v} is not an allowed exclusion")
-        return v
-
-    @validator('inclusions', each_item=True)
-    def check_inclusions(cls, v):
-        if v not in TYPICAL_MEASURE_TYPES + SPECIFIC_MEASURES:
-            raise ValueError(f"{v} is not an allowed inclusion")
-        return v
-
-    # Validator to ensure that the goal is within the pre-defined possibilities
-    @validator('goal')
-    def check_goal(cls, v):
-        if v not in cls._allowed_goals:
-            raise ValueError(f"{v} is not a valid goal")
-        return v
-
-    # Validator to ensure that the housing type is within the pre-defined possibilities
-    @validator('housing_type')
-    def check_housing_type(cls, v):
-        if v not in cls._allowed_housing_types:
-            raise ValueError(f"{v} is not a valid housing type")
-        return v
-
-
-class MdsRequest(PlanTriggerRequest):
-    # When creating the mds report, we allow an optional list of measures to select from. If this is passed, it will
-    # cause the service to select the optimal package from the list of measures
-    measures: Optional[conlist(str, min_items=1)] = None
+    optimise: Optional[bool] = True
+    default_u_values: Optional[bool] = True

backend/docker/Dockerfile

@@ -1,5 +1,5 @@
 # Pull base image
-FROM python:3.10.12-slim-buster as build-image
+FROM python:3.11.10-slim-bullseye as build-image
 
 # Set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -12,10 +12,10 @@ WORKDIR var/task/Model
 RUN #apt-get update && apt-get install -y netcat-openbsd
 
 # Install python dependencies
-COPY ./backend/requirements/base.txt ./backend/requirements/base.txt
+COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
 RUN pip install --upgrade pip
 # Install and clean up temp caches
-RUN pip install -r backend/requirements/base.txt && rm -rf /root/.cache
+RUN pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
 
 # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
 # interface client
@@ -35,16 +35,12 @@ COPY --from=build-image /usr/local/lib/python3.10/site-packages/ /usr/local/lib/
 # Copy project files
 COPY ./backend/ ./backend
 COPY ./recommendations/ ./recommendations
-COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py
-COPY ./model_data/config.py ./model_data/config.py
-COPY ./model_data/optimiser/ ./model_data/optimiser/
-COPY ./model_data/__init__.py ./model_data/__init__.py
-COPY ./model_data/EpcClean.py ./model_data/EpcClean.py
-COPT ./model_data/simulation_system/core/ ./model_data/simulation_system/core/
-COPY ./model_data/utils.py ./model_data/utils.py
-COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/
-COPY ./datatypes/ ./datatypes/
 COPY ./utils/ ./utils/
+COPY ./etl/epc/ ./etl/epc/
+COPY ./etl/epc_clean/ ./etl/epc_clean/
+COPY ./etl/bill_savings/ ./etl/bill_savings/
+COPY ./etl/spatial/ ./etl/spatial/
+COPY ./datatypes/ ./datatypes/
 
 # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler
 ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]

backend/docker/lambda.Dockerfile

@@ -1,5 +1,5 @@
 # Pull base image
-FROM python:3.10.12-slim-buster as build-image
+FROM python:3.11.10-slim-bullseye as build-image
 
 # Set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -12,10 +12,10 @@ WORKDIR var/task/Model
 #RUN apt-get update && apt-get install -y netcat-openbsd
 
 # Install python dependencies
-COPY ./backend/requirements/base.txt ./backend/requirements/base.txt
+COPY ./backend/requirements/requirements.txt ./backend/requirements/requirements.txt
 # Install and clean up temp caches
 RUN pip install --upgrade pip \
-    && pip install -r backend/requirements/base.txt && rm -rf /root/.cache
+    && pip install -r backend/requirements/requirements.txt && rm -rf /root/.cache
 
 # Since we are not using a base AWS image, there is some additional setup required. We need to set up the runtime
 # interface client
@@ -24,28 +24,28 @@ RUN pip install --upgrade pip \
 RUN pip install awslambdaric
 
 # Second stage: "runtime-image"
-FROM python:3.10.12-slim-buster
+FROM python:3.11.10-slim-bullseye
+
+# Create the extensions directory to avoid warnings with RIE
+RUN mkdir -p /opt/extensions
 
 # Set work directory to the root of your project
 WORKDIR /var/task/Model
 
 # Copy the python dependencies from the build-image
-COPY --from=build-image /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
+COPY --from=build-image /usr/local/lib/python3.11/site-packages/ /usr/local/lib/python3.11/site-packages/
 
 # Copy project files
 COPY ./backend/ ./backend
 COPY ./recommendations/ ./recommendations
-COPY ./model_data/BaseUtility.py ./model_data/BaseUtility.py
-COPY ./model_data/config.py ./model_data/config.py
-COPY ./model_data/optimiser/ ./model_data/optimiser/
-COPY ./model_data/__init__.py ./model_data/__init__.py
-COPY ./model_data/EpcClean.py ./model_data/EpcClean.py
-COPY ./model_data/utils.py ./model_data/utils.py
-COPY ./model_data/epc_attributes/ ./model_data/epc_attributes/
-COPY ./model_data/simulation_system/core/DataProcessor.py ./model_data/simulation_system/core/DataProcessor.py
-COPY ./model_data/simulation_system/core/Settings.py ./model_data/simulation_system/core/Settings.py
-COPY ./datatypes/ ./datatypes/
 COPY ./utils/ ./utils/
+COPY ./etl/epc/ ./etl/epc/
+COPY ./etl/epc_clean/ ./etl/epc_clean/
+COPY ./etl/bill_savings/ ./etl/bill_savings/
+COPY ./etl/spatial/ ./etl/spatial/
+COPY ./BaseUtility.py ./BaseUtility.py
+COPY ./datatypes/ ./datatypes/
+
 
 # Set the ENTRYPOINT to the AWS Lambda RIC and CMD to your function handler
 ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ]

backend/ml_models/Valuation.py

@@ -1,4 +1,5 @@
 import numpy as np
+from scipy.constants import value
 
 
 class PropertyValuation:
@@ -103,6 +104,8 @@ class PropertyValuation:
         # Vander Elliot Intrusive surveys
         12103116: 1_537_000,
         12103117: 1_404_000,
+        # GLA Proposal
+        100020606627: 409_000
     }
 
     # We base our valuation uplifts on a number of sources
@@ -201,9 +204,12 @@ class PropertyValuation:
 
     @classmethod
     def estimate(cls, property_instance, target_epc):
-        value = cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn)
+        current_value = (
+            property_instance.valuation if property_instance.valuation else
+            cls.UPRN_VALUE_LOOKUP.get(property_instance.uprn)
+        )
 
-        if not value:
+        if not current_value:
             return {
                 "current_value": 0,
                 "lower_bound_increased_value": 0,
@@ -233,12 +239,13 @@ class PropertyValuation:
 
         max_increase = max(all_increases)
         min_increase = min(all_increases)
+
         avg_increase = np.mean(all_increases)
 
         return {
-            "current_value": value,
-            "lower_bound_increased_value": value * (1 + min_increase),
-            "upper_bound_increased_value": value * (1 + max_increase),
-            "average_increased_value": value * (1 + avg_increase),
-            "average_increase": value * (1 + avg_increase) - value
+            "current_value": current_value,
+            "lower_bound_increased_value": float(current_value * (1 + min_increase)),
+            "upper_bound_increased_value": float(current_value * (1 + max_increase)),
+            "average_increased_value": float(current_value * (1 + avg_increase)),
+            "average_increase": float(current_value * (1 + avg_increase) - current_value)
         }

backend/ml_models/api.py

@@ -1,3 +1,5 @@
+import aiohttp
+import asyncio
 import pandas as pd
 from tqdm import tqdm
 import requests
@@ -18,6 +20,8 @@ class ModelApi:
         # "hot_water_cost_predictions",
     ]
 
+    KWH_MODEL_PREFIXES = ["heating_kwh_predictions", "hotwater_kwh_predictions"]
+
     MODEL_URLS = {
         "sap_change_predictions": "sapmodel",
         "heat_demand_predictions": "heatmodel",
@@ -120,6 +124,28 @@ class ModelApi:
             # depending on how you want to handle errors in your application
             return None
 
+    async def predict_async(self, file_location, model_prefix: str):
+        """Makes an asynchronous POST request to the Model API with the provided parameters."""
+        logger.info(f"Making request to {model_prefix} change api")
+        url = f"{self.base_url}/{self.MODEL_URLS[model_prefix]}/predict"
+        payload = {
+            "file_location": file_location,
+            "property_id": "",  # This should get removed
+            "portfolio_id": self.portfolio_id,
+            "created_at": self.timestamp
+        }
+
+        async with aiohttp.ClientSession() as session:
+            try:
+                async with session.post(
+                    url, json=payload, headers={"Content-Type": "application/json"}, timeout=120
+                ) as response:
+                    response.raise_for_status()
+                    return await response.json()
+            except aiohttp.ClientError as e:
+                logger.error(f"An error occurred: {e}")
+                return None
+
     @staticmethod
     def extract_phase(recommendation_id):
         if 'phase=' in recommendation_id:
@@ -180,6 +206,43 @@ class ModelApi:
 
         return predictions
 
+    async def predict_all_async(self, df, bucket, model_prefixes=None, extract_ids=True) -> dict:
+        """Uploads data and makes asynchronous requests to the model APIs for predictions."""
+        model_prefixes = self.MODEL_PREFIXES if model_prefixes is None else model_prefixes
+
+        predictions = {}
+        tasks = []
+        async with aiohttp.ClientSession() as session:
+            for model_prefix in model_prefixes:
+                logger.info(f"Scoring for model prefix: {model_prefix}")
+                file_location = self.upload_scoring_data(df, bucket, model_prefix)
+                # Schedule the prediction request as a coroutine
+                tasks.append(
+                    self.predict_async(f"s3://{bucket}/" + file_location, model_prefix)
+                )
+
+            # Gather all asynchronous tasks (execute them concurrently)
+            responses = await asyncio.gather(*tasks, return_exceptions=True)
+
+            for model_prefix, response in zip(model_prefixes, responses):
+                if response:
+                    predictions_bucket = self.prediction_buckets[model_prefix]
+                    predictions_df = pd.DataFrame(
+                        read_dataframe_from_s3_parquet(
+                            bucket_name=predictions_bucket,
+                            file_key=response["storage_filepath"].split(predictions_bucket + "/")[1]
+                        )
+                    )
+                    predictions_df['predictions'] = predictions_df["predictions"].astype(float).round(1)
+                    if extract_ids:
+                        predictions_df[['property_id', 'recommendation_id']] = predictions_df['id'].str.split('+',
+                                                                                                              expand=True)
+                        predictions_df['phase'] = predictions_df['recommendation_id'].apply(self.extract_phase)
+
+                    predictions[model_prefix] = predictions_df
+
+        return predictions
+
     def paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True):
         all_predictions = self.predictions_template()
         to_loop_over = range(0, data.shape[0], batch_size)
@@ -196,3 +259,59 @@ class ModelApi:
                 all_predictions[key] = pd.concat([all_predictions[key], scored])
 
         return all_predictions
+
+    async def async_warm_up_lambdas(self, model_prefies=None):
+        """Send asynchronous pre-flight requests to each model endpoint to wake up the cold Lambdas without waiting
+        for responses."""
+        logger.info("Asynchronously warming up Lambda functions...")
+
+        model_prefixes = self.MODEL_PREFIXES if model_prefies is None else model_prefies
+
+        tasks = []
+        async with aiohttp.ClientSession() as session:
+            for model_prefix in model_prefixes:
+                url = f"{self.base_url}/{self.MODEL_URLS[model_prefix]}/predict"
+                # Create a coroutine for each warm-up request and add it to the tasks list
+                tasks.append(self._send_warm_up_request(session, url, model_prefix))
+
+            # Run all tasks concurrently but don't wait for the responses to finish
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    @staticmethod
+    async def _send_warm_up_request(session, url, model_prefix):
+        """Helper method to send a pre-flight request to a given model URL."""
+        try:
+            async with session.post(url, json={}, timeout=2) as response:
+                # Log success for monitoring but do not block on the response
+                logger.info(f"Warmed up {model_prefix} with status code: {response.status}")
+        except aiohttp.ClientError as e:
+            logger.warning(f"Failed to warm up {model_prefix}: {e}")
+
+        logger.info("Lambda functions are warmed up and ready to go!")
+
+    async def async_paginated_predictions(self, data, bucket, batch_size, model_prefixes=None, extract_ids=True):
+        all_predictions = self.predictions_template()
+        to_loop_over = range(0, data.shape[0], batch_size)
+
+        async def run_batches():
+            for chunk in tqdm(to_loop_over, total=len(to_loop_over)):
+                predictions_dict = await self.predict_all_async(
+                    df=data.iloc[chunk:chunk + batch_size],
+                    bucket=bucket,
+                    model_prefixes=model_prefixes,
+                    extract_ids=extract_ids
+                )
+
+                for key, scored in predictions_dict.items():
+                    all_predictions[key] = pd.concat([all_predictions[key], scored])
+
+        # Check if there is an existing event loop
+        try:
+            # If there is an existing event loop, await the coroutine directly
+            loop = asyncio.get_running_loop()
+            await run_batches()
+        except RuntimeError:  # No running event loop
+            # If no event loop is running, use asyncio.run()
+            asyncio.run(run_batches())
+
+        return all_predictions

backend/requirements/base.txt

@@ -1,42 +0,0 @@
-msgpack==1.0.5
-anyio==3.7.1
-cffi==1.15.1
-click==8.1.3
-cryptography==37.0.4
-ecdsa==0.18.0
-epc-api-python==1.0.2
-exceptiongroup==1.1.2
-fastapi==0.99.1
-h11==0.14.0
-httptools==0.5.0
-idna==3.4
-mangum==0.17.0
-pyasn1==0.5.0
-pycparser==2.21
-pydantic==1.10.11
-PyJWT==2.7.0
-python-dotenv==1.0.0
-python-jose==3.3.0
-PyYAML==6.0
-rsa==4.9
-six==1.16.0
-sniffio==1.3.0
-starlette==0.27.0
-typing_extensions==4.7.1
-uvicorn==0.22.0
-uvloop==0.17.0
-urllib3<2
-watchfiles==0.19.0
-websockets==11.0.3
-sqlalchemy==2.0.19
-psycopg2-binary
-pytz==2023.3
-mip==1.15.0
-boto3==1.28.3
-pandas==1.5.3
-pyarrow==12.0.1
-textblob
-usaddress==0.5.10
-
-# Requirements we may not need
-xgboost==1.7.6

backend/requirements/local.txt

@@ -1,28 +0,0 @@
-anyio==3.7.1
-cffi==1.15.1
-click==8.1.3
-cryptography==37.0.4
-ecdsa==0.18.0
-exceptiongroup==1.1.2
-fastapi==0.99.1
-h11==0.14.0
-httptools==0.5.0
-idna==3.4
-mangum==0.17.0
-pyasn1==0.5.0
-pycparser==2.21
-pydantic==1.10.11
-PyJWT==2.7.0
-python-dotenv==1.0.0
-python-jose==3.3.0
-PyYAML==6.0
-rsa==4.9
-six==1.16.0
-sniffio==1.3.0
-starlette==0.27.0
-typing_extensions==4.7.1
-uvicorn==0.22.0
-uvloop==0.17.0
-watchfiles==0.19.0
-websockets==11.0.3
-boto3

backend/requirements/requirements.txt

@@ -0,0 +1,31 @@
+# Pandas and numpy
+numpy==2.1.2
+pandas==2.2.3
+pytz==2024.2
+six==1.16.0
+# tqdm
+tqdm==4.66.5
+# fastapi
+fastapi==0.115.2
+sqlalchemy==2.0.36
+pydantic-settings==2.6.0
+psycopg2-binary==2.9.10
+python-jose==3.3.0
+cryptography==43.0.3
+mangum==0.19.0
+# AWS
+boto3==1.35.44
+# ML, Data Science
+usaddress==0.5.11
+epc-api-python==1.0.2
+fuzzywuzzy==0.18.0
+python-Levenshtein==0.26.0
+textblob==0.18.0.post0
+msgpack==1.1.0
+scikit-learn==1.5.2
+cffi==1.15.1
+mip==1.15.0
+# Data
+pyarrow==17.0.0
+fastparquet==2024.5.0
+aiohttp==3.10.10

backend/tests/test_property.py

@@ -1,9 +1,11 @@
+from datetime import datetime
 import pandas as pd
 import pytest
 from unittest.mock import Mock
 from backend.Property import Property
 from etl.epc_clean.EpcClean import EpcClean
 from etl.epc.Record import EPCRecord
+from etl.bill_savings.KwhData import KwhData
 
 # Define some test data
 mock_epc_response = {
@@ -17,12 +19,13 @@ mock_epc_response = {
             "built-form": "Detached",
             "inspection-date": "2023-06-01",
             'lodgement-datetime': '2023-06-01 20:29:01',
+            'lodgement-date': '2023-06-01',
             "some-other-key": "some-value",
             "roof-description": "pitched, no insulation",
             "walls-description": "Walls Description",
-            "windows-description": "Windows Description",
-            "mainheat-description": "Main Heating Description",
-            "hotwater-description": "Hot Water Description",
+            "windows-description": "Fully double glazed",
+            "mainheat-description": "Boiler and radiators, mains gas",
+            "hotwater-description": "From main system",
             "transaction-type": "rental",
             "lighting-description": "Good Lighting Efficiency",
             "energy-consumption-current": "50",
@@ -39,7 +42,10 @@ mock_epc_response = {
             "total-floor-area": 100,
             "construction-age-band": "England and Wales: 1967-1975",
             "floor-description": "Floor Description",
-            "floor-level": "Ground"
+            "floor-level": "Ground",
+            "lighting-cost-current": 123,
+            "heating-cost-current": 800,
+            "hot-water-cost-current": 200
         },
         {
             "lmk-key": 2,
@@ -49,12 +55,13 @@ mock_epc_response = {
             "built-form": "Detached",
             "inspection-date": "2023-05-01",
             'lodgement-datetime': '2023-05-01 20:29:01',
+            'lodgement-date': '2023-05-01',
             "some-other-key": "some-other-value",
             "roof-description": "Roof Description",
             "walls-description": "Walls Description",
-            "windows-description": "Windows Description",
-            "mainheat-description": "Main Heating Description",
-            "hotwater-description": "Hot Water Description",
+            "windows-description": "Fully double glazed",
+            "mainheat-description": "Boiler and radiators, mains gas",
+            "hotwater-description": "From main system",
             "transaction-type": "rental",
             "lighting-description": "Good Lighting Efficiency",
             "energy-consumption-current": "50",
@@ -71,98 +78,10 @@ mock_epc_response = {
             "total-floor-area": 100,
             "construction-age-band": "England and Wales: 1967-1975",
             "floor-description": "Floor Description",
-            "floor-level": "Ground"
-        }
-    ]
-}
-
-mock_epc_response_dupe = {
-    'rows': [
-        {
-            "lmk-key": 1,
-            "uprn": 1,
-            "number-habitable-rooms": 5,
-            "property-type": "House",
-            'inspection-date': '2023-06-01',
-            'lodgement-datetime': '2023-06-01 20:29:01',
-            'some-other-key': 'some-value', 'roof-description': 'Roof Description',
-            'walls-description': 'Walls Description', 'windows-description': 'Windows Description',
-            'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description',
-            "transaction-type": "rental",
-            "lighting-description": "Good Lighting Efficiency",
-            "energy-consumption-current": "50",
-            "co2-emissions-current": "123",
-            "mechanical-ventilation": "natural",
-            'photo-supply': 0,
-            "solar-water-heating-flag": "N",
-            "wind-turbine-count": 0,
-            "extension-count": 0,
-            "heat-loss-corridor": "no corridor",
-            "unheated-corridor-length": 0,
-            "mains-gas-flag": "Y",
-            "floor-height": 2.5,
-            "total-floor-area": 100,
-            "construction-age-band": "England and Wales: 1967-1975",
-            "floor-description": "Floor Description",
-            "floor-level": "Ground"
-        },
-        {
-            "lmk-key": 2,
-            "uprn": 2,
-            "number-habitable-rooms": 5,
-            "property-type": "House",
-            'inspection-date': '2023-05-01',
-            'lodgement-datetime': '2023-05-01 20:29:01',
-            'some-other-key': 'some-other-value',
-            'roof-description': 'Roof Description', 'walls-description': 'Walls Description',
-            'windows-description': 'Windows Description', 'mainheat-description': 'Main Heating Description',
-            'hotwater-description': 'Hot Water Description',
-            "transaction-type": "rental",
-            "lighting-description": "Good Lighting Efficiency",
-            "energy-consumption-current": "50",
-            "co2-emissions-current": "123",
-            "mechanical-ventilation": "natural",
-            'photo-supply': 0,
-            "solar-water-heating-flag": "N",
-            "wind-turbine-count": 0,
-            "extension-count": 0,
-            "heat-loss-corridor": "no corridor",
-            "unheated-corridor-length": 0,
-            "mains-gas-flag": "Y",
-            "floor-height": 2.5,
-            "total-floor-area": 100,
-            "construction-age-band": "England and Wales: 1967-1975",
-            "floor-description": "Floor Description",
-            "floor-level": "Ground"
-        },
-        {
-            "lmk-key": 3,
-            "uprn": 3,
-            "number-habitable-rooms": 5,
-            "property-type": "House",
-            'inspection-date': '2023-06-01',
-            'lodgement-datetime': '2023-06-01 20:29:01',
-            'some-other-key': 'duplicate-date',
-            'roof-description': 'Roof Description',
-            'walls-description': 'Walls Description', 'windows-description': 'Windows Description',
-            'mainheat-description': 'Main Heating Description', 'hotwater-description': 'Hot Water Description',
-            "transaction-type": "rental",
-            "lighting-description": "Good Lighting Efficiency",
-            "energy-consumption-current": "50",
-            "co2-emissions-current": "123",
-            "mechanical-ventilation": "natural",
-            'photo-supply': 0,
-            "solar-water-heating-flag": "N",
-            "wind-turbine-count": 0,
-            "extension-count": 0,
-            "heat-loss-corridor": "no corridor",
-            "unheated-corridor-length": 0,
-            "mains-gas-flag": "Y",
-            "floor-height": 2.5,
-            "total-floor-area": 100,
-            "construction-age-band": "England and Wales: 1967-1975",
-            "floor-description": "Floor Description",
-            "floor-level": "Ground"
+            "floor-level": "Ground",
+            "lighting-cost-current": 123,
+            "heating-cost-current": 800,
+            "hot-water-cost-current": 200
         }
     ]
 }
@@ -170,34 +89,14 @@ mock_epc_response_dupe = {
 
 class TestProperty:
 
-    @pytest.fixture(autouse=True)
-    def mock_photo_supply_lookup(self):
-        return pd.DataFrame(
-            [
-                dict(
-                    tenure="rental (social)",
-                    built_form="Detached",
-                    property_type="House",
-                    construction_age_band="England and Wales: 1967-1975",
-                    is_flat=False,
-                    is_pitched=True,
-                    is_roof_room=False,
-                    floor_area_decile=2,
-                    photo_supply_median=40
-                )
-            ]
-        )
-
-    @pytest.fixture(autouse=True)
-    def mock_floor_area_decile_thresholds(self):
-        return pd.DataFrame(
-            {"floor_area_decile_thresholds": [0, 10, 30, 50]}
-        )
-
     @pytest.fixture(autouse=True)
     def property_instance(self, mock_cleaner):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = mock_epc_response["rows"][0]
+        prepared_epc = mock_epc_response["rows"][0].copy()
+        # Replace hyphens with underscores
+        prepared_epc = {k.replace("-", "_"): v for k, v in prepared_epc.items()}
+        epc_record.prepared_epc = prepared_epc
+        epc_record.uprn = prepared_epc["uprn"]
 
         property_instance = Property(id=1, postcode="AB12CD", address="Test Address", epc_record=epc_record)
         property_instance.number_of_floors = 2
@@ -206,27 +105,6 @@ class TestProperty:
         property_instance.floor_height = 2.5
         return property_instance
 
-    @pytest.fixture(autouse=True)
-    def property_instance_dupe_data(self):
-        epc_record = EPCRecord()
-        epc_record.prepared_epc = mock_epc_response_dupe["rows"][0]
-        property_instance_dupe_data = Property(id=2, postcode="AB12CD", address="Test Address", epc_record=epc_record)
-        return property_instance_dupe_data
-
-    # @pytest.fixture
-    # def mock_epc_client(self):
-    #     mock_epc_client = Mock(spec=EpcClient(auth_token="mocked_auth_token"))
-    #     mock_epc_client.domestic.search.return_value = mock_epc_response.copy()
-    #     mock_epc_client.auth_token = "mocked_auth_token"
-    #     return mock_epc_client
-    #
-    # @pytest.fixture
-    # def mock_epc_client_dupe_data(self):
-    #     mock_epc_client_dupe_data = Mock(spec=EpcClient(auth_token="mocked_auth_token"))
-    #     mock_epc_client_dupe_data.domestic.search.return_value = mock_epc_response_dupe.copy()
-    #     mock_epc_client_dupe_data.auth_token = "mocked_auth_token"
-    #     return mock_epc_client_dupe_data
-
     @pytest.fixture
     def mock_cleaner(self):
         lighting_averages = [
@@ -270,15 +148,59 @@ class TestProperty:
                  "is_roof_room": False}
             ],
             "walls-description": [walls_data],
-            "windows-description": [{"original_description": "Windows Description"}],
-            "mainheat-description": [{"original_description": "Main Heating Description"}],
-            "hotwater-description": [{"original_description": "Hot Water Description"}],
+            "windows-description": [
+                {'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full',
+                 'glazing_type': 'double', 'no_data': False}
+            ],
+            "mainheat-description": [
+                {
+                    'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True,
+                    'has_fan_coil_units': False,
+                    'has_pipes_in_screed_above_insulation': False, 'has_pipes_in_insulated_timber_floor': False,
+                    'has_pipes_in_concrete_slab': False, 'has_boiler': True, 'has_air_source_heat_pump': False,
+                    'has_room_heaters': False, 'has_electric_storage_heaters': False, 'has_warm_air': False,
+                    'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+                    'has_community_scheme': False,
+                    'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+                    'has_portable_electric_heaters': False,
+                    'has_water_source_heat_pump': False, 'has_electric': False, 'has_mains_gas': True,
+                    'has_wood_logs': False,
+                    'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+                    'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False,
+                    'has_assumed': False,
+                    'has_electricaire': False, 'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False,
+                    "has_electric_heat_pumps": False,
+                    "has_micro-cogeneration": False
+                }
+            ],
+            "hotwater-description": [
+                {'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system',
+                 'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+                 'tariff_type': None,
+                 'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+                 'assumed': False, "appliance": None}
+            ],
             "lighting-description": [{"original_description": "Good Lighting Efficiency"}],
             "floor-description": [
                 {"original_description": "Floor Description", "is_suspended": True, "another_property_below": False}]
         }
         return mock_cleaner
 
+    @pytest.fixture
+    def kwh_client(self):
+        kwh_client = KwhData(bucket="retrofit-data-dev", read_consumption_data=False)
+        # We fix this pricing table for these tests
+        kwh_client.retail_price_comparison = pd.DataFrame(
+            [
+                {
+                    "Date": datetime.today().strftime("%Y-%m-%d"),
+                    'Average standard variable tariff (Large legacy suppliers)': 1
+                }
+            ]
+        )
+        kwh_client.retail_price_comparison["Date"] = pd.to_datetime(kwh_client.retail_price_comparison["Date"])
+        return kwh_client
+
     def test_init(self):
         epc_record = EPCRecord()
         epc_record.prepared_epc = {"uprn": 1}
@@ -292,13 +214,26 @@ class TestProperty:
         inst3 = Property(4, "AB12CD", "Test Address", epc_record=epc_record)
         assert inst3.data == {"uprn": 1}
 
-    def test_get_components(
-        self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds
+    def test_set_features(
+        self, property_instance, mock_cleaner, kwh_client,
     ):
-        property_instance.get_components(
+        kwh_predictions = {
+            "heating_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 12000}
+                ]
+            ),
+            "hotwater_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 3000}
+                ]
+            ),
+        }
+
+        property_instance.set_features(
             mock_cleaner.cleaned,
-            photo_supply_lookup=mock_photo_supply_lookup,
-            floor_area_decile_thresholds=mock_floor_area_decile_thresholds
+            kwh_client,
+            kwh_predictions
         )
 
         # Verify that the components are set correctly
@@ -318,9 +253,32 @@ class TestProperty:
             "is_sandstone_or_limestone": False,
             "is_granite_or_whinstone": False,
         }
-        assert property_instance.windows == {"original_description": "Windows Description"}
-        assert property_instance.main_heating == {"original_description": "Main Heating Description"}
-        assert property_instance.hotwater == {"original_description": "Hot Water Description"}
+        assert property_instance.windows == {
+            'original_description': 'Fully double glazed', 'has_glazing': True, 'glazing_coverage': 'full',
+            'glazing_type': 'double', 'no_data': False
+        }
+        assert property_instance.main_heating == {
+            'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True,
+            'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+            'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
+            'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+            'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+            'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+            'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False,
+            'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
+            'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
+            'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
+            'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False,
+            'has_micro-cogeneration': False
+        }
+
+        assert property_instance.hotwater == {
+            'original_description': 'From main system', 'heater_type': None,
+            'system_type': 'from main system', 'thermostat_characteristics': None,
+            'heating_scope': None, 'energy_recovery': None, 'tariff_type': None,
+            'extra_features': None, 'chp_systems': None, 'distribution_system': None,
+            'no_system_present': None, 'assumed': False, 'appliance': None
+        }
 
         assert property_instance.wall_type == "cavity"
 
@@ -330,11 +288,24 @@ class TestProperty:
 
         # Verify that ValueError is raised when EpcClean doesn't contain cleaned data
         with pytest.raises(ValueError, match="Cleaner does not contain cleaned data"):
-            property_instance.get_components(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame())
+            property_instance.set_features(mock_cleaner.cleaned, pd.DataFrame(), pd.DataFrame())
 
     def test_get_components_no_attributes(
-        self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds
+        self, property_instance, mock_cleaner, kwh_client
     ):
+        kwh_predictions = {
+            "heating_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 12000}
+                ]
+            ),
+            "hotwater_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 3000}
+                ]
+            ),
+        }
+
         # Modify the mock cleaner to have no attributes for a specific description
         mock_cleaner.cleaned = {
             "roof-description": []
@@ -351,23 +322,45 @@ class TestProperty:
             "is_sandstone_or_limestone": False,
             "is_granite_or_whinstone": False,
         }
-
         property_instance.floor = {
             "is_suspended": False,
             "another_property_below": False,
             "is_solid": True
         }
+        property_instance.main_heating = {
+            'original_description': 'Boiler and radiators, mains gas', 'has_radiators': True,
+            'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+            'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
+            'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+            'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+            'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+            'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric': False,
+            'has_mains_gas': True, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False,
+            'has_wood_pellets': False, 'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False,
+            'has_smokeless_fuel': False, 'has_lpg': False, 'has_assumed': False, 'has_electricaire': False,
+            'has_assumed_for_most_rooms': False, 'has_underfloor_heating': False, 'has_electric_heat_pumps': False,
+            'has_micro-cogeneration': False
+        }
+        property_instance.hotwater = {
+            'original_description': 'From main system', 'heater_type': None, 'system_type': 'from main system',
+            'thermostat_characteristics': None, 'heating_scope': None, 'energy_recovery': None,
+            'tariff_type': None,
+            'extra_features': None, 'chp_systems': None, 'distribution_system': None, 'no_system_present': None,
+            'assumed': False, "appliance": None
+        }
 
         # Assert backup cleaning has been applied
-        property_instance.get_components(
-            mock_cleaner.cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds
+        property_instance.set_features(
+            mock_cleaner.cleaned,
+            kwh_client,
+            kwh_predictions
         )
 
         assert property_instance.roof["clean_description"] == "Pitched, no insulation"
         assert property_instance.roof["is_pitched"]
 
     def test_get_components_multiple_attributes(
-        self, property_instance, mock_cleaner, mock_photo_supply_lookup, mock_floor_area_decile_thresholds
+        self, property_instance, mock_cleaner, kwh_client
     ):
         # This shouldn't happen - it would mean a cleaning error
         property_instance.data["roof-description"] = "Roof Description"
@@ -378,13 +371,27 @@ class TestProperty:
             ]
         }
 
+        kwh_predictions = {
+            "heating_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 12000}
+                ]
+            ),
+            "hotwater_kwh_predictions": pd.DataFrame(
+                [
+                    {"id": property_instance.uprn, "predictions": 3000}
+                ]
+            ),
+        }
+
         # Verify that ValueError is raised when multiple attributes are found
         with pytest.raises(ValueError, match="Either No attributes or multiple found for roof-description"):
-            property_instance.get_components(cleaned, mock_photo_supply_lookup, mock_floor_area_decile_thresholds)
+            property_instance.set_features(cleaned, kwh_client, kwh_predictions)
 
     def test_set_spatial(self):
         epc_record = EPCRecord()
         epc_record.prepared_epc = mock_epc_response["rows"][0]
+        epc_record.uprn = mock_epc_response["rows"][0]["uprn"]
         prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record)
 
         spatial1 = pd.DataFrame([{
@@ -418,6 +425,7 @@ class TestProperty:
         # floor, so we should set floor_level to 0
         epc_record = EPCRecord()
         epc_record.prepared_epc = {'floor-level': '01', 'property-type': 'Flat'}
+        epc_record.uprn = 1
         prop = Property(1, postcode="AB12CD", address="Test Address", epc_record=epc_record)
         prop.floor = {
             'original_description': 'Solid, no insulation (assumed)', 'clean_description': 'Solid, no insulation',

etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py

@@ -22,68 +22,78 @@ class AirSourceHeatPumpEfficiency:
     def create_dataset(self):
         logger.info("Creating solar photo supply dataset")
 
-        all_counts = []
+        heating_data = []
         for dir in tqdm(self.file_directories):
             filepath = dir / "certificates.csv"
             df = pd.read_csv(filepath, low_memory=False)
-            df = df[~pd.isnull(df["UPRN"])]
-            df["UPRN"] = df["UPRN"].astype(int).astype(str)
+            # df = df[~pd.isnull(df["UPRN"])]
+            # df["UPRN"] = df["UPRN"].astype(int).astype(str)
             # Take entries after SAP12
             df["LODGEMENT_DATE"] = pd.to_datetime(df["LODGEMENT_DATE"])
             df = df[df["LODGEMENT_DATE"] > EARLIEST_EPC_DATE]
 
-            df = df[
-                ~df["TENURE"].isin(
-                    [
-                        "unknown",
-                        "Not defined - use in the case of a new dwelling for which the intended tenure in not known. "
-                        "It is not to be used for an existing dwelling"
-                    ]
-                )
-            ]
+            # df = df[
+            #     ~df["TENURE"].isin(
+            #         [
+            #             "unknown",
+            #             "Not defined - use in the case of a new dwelling for which the intended tenure in not known. "
+            #             "It is not to be used for an existing dwelling"
+            #         ]
+            #     )
+            # ]
 
             # Take entries that contain an air source heat pump
             df = df[
-                df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False)
-            ]
+                (
+                    # Air source heat pumps
+                    (df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") &
+                    (df["MAINHEATCONT_DESCRIPTION"] == "Time and temperature zone control")
+                ) |
+                (
+                    # High heat retention storage
+                    df["MAINHEATCONT_DESCRIPTION"] == "Controls for high heat retention storage heaters"
+                )
+                ]
 
             # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
             for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
                 df = df[~pd.isnull(df[col])]
-            # Get the columns we're interested in
-            df = df[
-                [
-                    "PROPERTY_TYPE",
-                    "BUILT_FORM",
-                    "MAINHEAT_DESCRIPTION",
-                    "MAINHEAT_ENERGY_EFF",
-                    "MAINHEATCONT_DESCRIPTION",
-                    "MAINHEATC_ENERGY_EFF",
-                    "MAIN_FUEL",
-                    "HOTWATER_DESCRIPTION",
-                    "HOT_WATER_ENERGY_EFF",
-                    "MAINS_GAS_FLAG"
-                ]
+
+            heating_data.append(df)
+
+        # temp
+        # import pickle
+        # with open("heating_data - delete me.pkl", "wb") as f:
+        #     pickle.dump(heating_data, f)
+
+        heating_df = pd.concat(heating_data)
+        # Clean construction age band
+        from etl.epc.DataProcessor import EPCDataProcessor
+        heating_df["CONSTRUCTION_AGE_BAND_CLEAN"] = heating_df["CONSTRUCTION_AGE_BAND"].apply(
+            lambda x: EPCDataProcessor.clean_construction_age_band(x)
+        )
+
+        ashp_df = heating_df[
+            (heating_df["MAINHEAT_DESCRIPTION"] == "Air source heat pump, radiators, electric") &
+            # ~heating_df["CONSTRUCTION_AGE_BAND"].str.contains("England and Wales")
+            (~heating_df["CONSTRUCTION_AGE_BAND"].isin(["NO DATA!", "INVALID!"])) &
+            (heating_df["LODGEMENT_DATE"] >= pd.to_datetime("2019-01-01"))
             ]
-
-            counts = df.groupby(
+        ashp_efficiencies = (
+            ashp_df.groupby(
                 [
-                    "PROPERTY_TYPE",
-                    "BUILT_FORM",
-                    "MAINHEAT_DESCRIPTION",
+                    "CONSTRUCTION_AGE_BAND_CLEAN",
+                    # "WALLS_DESCRIPTION",
+                    # "ROOF_DESCRIPTION",
                     "MAINHEAT_ENERGY_EFF",
-                    "MAINHEATCONT_DESCRIPTION",
-                    "MAINHEATC_ENERGY_EFF",
-                    "MAIN_FUEL",
-                    "HOTWATER_DESCRIPTION",
-                    "HOT_WATER_ENERGY_EFF",
-                    "MAINS_GAS_FLAG"
                 ]
-            ).size().reset_index(name="count")
+            )["LMK_KEY"].count().reset_index()
+        )
 
-            all_counts.append(counts)
+        ashp_df["MAINHEAT_ENERGY_EFF"].value_counts()
 
-        all_counts = pd.concat(all_counts)
+        ashp_efficiencies["CONSTRUCTION_AGE_BAND_CLEAN"].value_counts()
+        ashp_efficiency_agg
 
         all_counts_agg = all_counts.groupby(
             [

etl/air_source_heat_pump/app.py

@@ -1,8 +1,10 @@
+import inspect
 from pathlib import Path
 from backend.app.plan.utils import get_cleaned
 from etl.air_source_heat_pump.AirSourceHeatPumpEfficiency import AirSourceHeatPumpEfficiency
 
-DATA_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates"
+file_src = inspect.getfile(lambda: None)
+DATA_DIRECTORY = Path(file_src).parent / "local_data" / "all-domestic-certificates"
 
 
 def app():

etl/bill_savings/EnergyConsumptionModel.py

@@ -1,6 +1,6 @@
 import pandas as pd
 import numpy as np
-from xgboost import XGBRegressor
+# from xgboost import XGBRegressor
 from sklearn.model_selection import train_test_split
 from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_percentage_error
 from sklearn.feature_selection import RFECV
@@ -79,13 +79,13 @@ class EnergyConsumptionModel:
             if x not in self.CATEGORICAL_COLUMNS
         })
 
-        if model_paths:
-            for target, path in model_paths.items():
-                # Read model
-                self.models[target] = read_pickle_from_s3(
-                    bucket_name=f"retrofit-model-directory-{environment}", s3_file_name=path
-                )
-                # Read dummy schema
+        # if model_paths:
+        #     for target, path in model_paths.items():
+        #         # Read model
+        #         self.models[target] = read_pickle_from_s3(
+        #             bucket_name=f"retrofit-model-directory-{environment}", s3_file_name=path
+        #         )
+        # Read dummy schema
 
         if dummy_schema_path:
             self.dummy_schema = read_pickle_from_s3(
@@ -278,33 +278,33 @@ class EnergyConsumptionModel:
 
         logger.info(f"Feature selection completed for target {target}")
 
-    def init_model(self, feature_selection=False):
-
-        if feature_selection:
-            # Set up a smaller model to work it
-            return XGBRegressor(
-                objective='reg:squarederror',
-                n_estimators=50,
-                learning_rate=0.05,
-                max_depth=6,
-                subsample=0.8,
-                colsample_bytree=0.8,
-                reg_alpha=0.1,
-                reg_lambda=0.1
-            )
-
-        return XGBRegressor(
-            objective='reg:squarederror',
-            n_estimators=1000,
-            learning_rate=0.05,
-            max_depth=6,
-            min_child_weight=3,
-            subsample=0.8,
-            colsample_bytree=0.8,
-            reg_alpha=0.1,
-            reg_lambda=0.1
-            # n_jobs=self.n_jobs
-        )
+    # def init_model(self, feature_selection=False):
+    #
+    #     if feature_selection:
+    #         # Set up a smaller model to work it
+    #         return XGBRegressor(
+    #             objective='reg:squarederror',
+    #             n_estimators=50,
+    #             learning_rate=0.05,
+    #             max_depth=6,
+    #             subsample=0.8,
+    #             colsample_bytree=0.8,
+    #             reg_alpha=0.1,
+    #             reg_lambda=0.1
+    #         )
+    #
+    #     return XGBRegressor(
+    #         objective='reg:squarederror',
+    #         n_estimators=1000,
+    #         learning_rate=0.05,
+    #         max_depth=6,
+    #         min_child_weight=3,
+    #         subsample=0.8,
+    #         colsample_bytree=0.8,
+    #         reg_alpha=0.1,
+    #         reg_lambda=0.1
+    #         # n_jobs=self.n_jobs
+    #     )
 
     def fit_model(self, target):
         """Fits the model to the training data and removes zero-importance features."""

etl/bill_savings/KwhData.py

@@ -259,6 +259,9 @@ class KwhData:
         # Create new features:
         data['estimate_annual_kwh'] = data['energy-consumption-current'] * data['total-floor-area']
 
+        # Ensure this is string, because we could have mixed types
+        data["lodgement-datetime"] = data["lodgement-datetime"].astype(str)
+
         if save:
             self.model_training_data_filepath = f"energy_consumption/{self.run_date}/training_data.parquet"
             logger.info(f"Storing energy consumption dataset in s3 at {self.consumption_data_filepath}")

etl/bill_savings/data_collection.py

@@ -100,9 +100,44 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e
     bills = address_res.find('div', {'id': 'bills-affected'})
     bills_list = bills.find_all('li')
     if not bills_list:
-        return None
-    heating_text = bills_list[0].text
-    hot_water_text = bills_list[1].text
+        # If this is the case, it's usually becaue the EPC was very old. Early EPCs did not have this information
+        heating_text = None
+        hot_water_text = None
+    else:
+        heating_text = bills_list[0].text
+        hot_water_text = bills_list[1].text
+
+    # Search for the assessment informaton
+    assessment_information = address_res.find('div', {'id': 'information'})
+    # Parse this information
+    rows = assessment_information.find_all('div', class_='govuk-summary-list__row')
+    # Create a dictionary to hold the parsed information
+    assessment_data = {}
+    for row in rows:
+        key = row.find('dt').text.strip()
+        if key == "Type of assessment":
+            # We dont reliably extract this
+            continue
+        value_tag = row.find('dd')
+
+        # Check if value contains a link (email)
+        if value_tag.find('a'):
+            value = value_tag.find('a').text.strip()
+        elif value_tag.find('summary'):
+            value = value_tag.find('span').text.strip()
+        else:
+            value = value_tag.text.strip()
+
+        assessment_data[key] = value
+
+    expected_keys = [
+        'Assessor’s name', 'Telephone', 'Email', 'Accreditation scheme', 'Assessor’s ID', 'Assessor’s declaration',
+        'Date of assessment', 'Date of certificate'
+    ]
+    # Check we have all the expected keys
+    for key in expected_keys:
+        if key not in assessment_data:
+            raise ValueError(f"Missing key: {key}")
 
     resulting_data = {
         'extracted_uprn': uprn,
@@ -114,6 +149,7 @@ def retrieve_find_my_epc_data(uprn: int, postcode: str, address: str, expected_e
         "potential_epc_efficiency": int(potential_rating.split(' ')[-1]),
         "heating_text": heating_text,
         "hot_water_text": hot_water_text,
+        **assessment_data
     }
 
     return resulting_data

etl/costs/app.py

@@ -11,7 +11,7 @@ import inspect
 
 src_file_path = inspect.getfile(lambda: None)
 
-DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240626 Hestia Materials.xlsx"
+DATA_DIRECTORY = Path(src_file_path).parent / "local_data" / "20240917 Hestia Materials.xlsx"
 # Environment file is at the same level as this file
 ENV_FILE = Path(src_file_path).parent / "etl" / "costs" / ".env"
 dotenv.load_dotenv(ENV_FILE)
@@ -46,6 +46,17 @@ def push_costs_to_db(engine, costs_df):
         session.commit()
 
 
+def set_current_costs_inactive(engine):
+    """
+    Set all current costs to inactive in the database.
+
+    :param engine: The SQLAlchemy engine connected to your database.
+    """
+    with Session(engine) as session:
+        session.query(Material).update({Material.is_active: False})
+        session.commit()
+
+
 def app():
     """
     This application uploads the cost data to our database
@@ -71,6 +82,7 @@ def app():
     db_engine = create_engine(db_string, pool_size=5, max_overflow=5)
 
     cwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="cavity_wall_insulation", header=0)
+    ventilation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="Ventilation", header=0)
     loft_insulation_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="loft_insulation", header=0)
     iwi_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="internal_wall_insulation", header=0)
     suspended_floor_costs = pd.read_excel(DATA_DIRECTORY, sheet_name="suspended_floor_insulation", header=0)
@@ -84,6 +96,7 @@ def app():
     costs = pd.concat(
         [
             cwi_costs,
+            ventilation_costs,
             loft_insulation_costs,
             iwi_costs,
             suspended_floor_costs,
@@ -108,6 +121,11 @@ def app():
         costs[col] = costs[col].fillna(0)
 
     # Push the costs to the database
+    # Since this is just uploading all of the new costs to the database, we make all of the current costs inactive
+    print("Setting all current costs to inactive")
+    set_current_costs_inactive(db_engine)
+
+    print("Pushing costs to db")
     push_costs_to_db(db_engine, costs)
 
 

etl/customers/Cleethorpes Portfolio/epc data.py

@@ -0,0 +1,97 @@
+import os
+import pandas as pd
+from backend.SearchEpc import SearchEpc
+from dotenv import load_dotenv
+from tqdm import tqdm
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+
+def app():
+    """
+    Simple script to pull the EPC data for the Cleethorpes Portfolio
+    :return:
+    """
+
+    asset_list = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Updated Tenancy Schedule "
+        "Portfolio.xlsx",
+    )
+    asset_list["row_id"] = asset_list.index
+    asset_list[" Street No."] = asset_list[" Street No."].astype(str)
+
+    epc_data = []
+    for _, property in tqdm(asset_list.iterrows(), total=len(asset_list)):
+
+        if property[" Street No."] == "Ground Floor Commercial":
+            continue
+        uprn = property["Uprn"]
+        if not pd.isnull(uprn):
+            searcher = SearchEpc(
+                address1="",
+                postcode="",
+                auth_token=EPC_AUTH_TOKEN,
+                os_api_key="",
+                uprn=int(uprn)
+            )
+            searcher.find_property(skip_os=True)
+        else:
+
+            if not pd.isnull(property[" Flat No."]) and property[" Flat No."] not in ["", " "]:
+                address1 = property[" Flat No."].strip() + ", " + property[" Street No."].strip()
+            else:
+                address1 = property[" Street No."].strip()
+
+            if address1 == "1a Mews House  30":
+                address1 = "1a Rear of"
+            searcher = SearchEpc(
+                address1=address1,
+                postcode=property[" Postcode"].strip(),
+                auth_token=EPC_AUTH_TOKEN,
+                os_api_key="",
+                uprn=None,
+            )
+            searcher.get_epc()
+            # Get the newest record on lodgement-date
+            sorted_epcs = sorted(
+                searcher.data["rows"], key=lambda x: x["lodgement-date"]
+            )
+            searcher.newest_epc = sorted_epcs[-1]
+
+        if searcher.newest_epc is None:
+            raise ValueError(f"No EPC found for UPRN: {uprn}")
+
+        epc_data.append(
+            {
+                "row_id": property["row_id"],
+                **searcher.newest_epc
+            }
+        )
+
+    epc_df = pd.DataFrame(epc_data)
+
+    # Merge on data
+    asset_list_with_epc = asset_list.merge(
+        epc_df[["row_id", "address", "current-energy-rating", "current-energy-efficiency", "lodgement-date"]],
+        how="left",
+        left_on="row_id",
+        right_on="row_id",
+    ).rename(
+        columns={
+            "address": "EPC Address",
+            "current-energy-rating": "Current EPC Rating",
+            "current-energy-efficiency": "Current SAP Score",
+            "lodgement-date": "EPC Date"
+        }
+    )
+
+    asset_list_with_epc.to_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/Portfolio with EPCs.xlsx",
+        index=False
+    )
+
+    epc_df.to_csv(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/Cleethorpes Portoflio/epc_data.csv",
+        index=False
+    )

etl/customers/aiha/epc_data_pull.py

@@ -0,0 +1,791 @@
+import os
+from tqdm import tqdm
+from dotenv import load_dotenv
+import pandas as pd
+import numpy as np
+import msgpack
+from utils.s3 import read_from_s3
+from backend.SearchEpc import SearchEpc
+from etl.spatial.OpenUprnClient import OpenUprnClient
+
+load_dotenv(dotenv_path="backend/.env")
+EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
+
+pd.set_option('display.max_rows', 500)
+pd.set_option('display.max_columns', 500)
+pd.set_option('display.width', 1000)
+
+
+def app():
+    # Retrieve EPC data for the SHDF AIHA portfolio
+
+    data = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Khalim Review - 240902 - KSQ - AIHA - SHDF Wave "
+        "3 bid - Supplementary information.xlsx",
+        sheet_name="All units information",
+        header=3
+    )
+
+    # Remove the .eg row
+    data = data.tail(-1)
+
+    # Remove the bottom 2 rows
+    data = data.head(-2)
+    data = data.reset_index(drop=True)
+    data["row_id"] = data.index
+
+    ammendments = {
+        "12 11-18 Schonfeld Square": "12 Schonfeld Square",
+        "35 35-37 Schonfeld Square": "35 Schonfeld Square",
+        '77 Schonfeld Square': '77 Lordship Road',
+        "83 Lordship Road (Schonfeld Square)": "83 Lordship Road",
+        "A 80 Bethune Road": "80A Bethune Road",
+        "86B Bethune Road": "Flat B, 86 Bethune Road",
+        "22 Glendale Road": "22 Glendale Avenue",
+        "121 Southbourne Road": "121 Southbourne Grove",
+    }
+
+    no_epc = [
+        "80B Bethune Road",
+        "89B Manor Road",
+        "12 Monkville Avenue",
+        "9 Greenview",
+    ]
+
+    property_type_map = {
+        "House, mid-terrace": "House",
+        "House, end terrace": "House",
+        "House, semi-detached": "House",
+        "House, detached": "House",
+        "Flat": "Flat",
+    }
+
+    epc_data = []
+    epc_metadata = []
+    for _, home in tqdm(data.iterrows(), total=len(data)):
+
+        # Build address 1 based on if there is:
+        # 1) Address letter or number
+        # 2) Street address
+
+        modified = False
+        address1 = ""
+        address1_backup = ""
+
+        if home["Address letter or number"] in ["A", "B", "C"]:
+
+            house_no = home['Street address'].split(' ')[0]
+            street = ' '.join(home['Street address'].split(' ')[1:])
+            address1 = f"{house_no}{home['Address letter or number']} {street}"
+
+            address1_backup = f"Flat {home['Address letter or number']} {house_no} {street}"
+            modified = True
+
+        else:
+            if not pd.isnull(home["Address letter or number"]):
+                address1 += f"{home['Address letter or number']} "
+            if not pd.isnull(home["Street address"]):
+                address1 += f"{home['Street address']}"
+        address1 = address1.strip()
+
+        if address1.split(" ")[-1].lower() == "rd":
+            # Replace with road
+            address1 = address1.lower().replace(" rd", " road")
+
+        # Specific ammendments
+        if address1 in ammendments:
+            address1 = ammendments[address1]
+
+        if address1 in no_epc:
+            continue
+
+        searcher = SearchEpc(
+            address1=address1,
+            postcode=home["Postcode"],
+            auth_token=EPC_AUTH_TOKEN,
+            os_api_key="",
+            property_type=property_type_map[home["Property type"]]
+        )
+        searcher.find_property(skip_os=True)
+
+        if searcher.newest_epc is None and modified:
+            searcher = SearchEpc(
+                address1=address1_backup,
+                postcode=home["Postcode"],
+                auth_token=EPC_AUTH_TOKEN,
+                os_api_key="",
+                property_type=property_type_map[home["Property type"]]
+            )
+            searcher.find_property(skip_os=True)
+
+        if searcher.newest_epc is None:
+            raise Exception("Not found")
+
+        epc_data.append(
+            {
+                "row_id": home["row_id"],
+                **searcher.newest_epc
+            }
+        )
+
+        searcher.get_metadata()
+
+        epc_metadata.append(
+            {
+                "row_id": home["row_id"],
+                "address": address1,
+                "postcode": home["Postcode"],
+                **searcher.metadata
+            }
+        )
+
+    epc_metadata = pd.DataFrame(epc_metadata)
+    epc_data = pd.DataFrame(epc_data)
+
+    # Check matched addresses
+    matched_addresses = epc_metadata[["row_id", "address", "postcode"]].copy()
+    matched_addresses = matched_addresses.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    )
+
+    # We look for differences between the asset list and the EPC data
+    comparison_cols = {
+        "Property type": [
+            {
+                "epc_col": "property-type",
+                "map": property_type_map
+            },
+            {
+                "epc_col": "built-form",
+                "map": {
+                    "House, mid-terrace": "Mid-Terrace",
+                    "House, end terrace": "End-Terrace",
+                    "House, semi-detached": "Semi-Detached",
+                    "House, detached": "Detached",
+                    "Flat": "Flat",
+                }
+            }
+        ],
+        "Energy starting band (EPC)": [
+            {
+                "epc_col": "current-energy-rating",
+                "map": {}
+            }
+        ],
+        "Wall type": [
+            {
+                "epc_col": "walls-description",
+                "search_terms": {
+                    "solid": "Solid brick",
+                    "cavity": "Cavity wall",
+                    "solid - internal lining": "Solid brick",
+                }
+            }
+        ],
+        "Roof type": [
+            {
+                "epc_col": "roof-description",
+                "search_terms": {
+                    "pitched": "Pitched",
+                    "n/a - (flat above)": "another dwelling above"
+                }
+            }
+        ],
+        "Floor type": [
+            {
+                "epc_col": "floor-description",
+                "search_terms": {
+                    "solid": "Solid",
+                    "suspended": "Suspended",
+                    "solid - floating floor for services": "Solid"
+                }
+            }
+        ],
+    }
+
+    import re
+    differences = []
+    for asset_list_col, list_of_configs in comparison_cols.items():
+
+        if asset_list_col in ["Wall type", "Roof type", "Floor type"]:
+            config = list_of_configs[0]
+            # We handle this differently
+            remapped = data[["row_id", asset_list_col]].copy()
+            # Strip the asset list col incase of leading/trailing spaces
+            remapped[asset_list_col] = remapped[asset_list_col].str.strip()
+            remapped[asset_list_col] = remapped[asset_list_col].str.lower()
+            remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner")
+            # We do a search term check
+            remapped["Match"] = None
+            for search_term, epc_term in config["search_terms"].items():
+                if "/" in search_term:
+                    escaped_search_term = re.escape(search_term)
+                    remapped.loc[remapped[asset_list_col].str.contains(escaped_search_term), "Match"] = (
+                        remapped.loc[
+                            remapped[asset_list_col].str.contains(escaped_search_term), config["epc_col"]
+                        ].str.contains(epc_term)
+                    )
+                else:
+                    remapped.loc[remapped[asset_list_col].str.contains(search_term), "Match"] = (
+                        remapped.loc[
+                            remapped[asset_list_col].str.contains(search_term), config["epc_col"]
+                        ].str.contains(epc_term)
+                    )
+
+            if pd.isnull(remapped["Match"]).sum():
+                raise Exception("Not all matched")
+
+            remapped["Match"] = remapped["Match"].astype(bool)
+
+            if not all(remapped["Match"]):
+                differences.append(
+                    {
+                        "Column": asset_list_col,
+                        "Differences": remapped[~remapped["Match"]],
+                    }
+                )
+
+            continue
+
+        for config in list_of_configs:
+
+            remapped = data[["row_id", asset_list_col]].copy()
+            if config["map"]:
+                remapped[asset_list_col] = remapped[asset_list_col].map(config["map"])
+
+            # Merge on
+            remapped = remapped.merge(epc_data[["row_id", config["epc_col"]]], on="row_id", how="inner")
+            remapped["Match"] = remapped[asset_list_col] == remapped[config["epc_col"]]
+            if not all(remapped["Match"]):
+                differences.append(
+                    {
+                        "Column": asset_list_col,
+                        "Differences": remapped[~remapped["Match"]],
+                    }
+                )
+
+    # Check for property type
+    property_type_differences = differences[0]["Differences"].copy()
+    property_type_differences = property_type_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    )
+    print(property_type_differences)
+
+    # Check for built form
+    built_form_differences = differences[1]["Differences"].copy()
+    built_form_differences = built_form_differences[built_form_differences["Property type"] != "Flat"]
+    built_form_differences = built_form_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    )
+    print(built_form_differences)
+
+    # Check for energy rating
+    energy_rating_differences = differences[2]["Differences"].copy()
+    energy_rating_differences = energy_rating_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    ).merge(
+        epc_data[["row_id", "uprn"]], on="row_id", how="inner"
+    )
+    print(energy_rating_differences)
+
+    # Check for wall type
+    wall_type_differences = differences[3]["Differences"].copy()
+    wall_type_differences = wall_type_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    ).merge(
+        epc_data[["row_id", "uprn"]], on="row_id", how="inner"
+    )
+    print(wall_type_differences)  # Many wall type differences
+
+    # Check for roof type
+    roof_type_differences = differences[4]["Differences"].copy()
+    roof_type_differences = roof_type_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    ).merge(
+        epc_data[["row_id", "uprn"]], on="row_id", how="inner"
+    )
+    print(roof_type_differences)  # Many roof type differences
+
+    # Check for floor type
+    floor_type_differences = differences[5]["Differences"].copy()
+    floor_type_differences = floor_type_differences.merge(
+        data[["row_id", "Address letter or number", "Street address"]], on="row_id", how="inner"
+    ).merge(
+        epc_data[["row_id", "uprn"]], on="row_id", how="inner"
+    )
+    print(floor_type_differences)  # Many floor type differences
+
+    # TODO: 47 Ashtead Road [100021024699] shows solid brick wall on EPC - is probably cavity wall
+
+    # We have the EPC data. Let's check conservation area/historic/listed building status
+    portfolio_spatial_data = OpenUprnClient.get_spatial_data(
+        epc_data["uprn"].unique().tolist(), bucket_name="retrofit-data-dev"
+    )
+
+    portfolio_spatial_data["UPRN"] = portfolio_spatial_data["UPRN"].astype(str)
+
+    spatial_data = data[["row_id", "Planning constraints"]].merge(
+        epc_data[["row_id", "uprn"]], on="row_id", how="left",
+
+    ).merge(
+        portfolio_spatial_data[["UPRN", "conservation_status", "is_listed_building", "is_heritage_building"]],
+        left_on="uprn",
+        right_on="UPRN", how="left"
+    )
+
+    spatial_data[
+        (spatial_data["Planning constraints"] == "None")
+    ]["conservation_status"].value_counts()
+
+    # One property is in a conservation area, that was not picked up in the asset data
+    print(spatial_data[
+        (spatial_data["Planning constraints"] == "None") &
+        (spatial_data["conservation_status"] == True)
+        ].merge(
+        data[["row_id", "Address letter or number", "Street address", "Postcode"]], on="row_id", how="left"
+    ))
+
+    # All properties match up apart from one where the asset data indicates it's in a conservation area, however
+    # the sparital data indicates it's not. There do not appear to be any listed/heritage buildings in the portfolio
+
+    ################################################################
+    # Draft archetyping
+    ################################################################
+
+    cleaned = read_from_s3(
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name="retrofit-data-dev"
+    )
+    cleaned = msgpack.unpackb(cleaned, raw=False)
+
+    epc_data = epc_data.merge(
+        pd.DataFrame(cleaned["walls-description"])[
+            ['original_description',
+             'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick', 'is_system_built', 'is_timber_frame',
+             'is_as_built', 'is_assumed', 'insulation_thickness']
+
+        ].rename(
+            columns={
+                "is_solid_brick": "is_solid_brick_wall",
+                "is_system_built": "is_system_built_wall",
+                "is_timber_frame": "is_timber_frame_wall",
+                "is_assumed": "is_assumed_wall",
+                "insulation_thickness": "insulation_thickness_wall"
+            }
+        ),
+        left_on="walls-description",
+        right_on="original_description"
+    ).merge(
+        pd.DataFrame(cleaned["roof-description"])[
+            [
+                'original_description', 'is_pitched', 'is_roof_room', 'is_loft',
+                'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed',
+                'has_dwelling_above', 'insulation_thickness'
+            ]
+        ].rename(
+            columns={
+                "is_assumed": "is_assumed_roof",
+            }
+        ),
+        left_on="roof-description",
+        right_on="original_description"
+    ).merge(
+        pd.DataFrame(cleaned["floor-description"])[
+            [
+                'original_description', 'is_solid', 'is_suspended', 'is_assumed',
+                'insulation_thickness'
+            ]
+        ].rename(
+            columns={
+                "is_assumed": "is_assumed_floor",
+                "insulation_thickness": "insulation_thickness_floor"
+            }
+        ),
+        left_on="floor-description",
+        right_on="original_description"
+    )
+
+    archetyping_data = data[
+        [
+            "row_id",
+            "Energy starting band (EPC)",
+            "Property type",
+            "Property year built",
+            "Gross internal area (sqm)",
+            "Current heating system type",
+            "Wall type",
+            "Floor type",
+            "Roof type",
+            "Window type",
+            "Location (Floor)",
+        ]
+    ].merge(
+        epc_metadata[["row_id", "floor"]],
+        how="left",
+        on="row_id"
+    ).merge(
+        epc_data[
+            [
+                "row_id", "uprn", "current-energy-rating", "property-type", "built-form", "total-floor-area",
+                'is_cavity_wall', 'is_filled_cavity', 'is_solid_brick_wall', 'is_system_built_wall',
+                'is_timber_frame_wall', 'is_as_built', 'is_assumed_wall', 'insulation_thickness_wall',
+                'is_solid', 'is_suspended', 'is_assumed_floor', 'insulation_thickness_floor',
+                'is_pitched', 'is_roof_room', 'is_loft',
+                'is_flat', 'is_thatched', 'is_at_rafters', 'is_assumed_roof',
+                'has_dwelling_above', 'insulation_thickness', "mainheat-description",
+                "local-authority-label"
+            ]
+        ],
+        how="left",
+        on="row_id"
+    ).merge(
+        spatial_data[["row_id", "conservation_status", ]],
+        on="row_id",
+        how="left"
+    )
+
+    if archetyping_data.shape[0] != data.shape[0]:
+        raise Exception("Mismatch in data")
+
+    # We create groups analogous to the Energy Company Obligation
+    # 0 - 72, 73 - 97, 98 - 199, 200+
+    archetyping_data["Floor_area_category"] = pd.cut(
+        archetyping_data["Gross internal area (sqm)"],
+        bins=[0, 72, 97, 199, 1000],
+        labels=["0-72", "73-97", "98-199", "200+"]
+    )
+    archetyping_data["Floor_area_category_backup"] = pd.cut(
+        archetyping_data["total-floor-area"].astype(float),
+        bins=[0, 72, 97, 199, 1000],
+        labels=["0-72", "73-97", "98-199", "200+"]
+    )
+    archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].fillna(
+        archetyping_data["Floor_area_category_backup"]
+    )
+    archetyping_data["Floor_area_category"] = archetyping_data["Floor_area_category"].astype(str)
+    archetyping_data["Floor_area_category"] = np.where(
+        pd.isnull(archetyping_data["Floor_area_category"]),
+        "Unknown",
+        archetyping_data["Floor_area_category"]
+    )
+    archetyping_data = archetyping_data.drop(columns=["Floor_area_category_backup"])
+
+    archetyping_data["property-type-reduced"] = np.where(
+        archetyping_data["property-type"].isin(["Flat", "Maisionette"]),
+        "Flat/Maisonette",
+        archetyping_data["property-type"]
+    )
+
+    archetyping_data["built-form-reduced"] = np.where(
+        archetyping_data["built-form"].isin(["End-Terrace", "Semi-Detached"]),
+        "End-Terrace/Semi-Detached",
+        archetyping_data["built-form"]
+    )
+    archetyping_data["built-form-reduced"] = np.where(
+        archetyping_data["property-type-reduced"] == "Flat/Maisonette",
+        "Flat/Maisonette",
+        archetyping_data["built-form-reduced"]
+    )
+
+    archetyping_data["Wall type"] = np.where(
+        archetyping_data["Wall type"].isin(['Solid ', 'Solid - internal lining ']),
+        "Solid",
+        archetyping_data["Wall type"]
+    )
+    archetyping_data["Wall type"] = np.where(
+        archetyping_data["Wall type"].isin(['Cavity ', 'cavity ']),
+        "Cavity",
+        archetyping_data["Wall type"]
+    )
+
+    # Proposed remaps based on discoveries
+    value_remaps = {
+        # 8 Filey Avenue
+        "100021040744": {
+            "variable": "Property type",
+            "newvalue": "House, mid-terrace",
+        },
+        # 7	Yetev Lev Court
+        "100021032043": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        # 14 Yetev Lev Court
+        "100021032050": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        # 23 Yetev Lev Court
+        "100021032059": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        # 30 Yetev Lev Court
+        "100021032066": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        # 34 Yetev Lev Court
+        "100021032070": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        # B	86 Bethune Road
+        "100021026285": {
+            "variable": "Wall type",
+            "newvalue": "Solid",
+        },
+        # A	80 Bethune Road
+        "100021026277": {
+            "variable": "Wall type",
+            "newvalue": "Solid",
+        },
+        # 140 Kyverdale Road
+        "100021052262": {
+            "variable": "Property type",
+            "newvalue": "House, mid-terrace",
+        },
+        # 6 Leabourne Road
+        "100021053799": {
+            "variable": "Wall type",
+            "newvalue": "Solid",
+        },
+        # 22 Britannia Gardens - needs confirmation
+        # 7 Satanita Road - needs confirmation
+        # 12 Cheltenham Crescent
+        "100011402969": {
+            "variable": "Wall type",
+            "newvalue": "Cavity",
+        },
+        "100021031752": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        # 79 Craven Park Road
+        "100021169682": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        # 88 Darenth Road
+        "100021036148": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021036165": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021036167": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021053849": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021054353": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021054560": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021059839": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        },
+        "100021059848": {
+            "variable": "Roof type",
+            "newvalue": "Room Roof"
+        }
+    }
+
+    # Perform the remaps
+    for uprn, config in value_remaps.items():
+        archetyping_data[config["variable"]] = np.where(
+            archetyping_data["uprn"].astype(str) == uprn, config["newvalue"], archetyping_data[config["variable"]]
+        )
+
+    # row_id = data[
+    #     # (data["Address letter or number"] == "C") &
+    #     (data["Street address"].str.strip() == "41 Moresby Road")
+    # ]["row_id"]
+    # if len(row_id) != 1:
+    #     raise Exception("Fail")
+    # print(epc_data[epc_data["row_id"] == row_id.values[0]]["uprn"])
+
+    # Map the year to the age band
+    def categorize_year(year):
+        if isinstance(year, str):
+            # Handle the case where year is in the format '1930s'
+            if 's' in year:
+                year = int(year[:4])
+            else:
+                year = int(year)
+        else:
+            year = int(year)
+
+        # Categorize based on year ranges
+        if year < 1900:
+            return 'A'
+        elif 1900 <= year <= 1929:
+            return 'B'
+        elif 1930 <= year <= 1949:
+            return 'C'
+        elif 1950 <= year <= 1966:
+            return 'D'
+        elif 1967 <= year <= 1975:
+            return 'E'
+        elif 1976 <= year <= 1982:
+            return 'F'
+        elif 1983 <= year <= 1990:
+            return 'G'
+        elif 1991 <= year <= 1995:
+            return 'H'
+        elif 1996 <= year <= 2002:
+            return 'I'
+        elif 2003 <= year <= 2006:
+            return 'J'
+        elif 2007 <= year <= 2011:
+            return 'K'
+        else:  # year >= 2012
+            return 'L'
+
+    archetyping_data["SAP_age_band"] = archetyping_data["Property year built"].apply(
+        categorize_year
+    )
+
+    # Flag if the property is in London/Manchester
+    archetyping_data["Location"] = np.where(
+        archetyping_data["local-authority-label"].isin(
+            ["Hackney", "Barnet", "Haringey"]
+        ),
+        "London",
+        np.where(
+            archetyping_data["local-authority-label"].isin(
+                ["Salford", "Bury"]
+            ),
+            "Manchester",
+            "Southend"
+        )
+    )
+    # 9 Greenview is in manchester
+    archetyping_data["Location"] = np.where(
+        archetyping_data["row_id"] == data[data["Street address"] == "9 Greenview"]["row_id"].values[0],
+        "Manchester",
+        archetyping_data["Location"]
+    )
+    # We fix the location for B 80 Bethune Road
+    archetyping_data["Location"] = np.where(
+        (
+            archetyping_data["row_id"].isin(
+                data[
+                    data["Street address"] == "80 Bethune Road"
+                    ]["row_id"].values.tolist()
+            )
+        ) & (
+            archetyping_data["row_id"].isin(
+                data[
+                    data["Address letter or number"] == "B"
+                    ]["row_id"].values.tolist()
+            )
+        ),
+        "London",
+        archetyping_data["Location"]
+    )
+
+    # Hackney            73 - London
+    # Southend-on-Sea     6 - Southend
+    # Barnet              4 - London
+    # Castle Point        4 - Southend
+    # Haringey            3 - London
+    # Salford             2 - Manchester
+    # Bury                1 - Manchester
+
+    primary_archetyping_cols = [
+        'Property type',
+        "Location (Floor)",
+        'Current heating system type',
+        'Wall type',
+        'Roof type',
+        # "Location",
+        # 'current-energy-rating', 'property-type-reduced', 'built-form-reduced', 'is_cavity_wall',
+        # 'is_solid_brick_wall', 'is_system_built_wall', 'is_timber_frame_wall', 'is_as_built',
+        # 'is_solid', 'is_roof_room',
+        # 'is_loft', 'is_flat', 'is_thatched',
+        # 'is_at_rafters', 'has_dwelling_above',
+        # 'conservation_status',
+    ]
+
+    secondary_cols = [
+        'SAP_age_band',
+        'is_filled_cavity',
+        'insulation_thickness_wall'
+        'insulation_thickness_floor'
+        'insulation_thickness',
+        'is_assumed_wall',
+        'is_assumed_roof',
+        'Floor_area_category'
+    ]
+
+    archetypes = archetyping_data[primary_archetyping_cols].drop_duplicates()
+    # Hash the variables
+    archetypes["archetype_hash"] = archetypes.apply(
+        lambda x: hash(tuple(x.values)),
+        axis=1
+    )
+    archetypes = archetypes.sort_values("archetype_hash", ascending=True)
+    archetypes = archetypes.reset_index(drop=True)
+    archetypes["archetype_id"] = archetypes.index
+
+    archetypes.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/basic-archetypes.csv", index=False)
+
+    # We match properties to archetypes
+    archetyping_data = archetyping_data.merge(
+        archetypes,
+        on=primary_archetyping_cols,
+        how="left"
+    )
+
+    # We should choose a representative property for each archetype
+    archetyping_data = archetyping_data.merge(
+        epc_metadata[["row_id", "days_since_last_epc"]],
+        how="left",
+        on="row_id"
+    )
+
+    # Mark the property with the oldest EPC as the representative property
+    representative_properties = archetyping_data.sort_values(
+        ["archetype_id", "days_since_last_epc"], ascending=[True, False]
+    ).drop_duplicates("archetype_id")
+
+    archetyping_data["for_sample"] = np.where(
+        archetyping_data["row_id"].isin(representative_properties["row_id"]),
+        True,
+        False
+    )
+
+    # We save the archetyping data
+    archetyping_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv",
+                            index=False)
+    # Save the EPC data
+    epc_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv", index=False)
+    # Save the spatial data
+    spatial_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge(
+        spatial_data,
+        on="row_id",
+        how="left"
+    )
+    spatial_data.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/spatial_data.csv", index=False)
+
+    # Save archetyping data
+    archetyping_data = data[["row_id", "Address letter or number", "Street address", "Postcode"]].merge(
+        archetyping_data,
+        on="row_id",
+        how="left"
+    )
+    archetyping_data.to_csv(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/archetyping_data.csv",
+        index=False
+    )

etl/customers/aiha/epc_surveyor_list.py

@@ -0,0 +1,62 @@
+import pandas as pd
+import numpy as np
+import time
+from tqdm import tqdm
+from etl.bill_savings.data_collection import retrieve_find_my_epc_data, calculate_expiry_date
+
+
+def main():
+    """
+    This script handles pulling the surveyor names and acreditation details for Surveyors who have completed
+    the newest EPC for AIHA's properties
+    """
+
+    epc_data = pd.read_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/epc_data.csv")
+    epc_data = epc_data[["uprn", "address", "address1", "postcode", "lodgement-date"]]
+
+    epc_collected_data = []
+    for _, unit in tqdm(epc_data.iterrows(), total=len(epc_data)):
+        time.sleep(np.random.uniform(0.2, 1.5))
+        uprn = int(unit["uprn"])
+        address = unit["address1"]
+        postcode = unit["postcode"]
+        expected_expiry_date = calculate_expiry_date(unit["lodgement-date"])
+
+        response = retrieve_find_my_epc_data(
+            uprn=uprn,
+            postcode=postcode,
+            address=address,
+            expected_expiry_date=expected_expiry_date
+        )
+        if response is None:
+            raise Exception("fix me")
+        epc_collected_data.append(response)
+
+    epc_collected_data = pd.DataFrame(epc_collected_data)
+
+    epc_collected_data = epc_data[["uprn", "address", "address1", "postcode"]].merge(
+        epc_collected_data, left_on="uprn", right_on="extracted_uprn"
+    )
+
+    elmhurst_surveys = epc_collected_data[
+        epc_collected_data["Accreditation scheme"].isin(
+            ["NHER", "Stroma Certification Ltd", "Elmhurst Energy Systems Ltd"]
+        )
+    ]
+
+    quidos_surveys = epc_collected_data[
+        epc_collected_data["Accreditation scheme"].isin(
+            ["Quidos Limited"]
+        )
+    ]
+
+    ecmk_surveys = epc_collected_data[
+        epc_collected_data["Accreditation scheme"].isin(
+            ["ECMK"]
+        )
+    ]
+
+    # Store the data:
+    elmhurst_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Elmhurst Surveys.csv")
+    quidos_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/Quidos Surveys.csv")
+    ecmk_surveys.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/ECMK Surveys.csv")

etl/customers/bcc_tender/app.py

@@ -102,7 +102,7 @@ analysis_epcs = analysis_epcs[
     [
         "UPRN", "TENURE", "CURRENT_ENERGY_RATING", "WALLS_DESCRIPTION", "ROOF_DESCRIPTION",
         "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA", "PROPERTY_TYPE", "BUILT_FORM", "MAINHEAT_DESCRIPTION",
-        "eligibility_type",
+        "eligibility_type", "PHOTO_SUPPLY", "ADDRESS1", "POSTCODE"
     ]
 ]
 analysis_epcs["grouped_epc_band"] = np.where(
@@ -110,6 +110,14 @@ analysis_epcs["grouped_epc_band"] = np.where(
     "EPC D",
     "EPC E-G"
 )
+
+analysis_epcs[pd.isnull(analysis_epcs["PHOTO_SUPPLY"])][["ADDRESS1", "POSTCODE"]].sample(1)
+
+analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].fillna(0)
+analysis_epcs["PHOTO_SUPPLY"] = analysis_epcs["PHOTO_SUPPLY"].astype(float)
+analysis_epcs["has_solar"] = np.where(analysis_epcs["PHOTO_SUPPLY"] > 0, 1, 0)
+analysis_epcs["has_solar"].value_counts()
+
 analysis_epcs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/bcc tender/analysis_epcs.csv", index=False)
 
 # Create aggregations and we store this information

etl/customers/eon/pilot_asset_list.py

@@ -2,12 +2,11 @@ import time
 
 import pandas as pd
 
-from utils.s3 import read_excel_from_s3
 from backend.SearchEpc import SearchEpc
 from dotenv import load_dotenv
 import os
 from tqdm import tqdm
-from utils.s3 import save_csv_to_s3
+from utils.s3 import save_csv_to_s3, read_excel_from_s3
 
 # Read in the .env file in backend
 load_dotenv(dotenv_path="backend/.env")
@@ -172,9 +171,6 @@ def app():
     # Let's just pull the full EPC data for this
     asset_list_with_uprn = []
     for row, property_meta in tqdm(raw_asset_list_base.iterrows(), total=raw_asset_list_base.shape[0]):
-        if row <= 104:
-            continue
-        time.sleep(1.1)
         searcher = SearchEpc(
             address1=property_meta["address"],
             postcode=property_meta["postcode"],
@@ -183,24 +179,26 @@ def app():
             full_address=", ".join([property_meta["address"], property_meta["postcode"]])
         )
 
-        # Let's just find the UPRN
-        searcher.ordnance_survey_client.get_places_api()
-
-        uprn = searcher.ordnance_survey_client.most_relevant_result["UPRN"]
-
+        searcher.find_property(skip_os=True)
+        if searcher.newest_epc["uprn-source"] == SearchEpc.UPRN_SOURCE_SIMULATED:
+            uprn = None
+        else:
+            uprn = searcher.uprn
         # searcher.find_property(skip_os=False)
 
         asset_list_with_uprn.append(
             {
                 **property_meta,
                 "uprn": uprn,
+                "matched_address": searcher.address1,
+                "matched_postcode": searcher.postcode
             }
         )
 
     # Store this as a backup
     # import pandas as pd
     # asset_list_with_uprn_df = pd.DataFrame(asset_list_with_uprn)
-    # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn.csv", index=False)
+    # asset_list_with_uprn_df.to_csv("eon_asset_list_with_uprn_2.csv", index=False)
     # Read in
     # asset_list_with_uprn = pd.read_csv("eon_asset_list_with_uprn.csv").to_dict(orient="records")
 

etl/customers/gla/example_model_outputs.py

@@ -0,0 +1,38 @@
+import pandas as pd
+from utils.s3 import save_csv_to_s3
+
+asset_list = [
+    {
+        "address": "4, King Henrys Drive",
+        "postcode": "CR0 0PA"
+    },
+]
+portfolio_id = 110
+user_id = 8
+
+asset_list = pd.DataFrame(asset_list)
+
+filename = f"{user_id}/{portfolio_id}/asset_list.csv"
+save_csv_to_s3(
+    dataframe=asset_list,
+    bucket_name="retrofit-plan-inputs-dev",
+    file_name=filename
+)
+
+body1 = {
+    "portfolio_id": str(portfolio_id),
+    "housing_type": "Private",
+    "goal": "Increasing EPC",
+    "goal_value": "A",
+    "trigger_file_path": filename,
+    "already_installed_file_path": "",
+    "patches_file_path": "",
+    "non_invasive_recommendations_file_path": "",
+    "inclusions": [
+        "cavity_wall_insulation", "loft_insulation", "air_source_heat_pump", "solar_pv"
+    ],
+    "budget": None,
+    "scenario_name": "Whole House",
+    "multi_plan": False,
+}
+print(body1)

etl/customers/gla/proposal_investigation.py

@@ -0,0 +1,173 @@
+"""
+This script performs some basic analysis to identify EPC data for postcodes specified in the Warmer Homes Local Grant
+"""
+
+import inspect
+import requests
+import json
+import pandas as pd
+from pathlib import Path
+from etl.ownership.Ownership import Ownership
+
+postcodes = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Downloads/WHLG-eligible-postcodes_RP edit.xlsx", sheet_name='Eligible postcodes'
+)
+# Take just the first three columns
+postcodes = postcodes[
+    ['List of eligible postcodes via the IMD Income Decile 1-2 pathway', 'Unnamed: 1', 'Unnamed: 2']
+]
+
+postcodes.columns = ['postcode', 'Local Authority', 'London Borough?']
+# Drop the first row
+postcodes = postcodes.drop([0, 1])
+# Take just the London Boroughs
+postcodes = postcodes[postcodes["London Borough?"] == "Yes"]
+# Since there are a large number of potcodes (425k), let's just take a few examples
+# Take postcodes that begin with "BN15"
+# postcodes = postcodes[postcodes["postcode"].str.startswith("BN15")]
+
+# The Local Authority is Adur, so let's get the EPC data for this area
+# epc_data = pd.read_csv(
+#     "/Users/khalimconn-kowlessar/Documents/hestia/Model/local_data/all-domestic-certificates/domestic-E07000223-Adur"
+#     "/certificates.csv", low_memory=False
+# )
+# # Filter on these postcodes
+# epc_data = epc_data[epc_data["POSTCODE"].str.lower().isin(postcodes["postcode"].str.lower())]
+# epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
+# # Take the newest EPC for each UPRN, based on LODGEMENT_DATE
+# epc_data["LODGEMENT_DATE"] = pd.to_datetime(epc_data["LODGEMENT_DATE"])
+# epc_data = epc_data.sort_values("LODGEMENT_DATE", ascending=False).drop_duplicates("UPRN")
+#
+# # Let's look at the breakdown of EPC ratings. We want the count and the % of the total
+# ratings_distribution = epc_data.groupby("CURRENT_ENERGY_RATING").size().reset_index()
+# ratings_distribution.columns = ["Rating", "Count"]
+# ratings_distribution["Percentage"] = ratings_distribution["Count"] / ratings_distribution["Count"].sum() * 100
+
+# Can we identify the owners of these units so we can contact them?
+
+file_src = inspect.getfile(lambda x: None)
+DATA_DIRECTORY = Path(file_src).parent / "local_data" / "all-domestic-certificates"
+epc_paths = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()]
+epc_paths = [str(entry / "certificates.csv") for entry in epc_paths]
+
+ownership = Ownership(
+    epc_paths=epc_paths,
+    domestic_ownership_path="/Users/khalimconn-kowlessar/Downloads/CCOD_FULL_2024_07.csv",
+    overseas_ownership_path="/Users/khalimconn-kowlessar/Downloads/OCOD_FULL_2024_07.csv",
+    land_registry_path="/Users/khalimconn-kowlessar/Downloads/pp-complete.csv",
+    project_name="gla-proposal",
+    bucket="retrofit-data-dev",
+    average_property_value=0,
+    portfolio_value=0,
+    excluded_owners=[],
+    excluded_uprns=[],
+    save=True
+)
+
+# Data will be found at ownership/gla-proposal
+ownership.source_epc_properties(column_filters={}, postcodes=postcodes["postcode"].str.lower().tolist())
+
+# Step 2: Get company ownership data
+ownership.load_company_ownership()
+
+# Step 3: Prepare data for matching
+ownership.prepare_for_matching()
+
+# Step 4: Match EPC data to ownership data
+ownership.match()
+
+from utils.s3 import save_excel_to_s3, read_excel_from_s3
+
+# Save the data to S3
+# save_excel_to_s3(
+#     df=ownership.matched_addresses,
+#     bucket_name=ownership.bucket,
+#     file_key=ownership.matched_addresses_pre_filter_filepath
+# )
+
+# Read in matches
+matches = read_excel_from_s3(
+    bucket_name=ownership.bucket,
+    file_key="ownership/gla-proposal/2024-10-10 19:02:34.131365/matched_addresses_pre_filter.xlsx",
+    header_row=0
+)
+
+# We have the matches, which we now need to match to the postcodes
+matches = ownership.matched_addresses.copy()
+# filter matches on the postcodes we're interested in
+matches = matches[matches["epc_postcode"].str.lower().isin(postcodes["postcode"].str.lower())]
+# Remove any social transactions
+matches = matches[~matches["TENURE"].isin(
+    ["Rented (social)", "rental (social)",
+     "Not defined - use in the case of a new dwelling for which the intended tenure in not known. It is not to be "
+     "used for an existing dwelling", "NO DATA!"])
+]
+matches["is_prs"] = matches["TENURE"].isin(["rental (private)", "Rented (private)"])
+# Look at the EPC ratings
+epc_ratings = matches.groupby(["CURRENT_ENERGY_RATING"]).size().reset_index()
+epc_ratings.columns = ["EPC Rating", "Count"]
+epc_ratings["Percentage"] = epc_ratings["Count"] / epc_ratings["Count"].sum() * 100
+
+# Take properties that are below an EPC C rating, as defined by the guidance and remove any new builds
+matches = matches[matches["CURRENT_ENERGY_RATING"].isin(["D", "E", "F", "G"])]
+# 11,694 properties
+matches["epc_postcode"].nunique()
+# 6899
+
+owners_count = matches.groupby(['Proprietor Name (1)', 'Company Registration No. (1)']).size().reset_index()
+owners_count.columns = ['Owner', 'Owner Registration #', 'Count']
+owners_count = owners_count.sort_values('Count', ascending=False)
+owners_count["Percentage"] = owners_count["Count"] / owners_count["Count"].sum() * 100
+
+# Take an example postal region
+matches = matches.sort_values("epc_postcode", ascending=True)
+# BR1, BR5
+example = matches[matches["epc_postcode"].str.startswith("CR0 ")].copy()
+example = example[example["TENURE"].isin(["rental (private)", "Rented (private)"])]
+
+pd.set_option('display.max_rows', 500)
+pd.set_option('display.max_columns', 500)
+pd.set_option('display.width', 1000)
+example[
+    ["epc_address", "epc_postcode", "CURRENT_ENERGY_RATING", "CURRENT_ENERGY_EFFICIENCY", "Proprietor Name (1)",
+     "Company Registration No. (1)"]
+].head(4)
+
+ownership.epc_data["UPRN"] = ownership.epc_data["UPRN"].astype(int)
+example = example.merge(
+    ownership.epc_data[["UPRN", "BUILT_FORM", "PROPERTY_TYPE", "WALLS_DESCRIPTION", "ROOF_DESCRIPTION"]],
+    on="UPRN",
+    how="left"
+)
+z = example[example["CURRENT_ENERGY_RATING"] == "E"]
+z = z[z["TENURE"].isin(["rental (private)", "Rented (private)"])]
+
+companies_house_api_key = "1d9c2877-3271-4642-80ed-a6170971653f"
+
+company_number = example.head(1)["Company Registration No. (1)"].values[0]
+url = f'https://api.company-information.service.gov.uk/company/{company_number}'
+
+# Make the API request
+response = requests.get(url, auth=(companies_house_api_key, ''))
+
+# Check if the request was successful
+if response.status_code == 200:
+    company_data = response.json()
+    # Pretty-print the fetched data
+    print(json.dumps(company_data, indent=4))
+else:
+    print(f"Failed to fetch data. Status code: {response.status_code}")
+    # Try appending a zero the beginning of the company number
+    company_number = f"0{company_number}"
+    url = f'https://api.company-information.service.gov.uk/company/{company_number}'
+    response = requests.get(url, auth=(companies_house_api_key, ''))
+    company_data = response.json()
+
+from pprint import pprint
+
+pprint(company_data)
+
+psc_url = f'https://api.company-information.service.gov.uk/company/{company_number}/persons-with-significant-control'
+psc_response = requests.get(psc_url, auth=(companies_house_api_key, ''))
+psc_data = psc_response.json()
+pprint(psc_data)

etl/customers/newhaven/slides.py

@@ -5,6 +5,7 @@ from sqlalchemy.orm import sessionmaker
 from backend.app.db.connection import db_engine
 from backend.app.db.models.recommendations import Recommendation, Plan, PlanRecommendations, Scenario
 from backend.app.db.models.portfolio import PropertyModel, PropertyDetailsEpcModel
+from utils.s3 import read_csv_from_s3
 
 
 def get_data(portfolio_id, scenario_ids):
@@ -415,3 +416,396 @@ def slides():
     pd.set_option('display.max_rows', None)
     # Show more characters in a column
     pd.set_option('display.max_colwidth', None)
+
+
+def lewes_outputs():
+    """
+    preparing of this data for the following 2 needs:
+    1) dataset to share with Nextgen heating
+    2) Breakdown of results by property type
+    :return:
+    """
+
+    # get the asset list
+    asset_list = read_csv_from_s3(bucket_name="retrofit-plan-inputs-dev", filepath="8/90/pilot.csv")
+    asset_list = pd.DataFrame(asset_list)
+    # Get non-invasive recommendations
+    non_intrusive_recommendations = read_csv_from_s3(
+        bucket_name="retrofit-plan-inputs-dev",
+        filepath="8/90/non_invasive_recommendations.csv"
+    )
+    non_intrusive_recommendations = pd.DataFrame(non_intrusive_recommendations)
+
+    # Right now this is the second version of the nehaven portfolio
+    portfolio_id = 90
+    # Look at one scenario at a time, otherwise this is agony
+    scenario_ids = [47, 48, 49, 50, 51]
+    properties_data, plans_data, recommendations_data = get_data(portfolio_id, scenario_ids)
+    properties_df = pd.DataFrame(properties_data)
+    recommendations_df = pd.DataFrame(recommendations_data)
+
+    # Unnest this
+    import ast
+    survey_recs = []
+    for _, row in non_intrusive_recommendations.iterrows():
+        recs = ast.literal_eval(row["recommendations"])
+        ashp_rec = next((r for r in recs if r["type"] == "air_source_heat_pump"), None)
+        solar_rec = next((r for r in recs if r["type"] == "solar_pv"), None)
+        to_append = {
+            "uprn": row["uprn"]
+        }
+        if ashp_rec["suitable"]:
+            to_append = {
+                **to_append,
+                "ashp_suitable": True,
+                "ashp_size_kw": ashp_rec["size"],
+                "ashp_cost": ashp_rec["cost"],
+            }
+
+        if solar_rec["suitable"]:
+            to_append = {
+                **to_append,
+                "solar_suitable": True,
+                "solar_size_kwp": solar_rec["array_wattage"],
+                "solar_cost": solar_rec["cost"],
+            }
+        survey_recs.append(to_append)
+    survey_recs = pd.DataFrame(survey_recs)
+
+    asset_list["uprn"] = asset_list["uprn"].astype(int)
+    survey_recs["uprn"] = survey_recs["uprn"].astype(int)
+
+    vital_kwh = 7597
+    domna_kwh = 10850
+    scaling_factor = vital_kwh / domna_kwh
+
+    next_gen_dataset = properties_df[[
+        "uprn", "address", "postcode",
+        "property_type", "built_form", "current_energy_demand_heating_hotwater",
+        "mainfuel", "total_floor_area", "floor_height"
+    ]].rename(
+        columns={
+            "mainfuel": "primary_fuel_type",
+            "total_floor_area": "gross_floor_area",
+            "current_energy_demand_heating_hotwater": "estimated_heating_hotwater_kwh"
+        }
+    ).merge(
+        asset_list[["uprn", "number_of_floors"]],
+        how="left",
+        on="uprn"
+    ).merge(
+        survey_recs,
+        how="left",
+        on="uprn"
+    )
+    next_gen_dataset["estimated_heating_hotwater_kwh_scaled"] = (
+        next_gen_dataset["estimated_heating_hotwater_kwh"] * scaling_factor
+    )
+
+    next_gen_dataset["ashp_suitable"] = next_gen_dataset["ashp_suitable"].fillna(False)
+    next_gen_dataset["solar_suitable"] = next_gen_dataset["solar_suitable"].fillna(False)
+
+    # We prepare the scenario outputs by property type
+    grouped_data = next_gen_dataset.copy()
+    grouped_data["property_sub_type"] = grouped_data["built_form"].copy()
+    # If a property is a flat, re-map sub_type just to flat
+    grouped_data.loc[grouped_data["property_type"] == "Flat", "property_sub_type"] = "Flat"
+    # Same for maisonettes
+    grouped_data.loc[grouped_data["property_type"] == "Maisonette", "property_sub_type"] = "Maisonette"
+
+    # We now pull out the recommendations impact by property type and sub type
+
+    # Exclude sealing open fireplaces
+    recommendations_df = recommendations_df[recommendations_df["type"] != "sealing_open_fireplace"]
+
+    # We update the type column so that if type == heating, and the description contains "air source heat pump",
+    # the type is "air_source_heat_pump", else if the description contains "high heat retention storage heaters", else
+    # if the description contains "condensing boiler, the type is updated to "boiler_upgrade"
+    recommendations_df["type"] = np.where(
+        recommendations_df["type"] == "heating",
+        np.where(
+            recommendations_df["description"].str.contains("air source heat pump"),
+            "Air Source Heat Pump",
+            np.where(
+                recommendations_df["description"].str.contains("high heat retention"),
+                "High Heat Retention Storage",
+                np.where(
+                    recommendations_df["description"].str.contains("condensing boiler"),
+                    "Boiler Upgrade",
+                    recommendations_df["type"]
+                )
+            )
+        ),
+        recommendations_df["type"]
+    )
+
+    recommendation_types = recommendations_df["type"].unique().tolist()
+    rename_dict = {
+        'hot_water_tank_insulation': 'Hot Water Tank Insulation',
+        'windows_glazing': 'Windows Glazing',
+        'secondary_heating': 'Secondary Heating',
+        'cavity_wall_insulation': 'Cavity Wall Insulation',
+        'flat_roof_insulation': 'Flat Roof Insulation',
+        'mechanical_ventilation': 'Mechanical Ventilation',
+        'loft_insulation': 'Loft Insulation',
+        'cylinder_thermostat': 'Cylinder Thermostat',
+        'room_roof_insulation': 'Room Roof Insulation',
+        'low_energy_lighting': 'Low Energy Lighting',
+        'external_wall_insulation': 'External Wall Insulation',
+        'solar_pv': 'Solar PV',
+        'heating_control': 'Heating Control',
+        'solid_floor_insulation': 'Solid Floor Insulation',
+        'suspended_floor_insulation': 'Suspended Floor Insulation',
+        'internal_wall_insulation': 'Internal Wall Insulation'
+    }
+
+    property_scenario_impact = []
+    for scenario_id in tqdm(scenario_ids):
+        # Get the recommendations for the scenario, default
+        scenario_recommendations = recommendations_df[
+            (recommendations_df["Scenario ID"] == scenario_id) &
+            (recommendations_df["default"] == True)
+            ].copy()
+
+        scenario_recommendations['Estimated Lighting kWh Savings'] = scenario_recommendations.apply(
+            lambda x: x['kwh_savings'] if x['type'] == 'low_energy_lighting' else 0,
+            axis=1)
+        scenario_recommendations['Estimated Solar kWh Savings'] = scenario_recommendations.apply(
+            lambda x: x['kwh_savings'] if x['type'] == 'solar_pv' else 0, axis=1)
+
+        # Set 'Estimated Kwh Savings' to zero where specific kwh columns are used
+        scenario_recommendations['Estimated Heating Demand kWh Savings'] = scenario_recommendations.apply(
+            lambda x: 0 if x['type'] in ['low_energy_lighting', 'solar_pv'] else x[
+                'kwh_savings'], axis=1)
+
+        scenario_grouped_data = scenario_recommendations.groupby(['property_id']).agg({
+            'Estimated Heating Demand kWh Savings': 'sum',
+            'Estimated Lighting kWh Savings': 'sum',
+            'Estimated Solar kWh Savings': 'sum',
+            "estimated_cost": "sum"
+        }).reset_index()
+
+        comparison = properties_df.drop_duplicates()[
+            ["uprn", "property_id", "current_energy_demand_heating_hotwater"]
+        ].merge(
+            scenario_grouped_data, on=["property_id"], how="left"
+        )
+        comparison["Estimated Heating Demand kWh Savings"] = (
+            comparison["Estimated Heating Demand kWh Savings"].fillna(0)
+        )
+        comparison["Estimated Lighting kWh Savings"] = (
+            comparison["Estimated Lighting kWh Savings"].fillna(0)
+        )
+        comparison["Estimated Solar kWh Savings"] = (
+            comparison["Estimated Solar kWh Savings"].fillna(0)
+        )
+        comparison["estimated_cost"] = comparison["estimated_cost"].fillna(0)
+
+        comparison["post_scenario_heating_hotwater_kwh"] = (
+            comparison["current_energy_demand_heating_hotwater"] - comparison["Estimated Heating Demand kWh Savings"]
+        )
+
+        # For each scenario, we create a measure matrix
+        measure_matrix = scenario_recommendations.pivot_table(
+            index='property_id',
+            columns='type',
+            values='id',  # Using 'id' just as a placeholder for the pivot
+            aggfunc=lambda x: True,  # If an ID exists for a given type, mark as True
+            fill_value=False  # Fill other entries as False
+        ).reset_index()
+
+        non_zero_heat_demand_impact = comparison[
+            (comparison["Estimated Heating Demand kWh Savings"] > 0) |
+            (comparison["Estimated Lighting kWh Savings"] > 0) |
+            (comparison["Estimated Solar kWh Savings"] > 0)
+            ]
+        measure_matrix = measure_matrix[
+            measure_matrix["property_id"].isin(non_zero_heat_demand_impact["property_id"].values)
+        ]
+        measure_matrix = measure_matrix.rename(columns=rename_dict)
+
+        comparison = comparison.merge(
+            measure_matrix, on="property_id", how="left"
+        )
+        comparison["scenario_id"] = scenario_id
+
+        property_scenario_impact.append(comparison)
+
+    property_scenario_impact = pd.concat(property_scenario_impact)
+    # property_scenario_impact = property_scenario_impact.drop(columns=["property_id", "Estimated Kwh Savings"])
+    for v in list(rename_dict.values()) + ["Air Source Heat Pump", "High Heat Retention Storage", "Boiler Upgrade"]:
+        # Fill NaNs with False
+        property_scenario_impact[v] = property_scenario_impact[v].fillna(False)
+
+    # Scale
+    property_scenario_impact["post_scenario_heating_hotwater_kwh_scaled"] = (
+        property_scenario_impact["post_scenario_heating_hotwater_kwh"] * scaling_factor
+    )
+
+    grouped_data = grouped_data.merge(
+        property_scenario_impact, how="left", on="uprn"
+    )
+
+    # Agg the data
+    grouped_data = grouped_data.groupby(["property_type", "property_sub_type", "scenario_id"]).agg({
+        "estimated_heating_hotwater_kwh": "mean",
+        "estimated_heating_hotwater_kwh_scaled": "mean",
+        "estimated_cost": "mean",
+        "post_scenario_heating_hotwater_kwh": "mean",
+        "post_scenario_heating_hotwater_kwh_scaled": "mean"
+    }).reset_index()
+
+    scenario_names = pd.DataFrame(
+        [
+            {
+                "scenario_id": 47,
+                "scenario": "Demand Reduction – cavity & roof insulation",
+            },
+            {
+                "scenario_id": 48,
+                "scenario": "Demand reduction – no solid wall, floors or heating/renewables",
+            },
+            {
+                "scenario_id": 49,
+                "scenario": "Demand reduction – no decant"
+            },
+            {
+                "scenario_id": 50,
+                "scenario": "Demand reduction – no decant + heating & solar",
+            },
+            {
+                "scenario_id": 51,
+                "scenario": "Whole house retrofit"
+            }
+        ]
+
+    )
+
+    grouped_data = grouped_data.merge(
+        scenario_names, how="left", on="scenario_id"
+    )
+
+    if not grouped_data[
+        grouped_data["estimated_heating_hotwater_kwh"] < grouped_data["post_scenario_heating_hotwater_kwh"]].empty:
+        raise Exception("someting went wrong")
+
+    if not grouped_data[grouped_data["estimated_heating_hotwater_kwh_scaled"] < grouped_data[
+        "post_scenario_heating_hotwater_kwh_scaled"]].empty:
+        raise Exception("someting went wrong")
+
+    # Reorder the columns
+    grouped_data = grouped_data[
+        [
+            'property_type',
+            'property_sub_type',
+            'scenario',
+            'estimated_heating_hotwater_kwh',
+            'post_scenario_heating_hotwater_kwh',
+            'estimated_heating_hotwater_kwh_scaled',
+            'post_scenario_heating_hotwater_kwh_scaled',
+            'estimated_cost',
+        ]
+    ]
+
+    grouped_data = grouped_data.rename(
+        columns={
+            "property_type": "Property Type",
+            "property_sub_type": "Property Sub Type",
+            "scenario": "Scenario",
+            "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh",
+            "post_scenario_heating_hotwater_kwh": "Post Scenario Heating & Hot Water kwh",
+            "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh (scaled)",
+            "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh (scaled)",
+            "estimated_cost": "Estimated Cost or Retrofit",
+        }
+    )
+
+    # grouped_data.to_excel(
+    #     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Scenario kWh Impact by Property "
+    #     "Type.xlsx",
+    #     index=False
+    # )
+
+    property_scenario_impact = property_scenario_impact.merge(
+        scenario_names, how="left", on="scenario_id"
+    )
+
+    lewes_data = next_gen_dataset.merge(
+        property_scenario_impact, how="left", on="uprn"
+    )
+
+    lewes_data = lewes_data.sort_values(
+        ["postcode", "uprn", "scenario_id"], ascending=True
+    )
+
+    # Rearrange,  rename columns and drop what we don't need
+    # TODO - remap the heating type
+    lewes_data = lewes_data[
+        [
+            'uprn', 'address', 'postcode', 'property_type', 'built_form',
+            # 'estimated_heating_hotwater_kwh',
+            'primary_fuel_type', 'gross_floor_area', 'floor_height', 'number_of_floors', 'ashp_suitable',
+            'ashp_size_kw',
+            'ashp_cost', 'solar_suitable', 'solar_size_kwp', 'solar_cost',
+            'scenario',
+            'estimated_heating_hotwater_kwh_scaled',
+            'post_scenario_heating_hotwater_kwh_scaled',
+            # 'property_id',  - dropped
+            # 'current_energy_demand_heating_hotwater',
+            'Estimated Heating Demand kWh Savings',
+            'Estimated Lighting kWh Savings',
+            'Estimated Solar kWh Savings',
+            'estimated_cost',
+            'post_scenario_heating_hotwater_kwh', 'Cavity Wall Insulation', 'Cylinder Thermostat',
+            'Flat Roof Insulation',
+            'Hot Water Tank Insulation', 'Loft Insulation', 'Mechanical Ventilation', 'Room Roof Insulation',
+            # 'scenario_id', - dropped
+            'Low Energy Lighting', 'Secondary Heating', 'Windows Glazing', 'External Wall Insulation',
+            'Heating Control',
+            'Solar PV',
+            'Air Source Heat Pump', 'Boiler Upgrade', 'High Heat Retention Storage',
+            'Internal Wall Insulation',
+            'Solid Floor Insulation',
+            'Suspended Floor Insulation',
+        ]
+    ].rename(
+        columns={
+            "primary_fuel_type": "Primary Fuel Type",
+            "gross_floor_area": "Gross Floor Area",
+            "floor_height": "Floor Height",
+            "number_of_floors": "Number of Floors",
+            "ashp_suitable": "Is an ASHP Suitable?",
+            "ashp_size_kw": "ASHP Size (kW)",
+            "ashp_cost": "ASHP Cost",
+            "solar_suitable": "Is Solar PV Suitable?",
+            "solar_size_kwp": "Solar PV Size (kWp)",
+            "solar_cost": "Solar PV Cost",
+            # "estimated_heating_hotwater_kwh": "Estimated Heating & Hot Water kwh",
+            "estimated_heating_hotwater_kwh_scaled": "Estimated Heating & Hot Water kwh",
+            "post_scenario_heating_hotwater_kwh_scaled": "Post Scenario Heating & Hot Water kwh",
+            "estimated_cost": "Estimated Cost of Scenario"
+        }
+    )
+
+    # We save this dataset, which will be shared with Lewes Council
+    lewes_data.to_csv(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/Lewes property data.csv", index=False
+    )
+
+    df_pivot = property_scenario_impact.pivot_table(index='uprn', columns='scenario',
+                                                    values=['post_scenario_heating_hotwater_kwh',
+                                                            'post_scenario_heating_hotwater_kwh_scaled'])
+
+    # Flattening multi-index columns
+    df_pivot.columns = [f'{col[0]}_{col[1]}' for col in df_pivot.columns]
+
+    # Reset the index to have a clean dataframe
+    df_pivot.reset_index(inplace=True)
+
+    next_gen_dataset = next_gen_dataset.merge(
+        df_pivot, how="left", on="uprn"
+    )
+
+    next_gen_dataset.to_csv(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Newhaven/outputs/next_gen_dataset.csv", index=False
+    )

etl/customers/remote_assessments/app.py

@@ -0,0 +1,78 @@
+import pandas as pd
+from utils.s3 import save_csv_to_s3
+
+PORTFOLIO_ID = 111
+USER_ID = 8
+
+
+def app():
+    """
+    This application is used to initialise and run remote assessments
+    :return:
+    """
+
+    asset_list = [
+        {
+            "uprn": 100050770761,
+            "address": "12 Sheardown Street",
+            "postcode": "DN4 0BH"
+        }
+    ]
+    asset_list = pd.DataFrame(asset_list)
+
+    # Store the asset list in s3
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    non_invasive_recommendations = [
+        {
+            "uprn": 100050770761,
+            "recommendations": [
+                {
+                    "type": "extension_cavity_wall_insulation",
+                    "sap_points": 2,
+                }
+            ]
+        }
+    ]
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.csv"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    valuation_data = [
+        {
+            "uprn": 100050770761,
+            "value": 67_000
+        }
+    ]
+    # Store valuation data to s3
+    valuation_filename = f"{USER_ID}/{PORTFOLIO_ID}/valuation.csv"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(valuation_data),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=valuation_filename
+    )
+
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increasing EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "valuation_file_path": valuation_filename,
+        "scenario_name": "Full package remote assessment",
+        "multi_plan": True,
+        "budget": None,
+    }
+    print(body)

etl/customers/vectis/outputs.py

@@ -13,7 +13,7 @@ def app():
         "surveyor": "JAFFERSONS ENERGY CONSULTANTS",
         "project_code": "VEC001",
     }
-
+    # 5 Grove Mansions
     # These are the recommendations based on the on-site survey of the property.
     non_intrusive_recommendations = [
         {
@@ -22,17 +22,17 @@ def app():
             "recommendations": [
                 {
                     "type": "draught_proofing",
-                    "cost": 123,
+                    "cost": 100,
                     "survey": True,
                     "sap_points": 1
                 },
                 {
-                    "type": "mixed_glazing", "cost": 12345, "survey": True,
+                    "type": "mixed_glazing", "cost": 14632, "survey": True,
                     "description": "Install double glazing to north facing windows and secondary glazing to the "
                                    "remaining windows at the front of the building",
                     "sap_points": 3
                 },
-                {"type": "trickle_vents", "cost": 500, "survey": True},
+                {"type": "trickle_vents", "cost": 1000, "survey": True},
                 {"type": "suspended_floor_insulation", "cost": None, "survey": True, "sap_points": 2},
                 {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 5},
             ]
@@ -41,14 +41,14 @@ def app():
             # 8 Grove Mansions
             "uprn": 10024087855,
             "recommendations": [
-                {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 2},
+                {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 2},
                 {
-                    "type": "mixed_glazing", "cost": 12345, "survey": True,
+                    "type": "mixed_glazing", "cost": 7814, "survey": True,
                     "description": "Install double glazing to north facing windows and secondary glazing to the "
                                    "remaining windows at the front of the building",
                     "sap_points": 4
                 },
-                {"type": "trickle_vents", "cost": 500, "survey": True},
+                {"type": "trickle_vents", "cost": 700, "survey": True},
                 {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 0},
                 {"type": "internal_wall_insulation", "cost": None, "survey": True, 'sap_points': 5},
             ]
@@ -57,14 +57,14 @@ def app():
             # 9 Grove Mansions
             "uprn": 121016128,
             "recommendations": [
-                {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 1},
+                {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 1},
                 {
-                    "type": "mixed_glazing", "cost": 12345, "survey": True,
+                    "type": "mixed_glazing", "cost": 9740, "survey": True,
                     "description": "Install double glazing to north facing windows and secondary glazing to the "
                                    "remaining windows at the front of the building",
                     "sap_points": 3
                 },
-                {"type": "trickle_vents", "cost": 500, "survey": True},
+                {"type": "trickle_vents", "cost": 1000, "survey": True},
                 {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 1},
                 {"type": "suspended_floor_insulation", "cost": None, "sap_points": 1},
                 {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 6},
@@ -75,12 +75,12 @@ def app():
             "uprn": 121016124,
             "recommendations": [
                 {
-                    "type": "mixed_glazing", "cost": 12345, "survey": True,
+                    "type": "mixed_glazing", "cost": 12662, "survey": True,
                     "description": "Install double glazing to north facing windows and secondary glazing to the "
                                    "remaining windows at the front of the building",
                     "sap_points": 5
                 },
-                {"type": "trickle_vents", "cost": 500, "survey": True},
+                {"type": "trickle_vents", "cost": 1300, "survey": True},
                 {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 2},
                 {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 8},
             ]
@@ -89,14 +89,14 @@ def app():
             # 14 Grove Mansions
             "uprn": 121016117,
             "recommendations": [
-                {"type": "draught_proofing", "cost": 123, "survey": True, "sap_points": 1},
+                {"type": "draught_proofing", "cost": 100, "survey": True, "sap_points": 1},
                 {
-                    "type": "mixed_glazing", "cost": 12345, "survey": True,
+                    "type": "mixed_glazing", "cost": 10736, "survey": True,
                     "description": "Install double glazing to north facing windows and secondary glazing to the "
                                    "remaining windows at the front of the building",
                     "sap_points": 4
                 },
-                {"type": "trickle_vents", "cost": 500, "survey": True},
+                {"type": "trickle_vents", "cost": 1000, "survey": True},
                 {"type": "low_energy_lighting", "cost": None, "survey": True, "sap_points": 1},
                 {"type": "internal_wall_insulation", "cost": None, "survey": True, "sap_points": 6},
             ]
@@ -113,6 +113,7 @@ def app():
     ]
 
     asset_list = [
+        # These are properties where we've done a survey
         {
             "uprn": 121016121, "address": "", "postcode": ""
         },
@@ -131,6 +132,63 @@ def app():
         {
             "uprn": 10024087902, "address": "", "postcode": ""
         },
+        # These properties we just model with default data
+        # Flat 1
+        {
+            "uprn": 121016113, "address": "", "postcode": ""
+        },
+        # Flat 10
+        {
+            "uprn": 121016114, "address": "", "postcode": ""
+        },
+        # Flat 11
+        {
+            "uprn": 121016115, "address": "", "postcode": ""
+        },
+        # Flat 12
+        {
+            "uprn": 121016116, "address": "", "postcode": ""
+        },
+        # Flat 15
+        {
+            "uprn": 121016118, "address": "", "postcode": ""
+        },
+        # Flat 16
+        {
+            "uprn": 121016119, "address": "", "postcode": ""
+        },
+        # Flat 17
+        {
+            "address": "Flat 17 Grove Mansions", "postcode": "SW4 9SL"
+        },
+        # Flat 18
+        {
+            "uprn": 10024087901, "address": "", "postcode": ""
+        },
+        # Flat 3
+        {
+            "uprn": 121016122, "address": "", "postcode": ""
+        },
+        # Flat 4
+        {
+            "uprn": 121016123, "address": "", "postcode": ""
+        },
+        # Flat 6
+        {
+            "uprn": 121016125, "address": "", "postcode": ""
+        },
+        # Flat 7
+        {
+            "uprn": 10024087854, "address": "", "postcode": ""
+        },
+        # Flat 7A
+        {
+            "uprn": 10024087840, "address": "", "postcode": ""
+        },
+        # Flat 8A
+        {
+            "uprn": 10024087841, "address": "", "postcode": ""
+        },
     ]
     asset_list = pd.DataFrame(asset_list)
 
@@ -162,7 +220,7 @@ def app():
         "patches_file_path": "",
         "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
         "inclusions": [
-            "draught_proofing", "mixed_glazing", "trickle_vents", "low_energy_lighting",
+            "draught_proofing", "mixed_glazing", "trickle_vents", "low_energy_lighting", "windows"
         ],
         "budget": None,
         "scenario_name": "Quick wins - do now while tenanted",
@@ -185,7 +243,9 @@ def app():
             "trickle_vents",
             "low_energy_lighting",
             "suspended_floor_insulation",
-            "internal_wall_insulation"
+            "internal_wall_insulation",
+            "room_roof_insulation",
+            "windows"
         ],
         "budget": None,
         "scenario_name": "Do when void",

etl/epc/DataProcessor.py

@@ -263,7 +263,7 @@ class EPCDataProcessor:
 
         # Use replace function to map data (if exists in key), to corresponding value - i.e. Remove invalid values
         data = self.data.replace(data_anomaly_map)
-        data = data.replace(np.NAN, None)
+        data = data.replace(np.nan, None)
 
         self.data = data
 
@@ -384,7 +384,7 @@ class EPCDataProcessor:
             has_missings = pd.isnull(self.data[col]).sum()
             while has_missings:
                 self.data = apply_clean(
-                    data=self.data, matching_columns=matching_columns[0 : to_index + 1]
+                    data=self.data, matching_columns=matching_columns[0: to_index + 1]
                 )
                 has_missings = pd.isnull(self.data[col]).sum()
 
@@ -487,7 +487,7 @@ class EPCDataProcessor:
 
         filled_data = (
             self.data.groupby("UPRN", group_keys=True)[columns_to_fill]
-            .apply(lambda group: group.fillna(method="bfill").fillna(method="ffill"))
+            .apply(lambda group: group.bfill().ffill().infer_objects(copy=False))
             .reset_index()
             .set_index("level_1")
             .sort_index()
@@ -791,7 +791,7 @@ class EPCDataProcessor:
         We fill photo supply with zeros where it's missing
         """
 
-        self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].fillna(0)
+        self.data["PHOTO_SUPPLY"] = self.data["PHOTO_SUPPLY"].astype("Int64").fillna(0)
 
     @staticmethod
     def apply_averages_cleaning(
@@ -858,12 +858,12 @@ class EPCDataProcessor:
 
         # Fill NaN values with averages
         for col in cols_to_clean:
-            data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"], inplace=True)
-            data_to_clean.drop(columns=[f"{col}_AVERAGE"], inplace=True)
+            data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[f"{col}_AVERAGE"])
+            data_to_clean = data_to_clean.drop(columns=[f"{col}_AVERAGE"])
             # If we still have missings
-            data_to_clean[col].fillna(data_to_clean[col].mean(), inplace=True)
+            data_to_clean[col] = data_to_clean[col].fillna(data_to_clean[col].mean())
             # Final step if we still have missings - use global mean
-            data_to_clean[col].fillna(global_averages[col], inplace=True)
+            data_to_clean[col] = data_to_clean[col].fillna(global_averages[col])
 
         return data_to_clean
 

etl/epc/Dataset.py

@@ -203,11 +203,11 @@ class TrainingDataset(BaseDataset):
         common_cols = [[col + "_starting", col + "_ending"] for col in common_cols]
 
         self.df = self.df.loc[
-                  :,
-                  no_suffix_cols
-                  + only_ending_cols
-                  + [col for cols in common_cols for col in cols],
-                  ]
+            :,
+            no_suffix_cols
+            + only_ending_cols
+            + [col for cols in common_cols for col in cols],
+        ]
 
     def _remove_abnormal_change_in_floor_area(self):
         """
@@ -511,7 +511,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["is_sandstone_or_limestone"]
                     == expanded_df["is_sandstone_or_limestone_ending"]
                 )
-                ]
+            ]
         elif component == "floor":
             expanded_df = expanded_df[
                 (expanded_df["is_suspended"] == expanded_df["is_suspended_ending"])
@@ -528,7 +528,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["is_to_external_air"]
                     == expanded_df["is_to_external_air_ending"]
                 )
-                ]
+            ]
         elif component == "roof":
             expanded_df = expanded_df[
                 (expanded_df["is_pitched"] == expanded_df["is_pitched_ending"])
@@ -541,7 +541,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["has_dwelling_above"]
                     == expanded_df["has_dwelling_above_ending"]
                 )
-                ]
+            ]
 
         return expanded_df
 

etl/epc/Record.py

@@ -575,6 +575,8 @@ class EPCRecord:
         mains_gas_map = {
             "Y": True,
             "N": False,
+            True: True,
+            False: False
         }
 
         self.prepared_epc["mains-gas-flag"] = (

etl/epc/settings.py

@@ -182,7 +182,6 @@ EFFICIENCY_FEATURES = [
 
 ROOM_FEATURES = ["number_habitable_rooms", "number_heated_rooms"]
 
-
 COMPONENT_FEATURES = CORE_COMPONENT_FEATURES + [
     "TRANSACTION_TYPE",
     "ENERGY_TARIFF",  # Not sure if this is relevant
@@ -241,7 +240,11 @@ BUILT_FORM_REMAP = {
 DATA_PROCESSOR_SETTINGS = {
     "low_memory": False,
     "epc_minimum_count": 1,
-    "column_mappings": {"UPRN": [int, str]},
+    "column_mappings": {
+        "UPRN": [int, str],
+        "NUMBER_HEATED_ROOMS": [float],
+        "NUMBER_HABITABLE_ROOMS": [float],
+    },
 }
 
 # This has a manual mapping of the column types required

etl/epc_clean/app.py

@@ -44,7 +44,7 @@ def app():
         # Rename the columns to the same format as the api returns
         data.columns = [c.replace("_", "-").lower() for c in data.columns]
         # Take just date before the date threshold
-        data = data[data["lodgement-date"] >= EARLIEST_EPC_DATE]
+        data = data[data["lodgement-date"] >= "2011-01-01"]
 
         # Convert to list of dictioaries as returned by the api
         data = data.to_dict("records")

etl/epc_clean/epc_attributes/FloorAttributes.py

@@ -11,7 +11,7 @@ class FloorAttributes(Definitions):
     # For the short term, while we are still exploring the data, we maintain a list of error cases which
     # we want to ignore and consider as no data.
 
-    OBSERVED_ERRORS = ["Conservatory"]
+    OBSERVED_ERRORS = ["Conservatory", "insulated"]
 
     WELSH_TEXT = {
         "(anheddiad arall islaw)": "(another dwelling below)",
@@ -30,8 +30,10 @@ class FloorAttributes(Definitions):
         "i ofod heb ei wresogi, wedigçöi inswleiddio": "to unheated space, insulated",
         "solet, wedigçöi inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)",
         "solet, wedigçöi inswleiddio": "solid, insulated",
+        "solet, wedi???i inswleiddio (rhagdybiaeth)": "solid, insulated (assumed)",
         "i ofod heb ei wresogi, dim inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
-        "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation"
+        "i ofod heb ei wresogi, heb ei inswleiddio (rhagdybiaeth)": "to unheated space, no insulation (assumed)",
+        "i ofod heb ei wresogi, dim inswleiddio": "to unheated space, no insulation",
     }
 
     def __init__(self, description: str):

etl/epc_clean/epc_attributes/HotWaterAttributes.py

@@ -19,6 +19,7 @@ class HotWaterAttributes(Definitions):
         'solid fuel boiler',  # burns solid materials to generate heat for water heating and/or space heating
         'solid fuel range cooker',
         'room heaters',  # Generic/unspecified category
+        'electric multipoint',
     ]
 
     # SYSTEM_TYPES refer to the larger system within which the heater operates.
@@ -96,9 +97,11 @@ class HotWaterAttributes(Definitions):
 
     WELSH_TEXT = {
         "ogçör brif system": "from main system",
+        "o r brif system": "from main system",
         "ogçör brif system, adfer gwres nwyon ffliw": "from main system, flue gas heat recovery",
         "bwyler/cylchredydd nwy": "gas boiler/circulator",
         "ogçör brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat",
+        "o r brif system, dim thermostat ar y silindr": "from main system, no cylinder thermostat",
         "twymwr tanddwr, an-frig": "electric immersion, off-peak",
         "ogçör brif system, gydag ynnigçör haul": "from main system, plus solar",
         "twymwr tanddwr, tarriff safonol": "electric immersion, standard tariff",
@@ -124,13 +127,21 @@ class HotWaterAttributes(Definitions):
                                                                                    "thermostat, flue gas heat recovery",
         "ogçör brif system, gydag ynnigçör haul, adfer gwres nwyon ffliw": "from main system, plus solar, flue gas "
                                                                            "heat recovery",
+        "o r brif system, gydag ynni r haul, dim thermostat ar y silindr": "from main system, plus solar, no cylinder "
+                                                                           "thermostat",
+        "o r brif system, gydag ynni r haul": "from main system, plus solar",
     }
 
+    NODATA_DESCRIPTIONS = [
+        "sap05 hot-water",
+        "sap hot-water"
+    ]
+
     def __init__(self, description: str):
         self.description: str = clean_description(description.lower()).strip()
 
         self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or (
-            self.description == "sap05 hot-water"
+            self.description in self.NODATA_DESCRIPTIONS
         )
 
         translation = self.WELSH_TEXT.get(self.description)

etl/epc_clean/epc_attributes/LightingAttributes.py

@@ -7,6 +7,7 @@ from etl.epc_clean.utils import correct_spelling
 class LightingAttributes(Definitions):
     WELSH_TEXT = {
         "goleuadau ynni-isel ym mhob un ogçör mannau gosod": "low energy lighting in all fixed outlets",
+        "goleuadau ynni-isel ym mhob un o r mannau gosod": "low energy lighting in all fixed outlets",
         "dim goleuadau ynni-isel": "no low energy lighting",
         "goleuadau ynni-isel ym mhob un o'r mannau gosod": 'Low energy lighting in all fixed outlets'
     }

etl/epc_clean/epc_attributes/MainFuelAttributes.py

@@ -50,7 +50,8 @@ class MainFuelAttributes(Definitions):
 
     NO_INDIVIDUAL_HEATING_OR_COMMUNITY_NETWORK = [
         'to be used only when there is no heatinghotwater system or data is from a community network',
-        'to be used only when there is no heatinghotwater system'
+        'to be used only when there is no heatinghotwater system',
+        'community heating schemes waste heat from power stations',
     ]
 
     def __init__(self, description: str):

etl/epc_clean/epc_attributes/MainheatAttributes.py

@@ -34,7 +34,10 @@ class MainHeatAttributes(Definitions):
         "gwresogyddion ystafell, trydan": "room heaters, electric",
         "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan": "air source heat pump, underfloor heating, "
                                                                     "electric",
+        "pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, trydan, pwmp gwres sygçön tarddu yn yr awyr, dan y llawr, "
+        "trydan": "air source heat pump, underfloor heating, electric",
         "cynllun cymunedol": "community scheme",
+        "cynllun cymunedol, heat from boilers - mains gas": "community scheme",
         "bwyler a gwres dan y llawr, nwy prif gyflenwad": "boiler and underfloor heating, mains gas",
         "bwyler a rheiddiaduron, logiau coed": 'boiler and radiators, wood logs',
         "bwyler a rheiddiaduron, tanwydd di-fwg": "boiler and radiators, smokeless fuel",
@@ -59,6 +62,16 @@ class MainHeatAttributes(Definitions):
         "bwyler a rheiddiaduron, olew, st+¦r wresogyddion trydan": "boiler and radiators, oil, electric storage "
                                                                    "heaters",
         "pwmp gwres sygçön tarddu yn yr awyr, awyr gynnes, trydan": "air source heat pump, warm air, electric",
+        "stor wresogyddion trydan": "electric storage heaters",
+        # Not 100% certain - the translation is "bottled gas"
+        "bwyler a rheiddiaduron, nwy potel": "boiler and radiators, lpg",
+        "gwresogyddion trydan cludadwy wedi i ragdybio ar gyfer y rhan fwyaf o r ystafelloedd": "portable electric "
+                                                                                                "heaters assumed for "
+                                                                                                "most rooms",
+        "st r wresogyddion trydan": "electric storage heaters",
+        "dim system ar gael, rhagdybir bod gwresogyddion trydan, trydan": "no system present, electric heaters assumed",
+        # Should be handled by edge cases
+        ", trydan": ", electric",
     }
 
     REMAP = {
@@ -66,6 +79,13 @@ class MainHeatAttributes(Definitions):
         "electric heat pumps": "electric heat pump",
         "solar-assisted heat pump": "solar assisted heat pump",
         "portable electric heating": "portable electric heaters",
+        "portable electric heating assumed for most rooms": "portable electric heaters assumed for most rooms",
+        "electric storage, electric": "electric storage heaters",
+        "radiator heating, electric": "room heaters, electric",
+        "hot-water-only systems, gas": "no system present, electric heaters assumed",
+        "gas-fired heat pumps, electric": "air source heat pump, electric",
+        "radiator heating, heat from boilers - gas": "boiler and radiators, mains gas",
+        "heat pump, warm air, mains gas": "air source heat pump, warm air, mains gas",
     }
 
     edge_case_result = {}
@@ -97,6 +117,10 @@ class MainHeatAttributes(Definitions):
 
         self.description = remapped
 
+        backup_remap = self.REMAP.get(self.description)
+        if backup_remap:
+            self.description = backup_remap
+
         self.process_edge_cases()
 
         if not self.nodata:
@@ -138,6 +162,21 @@ class MainHeatAttributes(Definitions):
             self.is_edge_case = True
             return
 
+        if self.description == ', electric':
+            self.edge_case_result['has_electric'] = True
+            self.is_edge_case = True
+            return
+
+        if self.description == ', mains gas':
+            self.edge_case_result['has_mains_gas'] = True
+            self.is_edge_case = True
+            return
+
+        if self.description == 'community, community':
+            self.edge_case_result['has_community_scheme'] = True
+            self.is_edge_case = True
+            return
+
     def process(self) -> Dict[str, Union[str, bool]]:
 
         result: Dict[str, Union[str, bool]] = {f'has_{ds.replace(" ", "_")}': False for ds in self.DISTRIBUTION_SYSTEMS}

etl/epc_clean/epc_attributes/MainheatControlAttributes.py

@@ -75,6 +75,8 @@ class MainheatControlAttributes(Definitions):
     TO_REMAP = {
         "celect control": 'celect-type control',
         "celect controls": 'celect-type control',
+        "trv's, program & flow switch": 'trvs, programmer & flow switch',
+        'appliance thermostat': 'appliance thermostats',
     }
 
     WELSH_TEXT = {
@@ -113,12 +115,20 @@ class MainheatControlAttributes(Definitions):
         't+ól un gyfradd, trvs': 'single rate heating, trvs',
         'trvs a falf osgoi': 'trvs and bypass',
         'rheolaeth celect': 'celect-type control',
+        'rheoli r tal a llaw': 'manual charge control',
+        'tal un gyfradd, thermostat ystafell yn unig': 'flat rate charging, room thermostat only',
+        "rheoli'r t l llaw": "manual charge control",
     }
 
+    NO_DATA_DESCRIPTIONS = [
+        "SAP05:Main-Heating-Controls",
+        "SAP:Main-Heating-Controls",
+    ]
+
     def __init__(self, description: str):
         self.description: str = clean_description(description.lower()).strip()
         self.nodata = not self.description or description in self.DATA_ANOMALY_MATCHES or (
-            description == "SAP05:Main-Heating-Controls"
+            description in self.NO_DATA_DESCRIPTIONS
         )
 
         translation = self.WELSH_TEXT.get(self.description)

etl/epc_clean/epc_attributes/RoofAttributes.py

@@ -6,31 +6,40 @@ from etl.epc_clean.epc_attributes.attribute_utils import extract_component_types
 
 class RoofAttributes(Definitions):
     ROOF_TYPES = ['pitched', 'roof room', 'loft', 'flat', 'thatched', 'at rafters', 'assumed']
-    DWELLING_ABOVE = ["another dwelling above", "other premises above"]
+    DWELLING_ABOVE = ["another dwelling above", "other premises above", "other dwelling above"]
 
     WELSH_TEXT = {
         "ar oleddf, dim inswleiddio": "pitched, no insulation",
         "ar oleddf, dim inswleiddio (rhagdybiaeth)": "pitched, no insulation (assumed)",
         "ar oleddf, wedigçöi inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)",
+        "ar oleddf, wedi?i inswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)",
+        "ar oleddf, wedigçöi hinswleiddio (rhagdybiaeth)": "pitched, insulated (assumed)",
         "ar oleddf, wedigçöi inswleiddio": "pitched, insulated",
+        "ar oleddf, wedi?i inswleiddio": "pitched, insulated",
         "ar oleddf, inswleiddio cyfyngedig (rhagdybiaeth)": "pitched, limited insulation (assumed)",
         "ar oleddf, inswleiddio cyfyngedig": "pitched, limited insulation",
         "ar oleddf, wedigçöi inswleiddio wrth y trawstiau": 'pitched, insulated at rafters',
+        "ar oleddf, wedi?i inswleiddio wrth y trawstiau": 'pitched, insulated at rafters',
+        "ar oleddf, wedi?i inswleiddio wrth y trawstia": 'pitched, insulated at rafters',
+        "ar oleddf, wedigçöi inswleiddio wrth y trawstia": 'pitched, insulated at rafters',
         "yn wastad, inswleiddio cyfyngedig (rhagdybiaeth)": "flat, limited insulation (assumed)",
         "yn wastad, inswleiddio cyfyngedig": "flat, limited insulation",
         "yn wastad, dim inswleiddio (rhagdybiaeth)": "flat, no insulation (assumed)",
         "yn wastad, dim inswleiddio": "flat, no insulation",
         "yn wastad, wedigçöi inswleiddio (rhagdybiaeth)": "flat, insulated (assumed)",
+        "yn wastad, wedi?i hinswleiddio (rhagdybiaeth)": "flat, insulated (assumed)",
         "yn wastad, wedigçöi inswleiddio": "flat, insulated",
         "(eiddo arall uwchben)": "(another dwelling above)",
         "(annedd arall uwchben)": "(another dwelling above)",
         "ystafell(oedd) to, wedigçöi hinswleiddio": "roof room(s), insulated",
+        "ystafell(oedd) to, wedi?i hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)",
         "ystafell(oedd) to, wedigçöi hinswleiddio (rhagdybiaeth)": "roof room(s), insulated (assumed)",
         "ystafell(oedd) to, inswleiddio cyfyngedig (rhagdybiaeth)": "roof room(s), limited insulation (assumed)",
         "ystafell(oedd) to, inswleiddio cyfyngedig": "roof room(s), limited insulation",
         "ystafell(oedd) to, nenfwd wedigçöi inswleiddio": "roof room(s), ceiling insulated",
         "ystafell(oedd) to, dim inswleiddio (rhagdybiaeth)": "roof room(s), no insulation (assumed)",
         "ystafell(oedd) to, dim inswleiddio": "roof room(s), no insulation",
+        "to gwellt, gydag inswleiddio ychwanegol": "thatched, with additional insulation",
     }
 
     DEFAULT_KEYS = [
@@ -62,10 +71,18 @@ class RoofAttributes(Definitions):
         search for regular expressions and translate
         """
 
-        loft_insulation_thickness_match = re.search(r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft", self.description)
-        loft_insulation_thickness_match2 = re.search(r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof", self.description)
-        loft_insulation_thickness_match3 = re.search(r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof",
-                                                     self.description)
+        loft_insulation_regexes = [
+            r"ar oleddf, (\d+ mm) o inswleiddio yn y llofft",
+            r"ar oleddf, (\d+ mm) lo inswleiddio yn y llof",
+            r"ar oleddf, (\d+\+ mm) lo inswleiddio yn y llof",
+            r"ar oleddf, (\d+mm) o inswleiddio yn y llofft",
+            r"ar oleddf, (\d+\+ mm) o inswleiddio yn y llofft"
+        ]
+        li_thickness_match = None
+        for regex in loft_insulation_regexes:
+            li_thickness_match = re.search(regex, self.description)
+            if li_thickness_match:
+                break
 
         uvalue_search = re.search(r"trawsyriannedd thermol cyfartalog (\d+(\.\d+)?)\s*w/m-¦k", self.description)
         uvalue_search2 = re.search(
@@ -73,15 +90,8 @@ class RoofAttributes(Definitions):
         )
 
         # Step 2: Generalized translation with placeholder
-        if (loft_insulation_thickness_match is not None) | \
-            (loft_insulation_thickness_match2 is not None) | \
-            (loft_insulation_thickness_match3 is not None):
-            if loft_insulation_thickness_match is not None:
-                insulation_thickness = loft_insulation_thickness_match.group(1)
-            elif loft_insulation_thickness_match2 is not None:
-                insulation_thickness = loft_insulation_thickness_match2.group(1)
-            else:
-                insulation_thickness = loft_insulation_thickness_match3.group(1)
+        if li_thickness_match is not None:
+            insulation_thickness = li_thickness_match.group(1)
 
             self.description = f"pitched, {insulation_thickness} loft insulation"
         elif uvalue_search is not None or uvalue_search2 is not None:
@@ -113,9 +123,8 @@ class RoofAttributes(Definitions):
         # roof type
         result, description = extract_component_types(result, description, list_of_components=self.ROOF_TYPES)
 
-        result["has_dwelling_above"] = (
-            "another dwelling above" in description or "other premises above" in description
-        )
+        result["has_dwelling_above"] = any([x in description for x in self.DWELLING_ABOVE])
+
         for dwelling_above in self.DWELLING_ABOVE:
             description = description.replace(dwelling_above, "")
 

etl/epc_clean/epc_attributes/WindowAttributes.py

@@ -27,18 +27,26 @@ class WindowAttributes(Definitions):
         "gwydrau triphlyg llawn": "fully triple glazed",
         "gwydrau triphlyg rhannol": "partial triple glazed",
         "gwydrau triphlyg mwyaf": "mostly triple glazed",
+        "gwydrau triphlyg gan mwyaf": "mostly triple glazed",
         "gwydrau eilaidd llawn": "full secondary glazing",
         "gwydrau eilaidd mwyaf": "mostly secondary glazing",
         "gwydrau eilaidd rhannol": "partial secondary glazing",
         "gwydrau lluosog ym mhobman": "multiple glazing throughout",
     }
 
+    # These are observed data anomalies that we want to ignore
+    NO_DATA_CASES = [
+        "SAP05:Windows",
+        "Solid, no insulation (assumed)",  # A description typically associated with floors, not windows
+        "Suspended, no insulation (assumed)",  # A description typically associated with floors, not windows
+    ]
+
     def __init__(self, description: str):
         self.description: str = clean_description(description.lower())
 
         # In the case of an empty description, we want to return a dictionary with all values set to False
         # and indicate there was no data
-        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description == "SAP05:Windows"
+        self.nodata = not description or description in self.DATA_ANOMALY_MATCHES or description in self.NO_DATA_CASES
 
         translation = self.WELSH_TEXT.get(self.description)
         if translation:

etl/epc_clean/tests/test_data/test_floor_attributes_cases.py

@@ -367,7 +367,7 @@ clean_floor_cases = [
      'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True, 'is_to_external_air': False,
      'is_suspended': False, 'is_solid': False, 'insulation_thickness': 'none', "another_property_below": False},
     {'original_description': "Average thermal transmittance 1.10 W/m+é-¦K", 'thermal_transmittance': 1.1,
-     'thermal_transmittance_unit': 'w/m+é-¦k', 'is_assumed': False,
+     'thermal_transmittance_unit': 'w/m-¦k', 'is_assumed': False,
      'is_to_unheated_space': False, 'is_to_external_air': False, 'is_suspended': False, 'is_solid': False,
      'another_property_below': False, 'insulation_thickness': None},
     {

etl/epc_clean/tests/test_data/test_hot_water_attributes_cases.py

@@ -219,4 +219,9 @@ hotwater_cases = [
      'heater_type': 'electric instantaneous', 'system_type': None, 'thermostat_characteristics': None,
      'heating_scope': None, 'energy_recovery': 'waste water heat recovery', 'tariff_type': None, 'extra_features': None,
      'chp_systems': None, 'distribution_system': None, 'no_system_present': None, 'assumed': False, "appliance": None},
+    {'original_description': 'Electric multipoint', 'heater_type': 'electric multipoint', 'system_type': None,
+     'thermostat_characteristics': None,
+     'heating_scope': None, 'energy_recovery': None, 'tariff_type': None, 'extra_features': None, 'chp_systems': None,
+     'distribution_system': None, 'no_system_present': None, 'appliance': None, 'assumed': False}
+
 ]

etl/epc_clean/tests/test_data/test_mainheat_attributes_cases.py

@@ -1658,11 +1658,55 @@ mainheat_cases = [
      'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
      'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
      'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
-     'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+     'has_portable_electric_heaters': True, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
      'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
-     'has_community_heat_pump': False, 'has_portable_electric_heating': True, 'has_electric': True,
+     'has_community_heat_pump': False, 'has_electric': True,
      'has_mains_gas': False, 'has_wood_logs': False, 'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
      'has_anthracite': False, 'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False,
      'has_b30k': False, 'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': True,
-     'has_underfloor_heating': False}
+     'has_underfloor_heating': False},
+    {'original_description': 'Radiator heating, electric', 'has_radiators': False, 'has_fan_coil_units': False,
+     'has_pipes_in_screed_above_insulation': False,
+     'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+     'has_air_source_heat_pump': False, 'has_room_heaters': True, 'has_electric_storage_heaters': False,
+     'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+     'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+     'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+     'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+     'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False,
+     'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+     'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+     'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False,
+     'has_underfloor_heating': False},
+    {
+        'original_description': 'Hot-Water-Only Systems, gas',
+        'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+        'has_air_source_heat_pump': False, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+        'has_warm_air': False, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': True,
+        'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+        'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+        'has_community_heat_pump': False, 'has_electric': True, 'has_mains_gas': False, 'has_wood_logs': False,
+        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+        'has_assumed': True, 'has_electricaire': False, 'has_assumed_for_most_rooms': False,
+        'has_underfloor_heating': False
+    },
+    {
+        "original_description": "heat pump, warm air, mains gas",  # This gets remapped to air source heat pump
+        'has_radiators': False, 'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
+        'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': False,
+        'has_air_source_heat_pump': True, 'has_room_heaters': False, 'has_electric_storage_heaters': False,
+        'has_warm_air': True, 'has_electric_underfloor_heating': False, 'has_electric_ceiling_heating': False,
+        'has_community_scheme': False, 'has_ground_source_heat_pump': False, 'has_no_system_present': False,
+        'has_portable_electric_heaters': False, 'has_water_source_heat_pump': False, 'has_electric_heat_pump': False,
+        'has_micro-cogeneration': False, 'has_solar_assisted_heat_pump': False, 'has_exhaust_source_heat_pump': False,
+        'has_community_heat_pump': False, 'has_electric': False, 'has_mains_gas': True, 'has_wood_logs': False,
+        'has_coal': False, 'has_oil': False, 'has_wood_pellets': False, 'has_anthracite': False,
+        'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False, 'has_lpg': False, 'has_b30k': False,
+        'has_assumed': False, 'has_electricaire': False, 'has_assumed_for_most_rooms': False,
+        'has_underfloor_heating': False
+
+    }
 ]

etl/epc_clean/tests/test_data/test_roof_attributes_cases.py

@@ -397,7 +397,7 @@ clean_roof_test_cases = [
      'is_thatched': False, 'is_at_rafters': False, 'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
      'insulation_thickness': 'none'},
     {'original_description': 'Average thermal transmittance 0.80 W/m+é-¦K', 'thermal_transmittance': 0.8,
-     'thermal_transmittance_unit': 'w/m+é-¦k', 'is_pitched': False, 'is_roof_room': False,
+     'thermal_transmittance_unit': 'w/m-¦k', 'is_pitched': False, 'is_roof_room': False,
      'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False, 'is_assumed': False,
      'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': None}
 ]

etl/epc_clean/tests/test_data/test_wall_attributes_cases.py

@@ -1,5 +1,5 @@
 wall_cases = [
-    {'original_description': 'Average thermal transmittance -4.67 W/m-¦K', 'thermal_transmittance': -4.67,
+    {'original_description': 'Average thermal transmittance -4.67 W/m-¦K', 'thermal_transmittance': 4.67,
      'thermal_transmittance_unit': 'w/m-¦k', 'is_cavity_wall': False, 'is_filled_cavity': False,
      'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False, 'is_granite_or_whinstone': False,
      'is_as_built': False, 'is_cob': False, 'is_assumed': False, 'is_sandstone_or_limestone': False,
@@ -692,7 +692,7 @@ wall_cases = [
      'is_cob': False, 'is_assumed': True, 'is_sandstone_or_limestone': False, 'insulation_thickness': 'none',
      'external_insulation': False, 'internal_insulation': False},
     {'original_description': 'Average thermal transmittance 1.60 W/m+é-¦K',
-     'thermal_transmittance': 1.6, 'thermal_transmittance_unit': 'w/m+é-¦k', 'is_cavity_wall': False,
+     'thermal_transmittance': 1.6, 'thermal_transmittance_unit': 'w/m-¦k', 'is_cavity_wall': False,
      'is_filled_cavity': False, 'is_solid_brick': False, 'is_system_built': False, 'is_timber_frame': False,
      'is_granite_or_whinstone': False, 'is_as_built': False, 'is_cob': False, 'is_assumed': False,
      'is_sandstone_or_limestone': False, 'insulation_thickness': None, 'external_insulation': False,

etl/epc_clean/tests/test_mainheat_attributes.py

@@ -11,10 +11,6 @@ class TestMainHeatAttributes:
         floor_attr = MainHeatAttributes(valid_description)
         assert floor_attr.description == valid_description.lower()
 
-        # Test initialization with an empty description
-        with pytest.raises(ValueError):
-            MainHeatAttributes('')
-
         # Test initialization with a description that contains none of the keywords
         with pytest.raises(ValueError):
             MainHeatAttributes('description without keywords')
@@ -38,7 +34,6 @@ class TestMainHeatAttributes:
     def test_invalid_description(self):
         # Test that invalid descriptions raise a ValueError
         invalid_descriptions = [
-            "",
             "invalid description",
             "description with no known heating data_types",
         ]

etl/epc_clean/tests/test_wall_attributes.py

@@ -16,7 +16,7 @@ class TestWallAttributes:
         description = 'average thermal transmittance -4.67 w/m-¦k'
         wa = wall_attr(description)
         result = wa.process()
-        assert result['thermal_transmittance'] == -4.67
+        assert result['thermal_transmittance'] == 4.67
         assert result['thermal_transmittance_unit'] == 'w/m-¦k'
 
     def test_wall_types(self, wall_attr):

etl/ownership/Ownership.py

@@ -61,6 +61,7 @@ class Ownership:
         portfolio_value: float,
         excluded_owners: List[str] = None,
         excluded_uprns: List[int] = None,
+        save=True
     ):
         """
 
@@ -115,6 +116,8 @@ class Ownership:
             f"ownership/{self.project_name}/{self.run_timestamp}/portfolio_epc_data.xlsx"
         )
 
+        self.save = save
+
         # Data
         self.epc_data = None
         self.ownership_data = None
@@ -158,21 +161,22 @@ class Ownership:
         # Step 5: Match land registry data to existing matches
         self.match_with_land_registry()
         # We store this data in s3 before we perform any filtering
-        save_excel_to_s3(
-            df=self.matched_addresses,
-            bucket_name=self.bucket,
-            file_key=self.matched_addresses_pre_filter_filepath
-        )
-        save_excel_to_s3(
-            df=self.combined_matching_lookup,
-            bucket_name=self.bucket,
-            file_key=self.combined_matching_lookup_pre_filter_filepath
-        )
+        if self.save:
+            save_excel_to_s3(
+                df=self.matched_addresses,
+                bucket_name=self.bucket,
+                file_key=self.matched_addresses_pre_filter_filepath
+            )
+            save_excel_to_s3(
+                df=self.combined_matching_lookup,
+                bucket_name=self.bucket,
+                file_key=self.combined_matching_lookup_pre_filter_filepath
+            )
 
         # Prepare the final outputs:
         self.create_final_matches()
 
-    def source_epc_properties(self, column_filters=None):
+    def source_epc_properties(self, column_filters=None, postcodes=None):
         """
         This function will filter the epc data as specified by column filters, searching across all of the EPC tables
         :param column_filters: Dictionary with column names as keys and list of acceptable values as values. This
@@ -180,6 +184,7 @@ class Ownership:
                                 {"column_name": ["value1", "value2", ...]}, where column_name is the name of the column
                                 in the EPC data and ["value1", "value2", ...] is a list of acceptable values for that
                                 column. If a column is not found in the EPC data, an exception is raised.
+        :param postcodes: A list of postcodes to filter the data on
         """
 
         column_filters = {} if column_filters is None else column_filters
@@ -203,6 +208,11 @@ class Ownership:
                 else:
                     raise Exception(f"Column {column} not found in data. column_filters is malformed")
 
+            if postcodes is not None:
+                epc_data = epc_data[epc_data["POSTCODE"].str.lower().isin(postcodes)]
+            if epc_data.empty:
+                continue
+
             data.append(epc_data)
 
         self.epc_data = pd.concat(data, ignore_index=True)
@@ -210,12 +220,13 @@ class Ownership:
         if self.excluded_uprns:
             self.epc_data = self.epc_data[~self.epc_data["UPRN"].astype(float).isin(self.excluded_uprns)]
 
-        # We now store the data in s3
-        save_excel_to_s3(
-            df=self.epc_data,
-            bucket_name=self.bucket,
-            file_key=self.epc_data_filepath
-        )
+        if self.save:
+            # We now store the data in s3
+            save_excel_to_s3(
+                df=self.epc_data,
+                bucket_name=self.bucket,
+                file_key=self.epc_data_filepath
+            )
 
     def load_company_ownership(self):
         """
@@ -484,11 +495,11 @@ class Ownership:
                 house_no = house_no.replace(",", "")
 
             if house_no is None:
-                # It's hard for us to get a reliable match
-                # filtered = filtered[filtered["Property Address"].str.contains(address["ADDRESS1"])]
-                # if filtered.shape[0] > 1:
-                #     raise Exception("No valid - maybe we should do levenstein?")
-                continue
+                # If the house number is missing, it means that we usually have a named property so we look for an
+                # exact match on that name
+                filtered = filtered[filtered["Property Address"].str.lower().str.contains(address["ADDRESS"].lower())]
+                if filtered.shape[0] != 1:
+                    continue
 
             else:
 
@@ -590,7 +601,8 @@ class Ownership:
                     "CURRENT_ENERGY_RATING",
                     "POSTCODE",
                     "LODGEMENT_DATE",
-                    "TRANSACTION_TYPE"
+                    "TRANSACTION_TYPE",
+                    "TENURE",
                 ]
             ].rename(
                 columns={
@@ -1002,25 +1014,26 @@ class Ownership:
         if self.portfolio_properties["UPRN"].nunique() != self.portfolio_epc_data["UPRN"].nunique():
             raise ValueError("Portfolio properties and epc data don't match")
 
-        logger.info("Storing final outpus")
-        # Store data
-        save_excel_to_s3(
-            df=self.portfolio_owners,
-            bucket_name=self.bucket,
-            file_key=self.portfolio_owners_filepath,
-        )
+        if self.save:
+            logger.info("Storing final outpus")
+            # Store data
+            save_excel_to_s3(
+                df=self.portfolio_owners,
+                bucket_name=self.bucket,
+                file_key=self.portfolio_owners_filepath,
+            )
 
-        save_excel_to_s3(
-            df=self.portfolio_properties,
-            bucket_name=self.bucket,
-            file_key=self.portfolio_properties_filepath,
-        )
+            save_excel_to_s3(
+                df=self.portfolio_properties,
+                bucket_name=self.bucket,
+                file_key=self.portfolio_properties_filepath,
+            )
 
-        save_excel_to_s3(
-            df=self.portfolio_epc_data,
-            bucket_name=self.bucket,
-            file_key=self.portfolio_epc_data_filepath,
-        )
+            save_excel_to_s3(
+                df=self.portfolio_epc_data,
+                bucket_name=self.bucket,
+                file_key=self.portfolio_epc_data_filepath,
+            )
 
     def get_asset_list(self):
         """

etl/sfr/midlands_portfolio_asset_list.py

@@ -0,0 +1,52 @@
+import pandas as pd
+from utils.s3 import save_csv_to_s3
+
+
+def app():
+    """
+    This script sets up
+    :return:
+    """
+
+    portfolio_id = 108
+
+    # Read in the portfolio EPC data
+    epc_data = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/20240820 portfolio_epc_data.xlsx"
+    )
+
+    asset_list = epc_data[
+        [
+            "ADDRESS1", "POSTCODE", "UPRN"
+        ]
+    ].copy().rename(
+        columns={
+            "ADDRESS1": "address",
+            "POSTCODE": "postcode",
+            "UPRN": "uprn"
+        }
+    )
+
+    # Store data and prepare payload
+
+    filename = f"{8}/{portfolio_id}/asset_list.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    body = {
+        "portfolio_id": str(portfolio_id),
+        "housing_type": "Private",
+        "goal": "Increasing EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": "",
+        "patches_file_path": "",
+        "non_invasive_recommendations_file_path": "",
+        "budget": None,
+        "scenario_name": "EPC C Package",
+        "multi_plan": True,
+    }
+    print(body)

etl/sfr/midlands_portfolio_est_funding.py

@@ -0,0 +1,209 @@
+import msgpack
+
+import pandas as pd
+from utils.s3 import read_from_s3
+from recommendations.recommendation_utils import (
+    estimate_number_of_floors, esimtate_pitched_roof_area, estimate_external_wall_area, estimate_perimeter
+)
+
+
+def app():
+    """
+    Aims to estimate the amount of GBIS funding eligible
+    :return:
+    """
+
+    cleaned = read_from_s3(
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name="retrofit-data-dev"
+    )
+
+    cleaned = msgpack.unpackb(cleaned, raw=False)
+
+    epc_data = pd.read_excel(
+        "/Users/khalimconn-kowlessar/Downloads/20240820 portfolio_epc_data.xlsx"
+    )
+
+    # For simplicity, get roofs or cavities
+    epc_data = epc_data.merge(
+        pd.DataFrame(cleaned["roof-description"]),
+        how="left",
+        left_on="ROOF_DESCRIPTION",
+        right_on="original_description"
+    )
+
+    epc_data["needs_roof_work"] = epc_data["insulation_thickness"].isin(
+        [
+            None,
+            "100",
+            '150',
+            '50',
+            '75',
+            'below average',
+            '25',
+            '12'
+        ]
+    ) & (epc_data["is_flat"] | epc_data["is_pitched"])
+
+    epc_data = epc_data.merge(
+        pd.DataFrame(cleaned["walls-description"]),
+        how="left",
+        left_on="WALLS_DESCRIPTION",
+        right_on="original_description",
+        suffixes=("", "_wall")
+    )
+
+    epc_data["needs_cavity_done"] = epc_data["is_cavity_wall"] & epc_data["insulation_thickness_wall"].isin(
+        ['none', "below average"]
+    )
+
+    epc_data["needs_solid_wall"] = (epc_data["is_solid_brick"] | epc_data["is_system_built"]) & epc_data[
+        "insulation_thickness_wall"].isin(['none', "below average"])
+
+    epc_data["could_take_solar"] = (epc_data["is_flat"] | epc_data["is_pitched"])
+
+    loft_insulation_per_m2 = 16.07
+    flat_roof_insulation_per_m2 = 195
+    cwi_per_m2 = 14.21
+    ewi_per_m2 = 200
+    gbis_abs = 30
+    eco4_abs = 24
+    solar_pv_cost = 4009
+
+    # We assume the work will take the home from a high D to a low D
+    def get_abs(floor_area):
+        if floor_area <= 72:
+            return 155
+
+        if floor_area <= 97:
+            return 169
+
+        if floor_area <= 199:
+            return 196.4
+
+        return 350.1
+
+    # We assume the work will take the home from a high E to a high C
+    def get_eco4_abs(floor_area):
+        if floor_area <= 72:
+            return 596.6
+
+        if floor_area <= 97:
+            return 650.2
+
+        if floor_area <= 199:
+            return 755.8
+
+        return 1347.1
+
+    estimated_costs = []
+    for _, home in epc_data.iterrows():
+        to_append = {
+            "uprn": home["UPRN"],
+            "address": home["ADDRESS"],
+            "postcode": home["POSTCODE"],
+        }
+
+        project_abs = get_abs(home["TOTAL_FLOOR_AREA"])
+        available_funding = project_abs * gbis_abs
+
+        n_floors = estimate_number_of_floors(home["PROPERTY_TYPE"])
+        floor_height = float(home["FLOOR_HEIGHT"]) if not pd.isnull(home["FLOOR_HEIGHT"]) else 2.5
+
+        # We estimate the amount of insulation required
+        est_perimeter = estimate_perimeter(
+            floor_area=float(home["TOTAL_FLOOR_AREA"]) / n_floors,
+            num_rooms=float(home["NUMBER_HABITABLE_ROOMS"]) / n_floors
+        )
+
+        insulation_needed = estimate_external_wall_area(
+            num_floors=n_floors,
+            floor_height=floor_height,
+            perimeter=est_perimeter,
+            built_form=home["BUILT_FORM"],
+        )
+
+        # At the very least we'll need solid wall + solar
+        if home["needs_solid_wall"] and home["could_take_solar"]:
+            measure = "EWI + Solar"
+
+            total_cost = insulation_needed * ewi_per_m2 + solar_pv_cost
+
+            eco4_project_abs = get_eco4_abs(home["TOTAL_FLOOR_AREA"])
+            eco4_available_funding = eco4_project_abs * eco4_abs
+
+            cost_of_work_after_funding = total_cost - eco4_available_funding
+            cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding
+
+            to_append = {
+                **to_append,
+                "scheme": "eco4",
+                "available_funding": eco4_available_funding,
+                "measure": measure,
+                "project_abs": eco4_project_abs,
+                "cost_of_work": total_cost,
+                "cost_of_work_after_funding": cost_of_work_after_funding,
+            }
+
+            estimated_costs.append(to_append)
+            continue
+
+        # Check if it needs the walls done
+        if home["needs_cavity_done"]:
+            cost_of_insulation = insulation_needed * cwi_per_m2
+
+            cost_of_work_after_funding = cost_of_insulation - available_funding
+            cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding
+
+            to_append = {
+                **to_append,
+                "scheme": "gbis",
+                "available_funding": available_funding,
+                "measure": "Cavity Wall Insulation",
+                "project_abs": project_abs,
+                "cost_of_work": cost_of_insulation,
+                "cost_of_work_after_funding": cost_of_work_after_funding
+            }
+
+            estimated_costs.append(to_append)
+            continue
+
+        if home["needs_roof_work"]:
+            # We estimate how much the cost of insulation would be
+            if home["is_pitched"]:
+                measure = "Loft Insulation"
+
+                roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors
+                cost_of_insulation = roof_area * loft_insulation_per_m2
+            else:
+                measure = "Flat Roof Insulation"
+                roof_area = float(home["TOTAL_FLOOR_AREA"]) / n_floors
+                cost_of_insulation = roof_area * flat_roof_insulation_per_m2
+
+            cost_of_work_after_funding = cost_of_insulation - available_funding
+            cost_of_work_after_funding = 0 if cost_of_work_after_funding < 0 else cost_of_work_after_funding
+
+            to_append = {
+                **to_append,
+                "scheme": "gbis",
+                "available_funding": available_funding,
+                "measure": measure,
+                "project_abs": project_abs,
+                "cost_of_work": cost_of_insulation,
+                "cost_of_work_after_funding": cost_of_work_after_funding
+            }
+
+            estimated_costs.append(to_append)
+            continue
+
+    estimated_costs = pd.DataFrame(estimated_costs)
+
+    estimated_costs.to_csv("/Users/khalimconn-kowlessar/Documents/hestia/Customers/sfr/estimated_costs_gbis.csv")
+
+    # epc_data[["UPRN", "ADDRESS", "POSTCODE"]].to_csv(
+    #     "/Users/khalimconn-kowlessar/Documents/hestia/sfr/council_tax_bands_sample.csv")
+
+    n_properties_for_ashp = epc_data[
+        (epc_data["PROPERTY_TYPE"] == "House") &
+        (epc_data["BUILT_FORM"].isin(["Detached", "Semi-Detached"]))
+        ].shape[0]

etl/spatial/OpenUprnClient.py

@@ -1,10 +1,10 @@
 import os
 from tqdm import tqdm
 import pandas as pd
-import geopandas as gpd
 from utils.logger import setup_logger
 from utils.s3 import read_io_from_s3, save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet
 from backend.Property import Property
+from backend.SearchEpc import SearchEpc
 
 logger = setup_logger()
 
@@ -85,17 +85,6 @@ class OpenUprnClient:
                 return filename
         return None
 
-    @staticmethod
-    def convert_bng_data_to_gpd(df):
-
-        gpd_data = gpd.GeoDataFrame(
-            df,
-            geometry=gpd.points_from_xy(df.X_COORDINATE, df.Y_COORDINATE),
-            crs="EPSG:27700"  # British National Grid
-        )
-
-        return gpd_data
-
     def save_filenames_to_s3(self, bucket_name):
         """
         Save the filenames to s3
@@ -151,7 +140,7 @@ class OpenUprnClient:
             bucket_name=bucket_name, file_key="spatial/filename_meta.parquet"
         )
 
-        uprns = [p.uprn for p in input_properties]
+        uprns = [p.uprn for p in input_properties if p.uprn_source != SearchEpc.UPRN_SOURCE_SIMULATED]
         uprn_map = cls.make_uprn_map(uprns, uprn_filenames)
 
         for filename, associated_uprn in tqdm(uprn_map.items(), total=len(uprn_map)):
@@ -165,6 +154,9 @@ class OpenUprnClient:
                 if p.uprn in associated_uprn:
                     p.set_spatial(spatial_df[spatial_df["UPRN"] == p.uprn])
 
+                if p.uprn_source == SearchEpc.UPRN_SOURCE_SIMULATED:
+                    p.set_spatial(cls.empty_spatial_df())
+
         # Perform a final check to ensure that all properties have spatial data
         for p in input_properties:
             if p.spatial is None:
@@ -172,6 +164,22 @@ class OpenUprnClient:
 
         return input_properties
 
+    @staticmethod
+    def empty_spatial_df():
+        return pd.DataFrame(
+            [
+                {
+                    "X_COORDINATE": None,
+                    "Y_COORDINATE": None,
+                    "LATITUDE": None,
+                    "LONGITUDE": None,
+                    "conservation_status": False,
+                    "is_listed_building": False,
+                    "is_heritage_building": False,
+                }
+            ]
+        )
+
     @classmethod
     def get_spatial_data(cls, uprns: list[int], bucket_name):
         """

etl/spatial/app.py

@@ -6,6 +6,7 @@ our database for querying from other services
 import os
 from tqdm import tqdm
 import pandas as pd
+import geopandas as gpd
 from etl.spatial.ConservationAreaClient import ConservationAreaClient
 from etl.spatial.OpenUprnClient import OpenUprnClient
 from etl.spatial.SpecialBuildingsClient import SpecialBuildingsClient
@@ -25,6 +26,16 @@ HISTORIC_ENGLAND_HERITAGE_BUILDINGS_PATHNAME = \
 logger = setup_logger()
 
 
+def convert_bng_data_to_gpd(df):
+    gpd_data = gpd.GeoDataFrame(
+        df,
+        geometry=gpd.points_from_xy(df.X_COORDINATE, df.Y_COORDINATE),
+        crs="EPSG:27700"  # British National Grid
+    )
+
+    return gpd_data
+
+
 def app():
     """
     This application uses the conservation area datasets to determine if a UPRN is
@@ -85,7 +96,7 @@ def app():
     to_loop_over = open_uprn_client.data.groupby("filename")
 
     for filename, uprn_df in tqdm(open_uprn_client.data.groupby("filename"), total=len(to_loop_over)):
-        uprn_gdf = OpenUprnClient.convert_bng_data_to_gpd(uprn_df)
+        uprn_gdf = convert_bng_data_to_gpd(uprn_df)
 
         uprn_gdf = conservation_area_client.is_in_conservation_area_vectorised(uprn_gdf=uprn_gdf)
         uprn_gdf = special_buildings_client.is_listed_building_vectorised(uprn_gdf=uprn_gdf)

model_data/requirements/requirements.txt

@@ -1,8 +1,9 @@
-pydantic==1.10.11
+pydantic==2.9.2
+pydantic-settings==2.6.0
 epc-api-python==1.0.2
-pandas==2.0.3
-numpy==1.25.1
-pytz==2023.3
+numpy==2.1.2
+pandas==2.2.3
+pytz==2024.2
 tzdata==2023.3
 tqdm
 mypy
@@ -20,4 +21,6 @@ pyspellchecker
 textblob
 boto3
 pyarrow
-msgpack==1.0.5
+msgpack==1.1.0
+
+

recommendations/Costs.py

@@ -37,6 +37,37 @@ MCS_SOLAR_PV_COST_DATA = {
     "average_cost_per_kwh-Northern Ireland": 1347,
 }
 
+INSTALLER_SOLAR_COSTS = [
+    {'n_panels': 4, 'array_kwp': 1.6, 'cost': 3040.00, 'installer': 'CEG'},
+    {'n_panels': 5, 'array_kwp': 2.1, 'cost': 3201.00, 'installer': 'CEG'},
+    {'n_panels': 6, 'array_kwp': 2.5, 'cost': 3363.00, 'installer': 'CEG'},
+    {'n_panels': 7, 'array_kwp': 2.9, 'cost': 3524.00, 'installer': 'CEG'},
+    {'n_panels': 8, 'array_kwp': 3.3, 'cost': 3686.00, 'installer': 'CEG'},
+    {'n_panels': 9, 'array_kwp': 3.7, 'cost': 3847.00, 'installer': 'CEG'},
+    {'n_panels': 10, 'array_kwp': 4.1, 'cost': 4009.00, 'installer': 'CEG'},
+    {'n_panels': 11, 'array_kwp': 4.5, 'cost': 4170.00, 'installer': 'CEG'},
+    {'n_panels': 12, 'array_kwp': 4.9, 'cost': 4332.00, 'installer': 'CEG'},
+    {'n_panels': 13, 'array_kwp': 5.3, 'cost': 4835.00, 'installer': 'CEG'},
+    {'n_panels': 14, 'array_kwp': 5.7, 'cost': 5015.00, 'installer': 'CEG'},
+    {'n_panels': 15, 'array_kwp': 6.2, 'cost': 5176.00, 'installer': 'CEG'},
+    {'n_panels': 16, 'array_kwp': 6.6, 'cost': 5338.00, 'installer': 'CEG'},
+    {'n_panels': 17, 'array_kwp': 7.0, 'cost': 5500.00, 'installer': 'CEG'},
+    {'n_panels': 18, 'array_kwp': 7.4, 'cost': 6021.00, 'installer': 'CEG'}
+]
+# This is the maximum number of panels that we have a cost from the installers for
+INSTALLER_MAX_PANELS = 18
+
+# CEG uses use Solshare as an inverter to provide solar PV to multiple flats. This costs £7500 for the inverter alone
+# https://midsummerwholesale.co.uk/buy/solshare
+INSTALLER_SOLAR_PV_INVERTER_COST = 7500
+INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST = 500  # Just a rough guess to labour costs
+
+INSTALLER_SCAFFOLDING_COSTS = [
+    {'stories': 1, 'description': '1 Story Scaffold', 'cost': 531.00, 'installer': 'CEG'},
+    {'stories': 2, 'description': '2 Story Scaffold', 'cost': 841.00, 'installer': 'CEG'},
+    {'stories': 3, 'description': '3 Story Scaffold', 'cost': 1077.00, 'installer': 'CEG'}
+]
+
 # This data is based on the MCS database, We use the larger figure between the 2023 and 2024 average,
 # to be conservative
 MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = {
@@ -54,10 +85,27 @@ MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA = {
     "Scotland": 12586,
     "Northern Ireland": 12000,  # There are hardly any air source heat pump installs going on in Northern Ireland
 }
+
+INSTALLER_ASHP_COSTS = [
+    {'capacity_kw': 5.0, 'brand': 'Mitsubishi', 'tank_size_liters': 150, 'cost': 10149.53, 'installer': 'CEG'},
+    {'capacity_kw': 6.0, 'brand': 'Mitsubishi', 'tank_size_liters': 170, 'cost': 10823.48, 'installer': 'CEG'},
+    {'capacity_kw': 8.5, 'brand': 'Mitsubishi', 'tank_size_liters': 200, 'cost': 11312.43, 'installer': 'CEG'},
+    {'capacity_kw': 11.2, 'brand': 'Mitsubishi', 'tank_size_liters': 250, 'cost': 12156.75, 'installer': 'CEG'},
+    {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'},
+    {'capacity_kw': 14.0, 'brand': 'Mitsubishi', 'tank_size_liters': 300, 'cost': 14405.54, 'installer': 'CEG'},
+    {'capacity_kw': 17.0, 'brand': 'Grant', 'tank_size_liters': 300, 'cost': 14445.00, 'installer': 'CEG'},
+    {'capacity_kw': 20.0, 'brand': 'Ecoforest', 'tank_size_liters': 400, 'cost': 21189.41, 'installer': 'CEG'},
+    {'capacity_kw': None, 'brand': '2 x cascaded ASHPs', 'tank_size_liters': 500, 'cost': 22950.00, 'installer': 'CEG'}
+]
+
 BOILER_UPGRADE_SCHEME_ASHP_VALUE = 7500
 
-# This is based on quotes from installers
-BATTERY_COST = 3500
+INSTALLER_SOLAR_BATTERY_COSTS = [
+    {'capacity_kwh': 5, 'description': 'Battery Add on', 'cost': 2700.00, 'installer': 'CEG'},
+    {'capacity_kwh': 10, 'description': 'Battery Add on', 'cost': 4300.00, 'installer': 'CEG'},
+    {'capacity_kwh': 5, 'description': 'Battery Retrofit existing system', 'cost': 4250.00, 'installer': 'CEG'},
+    {'capacity_kwh': 10, 'description': 'Battery Retrofit Existing system', 'cost': 5950.00, 'installer': 'CEG'}
+]
 
 # This is based on https://www.checkatrade.com/blog/cost-guides/cost-smart-thermostat/
 SMART_APPLIANCE_THERMOSTAT_COST = 400
@@ -168,9 +216,8 @@ class Costs:
     # https://www.greenmatch.co.uk/windows/double-glazing/cost
     SASH_WINDOW_INFLATION_FACTOR = 1.5
 
-    # Typically, secondary glazing can be installed for 25% of the cost of double glazed windows - to be conservative,
-    # we scale the cost by half
-    SECONDARY_GLAZING_SCALING_FACTOR = 0.5
+    # Based on relative costs from SCIS
+    SECONDARY_GLAZING_SCALING_FACTOR = 0.85
 
     def __init__(self, property_instance):
         """
@@ -210,7 +257,6 @@ class Costs:
 
         :return: A dictionary containing detailed cost breakdown.
         """
-
         # CWI usually takes 1 day
         labour_hours = 8
         labour_days = 1
@@ -225,118 +271,57 @@ class Costs:
                 "labour_days": labour_days,
             }
 
-        material_cost_per_m2 = material["material_cost"]
-
-        base_material_cost = material_cost_per_m2 * wall_area
-        labour_cost = material["labour_cost"] * wall_area * self.labour_adjustment_factor
-
-        subtotal_before_profit = base_material_cost + labour_cost
-
-        contingency_cost = subtotal_before_profit * self.CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-
-        total_cost = subtotal_before_vat + vat_cost
+        total_including_vat = material["total_cost"] * wall_area
 
         if is_extraction_and_refill:
-            # bump up the cost of the work
-            total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area
+            total_including_vat = CAVITY_EXTRACTION_COST * wall_area
             # Additional 2 days work
-            labour_hours = labour_hours + (2 * 8)
-            labour_days = labour_days + 2
+            labour_hours += + (2 * 8)
+            labour_days += + 2
+
+        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
+        vat_cost = total_including_vat - total_excluding_vat
 
         return {
-            "total": total_cost,
-            "subtotal": subtotal_before_vat,
+            "total": total_including_vat,
+            "subtotal": total_excluding_vat,
             "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": base_material_cost,
-            "profit": profit_cost,
             "labour_hours": labour_hours,
-            "labour_cost": labour_cost,
             "labour_days": labour_days
         }
 
-    def loft_insulation(self, floor_area, material):
+    def loft_and_flat_insulation(self, floor_area, material):
         """
-        Calculates the total cost for cavity wall insulation based on material and labor costs,
+        Calculates the total cost for loft/flat roof insulation based on material and labor costs,
         including contingency, preliminaries, profit, and VAT.
 
         :return: A dictionary containing detailed cost breakdown.
         """
 
-        labour_hours = material["labour_hours_per_unit"] * floor_area
-        # Assume a team of 1 person
-        labour_days = labour_hours / 8
-
         if material["is_installer_quote"]:
             total_cost = material["total_cost"] * floor_area
 
             return {
                 "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
+                "labour_hours": 8,
+                "labour_days": 1,
             }
 
-        material_cost_per_m2 = material["material_cost"]
-
-        # We inflate material costs due to recent price increases
-        material_cost_per_m2 = material_cost_per_m2 * 1.5
-
-        base_material_cost = material_cost_per_m2 * floor_area
-        labour_cost = material["labour_cost"] * floor_area * self.labour_adjustment_factor
-
-        subtotal_before_profit = base_material_cost + labour_cost
-
-        # We use high risk contingency because of the possibility of access issues and clearing existing insulation
-        contingency_cost = subtotal_before_profit * self.HIGH_RISK_CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-
-        total_cost = subtotal_before_vat + vat_cost
+        total_including_vat = material["total_cost"] * floor_area
+        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
+        vat_cost = total_including_vat - total_excluding_vat
 
         return {
-            "total": total_cost,
-            "subtotal": subtotal_before_vat,
+            "total": total_including_vat,
+            "subtotal": total_excluding_vat,
             "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": base_material_cost,
-            "profit": profit_cost,
-            "labour_hours": labour_hours,
-            "labour_cost": labour_cost,
-            "labour_days": labour_days
+            "labour_hours": 8,
+            "labour_days": 1
         }
 
-    def internal_wall_insulation(self, wall_area, material, non_insulation_materials):
+    def solid_wall_insulation(self, wall_area, material):
         """
-        Broadly speaking, the high level steps to an internal wall insulation job are the following:
-
-        1) Demolition: This involves removing existing wall linings, fittings, and any other obstacles.
-        It's important to factor in the disposal of debris and the potential need for additional protective
-        measures to ensure the safety of the work area.
-
-        2) Insulation Installation: This is the core part of the process where the chosen insulation material is
-        applied. The choice of insulation material will depend on several factors including thermal performance,
-        wall construction, and space constraints.
-
-        3) Vapour Barrier Installation: This is crucial for preventing moisture from penetrating the insulation,
-        which can compromise its effectiveness and lead to mold growth.
-
-        4) Re-decoration: This involves applying plaster to the wall and then painting.
-        The quality of finish here is important for both aesthetic and functional reasons.
-
-        5) Trim and Finishing Work: Post-insulation, tasks such as re-installing skirting boards, door frames,
-        or window sills might be necessary.
+        Implements costing methodology now that we have direct quotes from installers.
         :return:
         """
 
@@ -356,74 +341,25 @@ class Costs:
                 "labour_days": labour_days,
             }
 
-        # Extract and check the different types of data we'll need
-        demolition_data = [x for x in non_insulation_materials if x["type"] == "iwi_wall_demolition"]
-        vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "iwi_vapour_barrier"]
-        redecoration_data = [x for x in non_insulation_materials if x["type"] == "iwi_redecoration"]
-        if not demolition_data:
-            raise ValueError("No data found for iwi_wall_demolition")
-
-        if (len(vapour_barrier_data) != 1) or (len(redecoration_data) != 3):
-            raise ValueError("Incorrect number of data entries for non-insulation materials")
-
         # Break out the individual material costs
         # Since we don't know the exact wall construction, we take an average for demolition costs, since
         # the cost will depend on the type of wall construction
-        demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data])
-        insulation_material_costs = material["material_cost"] * wall_area
-        vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * wall_area
-        redecoration_material_costs = sum([x["material_cost"] * wall_area for x in redecoration_data])
 
-        demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data])
-
-        # Again for demolition, we average since we aren't sure which demolition process will be used
-        demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data])
-        insulation_labour_costs = material["labour_cost"] * wall_area
-        vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * wall_area
-        redecoration_labour_costs = sum([x["labour_cost"] * wall_area for x in redecoration_data])
-
-        labour_costs = (demolition_labour_costs + insulation_labour_costs + vapour_barrier_labour_costs +
-                        redecoration_labour_costs)
-
-        labour_costs = labour_costs * self.labour_adjustment_factor
-
-        materials_costs = (demolition_material_costs + insulation_material_costs + vapour_barrier_material_costs +
-                           redecoration_material_costs)
-
-        subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs
-
-        contingency_cost = subtotal_before_profit * self.IWI_CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-
-        total_cost = subtotal_before_vat + vat_cost
-
-        demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data])
-        insulation_labour_hours = material["labour_hours_per_unit"] * wall_area
-        vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * wall_area
-        redecoration_labour_hours = sum([x["labour_hours_per_unit"] * wall_area for x in redecoration_data])
-
-        labour_hours = (demolition_labour_hours + insulation_labour_hours + vapour_barrier_labour_hours +
-                        redecoration_labour_hours)
+        total_including_vat = material["total_cost"] * wall_area
+        total_excluding_vat = total_including_vat / (1 + self.VAT_RATE)
+        vat_cost = total_including_vat - total_excluding_vat
 
+        # We estimate 1 weeks worth of work
+        labour_hours = 160
         # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5 people
         labour_days = (labour_hours / 8) / 4
 
         return {
-            "total": total_cost,
-            "subtotal": subtotal_before_vat,
+            "total": total_including_vat,
+            "subtotal": total_excluding_vat,
             "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": materials_costs,
-            "profit": profit_cost,
             "labour_hours": labour_hours,
             "labour_days": labour_days,
-            "labour_cost": labour_costs
         }
 
     def suspended_floor_insulation(self, insulation_floor_area, material, non_insulation_materials):
@@ -640,151 +576,6 @@ class Costs:
             "labour_cost": labour_costs
         }
 
-    def external_wall_insulation(self, wall_area, material, non_insulation_materials):
-        """
-        We characterise external wall insulation as the following steps:
-
-        1) Preparation of the Area: Tidying up the surroundings, trimming back foliage, and laying down protective
-        sheets to protect the flooring and landscaping around the work area.
-
-        2) Scaffolding Setup (if needed): Erecting scaffolding for safe access to the walls of semi-detached or
-        detached houses. For terraced houses or lower-level work, scaffolding might not be necessary.
-
-        3) Wall Surface Preparation: Cleaning the wall surface, removing any loose or flaking material,
-        and possibly applying a primer. If the existing wall is weak or damaged, partial or full replacement
-        of the top surface may be necessary.
-
-        4) Applying Primer: If the existing wall is suitable, applying a primer to improve adhesion of the insulation
-        boards and stabilize the wall surface, especially if it's old or weathered.
-
-        5) Insulation Application: Attaching insulation boards to the primed wall using adhesive, mechanical fixings,
-        or a combination of both.
-
-        6) Basecoat and Mesh Application: Applying a basecoat embedded with a reinforcing mesh over the insulation.
-        This layer provides strength and helps prevent cracking.
-
-        7) Decorative Finish: Applying a decorative finish, such as render or cladding, which protects the insulation
-        and provides an aesthetic look.
-
-        8) Reinstalling Fixtures: Reattaching any fixtures like downpipes, satellite dishes, or lighting fixtures that
-        were removed during preparation. Extensions or adjustments may be required due to the increased wall thickness.
-
-        9) Inspection and Cleanup: Conducting a thorough inspection to ensure quality and integrity of the EWI system,
-        followed by cleaning up the site to remove all debris and materials.
-
-        In the actual materials data, at this point, we have costing for:
-        - wall preparation, hacking off existing wall finishes, linings, etc (ewi_wall_demolition)
-        - wall surface cleaning and priming (ewi_wall_preparation)
-        - insulation (external_wall_insulation)
-        - basecoat and mesh with decorative render topcoat finish (ewi_basecoat_and_mesh)
-
-        All of this data comes from SPONS, however there are some clear features missing. Because we could not find
-        suitable cost records in SPONS for steps like cleaning the area, setting up small scale scaffolding,
-        re-attaching any fitings and cleaning up the area afterwards, instead we have accounted for these steps by
-        increasing the preliminaries rate. It is acknowldeged though, that this is not ideal and that the cost of these
-        steps should be included in the materials data. We will look to improve this in the future, with data from
-        installers
-
-        :param wall_area:
-        :param material:
-        :param non_insulation_materials:
-        :return:
-        """
-
-        if material["is_installer_quote"]:
-            total_cost = material["total_cost"] * wall_area
-            # Add on a buffer for scaffolding
-            if self.property.data["property-type"] == "House":
-                total_cost += self.EWI_SCAFFOLDING_PRELIMINARIES * total_cost
-
-            labour_hours = material["labour_hours_per_unit"] * wall_area
-
-            # To install internal wall insulation, a small to medium size project might be conducted by a team of 3-5
-            # people
-            labour_days = (labour_hours / 8) / 4
-
-            return {
-                "total": total_cost,
-                "labour_hours": labour_hours,
-                "labour_days": labour_days,
-            }
-
-        # For semi detatched and detatched houses, as well as maisonettes, we price for scaffolding
-
-        if self.property.data["property-type"] == "House":
-            if self.property.data["built-form"] in ['Semi-Detached', 'Detached', "End-Terrace"]:
-                preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
-            else:
-                preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
-        elif self.property.data["property-type"] in ["Maisonette", "Flat"]:
-            preliminaries_rate = self.EWI_SCAFFOLDING_PRELIMINARIES
-        elif self.property.data["property-type"] == "Bungalow":
-            preliminaries_rate = self.EWI_NO_SCAFFOLDING_PRELIMINARIES
-
-        demolition_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_demolition"]
-        preparation_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_preparation"]
-        redecoration_data = [x for x in non_insulation_materials if x["type"] == "ewi_wall_redecoration"]
-
-        if (len(demolition_data) != 3) or (len(preparation_data) != 1) or (len(redecoration_data) != 1):
-            raise ValueError("Incorrect number of data entries for non-insulation materials")
-
-        # Break out the individual material costs
-        # Since we don't know the exact wall construction, we take an average for demolition costs, since
-        # the cost will depend on the type of wall construction
-        demolition_material_costs = np.mean([x["material_cost"] * wall_area for x in demolition_data])
-        insulation_material_costs = material["material_cost"] * wall_area
-        preparation_material_costs = preparation_data[0]["material_cost"] * wall_area
-        redecoration_material_costs = redecoration_data[0]["material_cost"] * wall_area
-
-        demolition_plant_costs = np.mean([x["plant_cost"] * wall_area for x in demolition_data])
-
-        demolition_labour_costs = np.mean([x["labour_cost"] * wall_area for x in demolition_data])
-        insulation_labour_costs = material["labour_cost"] * wall_area
-        preparation_labour_costs = preparation_data[0]["labour_cost"] * wall_area
-        redecoration_labour_costs = redecoration_data[0]["labour_cost"] * wall_area
-
-        labour_costs = (demolition_labour_costs + insulation_labour_costs + redecoration_labour_costs +
-                        preparation_labour_costs)
-
-        labour_costs = labour_costs * self.labour_adjustment_factor
-
-        materials_costs = (demolition_material_costs + insulation_material_costs + preparation_material_costs +
-                           redecoration_material_costs)
-
-        subtotal_before_profit = labour_costs + materials_costs + demolition_plant_costs
-
-        contingency_cost = subtotal_before_profit * self.CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * preliminaries_rate
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-        total_cost = subtotal_before_vat + vat_cost
-
-        demolition_labour_hours = np.mean([x["labour_hours_per_unit"] * wall_area for x in demolition_data])
-        insulation_labour_hours = material["labour_hours_per_unit"] * wall_area
-        preparation_labour_hours = preparation_data[0]["labour_hours_per_unit"] * wall_area
-        redecoration_labour_hours = redecoration_data[0]["labour_hours_per_unit"] * wall_area
-
-        labour_hours = (demolition_labour_hours + insulation_labour_hours + redecoration_labour_hours +
-                        preparation_labour_hours)
-
-        # Assume a team of 3-5 people for a small to medium size project
-        labour_days = (labour_hours / 8) / 4
-
-        return {
-            "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": materials_costs,
-            "profit": profit_cost,
-            "labour_hours": labour_hours,
-            "labour_days": labour_days,
-            "labour_cost": labour_costs
-        }
-
     def low_energy_lighting(self, number_of_lights, number_current_lel_lights, material):
 
         """
@@ -833,92 +624,6 @@ class Costs:
             "labour_cost": labour_cost
         }
 
-    def flat_roof_insulation(self, floor_area, material, non_insulation_materials):
-        """
-        A model of a warm, flat roof construction can be seen in this video:
-        https://www.youtube.com/watch?v=WZ6Ng6YI9OA
-        Warm, flat roof insulation will normally be 100-125mm in depth
-
-        We break this measure down into the following jobs to be done
-        1) Preparation of the room. This involves cleaning the existing roof surface, removing any debris and repairing
-        any damage. Additionally, an edge barrier will likely need to be installed, to protect the sides of the
-        roof from water ingress.
-        2) Primer Application. A layer of primer is applied to the clean roof surface to enhance the adhestia of
-        subsequent layers, and seal the existing roof surface.
-        3) Vapour Proof Layer Installation. Lay a vapour control layer to prevent moisture ingress from inside the
-        building, which is essential in warm roof construction.
-        4) Insulation Layer Application. Place and securely fix insulation boards over the roof. These could be rigid
-        boards like PIR (Polyisocyanurate).
-        5) Waterproofing Membrane Installation: Cover the insulation (and timber layer, if used) with a
-        waterproofing membrane, like EPDM, PVC, or bituminous felt.  Carefully seal all joints, edges, and around any
-        roof penetrations to ensure water tightness
-
-        :param floor_area: Area of the flat roof to be insulated, based on the area of the floor
-        :param material: Selected insulation material
-        :param non_insulation_materials: Non-insulation materials required for the job
-        :return:
-        """
-
-        preparation_data_m2 = [
-            x for x in non_insulation_materials if
-            (x["type"] == "flat_roof_preparation") and (x["cost_unit"] == "gbp_per_m2")
-        ]
-        vapour_barrier_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_vapour_barrier"]
-        waterproofing_data = [x for x in non_insulation_materials if x["type"] == "flat_roof_waterproofing"]
-
-        if (len(preparation_data_m2) != 2) or (len(vapour_barrier_data) != 1) or (
-            len(waterproofing_data) != 1):
-            raise ValueError("Incorrect number of data entries for non-insulation materials")
-
-        # Break out the individual material costs
-        preparation_m2_material_costs = sum([x["material_cost"] * floor_area for x in preparation_data_m2])
-        vapour_barrier_material_costs = vapour_barrier_data[0]["material_cost"] * floor_area
-        insulation_material_costs = material["material_cost"] * floor_area
-
-        preparation_m2_labour_costs = sum([x["labour_cost"] * floor_area for x in preparation_data_m2])
-        vapour_barrier_labour_costs = vapour_barrier_data[0]["labour_cost"] * floor_area
-
-        # For waterproofing and upstand, we only have a total cost
-        waterproofing_total_costs = waterproofing_data[0]["total_cost"] * floor_area
-
-        labour_costs = preparation_m2_labour_costs + vapour_barrier_labour_costs
-        labour_costs = labour_costs * self.labour_adjustment_factor
-
-        materials_costs = preparation_m2_material_costs + vapour_barrier_material_costs + insulation_material_costs
-
-        subtotal_before_profit = labour_costs + materials_costs + waterproofing_total_costs
-
-        contingency_cost = subtotal_before_profit * self.FLAT_ROOF_CONTINGENCY
-        preliminaries_cost = subtotal_before_profit * self.PRELIMINARIES
-        profit_cost = subtotal_before_profit * self.PROFIT_MARGIN
-
-        subtotal_before_vat = subtotal_before_profit + contingency_cost + preliminaries_cost + profit_cost
-        vat_cost = subtotal_before_vat * self.VAT_RATE
-        total_cost = subtotal_before_vat + vat_cost
-
-        preparation_m2_labour_hours = sum([x["labour_hours_per_unit"] * floor_area for x in preparation_data_m2])
-        vapour_barrier_labour_hours = vapour_barrier_data[0]["labour_hours_per_unit"] * floor_area
-        waterproofing_labour_hours = waterproofing_data[0]["labour_hours_per_unit"] * floor_area
-
-        labour_hours = preparation_m2_labour_hours + vapour_barrier_labour_hours + waterproofing_labour_hours
-
-        # To install flat roof insulation, assume a small/medium project might be conducted by a team of 2-4.
-        # We'll assume a team of 2 since a lot of the roofs will be on the smaller side and will review this later
-        labour_days = (labour_hours / 8) / 2
-
-        return {
-            "total": total_cost,
-            "subtotal": subtotal_before_vat,
-            "vat": vat_cost,
-            "contingency": contingency_cost,
-            "preliminaries": preliminaries_cost,
-            "material": materials_costs,
-            "profit": profit_cost,
-            "labour_hours": labour_hours,
-            "labour_days": labour_days,
-            "labour_cost": labour_costs
-        }
-
     def window_glazing(self, number_of_windows, material, is_secondary_glazing=False):
         """
         We characterise the jobs to be done for window glazing as the following:
@@ -1014,7 +719,15 @@ class Costs:
             "labour_days": labour_days
         }
 
-    def solar_pv(self, wattage: float, has_battery: bool = False, array_cost=None):
+    def solar_pv(
+        self,
+        n_panels: int | float,
+        has_battery: bool = False,
+        array_cost=None,
+        n_floors: int = 1,
+        battery_kwh: int = 5,
+        needs_inverter=False
+    ):
 
         """
         Calculates the total cost for solar PV based data provided by the MCS dashboard, which contains
@@ -1026,23 +739,40 @@ class Costs:
 
         Price can also be benchmarked against this checkatrade article:
         https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/
-        :param wattage: Peak wattage of the solar PV system]
+        :param n_panels: Number of solar panels
         :param has_battery: Bool, whether the system includes a battery
         :param array_cost: float, containing the cost of the solar PV array
+        :param n_floors: int, number of floors in the property, used to estimate the cost of scaffolding
+        :param battery_kwh: int, capacity of the battery in kWh. Defaulted to 5
+        :param needs_inverter: Bool, whether the system needs an inverter, where the solar panels are feeding multiple
+                               units
         """
 
-        # Get the cost data relevant to the region
-        regional_cost = MCS_SOLAR_PV_COST_DATA["-".join(["average_cost_per_kwh", self.region])]
-
-        if array_cost is not None:
-            total_cost = array_cost
+        if n_panels > INSTALLER_MAX_PANELS:
+            base_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == INSTALLER_MAX_PANELS][0]["cost"]
+            cost_per_panel = [
+                c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == (INSTALLER_MAX_PANELS - 1)
+            ][0]["cost"]
+            cost_per_panel = base_cost - cost_per_panel
+            system_cost = base_cost + (n_panels - INSTALLER_MAX_PANELS) * cost_per_panel
         else:
-            kw = wattage / 1000
-            total_cost = kw * regional_cost
+            system_cost = [c for c in INSTALLER_SOLAR_COSTS if c["n_panels"] == n_panels][0]["cost"]
+
+        total_cost = array_cost if array_cost is not None else system_cost
 
         if has_battery:
-            # The battery cost is based on the £3500 quote, recieved from installers
-            total_cost += BATTERY_COST
+            battery_cost = [c for c in INSTALLER_SOLAR_BATTERY_COSTS if c["capacity_kwh"] == battery_kwh][0]["cost"]
+            total_cost += battery_cost
+
+        scaffolding_cost = [c for c in INSTALLER_SCAFFOLDING_COSTS if c["stories"] == n_floors][0]["cost"]
+        total_cost += scaffolding_cost
+
+        if needs_inverter:
+            total_cost += INSTALLER_SOLAR_PV_INVERTER_COST
+            # We also add an additional labour cost
+            total_cost += INSTALLER_SOLAR_PV_INVERTER_LABOUR_COST
+
+        # We add an additional cost for scaffolding
 
         subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
 
@@ -1111,7 +841,7 @@ class Costs:
             "labour_days": labour_days,
         }
 
-    def high_heat_electric_storage_heaters(self, number_heated_rooms):
+    def high_heat_electric_storage_heaters(self, number_heated_rooms, needs_cylinder):
 
         """
         We base the estimates for the cost of electric storage heaters on the cost per room as estimated by the
@@ -1122,7 +852,12 @@ class Costs:
         :param number_heated_rooms: int, number of rooms to be heated
         """
 
-        total_cost = 1500 * number_heated_rooms
+        if needs_cylinder:
+            # 1000 is the cost of a new hot water cylinder
+            total_cost = 1200 * number_heated_rooms + 1000
+        else:
+            # 500 is the cost of a dual immersion heater - a rough estimate
+            total_cost = 1200 * number_heated_rooms + 500
         subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
         vat = total_cost - subtotal_before_vat
 
@@ -1413,7 +1148,7 @@ class Costs:
             "labour_days": labour_days,
         }
 
-    def air_source_heat_pump(self):
+    def air_source_heat_pump(self, ashp_size):
         """
         Based on the region and type of property, this function will produce a cost estimation for an air source heat
         pump. This cost will include the boiler upgrade scheme grant
@@ -1421,14 +1156,19 @@ class Costs:
         """
 
         # This is the average cost of a project, we'll add some additional contingency
-        regional_cost = MCS_AIR_SOURCE_HEAT_PUMP_COST_DATA[self.region]
 
-        total_cost = regional_cost * (1 + self.CONTINGENCY) - BOILER_UPGRADE_SCHEME_ASHP_VALUE
+        if ashp_size is None:
+            cost = [x for x in INSTALLER_ASHP_COSTS if x["capacity_kw"] is None][0]["cost"]
+        else:
+            cost = [x for x in INSTALLER_ASHP_COSTS if x][0]["cost"]
+
+        # We add some contingency since there are additional costs such as resizing radiators, that could be required
+        total_cost = cost * (1 + self.CONTINGENCY)
         subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
         vat = total_cost - subtotal_before_vat
 
-        # We assume 3 days installation
-        labour_days = 3
+        # We assume 5 days installation
+        labour_days = 5
         labour_hours = labour_days * 8
 
         return {

recommendations/DraughtProofingRecommendations.py

@@ -38,6 +38,7 @@ class DraughtProofingRecommendations:
                 "phase": None,
                 "parts": [],
                 "type": "draught_proofing",
+                "measure_type": "draught_proofing",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/FireplaceRecommendations.py

@@ -9,7 +9,7 @@ class FireplaceRecommendations(Definitions):
     """
 
     # This is our base assumption for the cost of the work
-    COST_OF_WORK = 300
+    COST_OF_WORK = 235
 
     def __init__(
         self,
@@ -41,6 +41,7 @@ class FireplaceRecommendations(Definitions):
                 "phase": phase,
                 "parts": [],
                 "type": "sealing_open_fireplace",
+                "measure_type": "sealing_open_fireplace",
                 "description": "Seal %s open fireplaces" % str(number_open_fireplaces),
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/FloorRecommendations.py

@@ -68,7 +68,8 @@ class FloorRecommendations(Definitions):
 
         measures = MEASURE_MAP["floor_insulation"] if measures is None else measures
 
-        if not measures:
+        # If we have no measures or none of the measures are relevant, we can't recommend anything
+        if not measures or not any(x in measures for x in MEASURE_MAP["floor_insulation"]):
             return
 
         u_value = self.property.floor["thermal_transmittance"]
@@ -223,7 +224,9 @@ class FloorRecommendations(Definitions):
 
                     simulation_config = {
                         **floor_simulation_config,
-                        "floor_thermal_transmittance_ending": new_u_value,
+                        # We don't simulate the impact using this U-value, but rather the average because this
+                        # variable is way too volatile. Will likely be removed from the model
+                        "floor_thermal_transmittance_ending": 0.685593,
                     }
 
                     self.recommendations.append(
@@ -238,6 +241,7 @@ class FloorRecommendations(Definitions):
                                 ),
                             ],
                             "type": material["type"],
+                            "measure_type": material["type"],  # This is distinct between suspended and solid floor
                             "description": self._make_floor_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,

recommendations/HeatingControlRecommender.py

@@ -12,8 +12,11 @@ class HeatingControlRecommender:
 
         self.recommendation = []
 
-    def recommend(self, heating_description):
+    def recommend(self, heating_description, description_prefix="", description_suffix=""):
 
+        # TODO: Many of these functions are quite similar. We can possibly create a single wrapper function that
+        #  takes in the heating description and the description prefix/suffix, and then creates the appropriate
+        #  output
         # Reset the recommendations
         self.recommendation = []
 
@@ -24,14 +27,14 @@ class HeatingControlRecommender:
             return
 
         if heating_description in ["Electric storage heaters", "Electric storage heaters, radiators"]:
-            self.recommend_high_heat_retention_controls()
+            self.recommend_high_heat_retention_controls(description_prefix=description_prefix)
             return
 
         if heating_description in ["Boiler and radiators, mains gas"]:
             # We can recommend roomstat programmer trvs
-            self.recommend_roomstat_programmer_trvs()
+            self.recommend_roomstat_programmer_trvs(description_suffix=description_suffix)
             # We can also recommend time and temperature zone controls
-            self.recommend_time_temperature_zone_controls()
+            self.recommend_time_temperature_zone_controls(description_suffix=description_suffix)
 
             return
 
@@ -94,16 +97,22 @@ class HeatingControlRecommender:
         # We don't implement any other recommendations right now
         return
 
-    def recommend_high_heat_retention_controls(self):
+    def recommend_high_heat_retention_controls(self, description_prefix=""):
         """
         When applicable, we recommend upgrading the heating controls to high heat retention controls. This is a
         specific type of control system that is designed to work with electric storage heaters. It is a more
         efficient control system than the standard controls that come with electric storage heaters.
 
         We can then consider the heating system itself
+
+        If there is a description prefix, this means there is a dual heating system and so we need to add this to the
+        description
+
         :return:
         """
         new_description = "Controls for high heat retention storage heaters"
+        if description_prefix:
+            new_description = f"{description_prefix}, {new_description}"
 
         # We recommend upgrading to Celect type controls
         ending_config = MainheatControlAttributes(new_description).process()
@@ -112,7 +121,10 @@ class HeatingControlRecommender:
             new_config=ending_config, old_config=self.property.main_heating_controls
         )
         # This upgrade will only take the heating system to average energy efficiency
-        simulation_config["mainheatc_energy_eff_ending"] = "Good"
+        if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average"]:
+            simulation_config["mainheatc_energy_eff_ending"] = "Good"
+        else:
+            simulation_config["mainheatc_energy_eff_ending"] = self.property.data["mainheatc-energy-eff"]
 
         description_simulation = {
             "mainheatcont-description": new_description,
@@ -121,7 +133,7 @@ class HeatingControlRecommender:
 
         self.recommendation.append(
             {
-                "description": "upgrade heating controls to High Heat Retention Storage Heater Controls",
+                "description": "Upgrade heating controls to High Heat Retention Storage Heater Controls",
                 **self.costs.celect_type_controls(),
                 "simulation_config": simulation_config,
                 "description_simulation": description_simulation
@@ -131,7 +143,7 @@ class HeatingControlRecommender:
         # We don't implement any other recommendations right now
         return
 
-    def recommend_roomstat_programmer_trvs(self):
+    def recommend_roomstat_programmer_trvs(self, description_suffix=""):
         """
         If the home has a boiler and radiators, mains gas, we start by identifying potential heating controls that could
         be upgraded, that would provide a practical impact.
@@ -163,6 +175,8 @@ class HeatingControlRecommender:
             return
 
         new_controls_description = "Programmer, room thermostat and TRVS"
+        if description_suffix:
+            new_controls_description = f"{new_controls_description}, {description_suffix}"
 
         ending_config = MainheatControlAttributes(new_controls_description).process()
         # We use this to determine how we should be updating the config
@@ -192,7 +206,7 @@ class HeatingControlRecommender:
             has_trvs=has_trvs
         )
 
-        description = "upgrade heating controls to Room thermostat, programmer and TRVs"
+        description = "Upgrade heating controls to Room thermostat, programmer and TRVs"
 
         already_installed = "heating_control" in self.property.already_installed
         if already_installed:
@@ -202,6 +216,7 @@ class HeatingControlRecommender:
         self.recommendation.append(
             {
                 "type": "heating_control",
+                "measure_type": "roomstat_programmer_trvs",
                 "parts": [],
                 "description": description,
                 **cost_result,
@@ -216,7 +231,7 @@ class HeatingControlRecommender:
 
         return
 
-    def recommend_time_temperature_zone_controls(self):
+    def recommend_time_temperature_zone_controls(self, description_suffix=""):
         """
         If the home has a boiler, we can recommend time and temperature zone controls. This is a more advanced
         and more efficient control system than the standard controls that come with a boiler. However, it may come
@@ -238,6 +253,8 @@ class HeatingControlRecommender:
             return
 
         new_controls_description = "Time and temperature zone control"
+        if description_suffix:
+            new_controls_description = f"{new_controls_description}, {description_suffix}"
 
         ending_config = MainheatControlAttributes(new_controls_description).process()
 
@@ -260,8 +277,10 @@ class HeatingControlRecommender:
             number_heated_rooms=int(self.property.data["number-heated-rooms"])
         )
 
-        description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & "
-                       "temperature zone control)")
+        description = (
+            "Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & "
+            "temperature zone control)"
+        )
 
         already_installed = "heating_control" in self.property.already_installed
         if already_installed:
@@ -271,6 +290,7 @@ class HeatingControlRecommender:
         self.recommendation.append(
             {
                 "type": "heating_control",
+                "measure_type": "time_temperature_zone_control",
                 "parts": [],
                 "description": description,
                 **cost_result,

recommendations/HeatingRecommender.py

@@ -1,5 +1,9 @@
+import re
+import backend.app.assumptions as assumptions
 from recommendations.Costs import Costs, BOILER_UPGRADE_SCHEME_ASHP_VALUE
-from recommendations.recommendation_utils import check_simulation_difference, override_costs
+from recommendations.recommendation_utils import (
+    check_simulation_difference, override_costs, combine_recommendation_configs
+)
 from backend.Property import Property
 from backend.app.plan.schemas import MEASURE_MAP
 from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
@@ -9,15 +13,53 @@ from recommendations.HeatingControlRecommender import HeatingControlRecommender
 
 
 class HeatingRecommender:
-    ELECTRIC_HEATING_DESCRIPTIONS = [
-        "Room heaters, electric",
-        "Electric storage heaters",
-        "Electric storage heaters, radiators",
-        "Portable electric heaters assumed for most rooms",
-    ]
-
     high_heat_retention_contols_desc = "Controls for high heat retention storage heaters"
 
+    DUAL_HEATING_DESCRIPTIONS = {
+        "Boiler and radiators, mains gas, electric storage heaters": {
+            "hhr": {
+                "mainheating_description": "Boiler and radiators, mains gas, Electric storage heaters",
+                "recommendation_description": "Install high heat retention electric storage heaters alongside the "
+                                              "boiler. The current electric heaters may be retrofit with high heat "
+                                              "retention storage controls"
+                                              " however this is dependent on the existing system and may not be "
+                                              "possible.",
+                "controls_prefix": "current_controls"
+            },
+            "boiler": {
+                "mainheating_description": "Boiler and radiators, mains gas, electric storage heaters",
+                "recommendation_description": "Upgrade the existing boiler to a new, more efficient condensing "
+                                              "boiler. ",
+                "controls_suffix": "Manual charge controls"
+            },
+            # These are the heating types we need to produce a dual heating recommendation
+            "dual": {
+                "recommendation_description": "Upgrade both the existing boiler to a new condensing boiler and"
+                                              " upgrade storage heaters to high heat retention storage heaters.",
+                "types": [
+                    # type 1
+                    "boiler_upgrade",
+                    # type 2
+                    "high_heat_retention_storage_heater",
+                ]
+            }
+        },
+        "Portable electric heaters assumed for most rooms, room heaters, electric": {
+            "hhr": {
+                "mainheating_description": "Electric storage heaters, radiators",
+                "recommendation_description": "Install high heat retention electric storage heaters.",
+                "controls_prefix": ""
+            },
+            "boiler": {
+                "mainheating_description": "Boiler and radiators, mains gas",
+                "recommendation_description": "Upgrade to a new condensing boiler.",
+                "controls_suffix": ""
+            },
+            # These are the heating types we need to produce a dual heating recommendation
+            "dual": None
+        }
+    }
+
     def __init__(self, property_instance: Property):
         self.property = property_instance
         self.costs = Costs(self.property)
@@ -26,25 +68,50 @@ class HeatingRecommender:
         self.heating_control_recommendations = []
 
         self.has_electric_heating_description = (
-            self.property.main_heating["clean_description"] in self.ELECTRIC_HEATING_DESCRIPTIONS
+            self.property.main_heating["has_electric"] or self.property.main_heating["has_electricaire"]
         )
+        self.has_ashp = self.property.main_heating["has_air_source_heat_pump"]
+        self.has_room_heaters = (
+            self.property.main_heating["has_room_heaters"] or
+            self.property.main_heating["has_portable_electric_heaters"]
+        )
+        self.has_boiler = self.property.main_heating["has_boiler"]
+
+        self.dual_heating = self.identify_dual_heating()
+
+    def identify_dual_heating(self):
+        # All heat systems are in here so we identify whether two of these are true
+        # MainHeatAttributes.HEAT_SYSTEMS
+
+        n_trues = 0
+        for heat_system in MainHeatAttributes.HEAT_SYSTEMS:
+            if self.property.main_heating[f"has_{heat_system.replace(' ', '_')}"]:
+                n_trues += 1
+
+        if n_trues > 2 or n_trues == 0:
+            raise Exception("Implement me")
+        if n_trues == 1:
+            return False
+
+        return True
 
     def is_high_heat_retention_valid(self, ashp_only_heating_recommendation, measures):
         """
         Check conditions if high heat retention storage is valid
+        If there's already an ASHP in place, we don't recommend HHR
         :return:
         """
 
-        # If the property has assumed electric heating, regardless of whether or not it has a mains connection, we
-        # can consider hhr storage heaters
-        electric_heating_assumed = (
-            self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"]
-        )
+        # We can also recommend hhr if the property doesn't have a mains has connection
+        no_mains = not self.property.data["mains-gas-flag"]
 
-        has_electric = self.has_electric_heating_description or electric_heating_assumed
+        # If the property already has room heaters then we recommend HHR as an option since the home already has
+        # a variation of room heaters
+
+        hhr_suitable = no_mains or self.has_electric_heating_description or self.has_room_heaters
 
         return (
-            has_electric and (not ashp_only_heating_recommendation) and
+            hhr_suitable and (not ashp_only_heating_recommendation) and not self.has_ashp and
             ("high_heat_retention_storage_heater" in measures)
         )
 
@@ -55,7 +122,8 @@ class HeatingRecommender:
         """
 
         # 1) if the property has mains heating with boiler and radiators, we recommend optimal heating controls
-        has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"]
+        # If it's NOT a gas boiler, we'll potentially recommend a boiler
+        has_gas_boiler = self.has_boiler and self.property.main_heating["has_mains_gas"]
 
         # 2) If the property doesn't have a heating system, but it has access to the mains gas
         no_heating_has_mains = self.property.main_heating["clean_description"] in [
@@ -63,33 +131,102 @@ class HeatingRecommender:
         ] and self.property.data["mains-gas-flag"]
 
         # The property is using portable heaters and has access to gas mains
-        has_room_heaters = (
-            self.property.main_heating["clean_description"] in ["Room heaters, mains gas", "Room heaters, electric"] and
-            self.property.data["mains-gas-flag"]
-        )
+        has_room_heaters = self.has_room_heaters and self.property.data["mains-gas-flag"]
 
         # We also check if the property has electric heating, but it has access to the mains gas
         electic_heating_has_mains = self.has_electric_heating_description and self.property.data["mains-gas-flag"]
 
         portable_heaters_has_mains = (
-            self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"]
-            and
+            self.property.main_heating["has_portable_electric_heaters"] and self.property.data["mains-gas-flag"]
+        )
+
+        # The next condition is if the home has a non-gas boiler, such as an oil boiler, with a mains gas connection
+        non_gas_boiler = (
+            self.property.main_heating["has_boiler"] and
+            not self.property.main_heating["has_mains_gas"] and
             self.property.data["mains-gas-flag"]
         )
+        # Additionally, if the property has a gas connection, is using gas heating but doesn't have a boiler,
+        # we recommend a boiler
+        non_boiler_gas_heating = (
+            self.property.data["mains-gas-flag"] and
+            self.property.main_heating["has_mains_gas"] and
+            not self.property.main_heating["has_boiler"]
+        )
 
         is_valid = (
             (
-                has_boiler or
+                has_gas_boiler or
                 no_heating_has_mains or
                 electic_heating_has_mains or
                 has_room_heaters or
-                portable_heaters_has_mains
+                portable_heaters_has_mains or
+                non_gas_boiler or
+                non_boiler_gas_heating
             ) and
             (not ashp_only_heating_recommendation) and
-            ("boiler_upgrade" in measures)
+            ("boiler_upgrade" in measures) and
+            (not self.has_ashp)
         )
 
-        return is_valid, has_boiler
+        return is_valid, has_gas_boiler
+
+    def recommend_dual_heating(self):
+
+        if self.property.main_heating["clean_description"] not in self.DUAL_HEATING_DESCRIPTIONS:
+            return
+
+        # if we have set dual to None, we do not produce a dual heating recommendation
+        if self.DUAL_HEATING_DESCRIPTIONS[
+            self.property.main_heating["clean_description"]
+        ]["dual"] is None:
+            return
+
+        dual_heating_description = self.DUAL_HEATING_DESCRIPTIONS[
+            self.property.main_heating["clean_description"]
+        ]["dual"]["types"]
+
+        recommendation_system_types = list(set([x["system_type"] for x in self.heating_recommendations]))
+
+        # We check if we have the required type
+        if not any([x in recommendation_system_types for x in dual_heating_description]):
+            return
+
+        type_1_recommendations = [
+            x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[0]
+        ]
+        type_2_recommendations = [
+            x for x in self.heating_recommendations if x["system_type"] == dual_heating_description[1]
+        ]
+        # we combine the two recommendations
+        combined_recommendations = []
+        for rec in type_1_recommendations:
+            for rec2 in type_2_recommendations:
+                combined_rec = rec.copy()
+                # Update the description
+                combined_rec["description"] = self.DUAL_HEATING_DESCRIPTIONS[
+                    self.property.main_heating["clean_description"]
+                ]["dual"]["recommendation_description"]
+
+                # Combine simulation_config
+                # Make sure we end up with the best efficiecy values
+                combined_rec["simulation_config"] = combine_recommendation_configs(
+                    rec["simulation_config"], rec2["simulation_config"]
+                )
+                # Combine description_simulation
+                combined_rec["description_simulation"] = combine_recommendation_configs(
+                    rec["description_simulation"], rec2["description_simulation"]
+                )
+
+                # Combine costs
+                for k in ["total", "subtotal", "vat", "labour_hours", "labour_days"]:
+                    combined_rec[k] = rec[k] + rec2[k]
+
+                combined_rec["measure_type"] = "+".join([rec["measure_type"], rec2["measure_type"]])
+
+                combined_recommendations.append(combined_rec)
+
+        self.heating_recommendations.extend(combined_recommendations)
 
     def recommend(self, has_cavity_or_loft_recommendations, phase=0, measures=None):
         """
@@ -130,26 +267,26 @@ class HeatingRecommender:
 
         if hhr_valid:
             # Recommend high heat retention storage heaters
-            # TODO: We need to allow for the possibility that the property aleady has storage heaters, but just
-            #       needs the controls
             self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False)
 
-        gas_boiler_suitable, has_boiler = self.is_boiler_upgrade_suitable(
+        gas_boiler_suitable, has_gas_boiler = self.is_boiler_upgrade_suitable(
             measures=measures, ashp_only_heating_recommendation=ashp_only_heating_recommendation
         )
 
         if gas_boiler_suitable:
             # This indicates that the home previously did not have a boiler in place and so would require
             # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler
-            system_change = not has_boiler
-            exising_room_heaters = self.property.main_heating["clean_description"] in [
-                "Room heaters, electric", "Room heaters, mains gas"
-            ]
+            system_change = not has_gas_boiler
+            exising_room_heaters = self.property.main_heating["has_room_heaters"]
 
             self.recommend_boiler_upgrades(
                 phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters
             )
 
+        # If we have dual heating and we allow for a combined recommendation, to upgrade both systems
+        if self.dual_heating:
+            self.recommend_dual_heating()
+
         # We recommend air source heat pumps
         # Heat pumps are suitable for all property types:
         # https://energysavingtrust.org.uk/from-flats-to-terraced-houses-heat-pumps-are-suitable-for-all-property-types/
@@ -157,7 +294,11 @@ class HeatingRecommender:
         # In the future, we'll allow overrides, so that non-intrusive surveys can contradict these conditions
         # and either allow or prevent the recommendation of an air source heat pump
 
-        if self.property.is_ashp_valid(measures=measures) and non_invasive_ashp_recommendation["suitable"]:
+        if (
+            self.property.is_ashp_valid(measures=measures) and
+            non_invasive_ashp_recommendation["suitable"] and
+            not self.has_ashp
+        ):
             self.recommend_air_source_heat_pump(
                 phase=phase,
                 has_cavity_or_loft_recommendations=has_cavity_or_loft_recommendations,
@@ -229,6 +370,75 @@ class HeatingRecommender:
             description = ("Replace the existing boiler and cylinder without a thermostat with a new electric combi "
                            "boiler")
 
+    def size_heat_pump(self):
+        """
+        Given the methodology by installers (SCIS) this function will perform a basic heat loss calculation and
+        produce a recommendation for the size of the heat pump
+        :return:
+        """
+
+        floor_area = self.property.floor_area
+
+        # We use the default heat loss W/m2 values are specified by the insaller, depending on the property type
+
+        def remap_to_heat_loss(construction_age_band):
+            if "before 1900" in construction_age_band:
+                return "Pre 1900 (solid stone)"
+            elif "1900-1929" in construction_age_band:
+                return "Early 1900s (solid brick)"
+            elif re.search(r'1930|1949|1950|1966|1967|1975', construction_age_band):
+                return "1950-1980 (cavity void)"
+            elif re.search(r'1976|1982|1983|1990', construction_age_band):
+                return "Post 1980 (cavity wall construction)"
+            elif re.search(r'1991|1995|1996|2002|2003|2011', construction_age_band):
+                return "2000-2018"
+            elif "2012 onwards" in construction_age_band:
+                return "New build (2018+)"
+            else:
+                return None
+
+        def select_heatpump_size(heat_loss_calculation):
+            """
+            This function calculates the size of the heat pump based on the heat loss calculation, mapping
+            the heat loss calculation to the size of the heat pump in KW
+            :param heat_loss_calculation: This is calcualted as the floor area multipled by the heat loss constant,
+            divided by 1000
+            """
+            if heat_loss_calculation < 5:
+                return 5
+            elif 5 <= heat_loss_calculation < 6:
+                return 6
+            elif 6 <= heat_loss_calculation < 8.5:
+                return 8.5
+            elif 8.5 <= heat_loss_calculation < 11.2:
+                return 11.2
+            elif 11.2 <= heat_loss_calculation < 14:
+                return 14
+            elif 14 <= heat_loss_calculation < 17:
+                return 17
+            elif 17 <= heat_loss_calculation < 20:
+                return 20
+            else:
+                return None
+
+        heat_loss_constants = {
+            "New build (2018+)": 35,
+            "2000-2018": 50,
+            "Post 1980 (cavity wall construction)": 60,
+            "1950-1980 (cavity void)": 70,
+            "Early 1900s (solid brick)": 80,
+            "Pre 1900 (solid stone)": 90
+        }
+
+        heat_loss_group = remap_to_heat_loss(self.property.construction_age_band)
+        heat_loss_constant = heat_loss_constants[heat_loss_group]
+
+        heat_loss_calculation = floor_area * heat_loss_constant / 1000
+
+        heat_pump_size = select_heatpump_size(heat_loss_calculation)
+
+        return heat_pump_size
+
     def recommend_air_source_heat_pump(self, phase, has_cavity_or_loft_recommendations, _return=False):
         """
         This method will implement the recommendation for an air source heat pump
@@ -244,8 +454,9 @@ class HeatingRecommender:
 
         controls_recommender = HeatingControlRecommender(self.property)
         controls_recommender.recommend(heating_description="Air source heat pump, radiators, electric")
+        ashp_size = self.size_heat_pump()
 
-        ashp_costs = self.costs.air_source_heat_pump()
+        ashp_costs = self.costs.air_source_heat_pump(ashp_size)
         if non_intrusive_recommendation:
             # Update with non-intrusive recommendation
             if non_intrusive_recommendation.get("cost"):
@@ -274,11 +485,13 @@ class HeatingRecommender:
         # This is a map from the heating controls description to the description of the air source heat pump set up
         ashp_descriptions = {
             "Time and temperature zone control": (
-                "Install an air source heat pump, and upgrade heating controls to Smart Thermostats, "
-                "room sensors and smart radiator valves (time & temperature zone control)."
+                f"Install a {ashp_size}KW air source heat pump, and upgrade heating controls to Smart Thermostats, "
+                "room sensors and smart radiator valves (time & temperature zone control). Ensure you have an 18 or "
+                "24 hour tariff"
             ),
             "Programmer, TRVs and bypass": (
-                "Install an air source heat pump, with programmer, TRVs and a Bypass valve."
+                f"Install a {ashp_size}KW air source heat pump, with programmer, TRVs and a Bypass valve. Ensure you "
+                "have an 18 or 24 hour tariff"
             ),
         }
 
@@ -295,7 +508,7 @@ class HeatingRecommender:
                         ashp_costs_with_controls[key] += controls_rec[key]
 
             if controls_rec is None:
-                description = "Install an air source heat pump."
+                description = f"Install a {ashp_size}KW Air source heat pump. Ensure you have an 18 or 24 hour tariff"
             elif already_installed:
                 description = "The property already has an air source heat pump, no further action needed."
             else:
@@ -304,22 +517,21 @@ class HeatingRecommender:
             # If the property does not have existing cavity and loft insulation, we include a note that the cost
             # includes the boiler upgrade scheme and that the cavity and loft need to be treated, to ensure access
             # to the funding
-            if not non_intrusive_recommendation:
+            if not non_intrusive_recommendation and self.property.data["tenure"] not in assumptions.SOCIAL_TENURES:
                 if has_cavity_or_loft_recommendations:
                     description = description + (
-                        f" The cost includes the £"
-                        f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant. "
-                        f"You must ensure that the property has an insulated cavity and "
-                        f"270mm+ loft insulation to qualify for the grant"
+                        f" You must ensure that the property has an insulated cavity and "
+                        f"270mm+ loft insulation to qualify for the grant, to claim £"
+                        f"{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding from the boiler upgrade scheme grant. "
                     )
                 else:
                     description = description + (
-                        f" The cost includes the £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} boiler upgrade scheme grant"
+                        f" £{BOILER_UPGRADE_SCHEME_ASHP_VALUE} of funding can be claimed from the boiler upgrade scheme"
                     )
 
             simulation_config = {
-                "mainheat_energy_eff_ending": "Good",
-                "hot_water_energy_eff_ending": "Good"
+                "mainheat_energy_eff_ending": "Very Good",
+                "hot_water_energy_eff_ending": "Very Good"
             }
             description_simulation = {
                 "mainheat-description": new_heating_description,
@@ -375,10 +587,9 @@ class HeatingRecommender:
 
             ashp_recommendation = {
                 "phase": phase,
-                "parts": [
-                    # TODO
-                ],
+                "parts": [],
                 "type": "heating",
+                "measure_type": "air_source_heat_pump",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -418,7 +629,9 @@ class HeatingRecommender:
         description,
         phase,
         heating_controls_only,
-        system_change
+        system_change,
+        system_type,
+        measure_type
     ):
         """
         Given a recommendation for heating controls, and a recommendation for the heating system, we combine the two
@@ -433,7 +646,9 @@ class HeatingRecommender:
         :param system_change: Indicates if we are recommending a different type of heating system, compared to the
         current system. If we have a system change and we have a heat control recommendation, we only recommend
         both heating and controls together
-        :return:
+        :param system_type: The type of heating system we are recommending
+        :param measure_type: The type of measure we are recommending - more granular than the "type" field, allowing us
+        to distinguish between different types of heating recommendations
         """
 
         # We produce recommendations with & without heating controls
@@ -467,12 +682,8 @@ class HeatingRecommender:
                 }
 
                 controls_description = controls_recommendations[0]['description']
-                # Make the first letter of the description lowercase
-                controls_description = (
-                    controls_description[0].lower() + controls_description[1:]
-                )
 
-                recommendation_description = f"{description} and {controls_description}"
+                recommendation_description = f"{description} {controls_description}"
 
             already_installed = "heating_controls" in self.property.already_installed
             if already_installed:
@@ -481,10 +692,9 @@ class HeatingRecommender:
 
             recommendation = {
                 "phase": phase,
-                "parts": [
-                    # TODO
-                ],
+                "parts": [],
                 "type": "heating",
+                "measure_type": measure_type,
                 "description": recommendation_description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -492,7 +702,9 @@ class HeatingRecommender:
                 "already_installed": already_installed,
                 **total_costs,
                 "simulation_config": recommendation_simulation_config,
-                "description_simulation": recommendation_description_simulation
+                "description_simulation": recommendation_description_simulation,
+                # We insert the heating system type here
+                "system_type": system_type
             }
 
             output.append(recommendation)
@@ -548,6 +760,14 @@ class HeatingRecommender:
         We will recommend upgrading to a high heat retention storage system, if the current system is not already
         high heat retention storage
 
+        If the property currently has electric storage heaters, with automatic charge control, we allow for a high
+        heat retention stoarage heaters recommendation. This is because the automatic charge control is not the same
+        as the high heat retention storage heaters. HHR storage heaters aren't guaranteed to be more efficient but
+        we can at least present the option to the end user and they can decide if they want to go ahead with the
+        recommendation or not. There's a useful guide by quidos, describing the differences between some of the
+        different storage heater options:
+        https://www.quidos.co.uk/wp-content/uploads/2017/04/Technical-Bulletin-010417-Storage-Heatersv2.pdf
+
         :param phase: The phase of the recommendation
         :param system_change: Indicates if we are recommending a different type of heating system, compared to the
         current system
@@ -562,7 +782,24 @@ class HeatingRecommender:
 
         # We only recommend Celect-type controls if the current heating system is not Celect-type controls
         if self.property.main_heating_controls["clean_description"] != self.high_heat_retention_contols_desc:
-            controls_recommender.recommend(heating_description="Electric storage heaters, radiators")
+            if self.dual_heating:
+
+                controls_prefix = self.DUAL_HEATING_DESCRIPTIONS[
+                    self.property.main_heating["clean_description"]
+                ]["hhr"]["controls_prefix"]
+
+                if controls_prefix == "current_controls":
+                    description_prefix = self.property.main_heating_controls["clean_description"]
+                elif controls_prefix == "":
+                    description_prefix = ""
+                else:
+                    raise NotImplementedError("Implement me")
+            else:
+                description_prefix = ""
+
+            controls_recommender.recommend(
+                heating_description="Electric storage heaters", description_prefix=description_prefix
+            )
 
         has_hhr = self.is_hhr_already_installed()
         # Conditions for not recommending electric storage heaters
@@ -570,15 +807,41 @@ class HeatingRecommender:
             # No recommendation needed
             return
 
-        new_heating_description = "Electric storage heaters, radiators"
+        # We check if the property has dual heating in place with a boiler and storage heaters
+        if self.dual_heating:
+            new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[
+                self.property.main_heating["clean_description"]
+            ]["hhr"]["mainheating_description"]
+            new_hot_water_description = self.property.hotwater["clean_description"]  # We keep the hot water system
+        else:
+            new_heating_description = "Electric storage heaters"
+            new_hot_water_description = "Electric immersion, off-peak"
 
         # Set up artefacts, suitable for the simulation and regardless of controls
         heating_ending_config = MainHeatAttributes(new_heating_description).process()
         heating_simulation_config = check_simulation_difference(
             new_config=heating_ending_config, old_config=self.property.main_heating
         )
+
+        hot_water_end_config = HotWaterAttributes(new_hot_water_description).process()
+        hot_water_simulation_config = check_simulation_difference(
+            new_config=hot_water_end_config, old_config=self.property.hotwater
+        )
+
+        heating_simulation_config = {
+            **heating_simulation_config,
+            **hot_water_simulation_config
+        }
         # This upgrade will only take the heating system to average energy efficiency
-        heating_simulation_config["mainheat_energy_eff_ending"] = "Average"
+        if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor"] and not self.dual_heating:
+            heating_simulation_config["mainheat_energy_eff_ending"] = "Average"
+        else:
+            heating_simulation_config["mainheat_energy_eff_ending"] = self.property.data["mainheat-energy-eff"]
+
+        if self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor"]:
+            heating_simulation_config["hot_water_energy_eff_ending"] = "Average"
+        else:
+            heating_simulation_config["hot_water_energy_eff_ending"] = self.property.data["hot-water-energy-eff"]
 
         # If the property is off-gas and has no heating system in place, the number of heated rooms will actually
         # be 0, so we use the number of rooms as the figure
@@ -589,15 +852,37 @@ class HeatingRecommender:
                 self.property.number_of_rooms
             )
         )
+        # To be conservative, we adjust if we still have 1 room
+        if (number_heated_rooms == 1) and (self.property.number_of_rooms > 2):
+            number_heated_rooms = self.property.number_of_rooms - 1
+
         # Upgrade to electric storage heaters
         costs = self.costs.high_heat_electric_storage_heaters(
-            number_heated_rooms=number_heated_rooms
+            number_heated_rooms=number_heated_rooms,
+            needs_cylinder=self.property.hotwater["system_type"] == "from main system"
         )
-        description = "Install high heat retention electric storage heaters"
+        if self.dual_heating:
+            description = self.DUAL_HEATING_DESCRIPTIONS[
+                self.property.main_heating["clean_description"]
+            ]["hhr"]["recommendation_description"]
+
+        else:
+            description = "Install high heat retention electric storage heaters with an appropriate off-peak tariff."
+
+        # We check the existing heating system and controls
+        if (
+            self.property.main_heating["has_electric_storage_heaters"] and
+            self.property.main_heating_controls["charging_system"] in
+            ["automatic charge control", "manual charge control"]
+        ):
+            description += (" The current electric heaters may be retrofit with high heat retention storage controls"
+                            " however this is dependent on the existing system and may not be possible.")
 
         heating_description_simulation = {
             "mainheat-description": new_heating_description,
             "mainheat-energy-eff": heating_simulation_config["mainheat_energy_eff_ending"],
+            "hotwater-description": new_hot_water_description,
+            "hot-water-energy-eff": heating_simulation_config["hot_water_energy_eff_ending"]
         }
 
         recommendations = self.combine_heating_and_controls(
@@ -608,7 +893,9 @@ class HeatingRecommender:
             description=description,
             phase=phase,
             heating_controls_only=heating_controls_only,
-            system_change=system_change
+            system_change=system_change,
+            system_type="high_heat_retention_storage_heater",
+            measure_type="high_heat_retention_storage_heater"
         )
         if _return:
             return recommendations
@@ -688,12 +975,13 @@ class HeatingRecommender:
 
         has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]
 
-        has_inefficient_mains_water = (
-            self.property.hotwater["clean_description"] in ["From main system"] and
+        # We check if there's a mains connection and the hot water is inefficient, as this will improve with a boiler
+        has_inefficient_water = (
+            self.property.data["mains-gas-flag"] and
             self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"]
         )
 
-        if has_inefficient_space_heating or has_inefficient_mains_water:
+        if has_inefficient_space_heating or has_inefficient_water:
             boiler_size = self.estimate_boiler_size(
                 property_type=self.property.data["property-type"],
                 built_form=self.property.data["built-form"],
@@ -702,11 +990,26 @@ class HeatingRecommender:
                 num_heated_rooms=self.property.data["number-heated-rooms"],
             )
 
-            description = "Upgrade to a new condensing boiler"
+            if self.dual_heating:
+                description = self.DUAL_HEATING_DESCRIPTIONS[
+                    self.property.main_heating["clean_description"]
+                ]["boiler"]["recommendation_description"]
+            else:
+                description = "Upgrade to a new condensing boiler."
+
+            new_heating_eff = (
+                "Good" if self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]
+                else self.property.data["mainheat-energy-eff"]
+            )
+
+            new_hotwater_eff = (
+                "Good" if self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"]
+                else self.property.data["hot-water-energy-eff"]
+            )
 
             simulation_config = {
-                "mainheat_energy_eff_ending": "Good",
-                "hot_water_energy_eff_ending": "Good"
+                "mainheat_energy_eff_ending": new_heating_eff,
+                "hot_water_energy_eff_ending": new_hotwater_eff
             }
 
             description_simulation = {
@@ -717,7 +1020,13 @@ class HeatingRecommender:
             if system_change:
                 # Installation of a boiler improves the hot water system so we need to reflect this in
                 # the outcome of the recommendation
-                new_heating_description = "Boiler and radiators, mains gas"
+                if self.dual_heating:
+                    new_heating_description = self.DUAL_HEATING_DESCRIPTIONS[
+                        self.property.main_heating["clean_description"]
+                    ]["boiler"]["mainheating_description"]
+                else:
+                    new_heating_description = "Boiler and radiators, mains gas"
+
                 new_hotwater_description = "From main system"
                 new_fuel_description = "mains gas (not community)"
 
@@ -764,10 +1073,9 @@ class HeatingRecommender:
 
             boiler_recommendation = {
                 "phase": recommendation_phase,
-                "parts": [
-                    # TODO
-                ],
+                "parts": [],
                 "type": "heating",
+                "measure_type": "boiler_upgrade",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -775,13 +1083,23 @@ class HeatingRecommender:
                 "already_installed": already_installed,
                 "simulation_config": simulation_config,
                 "description_simulation": description_simulation,
-                **boiler_costs
+                **boiler_costs,
+                "system_type": "boiler_upgrade",
             }
 
         # We recommend the heating controls
         # If the property did not previously have a boiler, we combine
         controls_recommender = HeatingControlRecommender(self.property)
-        controls_recommender.recommend(heating_description="Boiler and radiators, mains gas")
+        if self.dual_heating:
+            description_suffix = self.DUAL_HEATING_DESCRIPTIONS[
+                self.property.main_heating["clean_description"]
+            ]["boiler"]["controls_suffix"]
+        else:
+            description_suffix = ""
+        controls_recommender.recommend(
+            heating_description="Boiler and radiators, mains gas",
+            description_suffix=description_suffix
+        )
         # We may have 2 recommendations from the heating controls
 
         if not controls_recommender.recommendation and not boiler_recommendation:
@@ -803,24 +1121,25 @@ class HeatingRecommender:
                     description=boiler_recommendation["description"],
                     phase=recommendation_phase,
                     heating_controls_only=False,
-                    system_change=True
+                    system_change=True,
+                    system_type="boiler_upgrade",
+                    measure_type="boiler_upgrade",
                 )
                 combined_recommendations.extend(combined_recommendation)
 
             # Overwrite the existing boiler recommendation
             self.heating_recommendations.extend(combined_recommendations)
         else:
-            # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade
-            # but we'll only upgrade if we have a heating recommendation
-            has_heating_recommendation = any(
-                rec["type"] == "heating" for rec in self.heating_recommendations
-            )
-            if has_heating_recommendation:
-                recommendation_phase += 1
-            # The heating controls recommendation is distrinct from the boiler upgrade recommendation
-            # We insert phase into the recommendations for heating controls
+            # We consider a heating control upgrade as a measure which occures in the same phase as a boiler upgrade
+            # Namely, we have the following options within this phase
+            # 1) Boiler + heating controls
+            # 2) Boiler only
+            # 3) Heating controls only
+            # But they are options that are not mutually exclusive
+            # So, we actually set heating controls as a heating recommendation
             for recommendation in controls_recommender.recommendation:
                 recommendation["phase"] = recommendation_phase
+                # recommendation["type"] = "heating"
 
             self.heating_control_recommendations.extend(controls_recommender.recommendation)
 

recommendations/HotwaterRecommendations.py

@@ -58,10 +58,9 @@ class HotwaterRecommendations:
         self.recommendations.append(
             {
                 "phase": phase,
-                "parts": [
-                    # TODO
-                ],
+                "parts": [],
                 "type": "hot_water_tank_insulation",
+                "measure_type": "hot_water_tank_insulation",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -107,6 +106,7 @@ class HotwaterRecommendations:
                 "phase": phase,
                 "parts": [],
                 "type": "cylinder_thermostat",
+                "measure_type": "cylinder_thermostat",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/LightingRecommendations.py

@@ -1,3 +1,5 @@
+import pandas as pd
+
 from backend.Property import Property
 from typing import List
 from recommendations.Costs import Costs
@@ -9,6 +11,9 @@ class LightingRecommendations:
     # worth more than 2 points, but this is unlikely in the context of other upgrades that can be made to the property
     SAP_LIMIT = 2
 
+    # If more than 50% of the lighting is LEDs already, the limit is 1 SAP point
+    SAP_LOWER_LIMIT = 1
+
     def __init__(self, property_instance: Property, materials: List):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
@@ -27,6 +32,37 @@ class LightingRecommendations:
         self.material = material[0]
         self.recommendation = []
 
+    @classmethod
+    def get_sap_limit(cls, lighting_energy_efficiency: str, lighting_proportion: float):
+        """
+        Lighting seems to be a more straight forward measure to estimate SAP points for, based on the starting
+        energy efficiency rating.
+
+        We seem to have the following brackes based on % of LEDs in outlets
+        Very poor: 0 - 9%
+        Poor: 10 - 24%
+        Average: 25 - 44%
+        Good: 45 - 69%
+        Very good: 70 - 100%
+        :return:
+        """
+
+        if lighting_energy_efficiency == "Very Good":
+            return 0
+
+        if lighting_energy_efficiency in ["Good", "Average"]:
+            return cls.SAP_LOWER_LIMIT
+
+        # If lighting_energy_efficiency is missing, we'll use the proportion of low energy lighting
+        if not lighting_energy_efficiency or pd.isnull(lighting_energy_efficiency):
+            if lighting_proportion >= 0.7:
+                return 0
+            if lighting_proportion >= 0.25:
+                return cls.SAP_LOWER_LIMIT
+            return cls.SAP_LIMIT
+
+        return cls.SAP_LIMIT
+
     @staticmethod
     def estimate_lighting_impact(number_of_bulbs: int):
         """
@@ -116,6 +152,7 @@ class LightingRecommendations:
                 "phase": phase,
                 "parts": [],
                 "type": "low_energy_lighting",
+                "measure_type": "low_energy_lighting",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -128,6 +165,7 @@ class LightingRecommendations:
                 "description_simulation": {
                     "lighting-energy-eff": "Very Good",
                     "lighting-description": "Low energy lighting in all fixed outlets",
+                    "low-energy-lighting": 100,
                 },
                 **cost_result,
                 "survey": leds_recommendation_config.get("survey", False)

recommendations/Recommendations.py

@@ -18,9 +18,8 @@ from recommendations.DraughtProofingRecommendations import DraughtProofingRecomm
 from backend.ml_models.AnnualBillSavings import AnnualBillSavings
 from backend.apis.GoogleSolarApi import GoogleSolarApi
 import backend.app.assumptions as assumptions
-from backend.app.plan.schemas import TYPICAL_MEASURE_TYPES, SPECIFIC_MEASURES, MEASURE_MAP
+from backend.app.plan.schemas import SPECIFIC_MEASURES, MEASURE_MAP, NON_INVASIVE_SPECIFIC_MEASURES
 
-ASHP_COP = 3
 STARTING_DUMMY_ID_VALUE = -9999
 
 
@@ -35,6 +34,7 @@ class Recommendations:
         materials: List,
         exclusions: List[str] = None,
         inclusions: List[str] = None,
+        default_u_values: bool = False,
     ):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
@@ -43,15 +43,20 @@ class Recommendations:
                             None, meaning no exclusions to be applied
         :param inclusions: List of specific measures of measure types to include. Defaulted to None, meaning all
                            measures are included
+        :param default_u_values: Boolean, if True, the recommendations will use the default u-values for the property
         """
 
         self.property_instance = property_instance
         self.materials = materials
         self.exclusions = exclusions if exclusions else []
         self.inclusions = inclusions if inclusions else []
+        self.default_u_values = default_u_values
 
-        self.all_typical_measures = TYPICAL_MEASURE_TYPES
         self.all_specific_measures = SPECIFIC_MEASURES
+        self.all_non_invase_measures = NON_INVASIVE_SPECIFIC_MEASURES
+        self.non_invasive_recommendation_types = [
+            r["type"] for r in self.property_instance.non_invasive_recommendations
+        ]
 
         self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials)
         self.wall_recomender = WallRecommendations(property_instance=property_instance, materials=materials)
@@ -78,16 +83,29 @@ class Recommendations:
 
         inclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.inclusions]
         exclusions_full = [MEASURE_MAP[x] if x in MEASURE_MAP else x for x in self.exclusions]
+        # We need to unlist any lists, but we should check if they're lists first
+        inclusions_full = [
+            item for sublist in inclusions_full for item in (sublist if isinstance(sublist, list) else [sublist])
+        ]
+        exclusions_full = [
+            item for sublist in exclusions_full for item in (sublist if isinstance(sublist, list) else [sublist])
+        ]
 
-        if inclusions_full and exclusions_full:
-            # All typical measures
-            return self.all_specific_measures
+        # If inclusions and exclusions are empty, it means that nothing was specified, so we allow
+        # all recommendation types
+        if not inclusions_full and not exclusions_full:
+            # All typical measures - this does not include non-invasive measures inless they are specified
+            return self.all_specific_measures + self.non_invasive_recommendation_types
 
         if inclusions_full:
             return inclusions_full
 
         if exclusions_full:
-            return [m for m in self.all_specific_measures if m not in exclusions_full]
+            measures = [
+                m for m in self.all_specific_measures + self.non_invasive_recommendation_types
+                if m not in exclusions_full
+            ]
+            return measures
 
     def recommend(self):
 
@@ -102,14 +120,22 @@ class Recommendations:
         property_recommendations = []
         phase = 0
         measures = self.find_included_measures()
+        non_invasive_recommendation_types = [r["type"] for r in self.property_instance.non_invasive_recommendations]
 
         # Building Fabric
-        self.wall_recomender.recommend(phase=phase, measures=measures)
+        self.wall_recomender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values)
         if self.wall_recomender.recommendations:
             property_recommendations.append(self.wall_recomender.recommendations)
             phase += 1
 
-        self.roof_recommender.recommend(phase=phase, measures=measures)
+        # We handle recommendations covering specific non-invasive measures
+        new_phase = self.wall_recomender.recommend_extended(phase=phase, measures=measures)
+        if self.wall_recomender.extended_recommendations:
+            property_recommendations.append(self.wall_recomender.extended_recommendations)
+            # We don't have any phasing here
+            phase = new_phase
+
+        self.roof_recommender.recommend(phase=phase, measures=measures, default_u_values=self.default_u_values)
         if self.roof_recommender.recommendations:
             property_recommendations.append(self.roof_recommender.recommendations)
             phase += 1
@@ -143,14 +169,20 @@ class Recommendations:
             if self.draught_proofing_recommender.recommendation:
                 property_recommendations.append(self.draught_proofing_recommender.recommendation)
 
-        if "floor_insulation" in measures:
-            self.floor_recommender.recommend(phase=phase, measures=measures)
-            if self.floor_recommender.recommendations:
-                property_recommendations.append(self.floor_recommender.recommendations)
+        self.floor_recommender.recommend(phase=phase, measures=measures)
+        if self.floor_recommender.recommendations:
+            property_recommendations.append(self.floor_recommender.recommendations)
+            phase += 1
+
+        if "low_energy_lighting" in measures:
+            self.lighting_recommender.recommend(phase=phase)
+            if self.lighting_recommender.recommendation:
+                property_recommendations.append(self.lighting_recommender.recommendation)
                 phase += 1
 
-        if "windows" in measures:
-            self.windows_recommender.recommend(phase=phase)
+        if "mixed_glazing" not in non_invasive_recommendation_types:
+            # If we have a mixed glazing recommendation, we prioritise this over the windows recommendation
+            self.windows_recommender.recommend(phase=phase, measures=measures)
             if self.windows_recommender.recommendation:
                 property_recommendations.append(self.windows_recommender.recommendation)
                 phase += 1
@@ -224,12 +256,6 @@ class Recommendations:
                 property_recommendations.append(self.hotwater_recommender.recommendations)
                 phase += 1
 
-        if "low_energy_lighting" in measures:
-            self.lighting_recommender.recommend(phase=phase)
-            if self.lighting_recommender.recommendation:
-                property_recommendations.append(self.lighting_recommender.recommendation)
-                phase += 1
-
         if "secondary_heating" in measures:
             self.secondary_heating_recommender.recommend(phase=phase)
             if self.secondary_heating_recommender.recommendation:
@@ -251,6 +277,11 @@ class Recommendations:
             property_recommendations,
         )
 
+        # Check to make sure measure_type is populated
+        for recs in property_recommendations:
+            if any(pd.isnull(rec.get("measure_type")) for rec in recs):
+                raise ValueError("Measure type is not populated")
+
         return property_recommendations, property_representative_recommendations
 
     @staticmethod
@@ -446,9 +477,32 @@ class Recommendations:
         impact_summary = []
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
-                if rec["type"] in ["mechanical_ventilation", "trickle_vents", "draught_proofing"]:
+                if rec["type"] in [
+                    "mechanical_ventilation", "trickle_vents", "draught_proofing", "extension_cavity_wall_insulation"
+                ]:
                     # We don't have a percieved sap impact of mechanical ventilation or trickle vents, and we don't
                     # have the capacity to score draught proofing
+                    if rec["type"] == "extension_cavity_wall_insulation":
+
+                        previous_phase = [x for x in impact_summary if x["phase"] == (rec["phase"] - 1)]
+                        if previous_phase:
+                            sap = previous_phase[0]["sap"]
+                            carbon = previous_phase[0]["carbon"]
+                            heat_demand = previous_phase[0]["heat_demand"]
+                        else:
+                            sap = float(property_instance.data["current-energy-efficiency"])
+                            carbon = float(property_instance.data["co2-emissions-current"])
+                            heat_demand = float(property_instance.data["energy-consumption-current"])
+
+                        impact_summary.append(
+                            {
+                                "phase": rec["phase"],
+                                "recommendation_id": rec["recommendation_id"],
+                                "sap": sap + rec["sap_points"],
+                                "carbon": carbon - rec["co2_equivalent_savings"],
+                                "heat_demand": heat_demand - rec["heat_demand"],
+                            }
+                        )
                     continue
 
                 phase_energy_efficiency_metrics = {
@@ -529,11 +583,27 @@ class Recommendations:
 
                 # For the moment, we cap the number of SAP points that can be achieved by LEDs at 2
                 if rec["type"] == "low_energy_lighting":
-                    property_phase_impact["sap"] = min(property_phase_impact["sap"], LightingRecommendations.SAP_LIMIT)
+                    lighting_sap_limit = LightingRecommendations.get_sap_limit(
+                        property_instance.data["lighting-energy-eff"],
+                        property_instance.lighting["low_energy_proportion"]
+                    )
+
+                    property_phase_impact["sap"] = min(property_phase_impact["sap"], lighting_sap_limit)
                     property_phase_impact["carbon"] = min(
                         property_phase_impact["carbon"], rec["co2_equivalent_savings"]
                     )
 
+                if rec["type"] == "loft_insulation":
+                    # When we have a loft insulation recommendation, where there is an extension and the existing
+                    # amount of loft insulation is already good, we limit the SAP points
+                    # By limiting here, we don't change the value in current_phase_values. This means that the
+                    # future recommendations won't have an impact that is too large
+                    li_sap_limit = RoofRecommendations.get_loft_insulation_sap_limit(
+                        property_instance.data["roof-energy-eff"], property_instance.data["extension-count"]
+                    )
+                    if li_sap_limit is not None:
+                        property_phase_impact["sap"] = min(property_phase_impact["sap"], li_sap_limit)
+
                 # Insert this information into the recommendation.
                 if not rec.get("survey", False):
                     rec["sap_points"] = property_phase_impact["sap"]
@@ -635,7 +705,11 @@ class Recommendations:
             {
                 "phase": r["phase"],
                 "recommendation_id": r["recommendation_id"],
-                "solar_kwh_savings": r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_PROPORTION,
+                "solar_kwh_savings": (
+                    r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_PROPORTION
+                ) if not r["has_battery"] else (
+                    r["initial_ac_kwh_per_year"] * assumptions.SOLAR_CONSUMPTION_WITH_BATTERY_PROPORTION
+                ),
             } for recs in property_recommendations for r in recs if r["type"] == "solar_pv"
         ], columns=["phase", "recommendation_id", "solar_kwh_savings"])
 
@@ -656,8 +730,8 @@ class Recommendations:
                             "id": STARTING_DUMMY_ID_VALUE,
                             "phase": STARTING_DUMMY_ID_VALUE,
                             "recommendation_id": STARTING_DUMMY_ID_VALUE,
-                            "predictions_heating": property_kwh["heating"],
-                            "predictions_hotwater": property_kwh["hot_water"],
+                            "predictions_heating": float(property_kwh["heating"]),
+                            "predictions_hotwater": float(property_kwh["hot_water"]),
                         }
                     ]
                 ),
@@ -722,7 +796,9 @@ class Recommendations:
         # We now deduce if any of the recommendations result in a change of fuel type
         for recs in property_recommendations:
             for rec in recs:
-                if rec["type"] in ["mechanical_ventilation", "trickle_vents", "draught_proofing"]:
+                if rec["type"] in [
+                    "mechanical_ventilation", "trickle_vents", "draught_proofing", "extension_cavity_wall_insulation"
+                ]:
                     # We cannot score the impact on draught proofing
                     continue
 
@@ -776,13 +852,14 @@ class Recommendations:
 
         electricity_standing_charge = AnnualBillSavings.DAILY_STANDARD_CHARGE_ELECTRICITY * 365
 
-        current_energy_bill = (
-            starting_figures["heating_cost"] +
-            starting_figures["hotwater_cost"] +
-            property_instance.energy_cost_estimates["unadjusted"]["lighting"] +
-            property_instance.energy_cost_estimates["unadjusted"]["appliances"] +
-            gas_standing_charge +
-            electricity_standing_charge
-        )
+        # We return a dictionary that contains the individual costs, that can be stored to the database
+        current_energy_bill = {
+            "heating_cost_current": float(starting_figures["heating_cost"]),
+            "hot_water_cost_current": float(starting_figures["hotwater_cost"]),
+            "lighting_cost_current": float(property_instance.energy_cost_estimates["unadjusted"]["lighting"]),
+            "appliances_cost_current": float(property_instance.energy_cost_estimates["unadjusted"]["appliances"]),
+            "gas_standing_charge": float(gas_standing_charge),
+            "electricity_standing_charge": float(electricity_standing_charge),
+        }
 
         return current_energy_bill

recommendations/RoofRecommendations.py

@@ -44,20 +44,14 @@ class RoofRecommendations:
         self.recommendations = []
 
         self.loft_insulation_materials = [
-            part for part in materials if part["type"] == "loft_insulation"
+            part for part in materials if (part["type"] == "loft_insulation") and (part["is_installer_quote"])
         ]
-        self.loft_non_insulation_materials = []
 
+        # We don't have proper installer quotes for flat roof insulation
         self.flat_roof_insulation_materials = [
             part for part in materials if part["type"] == "flat_roof_insulation"
         ]
 
-        self.flat_roof_non_insulation_materials = [
-            part for part in materials if part["type"] in [
-                "flat_roof_preparation", "flat_roof_vapour_barrier", "flat_roof_waterproofing"
-            ]
-        ]
-
         # Extract the insulation thickness from the roof, which is used throughout this method
         self.insulation_thickness = convert_thickness_to_numeric(
             self.property.roof["insulation_thickness"],
@@ -65,6 +59,23 @@ class RoofRecommendations:
             self.property.roof["is_flat"]
         )
 
+    @classmethod
+    def get_loft_insulation_sap_limit(cls, roof_energy_eff, extension_count):
+        """
+        Get the SAP limit for loft insulation
+        :param roof_energy_eff:
+        :return:
+        """
+
+        if extension_count == 0:
+            # No limit
+            return None
+
+        if roof_energy_eff in ["Good", "Very Good"]:
+            return 1
+
+        return None
+
     def mds_loft_insulation(self, phase):
         """
         For usages within the mds report
@@ -90,12 +101,17 @@ class RoofRecommendations:
 
         return (self.insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]
 
-    def is_room_roof_insulated(self):
+    def is_room_roof_insulated_or_unsuitable(self, measures):
 
         """
         Check if the room roof is already insulated
         """
 
+        # If the roof is a room roof room roof is not included in the measures, we deem the recommendation unsuitable
+        unsuitable = "room_roof_insulation" not in measures and self.property.roof["is_roof_room"]
+        if unsuitable:
+            return True
+
         full_insulated_room_roof = (
             self.property.roof["is_roof_room"] and
             self.property.roof["insulation_thickness"] in ["average", "above_average"]
@@ -109,7 +125,7 @@ class RoofRecommendations:
 
         return full_insulated_room_roof or room_roof_insulated_at_rafters
 
-    def recommend(self, phase, measures=None):
+    def recommend(self, phase, measures=None, default_u_values=False):
 
         if self.property.roof["has_dwelling_above"]:
             return
@@ -129,7 +145,7 @@ class RoofRecommendations:
         if (self.insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]:
             return
 
-        if self.is_room_roof_insulated():
+        if self.is_room_roof_insulated_or_unsuitable(measures):
             return
 
         # If we have a u-value already, need to implement this
@@ -138,7 +154,7 @@ class RoofRecommendations:
                 # The Roof is already compliant
                 return
 
-            if self.property.data["transaction-type"] == "new dwelling":
+            if self.property.data["transaction-type"] in ["new dwelling", "not sale or rental"]:
                 return
             raise NotImplementedError("Implement me")
 
@@ -155,25 +171,48 @@ class RoofRecommendations:
         )
 
         self.estimated_u_value = u_value
-        if (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) or (
-            "loft_insulation" not in measures
+        if (u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) or all(
+            m not in measures for m in MEASURE_MAP["roof_insulation"]
         ):
             # The Roof is already compliant
             return
 
-        if (self.property.roof["is_pitched"] and "loft_insulation" in measures) or (
-            self.property.roof["is_flat"] and "flat_roof_insulation" in measures
+        non_invasive_recommendations = self.property.non_invasive_recommendations
+
+        # We firstly handle non-intrusive recommendations, which may override the normal roof insulation recommendations
+        if ("loft_insulation" in [x["type"] for x in non_invasive_recommendations]) or (
+            self.property.roof["is_pitched"] and "loft_insulation" in measures
         ):
-            insulation_thickness = 0 if "loft_insulation" not in measures else self.insulation_thickness
-            self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
+            self.recommend_roof_insulation(
+                u_value=u_value,
+                insulation_thickness=self.insulation_thickness,
+                phase=phase,
+                is_flat=False,
+                is_pitched=True,
+                default_u_values=default_u_values
+            )
+            return
+
+        if (
+            (self.property.roof["is_flat"] and "flat_roof_insulation" in measures) or
+            "flat_roof_insulation" in [x["type"] for x in non_invasive_recommendations]
+        ):
+            self.recommend_roof_insulation(
+                u_value=u_value,
+                insulation_thickness=0,
+                phase=phase,
+                is_flat=True,
+                is_pitched=False,
+                default_u_values=default_u_values
+            )
             return
 
         # There are cases where the property might have a room roof as the second roof, but we have a recommendation for
         # it, so we allow this override
         if self.property.roof["is_roof_room"] and ("room_roof_insulation" in measures) or (
-            "room_roof_insulation" in [x["type"] for x in self.property.non_invasive_recommendations]
+            "room_roof_insulation" in [x["type"] for x in non_invasive_recommendations]
         ):
-            self.recommend_room_roof_insulation(u_value, phase)
+            self.recommend_room_roof_insulation(u_value, phase, default_u_values)
             return
 
         raise NotImplementedError("Implement me")
@@ -195,7 +234,7 @@ class RoofRecommendations:
         raise ValueError("Invalid material type")
 
     def recommend_roof_insulation(
-        self, u_value, insulation_thickness, roof, phase
+        self, u_value, insulation_thickness, phase, is_pitched, is_flat, default_u_values
     ):
 
         """
@@ -218,7 +257,10 @@ class RoofRecommendations:
 
         :param u_value: U-value of the roof before any retrofit measures have been installed
         :param insulation_thickness: Existing Insulation thickness of the loft
-        :param roof: dictionary describing the make-up of the roof
+        :param phase: Phase of the recommendation
+        :param is_pitched: Is the roof pitched
+        :param is_flat: Is the roof flat
+        :param default_u_values: Use default u-values
         :return:
         """
 
@@ -226,12 +268,12 @@ class RoofRecommendations:
         # Therefore the price is 100mm + whatever thickness is rolled on top, rolled at a 90 degree angle
         # from the base layer
 
-        if roof["is_pitched"]:
+        if is_pitched:
             insulation_materials = self.loft_insulation_materials
-            non_insulation_materials = self.loft_non_insulation_materials
-        elif roof["is_flat"]:
+            measure_type = "loft_insulation"
+        elif is_flat:
             insulation_materials = self.flat_roof_insulation_materials
-            non_insulation_materials = self.flat_roof_non_insulation_materials
+            measure_type = "flat_roof_insulation"
         else:
             raise ValueError("Roof is not pitched or flat")
 
@@ -243,15 +285,13 @@ class RoofRecommendations:
         lowest_selected_u_value = None
         recommendations = []
         for _, insulation_material_group in insulation_materials.groupby("description"):
-
             for _, material in insulation_material_group.iterrows():
-
                 # We make sure we hit a depth of 270mm. We should factor in any existing insulation if the
                 # loft is already partially insulated.
                 # Note: This requirement is only for loft insulation
                 if (
-                    (material["depth"] + insulation_thickness) < self.MINIMUM_RECOMMENDED_LOFT_INSULATION
-                ) and roof["is_pitched"]:
+                    material["depth"] < self.MINIMUM_RECOMMENDED_LOFT_INSULATION
+                ) and is_pitched:
                     continue
 
                 part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
@@ -272,17 +312,22 @@ class RoofRecommendations:
 
                 # We allow a small tolerance for error so we don't discount the recommendation entirely
                 if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
+
                     lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
 
+                    cost_result = self.costs.loft_and_flat_insulation(
+                        floor_area=self.property.insulation_floor_area,
+                        material=material
+                    )
+
+                    already_installed = material["type"] in self.property.already_installed
+                    if already_installed:
+                        cost_result = override_costs(cost_result)
+
                     if material["type"] == "loft_insulation":
-                        cost_result = self.costs.loft_insulation(
-                            floor_area=self.property.insulation_floor_area,
-                            material=material
-                        )
-                        already_installed = "loft_insulation" in self.property.already_installed
-                        if already_installed:
-                            cost_result = override_costs(cost_result)
-                        new_thickness = insulation_thickness + material["depth"]
+                        # We take the new thickness as just the thickness of the insulation, to be conservative
+                        # and assume that any existing insulation will be replaced
+                        new_thickness = material["depth"]
 
                         # This is based on the values we have in the training data
                         valid_numeric_values = [
@@ -307,27 +352,45 @@ class RoofRecommendations:
                                 valid_numeric_values, key=lambda x: abs(x - proposed_depth)
                             )
 
-                        if proposed_depth >= 270:
+                        if proposed_depth >= 300:
                             new_efficiency = "Very Good"
                         else:
-                            if self.property.data["walls-energy-eff"] not in ["Good", "Very Good"]:
+                            if self.property.data["roof-energy-eff"] not in ["Good", "Very Good"]:
                                 new_efficiency = "Good"
                             else:
                                 new_efficiency = "Very Good"
 
                         new_description = f"Pitched, {int(proposed_depth)}mm loft insulation"
 
+                        if default_u_values:
+                            # We update the u-value with the default if we're using default u-values
+                            new_u_value = get_roof_u_value(
+                                insulation_thickness=str(int(new_thickness)),
+                                has_dwelling_above=self.property.roof["has_dwelling_above"],
+                                is_loft=self.property.roof["is_loft"],
+                                is_roof_room=self.property.roof["is_roof_room"],
+                                is_thatched=self.property.roof["is_thatched"],
+                                age_band=self.property.age_band,
+                                is_flat=self.property.roof["is_flat"],
+                                is_pitched=self.property.roof["is_pitched"],
+                                is_at_rafters=self.property.roof["is_at_rafters"],
+                            )
+
                     elif material["type"] == "flat_roof_insulation":
-                        cost_result = self.costs.flat_roof_insulation(
-                            floor_area=self.property.insulation_floor_area,
-                            material=material,
-                            non_insulation_materials=non_insulation_materials
-                        )
-                        already_installed = "flat_roof_insulation" in self.property.already_installed
-                        if already_installed:
-                            cost_result = override_costs(cost_result)
                         new_description = "Flat, insulated"
                         new_efficiency = "Good"
+                        if default_u_values:
+                            new_u_value = get_roof_u_value(
+                                insulation_thickness="100",
+                                has_dwelling_above=self.property.roof["has_dwelling_above"],
+                                is_loft=self.property.roof["is_loft"],
+                                is_roof_room=self.property.roof["is_roof_room"],
+                                is_thatched=self.property.roof["is_thatched"],
+                                age_band=self.property.age_band,
+                                is_flat=self.property.roof["is_flat"],
+                                is_pitched=self.property.roof["is_pitched"],
+                                is_at_rafters=self.property.roof["is_at_rafters"],
+                            )
                     else:
                         raise ValueError("Invalid material type")
 
@@ -354,6 +417,7 @@ class RoofRecommendations:
                                 )
                             ],
                             "type": material["type"],
+                            "measure_type": measure_type,
                             "description": self.make_roof_insulation_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
@@ -370,7 +434,7 @@ class RoofRecommendations:
 
         self.recommendations = recommendations
 
-    def recommend_room_roof_insulation(self, u_value, phase):
+    def recommend_room_roof_insulation(self, u_value, phase, default_u_values):
         """
         This method recommends room in roof insulation for properties that have been identified
         to possess a room in roof.
@@ -409,6 +473,8 @@ class RoofRecommendations:
          - Flat ceilings can be insulated like a standard loft.
 
         :param u_value:  Current u-value of the roof
+        :param phase: Phase of the recommendation
+        :param default_u_values: Use default u-values
         :return:
         """
 
@@ -438,20 +504,7 @@ class RoofRecommendations:
                 _, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
                 new_u_value = math.ceil(new_u_value * 100.0) / 100.0
 
-                # If I have a lowest U value and my new u value is higher than that but lower than the
-                # diminishing returns threshold, it can be considered
-
-                # If I have a lowest U value and my new u value is lower than the lowest value, it's
-                # further into the diminishing returns threshold and can shouldn't be
-
-                # if is_diminishing_returns(
-                #     recommendations, new_u_value, lowest_selected_u_value, self.DIMINISHING_RETURNS_U_VALUE
-                # ):
-                #     continue
-
                 # We allow a small tolerance for error so we don't discount the recommendation entirely
-                # if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
-                # lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
 
                 estimated_cost = (
                     cost_per_unit * self.property.insulation_floor_area if
@@ -462,7 +515,7 @@ class RoofRecommendations:
                 sap_points = rir_non_invasive_recommendation.get("sap_points", None)
 
                 # Could also be Roof room(s), ceiling insulated
-                new_descriptin = "Pitched, insulated at rafters"
+                new_descriptin = "Roof room(s), insulated"
                 roof_ending_config = RoofAttributes(new_descriptin).process()
                 roof_simulation_config = check_simulation_difference(
                     new_config=roof_ending_config, old_config=self.property.roof, prefix="roof_"
@@ -472,6 +525,19 @@ class RoofRecommendations:
                 else:
                     new_efficiency = self.property.data["roof-energy-eff"]
 
+                if default_u_values:
+                    new_u_value = get_roof_u_value(
+                        insulation_thickness="average",
+                        has_dwelling_above=self.property.roof["has_dwelling_above"],
+                        is_loft=self.property.roof["is_loft"],
+                        is_roof_room=self.property.roof["is_roof_room"],
+                        is_thatched=self.property.roof["is_thatched"],
+                        age_band=self.property.age_band,
+                        is_flat=self.property.roof["is_flat"],
+                        is_pitched=self.property.roof["is_pitched"],
+                        is_at_rafters=self.property.roof["is_at_rafters"],
+                    )
+
                 simulation_config = {
                     **roof_simulation_config,
                     "roof_thermal_transmittance_ending": new_u_value,
@@ -490,13 +556,12 @@ class RoofRecommendations:
                 recommendations.append(
                     {
                         "phase": phase,
-                        "parts": [
-                            # TODO
-                        ],
+                        "parts": [],
                         "type": "room_roof_insulation",
+                        "measure_type": "room_roof_insulation",
                         "description": "Insulate room in roof at rafters and re-decorate",
                         "starting_u_value": u_value,
-                        "new_u_value": None,
+                        "new_u_value": new_u_value,
                         "sap_points": sap_points,
                         "simulation_config": simulation_config,
                         "description_simulation": {

recommendations/SecondaryHeating.py

@@ -52,6 +52,7 @@ class SecondaryHeating:
                 "phase": phase,
                 "parts": [],
                 "type": "secondary_heating",
+                "measure_type": "secondary_heating",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/SolarPvRecommendations.py

@@ -2,7 +2,7 @@ import numpy as np
 import pandas as pd
 
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import override_costs, esimtate_pitched_roof_area
+from recommendations.recommendation_utils import override_costs, estimate_pitched_roof_area
 
 
 class SolarPvRecommendations:
@@ -17,6 +17,8 @@ class SolarPvRecommendations:
     MAX_SYSTEM_WATTAGE = 6000
     MIN_SYSTEM_WATTAGE = 1000
 
+    MAX_ROOF_AREA_PERCENTAGE = 0.7
+
     def __init__(self, property_instance):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
@@ -104,8 +106,13 @@ class SolarPvRecommendations:
                 roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / total_roof_area * 100)
             else:
                 raise Exception("IMPLEMENT ME")
-            # Spread the cost to the individual units - adding a 20% contingency
-            total_cost = recommendation_config["total_cost"] / n_units
+            total_cost = self.costs.solar_pv(
+                array_cost=recommendation_config.get("cost", None),
+                n_panels=recommendation_config["n_panels"],
+                n_floors=self.property.number_of_storeys["number_of_storeys"],
+                needs_inverter=True,
+            )["total"] / n_units
+
             kw = np.floor(recommendation_config["array_wattage"] / 100) / 10
             # Default to a weeks work for a team of 3 people doing 8 hour days
             labour_days = 5
@@ -121,6 +128,7 @@ class SolarPvRecommendations:
                     "phase": phase,
                     "parts": [],
                     "type": "solar_pv",
+                    "measure_type": "solar_pv",
                     "description": description,
                     "starting_u_value": None,
                     "new_u_value": None,
@@ -169,9 +177,7 @@ class SolarPvRecommendations:
             if self.property.roof["is_flat"]:
                 roof_area = self.property.insulation_floor_area
             else:
-                roof_area = esimtate_pitched_roof_area(
-                    floor_area=self.property.insulation_floor_area, floor_height=self.property.data["floor-height"]
-                )
+                roof_area = estimate_pitched_roof_area(floor_area=self.property.insulation_floor_area, )
             solar_configurations = pd.DataFrame(
                 [
                     {
@@ -183,20 +189,26 @@ class SolarPvRecommendations:
             )
         else:
             # TODO: There may be some instances where we don't want to use the solar API so we should cover for them
-            panel_performance = self.property.solar_panel_configuration["panel_performance"]
+            panel_performance = self.property.solar_panel_configuration["panel_performance"].copy()
+            # We don't allow for more than 70% of the roof to be covered
+            panel_performance = panel_performance[
+                panel_performance["panneled_roof_area"] / self.property.roof_area <= self.MAX_ROOF_AREA_PERCENTAGE
+                ]
+
             roof_area = self.property.roof_area
-            solar_configurations = panel_performance.head(3).reset_index(drop=True)
+            solar_configurations = panel_performance.head(6).reset_index(drop=True)
 
         # We combine each of these configurations with estimates with and without a battery
         for rank, recommendation_config in solar_configurations.iterrows():
             roof_coverage_percent = round(recommendation_config["panneled_roof_area"] / roof_area * 100)
-            # We round up to the nearest 10
-            roof_coverage_percent = np.ceil(roof_coverage_percent / 10) * 10
+            # We round up to the nearest 5
+            roof_coverage_percent = np.ceil(roof_coverage_percent / 5) * 5
             for has_battery in [False, True]:
                 cost_result = self.costs.solar_pv(
-                    wattage=recommendation_config["array_wattage"],
                     has_battery=has_battery,
-                    array_cost=non_invasive_recommendation.get("cost", None)
+                    array_cost=non_invasive_recommendation.get("cost", None),
+                    n_panels=recommendation_config["n_panels"],
+                    n_floors=self.property.number_of_floors
                 )
                 kw = np.floor(recommendation_config["array_wattage"] / 100) / 10
                 if has_battery:
@@ -215,6 +227,7 @@ class SolarPvRecommendations:
                         "phase": phase,
                         "parts": [],
                         "type": "solar_pv",
+                        "measure_type": "solar_pv",
                         "description": description,
                         "starting_u_value": None,
                         "new_u_value": None,

recommendations/VentilationRecommendations.py

@@ -52,7 +52,7 @@ class VentilationRecommendations(Definitions):
 
         already_installed = "cavity_wall_insulation" in self.property.already_installed
 
-        estimated_cost = n_units * part[0]["cost"] if not already_installed else 0
+        estimated_cost = n_units * part[0]["total_cost"] if not already_installed else 0
         labour_hours = 4 * n_units if not already_installed else 0
         labour_days = 4 * n_units / 8.0 if not already_installed else 0
 
@@ -66,6 +66,7 @@ class VentilationRecommendations(Definitions):
                 "phase": None,
                 "parts": part,
                 "type": part[0]["type"],
+                "measure_type": "mechanical_ventilation",
                 "description": f"Install {n_units} {part[0]['description']} units",
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -106,6 +107,7 @@ class VentilationRecommendations(Definitions):
                 "phase": None,
                 "parts": [],
                 "type": "trickle_vents",
+                "measure_type": "trickle_vents",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/WallRecommendations.py

@@ -1,6 +1,7 @@
 import math
 from typing import List
 
+import numpy as np
 import pandas as pd
 
 from datatypes.enums import QuantityUnits
@@ -69,6 +70,7 @@ class WallRecommendations(Definitions):
         "Timber frame, as built, no insulation": "Timber frame, with external insulation",
         'Timber frame, as built, partial insulation': 'Timber frame, with external insulation',
         "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with external insulation",
+        "Sandstone, as built, no insulation": "Sandstone, with external insulation",
     }
 
     # These are the ending descriptions we consider for walls with internal insulation
@@ -83,6 +85,7 @@ class WallRecommendations(Definitions):
         "Timber frame, as built, no insulation": "Timber frame, with internal insulation",
         'Timber frame, as built, partial insulation': 'Timber frame, with internal insulation',
         "Sandstone or limestone, as built, no insulation": "Sandstone or limestone, with internal insulation",
+        "Sandstone, as built, no insulation": "Sandstone, with internal insulation",
     }
 
     def __init__(
@@ -97,6 +100,8 @@ class WallRecommendations(Definitions):
 
         # Will contains a list of recommended measures
         self.recommendations = []
+        # Contains a list of extended recommendation measures, such as extension insulation
+        self.extended_recommendations = []
 
         self.cavity_wall_insulation_materials = [
             part for part in materials if part["type"] == "cavity_wall_insulation"
@@ -106,23 +111,10 @@ class WallRecommendations(Definitions):
             part for part in materials if part["type"] == "internal_wall_insulation"
         ]
 
-        self.internal_wall_non_insulation_materials = [
-            part
-            for part in materials
-            if part["type"]
-               in ["iwi_wall_demolition", "iwi_vapour_barrier", "iwi_redecoration"]
-        ]
-
         self.external_wall_insulation_materials = [
             part for part in materials if part["type"] == "external_wall_insulation"
         ]
 
-        self.external_wall_non_insulation_materials = [
-            part
-            for part in materials
-            if part["type"] in ["ewi_wall_demolition", "ewi_wall_preparation", "ewi_wall_redecoration"]
-        ]
-
     def ewi_valid(self):
         """
         This method check available data, to determine if a property is suitable for external wall insulation
@@ -185,13 +177,12 @@ class WallRecommendations(Definitions):
         ewi_recommendations = self._find_insulation(
             u_value=u_value,
             insulation_materials=pd.DataFrame(self.external_wall_insulation_materials),
-            non_insulation_materials=self.external_wall_non_insulation_materials,
             phase=phase
         )
 
         return ewi_recommendations
 
-    def recommend(self, phase=0, measures=None):
+    def recommend(self, phase=0, measures=None, default_u_values=False):
         # if building built after 1990 + we're able to identify U-value +
         # U-value less than 0.18 and if in or close to a conversation area,
         # recommend internal wall insulation as a possible measure
@@ -267,19 +258,104 @@ class WallRecommendations(Definitions):
         if (is_cavity_wall and "cavity_wall_insulation" in measures) or "cavity_extract_and_refill" in measures:
             if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
                 # Test filling cavity
-                self.find_cavity_insulation(u_value, insulation_thickness, phase, measures)
+                self.find_cavity_insulation(u_value, insulation_thickness, phase, measures, default_u_values)
 
             return
 
         # Remaining wall types are treated with IWI or EWI
         if (u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE) and self.is_suitable_for_solid_insulation():
-            self.find_insulation(u_value, phase, measures=measures)
+            self.find_insulation(u_value, phase, measures=measures, default_u_values=default_u_values)
             return
 
         # If the u-value is within regulations, we don't do anything
         return
 
-    def find_cavity_insulation(self, u_value, insulation_thickness, phase, measures):
+    def recommend_extended(self, phase, measures):
+        """
+        Where we have extended measures, such as extension insulation, which cannot typically be picked up
+        from the EPC api, we handle the recommendation of these here
+        :param measures:
+        :return:
+        """
+
+        # These are the measures that are covered by this function
+        extended_measures = ["extension_cavity_wall_insulation"]
+
+        measures_to_recommend = [measure for measure in measures if measure in extended_measures]
+        if not measures_to_recommend:
+            return phase
+
+        # We reset this to be empty
+        self.extended_recommendations = []
+
+        recommendation_phase = phase
+        for measure in measures_to_recommend:
+            if measure == "extension_cavity_wall_insulation":
+                recommendation = self.recommend_extension_cavity_wall_insulation(phase=recommendation_phase)
+            else:
+                raise NotImplementedError(f"Measure {measure} is not implemented")
+            recommendation_phase += 1
+
+            self.extended_recommendations.append(recommendation)
+
+        return recommendation_phase
+
+    def recommend_extension_cavity_wall_insulation(self, phase):
+        """
+        This function produces the recommendation for extension cavity wall insulation
+        :return:
+        """
+
+        # TODO: We aren't provided with carbon, heat or bill savings figures for this measure
+
+        extension_cavity_insulation_recommendation = [
+            r for r in self.property.non_invasive_recommendations if r["type"] == "extension_cavity_wall_insulation"
+        ][0]
+
+        # https://surreybuildingprojects.co.uk/how-much-does-a-24m2-extension-cost
+        average_extension_floor_area = 24
+        # https://assets.publishing.service.gov.uk/media/5f047a01d3bf7f2be8350262
+        # /Size_of_English_Homes_Fact_Sheet_EHS_2018.pdf
+        # This is rough
+        average_house_floor_area = 94
+
+        proposed_extension_floor_area = self.property.floor_area * (
+            average_extension_floor_area / average_house_floor_area
+        )
+        # assume 3 walls are external
+        proposed_extension_insulation_wall_area = (
+            np.sqrt(proposed_extension_floor_area) * self.property.floor_height * 3
+        )
+
+        cost_result = self.costs.cavity_wall_insulation(
+            wall_area=proposed_extension_insulation_wall_area,
+            material=self.cavity_wall_insulation_materials[0],
+        )
+
+        recommendation = {
+            "phase": phase,
+            "parts": [],
+            "type": "extension_cavity_wall_insulation",
+            "measure_type": "extension_cavity_wall_insulation",
+            "description": "Insulate the cavity walls of the extension",
+            "starting_u_value": None,
+            "new_u_value": None,
+            "sap_points": extension_cavity_insulation_recommendation["sap_points"],
+            "heat_demand": 0,
+            "kwh_savings": 0,
+            "energy_savings": 0,
+            "energy_cost_savings": 0,
+            "co2_equivalent_savings": 0,
+            "already_installed": False,
+            "simulation_config": {},
+            "description_simulation": {},
+            **cost_result,
+            "default": True,
+        }
+
+        return recommendation
+
+    def find_cavity_insulation(self, u_value, insulation_thickness, phase, measures, default_u_values):
         """
         This method tests different materials to fill the cavity wall, determining which
         material will give us the best U-value.
@@ -301,6 +377,7 @@ class WallRecommendations(Definitions):
         filled cavity wall
         :param phase: The phase of the recommendation
         :param measures: The measures we're considering
+        :param default_u_values: If we should use default u values
         """
 
         insulation_materials = pd.DataFrame(self.cavity_wall_insulation_materials)
@@ -356,7 +433,15 @@ class WallRecommendations(Definitions):
                     description = self._make_description(material)
 
                 # updated the new u-value with the best possible our installers have
-                new_u_value = max(0.31, new_u_value)
+                if default_u_values:
+                    new_u_value = get_wall_u_value(
+                        clean_description="Cavity wall, filled cavity",
+                        age_band="G",
+                        is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
+                        is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
+                    )
+                else:
+                    new_u_value = max(0.31, new_u_value)
 
                 wall_ending_config = WallAttributes("Cavity wall, filled cavity").process()
 
@@ -371,7 +456,7 @@ class WallRecommendations(Definitions):
                 simulation_config = {
                     **simulation_config,
                     **walls_simulation_config,
-                    "walls_thermal_transmittance_ending": new_u_value,
+                    "walls_thermal_transmittance_ending": new_u_value if not default_u_values else 0.7,
                 }
 
                 recommendations.append(
@@ -386,6 +471,7 @@ class WallRecommendations(Definitions):
                             )
                         ],
                         "type": "cavity_wall_insulation",
+                        "measure_type": "cavity_wall_insulation",
                         "description": description,
                         "starting_u_value": u_value,
                         "new_u_value": new_u_value,
@@ -450,7 +536,7 @@ class WallRecommendations(Definitions):
 
         return simulation_config
 
-    def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase):
+    def _find_insulation(self, u_value, insulation_materials, phase, default_u_values):
 
         lowest_selected_u_value = None
         recommendations = []
@@ -495,6 +581,15 @@ class WallRecommendations(Definitions):
                         lowest_selected_u_value, new_u_value
                     )
 
+                    cost_result = self.costs.solid_wall_insulation(
+                        wall_area=self.property.insulation_wall_area,
+                        material=material.to_dict(),
+                    )
+
+                    already_installed = material["type"] in self.property.already_installed
+                    if already_installed:
+                        cost_result = override_costs(cost_result)
+
                     if material["type"] == "internal_wall_insulation":
 
                         if iwi_non_invasive_recommendations.get("cost") is not None:
@@ -505,18 +600,6 @@ class WallRecommendations(Definitions):
                         sap_points = iwi_non_invasive_recommendations.get("sap_points", None)
                         survey = iwi_non_invasive_recommendations.get("survey", False)
 
-                        cost_result = self.costs.internal_wall_insulation(
-                            wall_area=self.property.insulation_wall_area,
-                            material=material.to_dict(),
-                            non_insulation_materials=non_insulation_materials,
-                        )
-                        already_installed = (
-                            "internal_wall_insulation"
-                            in self.property.already_installed
-                        )
-                        if already_installed:
-                            cost_result = override_costs(cost_result)
-
                         new_description = self.get_internal_external_wall_description(
                             self.INTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value
                         )
@@ -526,18 +609,6 @@ class WallRecommendations(Definitions):
                         sap_points = ewi_non_invasive_recommendations.get("sap_points", None)
                         survey = ewi_non_invasive_recommendations.get("survey", False)
 
-                        cost_result = self.costs.external_wall_insulation(
-                            wall_area=self.property.insulation_wall_area,
-                            material=material.to_dict(),
-                            non_insulation_materials=non_insulation_materials,
-                        )
-                        already_installed = (
-                            "external_wall_insulation"
-                            in self.property.already_installed
-                        )
-                        if already_installed:
-                            cost_result = override_costs(cost_result)
-
                         new_description = self.get_internal_external_wall_description(
                             self.EXTERNALLY_INSULATED_WALL_DESCRIPTIONS, new_u_value
                         )
@@ -560,6 +631,15 @@ class WallRecommendations(Definitions):
                         "walls_thermal_transmittance_ending": new_u_value
                     }
 
+                    if default_u_values:
+                        # If we're using default U-values, we overwrite new_u_value
+                        new_u_value = get_wall_u_value(
+                            clean_description=new_description,
+                            age_band=self.property.age_band,
+                            is_granite_or_whinstone=self.property.walls["is_granite_or_whinstone"],
+                            is_sandstone_or_limestone=self.property.walls["is_sandstone_or_limestone"],
+                        )
+
                     recommendations.append(
                         {
                             "phase": phase,
@@ -572,6 +652,7 @@ class WallRecommendations(Definitions):
                                 )
                             ],
                             "type": material["type"],
+                            "measure_type": material["type"],  # This is distinguished between EWI & IWI
                             "description": self._make_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
@@ -589,7 +670,7 @@ class WallRecommendations(Definitions):
 
         return recommendations
 
-    def find_insulation(self, u_value, phase, measures):
+    def find_insulation(self, u_value, phase, measures, default_u_values):
         """
         This function contains the logic for finding potential insulation measures for a property, depending
         on the parts available and whether the property can have external wall insulation installed
@@ -608,8 +689,8 @@ class WallRecommendations(Definitions):
                 insulation_materials=pd.DataFrame(
                     self.external_wall_insulation_materials
                 ),
-                non_insulation_materials=self.external_wall_non_insulation_materials,
                 phase=phase,
+                default_u_values=default_u_values
             )
 
         iwi_recommendations = []
@@ -617,8 +698,8 @@ class WallRecommendations(Definitions):
             iwi_recommendations = self._find_insulation(
                 u_value=u_value,
                 insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials),
-                non_insulation_materials=self.internal_wall_non_insulation_materials,
                 phase=phase,
+                default_u_values=default_u_values
             )
 
         self.recommendations += ewi_recommendations + iwi_recommendations

recommendations/WindowsRecommendations.py

@@ -3,6 +3,7 @@ from typing import List
 import numpy as np
 
 from backend.Property import Property
+from backend.app.plan.schemas import MEASURE_MAP
 from etl.epc_clean.epc_attributes.WindowAttributes import WindowAttributes
 from recommendations.Costs import Costs
 from recommendations.recommendation_utils import override_costs, check_simulation_difference
@@ -32,7 +33,7 @@ class WindowsRecommendations:
             raise ValueError("There should only be one window glazing material")
         self.glazing_material = self.glazing_material[0]
 
-    def recommend(self, phase=0):
+    def recommend(self, measures=None, phase=0):
         """
         This method will recommend the best possible glazing options for a property.
 
@@ -41,26 +42,43 @@ class WindowsRecommendations:
         :return:
         """
 
-        # If the property is in a conservation area or is a listed building, it becomes more difficult to install
-        # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it
-        # requires planning permission and might require a more expensive window type, such as timber.
+        measures = MEASURE_MAP["windows"] if measures is None else measures
 
-        number_of_windows = self.property.number_of_windows
-        is_secondary_glazing = self.property.restricted_measures or (
-            self.property.windows["glazing_type"] == "secondary"
-        )
-        windows_area = self.property.windows_area
+        # If we have no windows recs, leave
+        if not any(x in measures for x in MEASURE_MAP["windows"]):
+            return
 
-        if not number_of_windows:
-            raise ValueError("Number of windows not specified")
+        if self.property.windows["glazing_type"] in ["triple", "high performance"]:
+            # We don't make any recommendations in this case. The property already has outstanding glazing
+            return
 
         if self.property.windows["has_glazing"] & (
             self.property.windows["glazing_coverage"] == "full"
         ):
             return
 
+        # If the property is in a conservation area or is a listed building, it becomes more difficult to install
+        # double glazing. Therefore, we don't recommend it. It is still possible but is not practical as it
+        # requires planning permission and might require a more expensive window type, such as timber.
+
+        number_of_windows = self.property.number_of_windows
+
+        if "double_glazing" in measures and "secondary_glazing" not in measures:
+            is_secondary_glazing = False
+        elif "secondary_glazing" in measures and "double_glazing" not in measures:
+            is_secondary_glazing = True
+        else:
+            is_secondary_glazing = self.property.restricted_measures or (
+                self.property.windows["glazing_type"] == "secondary"
+            )
+        windows_area = self.property.windows_area
+
+        if not number_of_windows:
+            raise ValueError("Number of windows not specified")
+
         if windows_area is not None:
-            raise Exception("We have windows area, we should use this data for our recommendations!!!")
+            # TODO - we don't have a price for this so we can't recommend it
+            print("We have windows area, we should use this data for our recommendations!!!")
 
         # We scale the number of windows based on the proportion of existing glazing
         if self.property.data["multi-glaze-proportion"] != "":
@@ -108,11 +126,101 @@ class WindowsRecommendations:
                     ". Secondary glazing recommended due to conservation area status"
                 )
 
+        # Set up the simulation config
+        windows_energy_eff = "Good"
+        if self.property.windows["glazing_type"] == "multiple":
+            glazing_type_ending = "multiple"
+            glazed_type_ending = (
+                "secondary glazing" if is_secondary_glazing else "double glazing installed during or after 2002"
+            )
+            new_windows_description = "Multiple glazing throughout"
+
+        elif self.property.windows["glazing_type"] == "single":
+            # We will only recommend either secondary or double glazing
+            glazing_type_ending = (
+                "secondary" if is_secondary_glazing else "double"
+            )
+            glazed_type_ending = (
+                "secondary glazing" if is_secondary_glazing else "double glazing installed during or after 2002"
+            )
+
+            if is_secondary_glazing:
+                new_windows_description = "Full secondary glazing"
+            else:
+                new_windows_description = "Fully double glazed"
+
+        elif self.property.windows["glazing_type"] == "double":
+            glazing_type_ending = (
+                "multiple" if is_secondary_glazing else "double"
+            )
+
+            # We set glazed type depending on which window type is more prevalent. Since there is already double
+            # glazing in place, if we're recommending more double glazing, we set the glazed type to double glazing
+            # otherwise, if we're recommending secondary glazing and the proportion of glazing in place already that
+            # is double is less than 50% we set the glazed type to secondary glazing
+
+            if not is_secondary_glazing:
+                glazed_type_ending = "double glazing installed during or after 2002"
+                new_windows_description = "Fully double glazed"
+            else:
+                if self.property.data["multi-glaze-proportion"] < 50:
+                    glazed_type_ending = "secondary glazing"
+                else:
+                    glazed_type_ending = "double glazing installed during or after 2002"
+
+                new_windows_description = "Multiple glazing throughout"
+
+        elif self.property.windows["glazing_type"] == "secondary":
+            glazing_type_ending = (
+                "secondary" if is_secondary_glazing else "multiple"
+            )
+            # This is the opposite. If there is secondary glazing in place, and we're recommending double
+            # we set glazed_type_ending, depending on the proportion of glazing in place
+            if is_secondary_glazing:
+                glazed_type_ending = "secondary glazing"
+                new_windows_description = "Full secondary glazing"
+            else:
+                if self.property.data["multi-glaze-proportion"] < 50:
+                    glazed_type_ending = "double glazing installed during or after 2002"
+                else:
+                    glazed_type_ending = "secondary glazing"
+                new_windows_description = "Multiple glazing throughout"
+
+        else:
+            raise ValueError("Invalid glazing type - implement me")
+
+        if self.property.data["windows-energy-eff"] == "Very Good":
+            raise ValueError("Very Good energy efficiency is not supported")
+
+        # For post 2002 windows, the energy efficiency is "Good" and so for the simulation, we simulate with "Good"
+
+        windows_ending_config = WindowAttributes(new_windows_description).process()
+
+        windows_simulation_config = check_simulation_difference(
+            new_config=windows_ending_config, old_config=self.property.windows, prefix="windows_"
+        )
+
+        simulation_config = {
+            **windows_simulation_config,
+            "multi_glaze_proportion_ending": 100,
+            "windows_energy_eff_ending": windows_energy_eff,
+            "glazing_type_ending": glazing_type_ending,
+            "glazed_type_ending": glazed_type_ending,
+        }
+
+        description_simulation = {
+            "multi-glaze-proportion": 100,
+            "windows-energy-eff": windows_energy_eff,
+            "windows-description": new_windows_description,
+            "glazed-type": glazed_type_ending,
+        }
+
         self.recommendation = [
             {
                 "phase": phase,
                 "parts": [],
                 "type": "windows_glazing",
+                "measure_type": "double_glazing" if not is_secondary_glazing else "secondary_glazing",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
@@ -120,13 +228,8 @@ class WindowsRecommendations:
                 "already_installed": already_installed,
                 **cost_result,
                 "is_secondary_glazing": is_secondary_glazing,
-                # TODO: Make this condition on is_secondary_glazing
-                "description_simulation": {
-                    "multi-glaze-proportion": 100,
-                    "windows-energy-eff": "Average",
-                    "windows-description": "Fully double glazed",
-                    "glazed-type": "double glazing installed during or after 2002",
-                }
+                "description_simulation": description_simulation,
+                "simulation_config": simulation_config,
             }
         ]
 
@@ -167,6 +270,7 @@ class WindowsRecommendations:
                 "phase": phase,
                 "parts": [],
                 "type": "mixed_glazing",
+                "measure_type": "mixed_glazing",
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,

recommendations/rdsap_tables.py

@@ -340,6 +340,7 @@ s9_list = [
 s10_list = [
     {
         "Age_band": "A, B, C, D",
+        "Insulation_Thickness": "none",
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 2.3,
         "Pitched_slates_or_tiles_insulation_at_rafters": 2.3,
         "Flat_roof": 2.3,
@@ -350,6 +351,7 @@ s10_list = [
     },
     {
         "Age_band": "E",
+        "Insulation_Thickness": 12,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 1.5,
         "Pitched_slates_or_tiles_insulation_at_rafters": 1.5,
         "Flat_roof": 1.5,
@@ -360,6 +362,7 @@ s10_list = [
     },
     {
         "Age_band": "F",
+        "Insulation_Thickness": 50,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.68,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.68,
         "Flat_roof": 0.68,
@@ -370,6 +373,7 @@ s10_list = [
     },
     {
         "Age_band": "G",
+        "Insulation_Thickness": 100,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.40,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.40,
         "Flat_roof": 0.40,
@@ -380,6 +384,7 @@ s10_list = [
     },
     {
         "Age_band": "H",
+        "Insulation_Thickness": 150,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.30,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.35,
         "Flat_roof": 0.35,
@@ -390,6 +395,7 @@ s10_list = [
     },
     {
         "Age_band": "I",
+        "Insulation_Thickness": 150,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.26,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.35,
         "Flat_roof": 0.35,
@@ -400,6 +406,7 @@ s10_list = [
     },
     {
         "Age_band": "J",
+        "Insulation_Thickness": 270,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.20,
         "Flat_roof": 0.25,
@@ -410,6 +417,7 @@ s10_list = [
     },
     {
         "Age_band": "K",
+        "Insulation_Thickness": 270,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.20,
         "Flat_roof": 0.25,
@@ -420,6 +428,7 @@ s10_list = [
     },
     {
         "Age_band": "L",
+        "Insulation_Thickness": 270,
         "Pitched_slates_or_tiles_insulation_between_joists_or_unknown": 0.16,
         "Pitched_slates_or_tiles_insulation_at_rafters": 0.18,
         "Flat_roof": 0.18,

recommendations/recommendation_utils.py

@@ -7,10 +7,18 @@ import numpy as np
 import pandas as pd
 
 from recommendations.rdsap_tables import (
-    epc_wall_description_map, wall_uvalues_df, default_wall_thickness, table_s9 as s9, table_s10 as s10,
-    table_s11 as s11, table_s12 as s12
+    epc_wall_description_map,
+    wall_uvalues_df,
+    default_wall_thickness,
+    table_s9 as s9,
+    table_s10 as s10,
+    table_s11 as s11,
+    table_s12 as s12,
+)
+from recommendations.config import (
+    PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION,
+    PARTIAL_CAVITY_DESCRIPTIONS,
 )
-from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION, PARTIAL_CAVITY_DESCRIPTIONS
 
 
 def r_value_per_mm_to_u_value(depth_mm: int, r_value_per_mm: float):
@@ -62,7 +70,9 @@ def calculate_u_value_uplift(u_value, insulation_u_value):
     return u_value_uplift, new_u_value
 
 
-def is_diminishing_returns(recommendations, new_u_value, lowest_selected_u_value, diminishing_returns_u_value):
+def is_diminishing_returns(
+    recommendations, new_u_value, lowest_selected_u_value, diminishing_returns_u_value
+):
     """
     What are defines diminishing returns?
     1) The new u value is lower than the lowest selected u value
@@ -136,9 +146,15 @@ def apply_formula_s_5_1_1(is_granite_or_whinstone, is_sandstone_or_limestone, ag
     S.5.1.1
     """
 
-    stone_wall_thickness = [x for x in default_wall_thickness if x["type"] == "stone"][0]
+    stone_wall_thickness = [x for x in default_wall_thickness if x["type"] == "stone"][
+        0
+    ]
 
-    thickness = stone_wall_thickness["J_K_L"] if age_band in ["J", "L", "L"] else stone_wall_thickness[age_band]
+    thickness = (
+        stone_wall_thickness["J_K_L"]
+        if age_band in ["J", "L", "L"]
+        else stone_wall_thickness[age_band]
+    )
 
     if is_granite_or_whinstone:
         return 3.3 - 0.002 * thickness
@@ -146,7 +162,9 @@ def apply_formula_s_5_1_1(is_granite_or_whinstone, is_sandstone_or_limestone, ag
     if is_sandstone_or_limestone:
         return 3 - 0.002 * thickness
 
-    raise ValueError("This should only be called when is_granite_or_whinstone or is_sandstone_or_limestone is True")
+    raise ValueError(
+        "This should only be called when is_granite_or_whinstone or is_sandstone_or_limestone is True"
+    )
 
 
 def get_wall_u_value(
@@ -164,16 +182,30 @@ def get_wall_u_value(
     if clean_description in PARTIAL_CAVITY_DESCRIPTIONS:
         # If we have a partial cavity fill, we linearly interpolate the u-value. This isn't necessarily the perfect
         # method and how we do this should be explored, however we want to distinguish between the old
-        filled_uvalue = float(wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Filled cavity"][age_band].values[0])
-        unfilled_uvalue = float(wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Cavity as built"][age_band].values[0])
+        filled_uvalue = float(
+            wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Filled cavity"][
+                age_band
+            ].values[0]
+        )
+        unfilled_uvalue = float(
+            wall_uvalues_df[wall_uvalues_df["Wall_type"] == "Cavity as built"][
+                age_band
+            ].values[0]
+        )
 
         mapped_value = str(
-            unfilled_uvalue - (PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION * (unfilled_uvalue - filled_uvalue))
+            unfilled_uvalue
+            - (
+                PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
+                * (unfilled_uvalue - filled_uvalue)
+            )
         )
     else:
         mapped_description = epc_wall_description_map[clean_description]
 
-        mapped_value = wall_uvalues_df[wall_uvalues_df["Wall_type"] == mapped_description][age_band].values[0]
+        mapped_value = wall_uvalues_df[
+            wall_uvalues_df["Wall_type"] == mapped_description
+        ][age_band].values[0]
 
         if pd.isnull(mapped_value) and "Park home" in mapped_description:
             # We don't know enough in this case so we default to 0
@@ -185,17 +217,19 @@ def get_wall_u_value(
             apply_formula_s_5_1_1(
                 is_granite_or_whinstone=is_granite_or_whinstone,
                 is_sandstone_or_limestone=is_sandstone_or_limestone,
-                age_band=age_band
+                age_band=age_band,
             )
         )
 
     if "b" in mapped_value:
         potential_uvalue = float(mapped_value.replace("b", ""))
-        formula_uvalue = float(apply_formula_s_5_1_1(
-            is_granite_or_whinstone=is_granite_or_whinstone,
-            is_sandstone_or_limestone=is_sandstone_or_limestone,
-            age_band=age_band
-        ))
+        formula_uvalue = float(
+            apply_formula_s_5_1_1(
+                is_granite_or_whinstone=is_granite_or_whinstone,
+                is_sandstone_or_limestone=is_sandstone_or_limestone,
+                age_band=age_band,
+            )
+        )
         return min(potential_uvalue, formula_uvalue)
 
     if mapped_value == "s1.1.2":
@@ -205,38 +239,69 @@ def get_wall_u_value(
     return float(mapped_value)
 
 
-def get_u_value_from_s9(thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters):
-    """Get the U-value from table S9 based on the insulation thickness."""
-
-    # If the roof as pitched & insulated at the rafters, it's a room roof
+def extract_thickness(thickness, is_roof_room, is_at_rafters, is_loft, is_flat):
     if is_roof_room or is_at_rafters:
+        # TODO: We get None instead of a string none, this should be fixed
+        if thickness is None:
+            thickness = "none"
         # We re-map the thickness
         thickness_map = {
             "below average": "50",
             "average": "100",
-            "above average": "270",
+            "above average": "150",
             "none": "0",
         }
         thickness = thickness_map[thickness]
 
+    if is_flat:
+        try:
+            thickness = int(thickness)
+            return thickness
+        except (TypeError, ValueError):
+            # If thickness is not a valid number (could be a string or None), return None
+            return None
+
     if thickness in ["below average", "average", "above average", "none", None] or (
         not is_loft and not is_roof_room and not is_at_rafters
     ):
         return None
     elif thickness.endswith("+"):
         thickness = int(thickness[:-1])
+        return thickness
     else:
         try:
             thickness = int(thickness)
+            return thickness
         except ValueError:
             # If thickness is not a valid number (could be a string or None), return None
             return None
 
-    # Determine the column to refer based on the roof type
-    column = 'Thatched_roof_U_value_W_m2K' if is_thatched else 'Slates_or_tiles_U_value_W_m2K'
 
-    # Get the correct U-value based on the insulation thickness
-    return s9[s9['Insulation_thickness_mm'] >= thickness][column].iloc[0]
+def get_u_value_from_s9(
+    thickness, s9, is_loft, is_roof_room, is_thatched, is_at_rafters
+):
+    """Get the U-value from table S9 based on the insulation thickness."""
+
+    if thickness in ["below average", "average", "above average", "none", None] or (
+        not is_loft and not is_roof_room and not is_at_rafters
+    ):
+        return None
+
+    if thickness in [0, "0"] and (is_loft or is_roof_room):
+        return None
+
+    # Determine the column to refer based on the roof type
+    column = (
+        "Thatched_roof_U_value_W_m2K"
+        if is_thatched
+        else "Slates_or_tiles_U_value_W_m2K"
+    )
+
+    if thickness in [0, "0"] and is_roof_room:
+        return s9[pd.isnull(s9["Insulation_thickness_mm"])][column].iloc[0]
+    else:
+        # Get the correct U-value based on the insulation thickness
+        return s9[s9["Insulation_thickness_mm"] >= thickness][column].iloc[0]
 
 
 def get_roof_u_value(
@@ -249,7 +314,7 @@ def get_roof_u_value(
     is_flat,
     is_pitched,
     is_at_rafters,
-    **kwargs
+    **kwargs,
 ):
     """
     Determine the U-value for a roof based on the description dictionary and age band.
@@ -280,6 +345,14 @@ def get_roof_u_value(
     if has_dwelling_above:
         return 0.0
 
+    thickness = extract_thickness(
+        thickness=insulation_thickness,
+        is_roof_room=is_roof_room,
+        is_at_rafters=is_at_rafters,
+        is_loft=is_loft,
+        is_flat=is_flat,
+    )
+
     # Step 1: Try to get the U-value from table S9 based on the insulation thickness
     # The conditions for using table S9 are:
     # - The insulation thickness is known
@@ -287,12 +360,12 @@ def get_roof_u_value(
     # The criteria for using this table is predominately defined by insulation around joists which is predominately
     # a feature of lofts and roof rooms
     u_value = get_u_value_from_s9(
-        thickness=insulation_thickness,
+        thickness=thickness,
         s9=s9,
         is_loft=is_loft,
         is_roof_room=is_roof_room,
         is_thatched=is_thatched,
-        is_at_rafters=is_at_rafters
+        is_at_rafters=is_at_rafters,
     )
 
     if u_value is not None:
@@ -302,27 +375,52 @@ def get_roof_u_value(
 
     # Define the columns to be used based on the description details
     if is_flat:
-        column = 'Flat_roof'
+        column = "Flat_roof"
     elif is_thatched:
         if is_roof_room:
-            column = 'Thatched_roof_room_in_roof'
+            column = "Thatched_roof_room_in_roof"
         else:
-            column = 'Thatched_roof'
+            column = "Thatched_roof"
     elif is_roof_room:
-        column = 'Room_in_roof_slates_or_tiles'
+        column = "Room_in_roof_slates_or_tiles"
     elif is_pitched:
         if is_at_rafters:
-            column = 'Pitched_slates_or_tiles_insulation_at_rafters'
+            column = "Pitched_slates_or_tiles_insulation_at_rafters"
         else:
-            column = 'Pitched_slates_or_tiles_insulation_between_joists_or_unknown'
+            column = "Pitched_slates_or_tiles_insulation_between_joists_or_unknown"
     else:
         # Default to pitched roof with insulation between joists or unknown
-        column = 'Pitched_slates_or_tiles_insulation_between_joists_or_unknown'
+        column = "Pitched_slates_or_tiles_insulation_between_joists_or_unknown"
 
     # Get the U-value from table S10 based on the age band and the determined column
-    u_value = s10.loc[s10['Age_band'].str.contains(age_band), column].values[0]
+    if is_flat and thickness is not None:
+        u_value = s10.loc[
+            (s10["Insulation_Thickness"] == thickness)
+            | s10["Age_band"].str.contains(age_band),
+            column,
+        ].values.min()
+    else:
+        u_value = s10.loc[s10["Age_band"].str.contains(age_band), column].values[0]
 
-    return float(u_value)
+    u_value = float(u_value)
+
+    # As per the documentation here: https://bregroup.com/documents/d/bre-group/rdsap_2012_9-94-20-09-2019
+    # Table s.10
+    # "The value from the table applies for unknown and as built. If the roof is known to have more insulation than
+    # would normally be expected for the age band, either observed or on the basis of documentary evidence, use the
+    # lower of the value in the table and:
+    # 50 mm insulation 0.68
+    # 100 mm insulation: 0.40
+    # 150 mm or more insulation: 0.30"
+    if thickness is not None:
+        if thickness == 50:
+            u_value = min(u_value, 0.68)
+        if thickness == 100:
+            u_value = min(u_value, 0.40)
+        if thickness >= 150:
+            u_value = min(u_value, 0.30)
+
+    return u_value
 
 
 def estimate_number_of_floors(property_type):
@@ -397,10 +495,14 @@ def get_exposed_floor_uvalue(insulation_thickness_str, age_band):
     else:
         insulation_thickness = int(insulation_thickness_str.replace("mm", ""))
 
-    return s12[s12["age_band"] == age_band][f"insulation_{insulation_thickness}"].values[0]
+    return s12[s12["age_band"] == age_band][
+        f"insulation_{insulation_thickness}"
+    ].values[0]
 
 
-def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None):
+def get_floor_u_value(
+    floor_type, area, perimeter, age_band, wall_type, insulation_thickness=None
+):
     """
     Estimate the u-value of a suspended floor, based on RdSap methodology
     Default U-value for UNINSULATED suspended floor, based on RdSAP methodology
@@ -446,14 +548,19 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati
     Rsi = 0.17  # in m²K/W
     Rse = 0.04  # in m²K/W
     lambda_ins = 0.035  # thermal conductivity of floor insulation in W/m·K
-    wall_thickness = [x[age_band] for x in default_wall_thickness if x["type"] == wall_type][0]
+    wall_thickness = [
+        x[age_band] for x in default_wall_thickness if x["type"] == wall_type
+    ][0]
     if wall_thickness is None and wall_type == "park home":
         # We don't know enough and likely won't make recommendations
         return 0
     wall_thickness = wall_thickness / 1000
 
     if insulation_thickness is None:
-        insulation_lookup = s11[s11["Age_band"].str.contains(age_band) & s11["Floor_construction"] == floor_type]
+        insulation_lookup = s11[
+            s11["Age_band"].str.contains(age_band) & s11["Floor_construction"]
+            == floor_type
+        ]
         if insulation_lookup.empty:
             insulation_thickness = 0
         else:
@@ -465,7 +572,7 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati
     # Calculate B
     B = 2 * area / perimeter
 
-    if floor_type == 'solid':
+    if floor_type == "solid":
         # Calculate dt
         dt = wall_thickness + lambda_g * (Rsi + Rf + Rse)
 
@@ -475,7 +582,7 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati
         else:
             U = lambda_g / (0.457 * B + dt)
 
-    elif floor_type == 'suspended':
+    elif floor_type == "suspended":
         # Define additional constants for suspended floors
         h = 0.3  # height above external ground level in meters
         v = 5  # average wind speed at 10 m height in m/s
@@ -498,7 +605,9 @@ def get_floor_u_value(floor_type, area, perimeter, age_band, wall_type, insulati
 
         U = 1 / (2 * Rsi + Rf + 1 / (Ug + Ux))
     else:
-        raise ValueError("Invalid floor type. Acceptable values are 'solid' or 'suspended'.")
+        raise ValueError(
+            "Invalid floor type. Acceptable values are 'solid' or 'suspended'."
+        )
 
     return round(U, 2)  # rounding U value to two decimal places
 
@@ -509,7 +618,13 @@ def extract_insulation_thickness(insulation_thickness_str):
     :param insulation_thickness_str:
     :return:
     """
-    if insulation_thickness_str in ["none", "average", "below average", "above average", None]:
+    if insulation_thickness_str in [
+        "none",
+        "average",
+        "below average",
+        "above average",
+        None,
+    ]:
         return None
 
     if isinstance(insulation_thickness_str, (float, int)):
@@ -527,7 +642,7 @@ def get_wall_type(
     is_cob,
     is_system_built,
     is_park_home,
-    **kwargs
+    **kwargs,
 ) -> Union[str, None]:
     """
     Converts booleans to a string wall type, for querying the wall thickness table
@@ -573,10 +688,10 @@ def estimate_external_wall_area(num_floors, floor_height, perimeter, built_form)
     total_wall_area = wall_area_one_floor * num_floors
 
     number_exposed_walls = {
-        'End-Terrace': 3,
-        'Mid-Terrace': 2,
-        'Semi-Detached': 3,
-        'Detached': 4,
+        "End-Terrace": 3,
+        "Mid-Terrace": 2,
+        "Semi-Detached": 3,
+        "Detached": 4,
     }
 
     exposed_wall_area = total_wall_area * (number_exposed_walls.get(built_form, 3) / 4)
@@ -622,27 +737,12 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat):
         return 0
 
     if is_pitched:
-        lookup = {
-            "none": 0,
-            "below average": 50,
-            "average": 100,
-            "above average": 270
-        }
+        lookup = {"none": 0, "below average": 50, "average": 100, "above average": 270}
     elif is_flat:
         # For a flat roof, if it's below average, we assume it's 0 and requires a re-roof
-        lookup = {
-            "none": 0,
-            "below average": 0,
-            "average": 100,
-            "above average": 150
-        }
+        lookup = {"none": 0, "below average": 0, "average": 100, "above average": 150}
     else:
-        lookup = {
-            "none": 0,
-            "below average": 100,
-            "average": 270,
-            "above average": 270
-        }
+        lookup = {"none": 0, "below average": 100, "average": 270, "above average": 270}
 
     mapped = lookup.get(string_thickness)
 
@@ -655,34 +755,17 @@ def convert_thickness_to_numeric(string_thickness, is_pitched, is_flat):
     return int(string_thickness)
 
 
-def esimtate_pitched_roof_area(floor_area: float, floor_height: float) -> float:
+def estimate_pitched_roof_area(floor_area: float) -> float:
     """
-    This function will estimate the area of a pitched roof, given the floor area below the roof and the floor
-    height of the property.
-
-    Given limited information about the home, this is a very rough method to estimate the roof area and we
-    assume the the room is a gable roof.
-
-    We assume a roughly average pitch of 45 degrees
-
-    Note that both floor area and height should be in the same units. E.g. if floor area is meters squared,
-    floor height should be in meters
+    This function mimics the methodology for calculating floor area in Elmhurst, so that we can simulate the outcomes
+    in a way that is consistent with the Elmhurst methodology.
 
     :param floor_area:      area of the home's floor
-    :param floor_height:    height of the home's floors
     :return:                Numerical estimate of the surface area of the top of the pitched roof
     """
 
-    # We estimate the length of the wall by just modelling the house as a square
-    wall_width = np.sqrt(floor_area)
-
-    # We're modelling the roof as two triangles where we know two of the three sides.
-    # The floor height makes up one side and half of the wall width makes up the other side
-    slope = np.sqrt(np.square(wall_width / 2) + np.square(floor_height))
-
-    area = 2 * (slope * wall_width)
-
-    return area
+    scalar = 1.0571283428862048
+    return scalar * (floor_area / np.cos(np.radians(30)))
 
 
 def estimate_windows(
@@ -697,11 +780,16 @@ def estimate_windows(
     # Assuming most houses will have at least one kitchen and one bathroom
     # Scale non-habitable windows with the number of habitable rooms
     non_habitable_base = 2  # Base for kitchen and bathroom
-    extra_non_habitable = max(0, (number_habitable_rooms - 3) // 2)  # Extra for large houses
+    extra_non_habitable = max(
+        0, (number_habitable_rooms - 3) // 2
+    )  # Extra for large houses
     window_count += non_habitable_base + extra_non_habitable
 
     # Adjustments based on built form and property type
-    if property_type in ["House", "Bungalow"] and built_form in ["Semi-Detached", "Detached"]:
+    if property_type in ["House", "Bungalow"] and built_form in [
+        "Semi-Detached",
+        "Detached",
+    ]:
         built_form_lookup = {
             "Semi-Detached": 3,
             "Detached": 4,
@@ -728,7 +816,10 @@ def estimate_windows(
         window_count += 2
 
     # Adjust for construction age band
-    if construction_age_band in ["England and Wales: before 1900", "England and Wales: 1900-1929"]:
+    if construction_age_band in [
+        "England and Wales: before 1900",
+        "England and Wales: 1900-1929",
+    ]:
         # Older houses with smaller, more numerous windows
         window_count += 1
 
@@ -751,7 +842,11 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned):
     df = []
     for x in all_epcs:
         # Get the cleaned mapping
-        mapped = [y for y in cleaned["walls-description"] if y["original_description"] == x["walls-description"]]
+        mapped = [
+            y
+            for y in cleaned["walls-description"]
+            if y["original_description"] == x["walls-description"]
+        ]
         if not mapped:
             continue
         df.append(
@@ -768,7 +863,9 @@ def calculate_cavity_age(newest_epc, older_epcs, cleaned):
     return cavity_age
 
 
-def check_simulation_difference(old_config, new_config, prefix="", keys_with_prefix=None):
+def check_simulation_difference(
+    old_config, new_config, prefix="", keys_with_prefix=None
+):
     """
     Given two dictionaries, that describe the heating control configurations, this method will compare the two
     and pick out the differences. These differences will be things that have been added and things that have been
@@ -777,14 +874,17 @@ def check_simulation_difference(old_config, new_config, prefix="", keys_with_pre
     """
 
     keys_with_prefix = (
-        ["is_assumed", "thermal_transmittance", "insulation_thickness"] if keys_with_prefix is None
+        ["is_assumed", "thermal_transmittance", "insulation_thickness"]
+        if keys_with_prefix is None
         else keys_with_prefix
     )
 
     differences = {}
     for key in new_config:
         if old_config[key] != new_config[key]:
-            new_key = prefix + key + "_ending" if key in keys_with_prefix else key + "_ending"
+            new_key = (
+                prefix + key + "_ending" if key in keys_with_prefix else key + "_ending"
+            )
             differences[new_key] = new_config[key]
 
     return differences
@@ -800,3 +900,45 @@ def override_costs(costs):
         costs[k] = 0
 
     return costs
+
+
+def combine_recommendation_configs(recommendation_config1, recommendation_config2):
+    """
+    Given two simulation configs, this function will combine them into one
+    :param recommendation_config1:
+    :param recommendation_config2:
+    :return:
+    """
+    # Efficiency values - keys which contain _energy_eff_ending
+    eff_1 = {
+        k: v
+        for k, v in recommendation_config1.items()
+        if ("_energy_eff_ending" in k) or ("-energy-eff" in k)
+    }
+    eff_2 = {
+        k: v
+        for k, v in recommendation_config2.items()
+        if ("_energy_eff_ending" in k) or ("-energy-eff" in k)
+    }
+
+    # We combine the simulation configs
+    combined = {**recommendation_config1, **recommendation_config2}
+
+    # Find overlapping keys
+    overlapping_keys = set(eff_1.keys()).intersection(set(eff_2.keys()))
+    if overlapping_keys:
+        # We make sure we take the best value - map efficiency values to numbers
+        numerical_embedding = {
+            "Very poor": 1,
+            "Poor": 2,
+            "Average": 3,
+            "Good": 4,
+            "Very good": 5,
+        }
+        for key in overlapping_keys:
+            if numerical_embedding[eff_1[key]] >= numerical_embedding[eff_2[key]]:
+                combined[key] = eff_1[key]
+            else:
+                combined[key] = eff_2[key]
+
+    return combined

recommendations/tests/test_air_source_heat_pump.py

@@ -1,944 +0,0 @@
-import pandas as pd
-import msgpack
-from datetime import datetime
-
-from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
-from backend.Property import Property
-from recommendations.HeatingRecommender import HeatingRecommender
-from recommendations.Recommendations import Recommendations
-from etl.epc.Record import EPCRecord
-from etl.solar.SolarPhotoSupply import SolarPhotoSupply
-from backend.ml_models.api import ModelApi
-
-
-def find_examples():
-    """ Some scrappy helper code to find EPC examples"""
-    # Let's look for some testing data, where the only thing different pre and post is the installation of an
-    # air source heat pump
-    data = read_dataframe_from_s3_parquet(
-        bucket_name="retrofit-data-dev",
-        file_key="sap_change_model/2024-03-24-15-51-13/dataset_no_cleaning.parquet"
-    )
-
-    # Firstly, take records where before there was no air source heat pump and afterwards there was
-    data = data[
-        data["has_air_source_heat_pump_ending"] & ~data["has_air_source_heat_pump"]
-        ]
-
-    # Start with a property that has a boiler
-    data = data[data["has_boiler"]]
-
-    static_columns = [
-        # Walls
-        'walls_thermal_transmittance_ending',
-        'is_filled_cavity_ending',
-        'is_park_home_ending',
-        'walls_insulation_thickness_ending',
-        'external_insulation_ending',
-        'internal_insulation_ending',
-        # Floors
-        # 'floor_thermal_transmittance_ending',  # Don't subset on this, because it changes based on floor area
-        'floor_insulation_thickness_ending',
-        # Roof
-        'roof_thermal_transmittance_ending',
-        'is_at_rafters_ending',
-        'roof_insulation_thickness_ending',
-        # Hot water - air source heat pump will shange the hot water system (probably from whatever it was -> main)
-        # 'heater_type_ending',
-        # 'system_type_ending',
-        # 'thermostat_characteristics_ending',
-        # 'heating_scope_ending',
-        # 'energy_recovery_ending',
-        # 'hotwater_tariff_type_ending',
-        # 'extra_features_ending',
-        # 'chp_systems_ending',
-        # 'distribution_system_ending',
-        # 'no_system_present_ending',
-        # 'appliance_ending',
-        # Heating - Will change when installing an ASHP
-        # 'has_radiators_ending',
-        # 'has_fan_coil_units_ending',
-        # 'has_pipes_in_screed_above_insulation_ending',
-        # 'has_pipes_in_insulated_timber_floor_ending',
-        # 'has_pipes_in_concrete_slab_ending',
-        # 'has_boiler_ending',
-        # 'has_air_source_heat_pump_ending',  # We want the air source heat pump to change
-        # 'has_room_heaters_ending',
-        # 'has_electric_storage_heaters_ending',
-        # 'has_warm_air_ending',
-        # 'has_electric_underfloor_heating_ending',
-        # 'has_electric_ceiling_heating_ending',
-        # 'has_community_scheme_ending',
-        # 'has_ground_source_heat_pump_ending',
-        # 'has_no_system_present_ending',
-        # 'has_portable_electric_heaters_ending',
-        # 'has_water_source_heat_pump_ending',
-        # 'has_electric_heat_pump_ending',
-        # 'has_micro-cogeneration_ending',
-        # 'has_solar_assisted_heat_pump_ending',
-        # 'has_exhaust_source_heat_pump_ending',
-        # 'has_community_heat_pump_ending',
-        # 'has_electric_ending',
-        # 'has_mains_gas_ending',
-        # 'has_wood_logs_ending', 'has_coal_ending', 'has_oil_ending',
-        # 'has_wood_pellets_ending', 'has_anthracite_ending', 'has_dual_fuel_mineral_and_wood_ending',
-        # 'has_smokeless_fuel_ending', 'has_lpg_ending', 'has_b30k_ending', 'has_electricaire_ending',
-        # 'has_assumed_for_most_rooms_ending', 'has_underfloor_heating_ending',
-        # 'thermostatic_control_ending',
-        # 'charging_system_ending',
-        # 'switch_system_ending',
-        # 'no_control_ending',
-        # 'dhw_control_ending',
-        # 'community_heating_ending',
-        # 'multiple_room_thermostats_ending',
-        # 'auxiliary_systems_ending',
-        # 'trvs_ending',
-        # 'rate_control_ending',
-        # Window
-        'glazing_type_ending',
-        # Fuel - could change with ASHP
-        # 'fuel_type_ending',
-        # 'main-fuel_tariff_type_ending',
-        # 'is_community_ending',
-        # 'no_individual_heating_or_community_network_ending',
-        # 'complex_fuel_type_ending',
-
-        'mechanical_ventilation_ending', 'secondheat_description_ending', 'glazed_type_ending',
-        'multi_glaze_proportion_ending', 'low_energy_lighting_ending', 'number_open_fireplaces_ending',
-        'solar_water_heating_flag_ending',
-        'photo_supply_ending',
-        'energy_tariff_ending',
-        'extension_count_ending',
-        'total_floor_area_ending',
-        # 'hot_water_energy_eff_ending',
-        'floor_energy_eff_ending',
-        'windows_energy_eff_ending',
-        'walls_energy_eff_ending',
-        'sheating_energy_eff_ending',
-        'roof_energy_eff_ending',
-        # 'mainheat_energy_eff_ending',
-        # 'mainheatc_energy_eff_ending',
-        'lighting_energy_eff_ending',
-        'number_habitable_rooms_ending',
-        'number_heated_rooms_ending',
-    ]
-
-    for col in static_columns:
-
-        base_starting = col.split("_ending")[0]
-        if base_starting + "_starting" in data.columns:
-            starting_col = base_starting + "_starting"
-        else:
-            starting_col = base_starting
-        # Filter
-        print("Column: %s" % col)
-        print("Starting size: %s" % data.shape[0])
-        data = data[data[starting_col] == data[col]]
-        print("Ending size: %s" % data.shape[0])
-
-        z = data[['uprn', col, starting_col]]
-
-    # Great example UPRNs
-    # 100030969273
-    # 10034685399 - Completely transforms the heating and hot water systems in the home (goes from oil -> electricity)
-    # 100091200828 - goes from a liquid petroleum gas boiler to ashp
-
-    # Look for starting with a gas boiler
-    data[
-        data["has_boiler"] & data["has_radiators"] & data["has_mains_gas"] & ~data["has_boiler_ending"]
-        ]
-
-    # UPRN: 100011776843
-
-
-class TestAirSourceHeatPump:
-
-    def test_eligible(self):
-        # This tests a house, which will be suitable for an air source heat pump
-        epc_record = EPCRecord()
-        epc_record.prepared_epc = {
-            "county": "Broxbourne",
-            "mainheat-energy-eff": "Good",
-            "hot-water-energy-eff": "Good",
-            "mainheatc-energy-eff": "Good",
-            "number-heated-rooms": 5,
-            "property-type": "House",
-            "built-form": "Semi-Detached"
-        }
-
-        property_instance = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
-        property_instance.main_heating = {
-            'original_description': 'Boiler and radiators, mains gas',
-            "clean_description": "Boiler and radiators, mains gas",
-            'has_radiators': True,
-            'has_fan_coil_units': False, 'has_pipes_in_screed_above_insulation': False,
-            'has_pipes_in_insulated_timber_floor': False, 'has_pipes_in_concrete_slab': False, 'has_boiler': True,
-            'has_air_source_heat_pump': False,
-            'has_room_heaters': False, 'has_electric_storage_heaters': False,
-            'has_warm_air': False,
-            'has_electric_underfloor_heating': False,
-            'has_electric_ceiling_heating': False, 'has_community_scheme': False,
-            'has_ground_source_heat_pump': False, 'has_no_system_present': False,
-            'has_portable_electric_heaters': False,
-            'has_water_source_heat_pump': False, 'has_electric': False,
-            'has_mains_gas': True, 'has_wood_logs': False,
-            'has_coal': False, 'has_oil': False, 'has_wood_pellets': False,
-            'has_anthracite': False,
-            'has_dual_fuel_mineral_and_wood': False, 'has_smokeless_fuel': False,
-            'has_lpg': False, 'has_assumed': False,
-            'has_electricaire': False, 'has_assumed_for_most_rooms': False,
-            'has_underfloor_heating': False,
-            "has_electric_heat_pumps": False,
-            "has_micro-cogeneration": False
-        }
-        property_instance.main_fuel = {
-            'original_description': 'mains gas (not community)', 'fuel_type': 'mains gas',
-            'tariff_type': None,
-            'is_community': False, 'no_individual_heating_or_community_network': False,
-            'complex_fuel_type': None
-        }
-        property_instance.hotwater = {
-            'original_description': 'From main system',
-            'clean_description': 'From main system',
-            'heater_type': None,
-            'system_type': 'from main system',
-            'thermostat_characteristics': None, 'heating_scope': None,
-            'energy_recovery': None, 'tariff_type': None,
-            'extra_features': None, 'chp_systems': None, 'distribution_system': None,
-            'no_system_present': None,
-            'assumed': False, "appliance": None
-        }
-        property_instance.main_heating_controls = {
-            'original_description': 'Programmer, room thermostat and TRVs',
-            'thermostatic_control': 'room thermostat', 'charging_system': None, 'switch_system': 'programmer',
-            'no_control': None, 'dhw_control': None, 'community_heating': None, 'multiple_room_thermostats': False,
-            'auxiliary_systems': None, 'trvs': 'trvs', 'rate_control': None
-
-        }
-
-        recommender = HeatingRecommender(property_instance=property_instance)
-
-        assert not recommender.heating_recommendations
-
-        recommender.recommend(phase=0)
-
-        assert recommender.recommendation is None
-
-    def test_air_source_heat_pump_gas_boiler_starting(self):
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.62', 'heating-cost-potential': '599', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '67', 'construction-age-band': 'England and Wales: 1950-1966',
-            'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Good',
-            'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '72',
-            'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '913',
-            'address3': '', 'mainheatcont-description': 'Programmer, no room thermostat', 'sheating-energy-eff': 'N/A',
-            'property-type': 'House', 'local-authority-label': 'Wigan', 'fixed-lighting-outlets-count': '9',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '210',
-            'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N', 'constituency': 'E14001039',
-            'co2-emissions-potential': '2.6', 'number-heated-rooms': '4',
-            'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '180',
-            'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-02-15',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '78', 'address1': '430 Gidlow Lane',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan',
-            'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112',
-            'environment-impact-current': '38', 'co2-emissions-current': '6.2',
-            'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN',
-            'mainheatc-energy-eff': 'Very Poor', 'main-fuel': 'mains gas (not community)',
-            'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A',
-            'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
-            'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
-            'lighting-cost-potential': '67', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
-            'main-heating-controls': '', 'lodgement-datetime': '2022-02-23 16:39:41', 'flat-top-storey': '',
-            'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas',
-            'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843',
-            'current-energy-efficiency': '45', 'energy-consumption-current': '441',
-            'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '67',
-            'lodgement-date': '2022-02-23', 'extension-count': '1', 'mainheatc-env-eff': 'Very Poor',
-            'lmk-key': '46cb404438a6d88ddff8965cab8b3027ec15c32d93e0b6a5f0381a5109b9bb0d', 'wind-turbine-count': '0',
-            'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '77',
-            'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '100',
-            'walls-description': 'Cavity wall, filled cavity',
-            'hotwater-description': 'From main system, no cylinder thermostat'
-        }
-
-        ending_epc = {
-            'low-energy-fixed-light-count': '', 'address': '430 Gidlow Lane', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.62', 'heating-cost-potential': '803', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '292', 'construction-age-band': 'England and Wales: 1950-1966',
-            'potential-energy-rating': 'C', 'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Good',
-            'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '78',
-            'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '861',
-            'address3': '', 'mainheatcont-description': 'Time and temperature zone control',
-            'sheating-energy-eff': 'N/A', 'property-type': 'House', 'local-authority-label': 'Wigan',
-            'fixed-lighting-outlets-count': '9', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
-            'hot-water-cost-current': '434', 'county': '', 'postcode': 'WN6 8RG', 'solar-water-heating-flag': 'N',
-            'constituency': 'E14001039', 'co2-emissions-potential': '2.0', 'number-heated-rooms': '4',
-            'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '147',
-            'local-authority': 'E08000010', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2022-05-11',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '43', 'address1': '430 Gidlow Lane',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Wigan',
-            'roof-energy-eff': 'Very Poor', 'total-floor-area': '80.0', 'building-reference-number': '10002334112',
-            'environment-impact-current': '63', 'co2-emissions-current': '3.4',
-            'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'WIGAN',
-            'mainheatc-energy-eff': 'Very Good', 'main-fuel': 'electricity (not community)',
-            'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A',
-            'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
-            'roof-env-eff': 'Very Poor', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
-            'lighting-cost-potential': '67', 'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100',
-            'main-heating-controls': '', 'lodgement-datetime': '2022-06-06 13:01:20', 'flat-top-storey': '',
-            'current-energy-rating': 'E', 'secondheat-description': 'Room heaters, mains gas',
-            'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100011776843',
-            'current-energy-efficiency': '53', 'energy-consumption-current': '252',
-            'mainheat-description': 'Air source heat pump, radiators, electric', 'lighting-cost-current': '67',
-            'lodgement-date': '2022-06-06', 'extension-count': '1', 'mainheatc-env-eff': 'Very Good',
-            'lmk-key': '672d5947f3d4a55d97255af71651d6127a939418fa66a687070af77e0ba90df2', 'wind-turbine-count': '0',
-            'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '70',
-            'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '100',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-
-        # differences = []
-        # for k, v in ending_epc.items():
-        #     if v != starting_epc[k]:
-        #         differences.append(
-        #             {
-        #                 "variable": k,
-        #                 "starting_value": starting_epc[k],
-        #                 "ending_value": v
-        #             }
-        #         )
-        # differences = pd.DataFrame(differences)
-        #
-        # diffs = differences[
-        #     differences["variable"].isin(
-        #         [
-        #             "mainheat-energy-eff",
-        #             "mainheatcont-description",
-        #             "mainheatc-energy-eff",
-        #             "main-fuel",
-        #             "mainheat-env-eff",
-        #             "mainheat-description",
-        #             "hot-water-energy-eff",
-        #             "hotwater-description"
-        #         ]
-        #     )
-        # ]
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = HeatingRecommender(property_instance=home)
-        recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
-
-        # Patch - for this property, the hot water energy efficiency is very poor. it's not clear why this is,
-        # but we insert this for this test
-        recommender.heating_recommendations[0]["simulation_config"]["hot_water_energy_eff_ending"] = "Very Poor"
-
-        property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
-
-        assert len(recommender.heating_recommendations) == 1
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-        assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 52.2
-
-    def test_air_source_heat_pump_gas_boiler_starting_2(self):
-        """
-        This property seems to have miniscule movement in SAP - just 2 poins
-        :return:
-        """
-
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.3', 'heating-cost-potential': '394', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '48', 'construction-age-band': 'England and Wales: 1967-1975',
-            'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
-            'lighting-energy-eff': 'Good', 'environment-impact-potential': '87',
-            'glazed-type': 'double glazing, unknown install date', 'heating-cost-current': '487', 'address3': '',
-            'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
-            'property-type': 'Bungalow', 'local-authority-label': 'Calderdale', 'fixed-lighting-outlets-count': '5',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '86',
-            'county': '', 'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614',
-            'co2-emissions-potential': '0.8', 'number-heated-rooms': '2',
-            'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '105',
-            'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-25',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '56', 'address1': '31 Whinney Hill Park',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Calder Valley',
-            'roof-energy-eff': 'Good', 'total-floor-area': '44.0', 'building-reference-number': '10001772583',
-            'environment-impact-current': '62', 'co2-emissions-current': '2.5',
-            'roof-description': 'Pitched, 250 mm loft insulation', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '2', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'BRIGHOUSE',
-            'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Good',
-            'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good',
-            'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40',
-            'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2021-11-25 11:39:35', 'flat-top-storey': '', 'current-energy-rating': 'D',
-            'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
-            'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '62',
-            'energy-consumption-current': '322', 'mainheat-description': 'Boiler and radiators, mains gas',
-            'lighting-cost-current': '56', 'lodgement-date': '2021-11-25', 'extension-count': '0',
-            'mainheatc-env-eff': 'Good', 'lmk-key': '077f70657e9c3f1f0ce5392798398398616b159493b2a8ca2338961596631c27',
-            'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '',
-            'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '60',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-
-        ending_epc = {
-            'low-energy-fixed-light-count': '', 'address': '31 Whinney Hill Park',
-            'uprn-source': 'Energy Assessor', 'floor-height': '2.3', 'heating-cost-potential': '277',
-            'unheated-corridor-length': '', 'hot-water-cost-potential': '266',
-            'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B',
-            'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Good',
-            'environment-impact-potential': '90', 'glazed-type': 'double glazing, unknown install date',
-            'heating-cost-current': '331', 'address3': '',
-            'mainheatcont-description': 'Programmer and room thermostat', 'sheating-energy-eff': 'N/A',
-            'property-type': 'Bungalow', 'local-authority-label': 'Calderdale',
-            'fixed-lighting-outlets-count': '5', 'energy-tariff': 'Single',
-            'mechanical-ventilation': 'natural', 'hot-water-cost-current': '404', 'county': '',
-            'postcode': 'HD6 2PX', 'solar-water-heating-flag': 'N', 'constituency': 'E14000614',
-            'co2-emissions-potential': '0.7', 'number-heated-rooms': '2',
-            'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '92',
-            'local-authority': 'E08000033', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
-            'inspection-date': '2021-11-25', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '48',
-            'address1': '31 Whinney Hill Park', 'heat-loss-corridor': '', 'flat-storey-count': '',
-            'constituency-label': 'Calder Valley', 'roof-energy-eff': 'Good', 'total-floor-area': '44.0',
-            'building-reference-number': '10001772583', 'environment-impact-current': '68',
-            'co2-emissions-current': '2.1', 'roof-description': 'Pitched, 250 mm loft insulation',
-            'floor-energy-eff': 'N/A', 'number-habitable-rooms': '2', 'address2': '',
-            'hot-water-env-eff': 'Poor', 'posttown': 'BRIGHOUSE', 'mainheatc-energy-eff': 'Average',
-            'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good',
-            'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in 60% of fixed outlets', 'roof-env-eff': 'Good',
-            'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '40',
-            'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2022-03-23 16:06:21', 'flat-top-storey': '', 'current-energy-rating': 'D',
-            'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
-            'transaction-type': 'rental', 'uprn': '100051304421', 'current-energy-efficiency': '64',
-            'energy-consumption-current': '283',
-            'mainheat-description': 'Air source heat pump, radiators, electric',
-            'lighting-cost-current': '57', 'lodgement-date': '2022-03-23', 'extension-count': '0',
-            'mainheatc-env-eff': 'Average',
-            'lmk-key': '6296248141447b53426a40f1c39da17dad5f4786485db55ee38737891111a4d4',
-            'wind-turbine-count': '0', 'tenure': 'Rented (social)', 'floor-level': '',
-            'potential-energy-efficiency': '89', 'hot-water-energy-eff': 'Very Poor',
-            'low-energy-lighting': '60', 'walls-description': 'Cavity wall, filled cavity',
-            'hotwater-description': 'From main system'
-        }
-
-        # differences = []
-        # for k, v in ending_epc.items():
-        #     if v != starting_epc[k]:
-        #         differences.append(
-        #             {
-        #                 "variable": k,
-        #                 "starting_value": starting_epc[k],
-        #                 "ending_value": v
-        #             }
-        #         )
-        # differences = pd.DataFrame(differences)
-        #
-        # diffs = differences[
-        #     differences["variable"].isin(
-        #         [
-        #             "mainheat-energy-eff",
-        #             "mainheatcont-description",
-        #             "mainheatc-energy-eff",
-        #             "main-fuel",
-        #             "mainheat-env-eff",
-        #             "mainheat-description",
-        #             "hot-water-energy-eff",
-        #             "hotwater-description"
-        #         ]
-        #     )
-        # ]
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = HeatingRecommender(property_instance=home)
-        recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
-        property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
-
-        assert len(recommender.heating_recommendations) == 1
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-        assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 69.3
-
-        # In actuality with this property, the heating controls get downgraded, so we test a manual patch of this
-        patched_simulation_config = {
-            'mainheat_energy_eff_ending': "Very Good",
-            'hot_water_energy_eff_ending': 'Very Poor',
-            'has_boiler_ending': False,
-            'has_air_source_heat_pump_ending': True,
-            'has_electric_ending': True,
-            'has_mains_gas_ending': False,
-            'fuel_type_ending': 'electricity',
-            'trvs_ending': None,
-            "mainheatc_energy_eff_ending": 'Average'
-        }
-
-        # PATCHING
-        property_recommendations_patch = Recommendations.insert_temp_recommendation_id(
-            [recommender.heating_recommendations]
-        )
-        property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations_patch, []
-        )
-
-        scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict_patch = model_api.predict_all(
-            df=scoring_data_patch,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-        # The error is only 0.3, so the model is working
-        assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 64.3
-        assert ending_epc["current-energy-efficiency"] == '64'
-
-    def test_air_source_heat_pump_lpg_boiler(self):
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry',
-            'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '1628',
-            'unheated-corridor-length': '', 'hot-water-cost-potential': '175',
-            'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'D',
-            'mainheat-energy-eff': 'Poor', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average',
-            'environment-impact-potential': '70', 'glazed-type': 'double glazing, unknown install date',
-            'heating-cost-current': '2158', 'address3': 'Perry',
-            'mainheatcont-description': 'No time or thermostatic control of room temperature',
-            'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire',
-            'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
-            'hot-water-cost-current': '257', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX',
-            'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '3.3',
-            'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)',
-            'energy-consumption-potential': '128', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached',
-            'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
-            'inspection-date': '2023-08-31', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '51',
-            'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '',
-            'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0',
-            'building-reference-number': '10005199915', 'environment-impact-current': '50',
-            'co2-emissions-current': '5.9', 'roof-description': 'Pitched, 270 mm loft insulation',
-            'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive',
-            'hot-water-env-eff': 'Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Very Poor',
-            'main-fuel': 'LPG (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average',
-            'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good',
-            'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '166',
-            'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2023-10-30 13:46:54', 'flat-top-storey': '', 'current-energy-rating': 'F',
-            'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
-            'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '32',
-            'energy-consumption-current': '243', 'mainheat-description': 'Boiler and radiators, LPG',
-            'lighting-cost-current': '277', 'lodgement-date': '2023-10-30', 'extension-count': '0',
-            'mainheatc-env-eff': 'Very Poor',
-            'lmk-key': 'f1d3bd4b8b50bc9b006231ccb158537c408523b748b3f4ef7e98cd03b144afa5', 'wind-turbine-count': '0',
-            'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '56',
-            'hot-water-energy-eff': 'Poor', 'low-energy-lighting': '33',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-
-        ending_epc = {
-            'low-energy-fixed-light-count': '', 'address': 'Holly Lodge, The Drive, Perry',
-            'uprn-source': 'Energy Assessor', 'floor-height': '2.8', 'heating-cost-potential': '917',
-            'unheated-corridor-length': '', 'hot-water-cost-potential': '328',
-            'construction-age-band': 'England and Wales: 1950-1966', 'potential-energy-rating': 'A',
-            'mainheat-energy-eff': 'Very Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Average',
-            'environment-impact-potential': '96', 'glazed-type': 'double glazing, unknown install date',
-            'heating-cost-current': '1098', 'address3': 'Perry',
-            'mainheatcont-description': 'Programmer, TRVs and bypass', 'sheating-energy-eff': 'N/A',
-            'property-type': 'Bungalow', 'local-authority-label': 'Huntingdonshire',
-            'fixed-lighting-outlets-count': '12', 'energy-tariff': 'Single', 'mechanical-ventilation': 'natural',
-            'hot-water-cost-current': '328', 'county': 'Cambridgeshire', 'postcode': 'PE28 0SX',
-            'solar-water-heating-flag': 'N', 'constituency': 'E14000757', 'co2-emissions-potential': '0.3',
-            'number-heated-rooms': '5', 'floor-description': 'Solid, no insulation (assumed)',
-            'energy-consumption-potential': '16', 'local-authority': 'E07000011', 'built-form': 'Semi-Detached',
-            'number-open-fireplaces': '0', 'windows-description': 'Fully double glazed', 'glazed-area': 'Normal',
-            'inspection-date': '2023-10-05', 'mains-gas-flag': 'N', 'co2-emiss-curr-per-floor-area': '6',
-            'address1': 'Holly Lodge', 'heat-loss-corridor': '', 'flat-storey-count': '',
-            'constituency-label': 'Huntingdon', 'roof-energy-eff': 'Good', 'total-floor-area': '117.0',
-            'building-reference-number': '10005199915', 'environment-impact-current': '92',
-            'co2-emissions-current': '0.7', 'roof-description': 'Pitched, 270 mm loft insulation',
-            'floor-energy-eff': 'N/A', 'number-habitable-rooms': '5', 'address2': 'The Drive',
-            'hot-water-env-eff': 'Very Good', 'posttown': 'HUNTINGDON', 'mainheatc-energy-eff': 'Average',
-            'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Average', 'windows-energy-eff': 'Average',
-            'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in 33% of fixed outlets', 'roof-env-eff': 'Good',
-            'walls-energy-eff': 'Average', 'photo-supply': '', 'lighting-cost-potential': '166',
-            'mainheat-env-eff': 'Very Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2023-11-01 16:29:16', 'flat-top-storey': '', 'current-energy-rating': 'A',
-            'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
-            'transaction-type': 'ECO assessment', 'uprn': '100091200828', 'current-energy-efficiency': '92',
-            'energy-consumption-current': '37', 'mainheat-description': 'Air source heat pump, radiators, electric',
-            'lighting-cost-current': '277', 'lodgement-date': '2023-11-01', 'extension-count': '0',
-            'mainheatc-env-eff': 'Average',
-            'lmk-key': 'cb7f2838b727907767c8c2a385cd22f722b1e4745463391d910d228e52124515', 'wind-turbine-count': '0',
-            'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '95',
-            'hot-water-energy-eff': 'Good', 'low-energy-lighting': '33',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = HeatingRecommender(property_instance=home)
-        recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
-        property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
-
-        assert len(recommender.heating_recommendations) == 1
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-        # We predict a huge uplift but not quite as much as the EPC, due to some distinct differences between our
-        # recommendation and the EPC
-        assert predictions_dict["sap_change_predictions"]["predictions"].values[0] == 81.3
-        assert ending_epc['current-energy-efficiency'] == '92'
-
-        # PATCH
-        # We patch the simulation config, to reflect the ending EPC, to see if we get the ending EPC's config
-        patched_simulation_config = {
-            'mainheat_energy_eff_ending': "Very Good",
-            'hot_water_energy_eff_ending': 'Good',
-            'has_boiler_ending': False,
-            'has_air_source_heat_pump_ending': True,
-            'has_electric_ending': True,
-            'has_lpg_ending': False,
-            'fuel_type_ending': 'electricity',
-            'switch_system_ending': 'programmer',
-            'no_control_ending': None,
-            'auxiliary_systems_ending': 'bypass',
-            'trvs_ending': 'trvs',
-            "mainheatc_energy_eff_ending": 'Average'
-        }
-
-        # PATCHING
-        property_recommendations_patch = Recommendations.insert_temp_recommendation_id(
-            [recommender.heating_recommendations]
-        )
-        property_recommendations_patch[0][0]["simulation_config"] = patched_simulation_config
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations_patch, []
-        )
-
-        scoring_data_patch = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict_patch = model_api.predict_all(
-            df=scoring_data_patch,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-
-        assert predictions_dict_patch["sap_change_predictions"]["predictions"].values[0] == 88.9
-        # We still underpredict but the improvement is notable
-
-    def test_offgrid(self):
-        """
-        We test on a property we've worked with before, where we compare two options
-        a) Upgrading to a boiler
-        b) Upgrading to a heat pump
-        :return:
-        """
-
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': '6 Beech Road', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.4', 'heating-cost-potential': '612', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '123', 'construction-age-band': 'England and Wales: 1930-1949',
-            'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Very Poor', 'windows-env-eff': 'Good',
-            'lighting-energy-eff': 'Good', 'environment-impact-potential': '87',
-            'glazed-type': 'double glazing installed during or after 2002', 'heating-cost-current': '2278',
-            'address3': '', 'mainheatcont-description': 'Appliance thermostats', 'sheating-energy-eff': 'N/A',
-            'property-type': 'House', 'local-authority-label': 'Dudley', 'fixed-lighting-outlets-count': '9',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '604',
-            'county': '', 'postcode': 'DY1 4BP', 'solar-water-heating-flag': 'N', 'constituency': 'E14000671',
-            'co2-emissions-potential': '1.0', 'number-heated-rooms': '4',
-            'floor-description': 'Solid, no insulation (assumed)', 'energy-consumption-potential': '93',
-            'local-authority': 'E08000027', 'built-form': 'End-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2024-03-13',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '83', 'address1': '6 Beech Road',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Dudley North',
-            'roof-energy-eff': 'Very Poor', 'total-floor-area': '60.0', 'building-reference-number': '10005780080',
-            'environment-impact-current': '41', 'co2-emissions-current': '5.0',
-            'roof-description': 'Pitched, 12 mm loft insulation', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '4', 'address2': '', 'hot-water-env-eff': 'Poor', 'posttown': 'DUDLEY',
-            'mainheatc-energy-eff': 'Good', 'main-fuel': 'electricity (not community)', 'lighting-env-eff': 'Good',
-            'windows-energy-eff': 'Good', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in 67% of fixed outlets', 'roof-env-eff': 'Very Poor',
-            'walls-energy-eff': 'Average', 'photo-supply': '0.0', 'lighting-cost-potential': '113',
-            'mainheat-env-eff': 'Poor', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2024-03-13 11:29:11', 'flat-top-storey': '', 'current-energy-rating': 'F',
-            'secondheat-description': 'None', 'walls-env-eff': 'Average', 'transaction-type': 'rental',
-            'uprn': '90055152', 'current-energy-efficiency': '32', 'energy-consumption-current': '491',
-            'mainheat-description': 'Room heaters, electric', 'lighting-cost-current': '113',
-            'lodgement-date': '2024-03-13', 'extension-count': '1', 'mainheatc-env-eff': 'Good',
-            'lmk-key': '78ddf851b660e599a0894924d0e6b503980f5e0ad1aa711f8411718dc2989c44', 'wind-turbine-count': '0',
-            'tenure': 'Rented (social)', 'floor-level': '', 'potential-energy-efficiency': '87',
-            'hot-water-energy-eff': 'Very Poor', 'low-energy-lighting': '67',
-            'walls-description': 'Cavity wall, filled cavity',
-            'hotwater-description': 'Electric immersion, standard tariff'
-        }
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = HeatingRecommender(property_instance=home)
-        recommender.recommend_air_source_heat_pump(phase=0, has_cavity_or_loft_recommendations=False)
-        recommender.recommend_boiler_upgrades(phase=0, system_change=True, exising_room_heaters=False)
-
-        assert len(recommender.heating_recommendations) == 3
-
-        property_recommendations = Recommendations.insert_temp_recommendation_id([recommender.heating_recommendations])
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-
-        # The ASHP isn't better under SAP, compared to a gas boiler with good heat controls
-        assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [66.9, 65.5, 65.9]

recommendations/tests/test_costs.py

@@ -18,10 +18,9 @@ class TestCosts:
             "description": "cwi",
             "depth": 75,
             "thermal_conductivity": 0.037,
-            "prime_cost": 5.17,
-            "material_cost": 5.62,
-            "labour_cost": 1.125,
+            "total_cost": 14,
             "labour_hours_per_unit": 0.065,
+            "is_installer_quote": True
         }
 
         cwi_results = costs.cavity_wall_insulation(
@@ -29,12 +28,7 @@ class TestCosts:
             material=cwi_material,
         )
 
-        assert cwi_results == {
-            'total': 1065.0661223512907, 'subtotal': 887.5551019594088, 'vat': 177.51102039188177,
-            'contingency': 63.396792997100626, 'preliminaries': 63.396792997100626, 'material': 539.0166061175574,
-            'profit': 126.79358599420125, 'labour_hours': 6.234177828761786, 'labour_cost': 94.95132385344874,
-            'labour_days': 0.38963611429761164
-        }
+        assert cwi_results == {'total': 1342.7459938871539, 'labour_hours': 8, 'labour_days': 1}
 
     def test_loft_insulation(self):
         mock_property = Mock()
@@ -47,22 +41,17 @@ class TestCosts:
             "description": "Crown Loft Roll 44 glass fibre roll",
             "depth": 270,
             "thermal_conductivity": 0.044,
-            "prime_cost": None,
-            "material_cost": 5.91938,
-            "labour_cost": 1.96,
-            "labour_hours_per_unit": 0.11
+            "total_cost": 11,
+            "labour_hours_per_unit": 0.11,
+            "is_installer_quote": True,
         }
 
-        loft_results = costs.loft_insulation(
+        loft_results = costs.loft_and_flat_insulation(
             floor_area=33.5,
             material=loft_material,
         )
 
-        assert loft_results == {
-            'total': 639.4133610000001, 'subtotal': 532.8444675000001, 'vat': 106.56889350000002,
-            'contingency': 71.045929, 'preliminaries': 35.5229645, 'material': 297.448845, 'profit': 71.045929,
-            'labour_hours': 3.685, 'labour_cost': 57.7808, 'labour_days': 0.460625
-        }
+        assert loft_results == {'total': 368.5, 'labour_hours': 8, 'labour_days': 1}
 
     def test_internal_wall_insulation(self):
         mock_property = Mock()
@@ -71,87 +60,6 @@ class TestCosts:
         }
 
         costs = Costs(mock_property)
-        iwi_non_insulation_materials = [
-            {'type': 'iwi_wall_demolition',
-             'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.',
-             'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
-             'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 10.27,
-             'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55, 'link': 'SPONs', 'Notes': 0.0},
-            {'type': 'iwi_wall_demolition',
-             'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim',
-             'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
-             'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.23,
-             'labour_hours_per_unit': 0.2, 'plant_cost': 1.25, 'total_cost': 7.48, 'link': 'SPONs', 'Notes': 0.0},
-            {'type': 'iwi_wall_demolition',
-             'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and '
-                            'plaster',
-             'depth': 0.0, 'depth_unit': 0.0, 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.0,
-             'thermal_conductivity_unit': 0.0, 'prime_material_cost': 0.0, 'material_cost': 0.0, 'labour_cost': 6.85,
-             'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94, 'link': 'SPONs', 'Notes': 0.0},
-            {'Notes': "",
-             'cost_unit': "",
-             'depth': "",
-             'depth_unit': "",
-             'description': 'Visqueen High Performance Vapour Barrier',
-             'labour_cost': 0.48,
-             'labour_hours_per_unit': 0.02,
-             'link': 'SPONs',
-             'material_cost': 1.21,
-             'plant_cost': 0,
-             'prime_material_cost': 0.58,
-             'thermal_conductivity': "",
-             'thermal_conductivity_unit': "",
-             'total_cost': 1.69,
-             'type': 'iwi_vapour_barrier'},
-            {'Notes': "",
-             'cost_unit': "",
-             'depth': "",
-             'depth_unit': "",
-             'description': 'Plaster; one coat Thistle board finish or other equal; steel trowelled; 3 mm thick work '
-                            'to walls or ceilings; one coat; to plasterboard base; over 600mm wide',
-             'labour_cost': 6.58,
-             'labour_hours_per_unit': 0.25,
-             'link': "",
-             'material_cost': 0.06,
-             'plant_cost': 0,
-             'prime_material_cost': 0.0,
-             'thermal_conductivity': "",
-             'thermal_conductivity_unit': "",
-             'total_cost': 6.64,
-             'type': 'iwi_redecoration'},
-            {'Notes': "",
-             'cost_unit': "",
-             'depth': "",
-             'depth_unit': "",
-             'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - '
-                            '5m high',
-             'labour_cost': 0.0,
-             'labour_hours_per_unit': 0.21,
-             'link': "",
-             'material_cost': 0.41,
-             'plant_cost': 0,
-             'prime_material_cost': "",
-             'thermal_conductivity': "",
-             'thermal_conductivity_unit': "",
-             'total_cost': 4.34,
-             'type': 'iwi_redecoration'},
-            {'Notes': "",
-             'cost_unit': "",
-             'depth': "",
-             'depth_unit': "",
-             'description': 'Fitting existing softwood skirting or architrave to new '
-                            'frames; 150mm high',
-             'labour_cost': 4.87,
-             'labour_hours_per_unit': 0.01,
-             'link': "",
-             'material_cost': 4.86,
-             'plant_cost': 0,
-             'prime_material_cost': "",
-             'thermal_conductivity': "",
-             'thermal_conductivity_unit': "",
-             'total_cost': 4.88,
-             'type': 'iwi_redecoration'}
-        ]
 
         iwi_material = {
             "type": "internal_wall_insulation",
@@ -161,26 +69,19 @@ class TestCosts:
             "cost_unit": "gbp_per_m2",
             "thermal_conductivity": 0.022,
             "thermal_conductivity_unit": "watt_per_meter_kelvin",
-            "prime_material_cost": "",
-            "material_cost": 11.68,
-            "labour_cost": 3.12,
             "labour_hours_per_unit": 0.18,
-            "plant_cost": "",
-            "total_cost": 14.8,
-            "link": "SPONs"
+            "total_cost": 200,
+            "link": "link",
+            "is_installer_quote": True
         }
 
-        iwi_results = costs.internal_wall_insulation(
+        iwi_results = costs.solid_wall_insulation(
             wall_area=95.9104281347967,
             material=iwi_material,
-            non_insulation_materials=iwi_non_insulation_materials
         )
 
         assert iwi_results == {
-            'total': 6880.2304726777775, 'subtotal': 5733.525393898148, 'vat': 1146.7050787796295,
-            'contingency': 764.470052519753, 'preliminaries': 382.2350262598765, 'material': 1747.488000615996,
-            'profit': 764.470052519753, 'labour_hours': 88.23759388401297, 'labour_days': 2.757424808875405,
-            'labour_cost': 1927.1602026551818
+            'total': 19182.085626959342, 'labour_hours': 17.263877064263404, 'labour_days': 0.5394961582582314
         }
 
     def test_suspended_floor_insulation(self):
@@ -201,7 +102,8 @@ class TestCosts:
             'total_cost': 13.46, 'link': 'SPONs',
             'Notes': 'Spons did not contain labour costs so we use values for similar insulations. '
                      'We use the '
-                     'same values as in Crown loft roll 44, since it is also an insulation roll'
+                     'same values as in Crown loft roll 44, since it is also an insulation roll',
+            "is_installer_quote": False
         }
 
         sus_floor_non_insulation_materials = [
@@ -256,7 +158,7 @@ class TestCosts:
             'depth': 100.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.033,
             'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 0,
             'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19, 'plant_cost': 0,
-            'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0
+            'total_cost': 16.42, 'link': 'SPONs', 'Notes': 0, "is_installer_quote": False
         }
 
         sol_floor_non_insulation_materials = [
@@ -342,81 +244,18 @@ class TestCosts:
         ewi_material = {
             'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
             'depth': 150.0, 'depth_unit': 'mm', 'cost_unit': 'gbp_per_m2', 'thermal_conductivity': 0.022,
-            'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'prime_material_cost': 23.53,
-            'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4, 'plant_cost': 0,
-            'total_cost': 67.68, 'link': 'SPONs', 'Notes': 0
+            'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+            'labour_hours_per_unit': 1.4,
+            'total_cost': 300, 'link': 'SPONs', 'Notes': 0, "is_installer_quote": True
         }
-        ewi_non_insulation_materials = [
-            {'type': 'ewi_wall_demolition',
-             'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping '
-                            'hammer; plaster to walls.',
-             'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2',
-             'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
-             'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 10.27,
-             'labour_hours_per_unit': 0.33, 'plant_cost': 1.28, 'total_cost': 11.55,
-             'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_demolition',
-                                            'description': 'Stud walls: Remove wall linings '
-                                                           'including battening behind; '
-                                                           'plasterboard and skim',
-                                            'depth': 0, 'depth_unit': 0,
-                                            'cost_unit': 'gbp_per_m2',
-                                            'thermal_conductivity': 0,
-                                            'thermal_conductivity_unit': 0,
-                                            'prime_material_cost': 0, 'material_cost': 0,
-                                            'labour_cost': 6.23, 'labour_hours_per_unit': 0.2,
-                                            'plant_cost': 1.25, 'total_cost': 7.48,
-                                            'link': 'SPONs', 'Notes': 0},
-            {'type': 'ewi_wall_demolition',
-             'description': 'Lathe and Plaster walls: Remove wall linings including battening '
-                            'behind; wood lath and plaster',
-             'depth': 0, 'depth_unit': 0, 'cost_unit': 'gbp_per_m2',
-             'thermal_conductivity': 0, 'thermal_conductivity_unit': 0,
-             'prime_material_cost': 0, 'material_cost': 0, 'labour_cost': 6.85,
-             'labour_hours_per_unit': 0.22, 'plant_cost': 2.09, 'total_cost': 8.94,
-             'link': 'SPONs', 'Notes': 0}, {'type': 'ewi_wall_preparation',
-                                            'description': 'Clean and prepare surfaces, '
-                                                           'one coat Keim dilution, '
-                                                           'one coat primer and two coats '
-                                                           'of Keim Ecosil paint; Brick or '
-                                                           'block walls; over 300 mm girth',
-                                            'depth': 0, 'depth_unit': 0, 'cost_unit': 0,
-                                            'thermal_conductivity': 0,
-                                            'thermal_conductivity_unit': 0,
-                                            'prime_material_cost': 0, 'material_cost': 7.3,
-                                            'labour_cost': 5.62, 'labour_hours_per_unit': 0.3,
-                                            'plant_cost': 0, 'total_cost': 12.92,
-                                            'link': 'SPONs',
-                                            'Notes': 'This work covers the preparation and '
-                                                     'priming of the wall before insulating'},
-            {'type': 'ewi_wall_redecoration',
-             'description': 'EPS insulation fixed with adhesive to SFS structure (measured '
-                            'separately) with horizontal PVC intermediate track and vertical '
-                            'T-spines; with glassfibre mesh reinforcement embedded in Sto '
-                            'Armat Classic Basecoat Render and Stolit K 1.5 Decorative '
-                            'Topcoat Render (white)',
-             'depth': 0, 'depth_unit': 0, 'cost_unit': 0, 'thermal_conductivity': 0,
-             'thermal_conductivity_unit': 0, 'prime_material_cost': 0, 'material_cost': 0,
-             'labour_cost': 0, 'labour_hours_per_unit': 0, 'plant_cost': 0,
-             'total_cost': 69.94, 'link': 'SPONs',
-             'Notes': 'This material in SPONs is for 70mm EPS insulation, which comes in at a '
-                      'cost of 99.17 per meter square. This includes the cost of insulation. '
-                      'To get the costing for just the works and not the insulation, '
-                      'we subtract the cost of EPS insulation, using Ravathem 75mm insulation '
-                      'as an example, which costs £29.23 per meter square, giving us the cost '
-                      'of the remaining works without insulation. This material gives us a '
-                      'cost for basecoat, mesh application and a render finish'}]
 
-        ewi_results = costs.external_wall_insulation(
+        ewi_results = costs.solid_wall_insulation(
             wall_area=95.9104281347967,
             material=ewi_material,
-            non_insulation_materials=ewi_non_insulation_materials
         )
 
         assert ewi_results == {
-            'total': 15047.078622131372, 'subtotal': 12539.232185109477, 'vat': 2507.8464370218953,
-            'contingency': 808.9827216199662, 'preliminaries': 2022.4568040499155, 'material': 4020.565147410677,
-            'profit': 1617.9654432399325, 'labour_hours': 187.02533486285358, 'labour_days': 5.8445417144641745,
-            'labour_cost': 3921.5600094613983
+            'total': 28773.12844043901, 'labour_hours': 134.2745993887154, 'labour_days': 4.196081230897356
         }
 
     def test_flat_roof_insulation(self):
@@ -426,120 +265,47 @@ class TestCosts:
         }
 
         costs = Costs(mock_property)
-        flat_roof_material = {'id': 1225, 'type': 'flat_roof_insulation',
-                              'description': 'Kingspan Thermaroof TR21 zero OPD '
-                                             'urethene insulation board',
-                              'depth': 100.0, 'depth_unit': 'mm', 'cost': None,
-                              'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04,
-                              'r_value_unit': 'square_meter_kelvin_per_watt',
-                              'thermal_conductivity': 0.025,
-                              'thermal_conductivity_unit': 'watt_per_meter_kelvin',
-                              'link': 'SPONs',
-                              'created_at': "now", 'is_active': True,
-                              'prime_material_cost': None, 'material_cost': 50.95,
-                              'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
-                              'plant_cost': 0.0, 'total_cost': 61.61,
-                              'notes': "SPONs didn't have a labour hours so we use "
-                                       "0.48 which is similar to other materials"}
+        flat_roof_material = {
+            'id': 1225, 'type': 'flat_roof_insulation',
+            'description': 'Kingspan Thermaroof TR21 zero OPD '
+                           'urethene insulation board',
+            'depth': 100.0, 'depth_unit': 'mm', 'cost': None,
+            'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04,
+            'r_value_unit': 'square_meter_kelvin_per_watt',
+            'thermal_conductivity': 0.025,
+            'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+            'link': 'SPONs',
+            'created_at': "now", 'is_active': True,
+            'prime_material_cost': None, 'material_cost': 50.95,
+            'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
+            'plant_cost': 0.0, 'total_cost': 61.61,
+            'notes': "SPONs didn't have a labour hours so we use "
+                     "0.48 which is similar to other materials",
+            "is_installer_quote": False
+        }
 
-        flat_roof_non_insulation_materials = [
-            {'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', 'depth': None,
-             'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': None,
-             'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None,
-             'created_at': datetime.datetime(2023, 10, 18, 16, 39, 9, 827188), 'is_active': True,
-             'prime_material_cost': None,
-             'material_cost': None, 'labour_cost': None, 'labour_hours_per_unit': None, 'plant_cost': None,
-             'total_cost': None,
-             'notes': None},
-            {'id': 1221, 'type': 'flat_roof_preparation',
-             'description': 'clean surface to receive new damp-proof membrane',
-             'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None,
-             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
-             'thermal_conductivity_unit': None,
-             'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
-             'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14,
-             'plant_cost': 0.0, 'total_cost': 4.36,
-             'notes': 'This data is based on concrete however forms a decent baseline for a Bituminous Felt flat roof'},
-            {'id': 1223, 'type': 'flat_roof_preparation',
-             'description': 'One coat primer; on wood surfaces before fixing; General surfaces; over 300 mm girth',
-             'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None,
-             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
-             'thermal_conductivity_unit': None,
-             'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
-             'prime_material_cost': None, 'material_cost': 2.49, 'labour_cost': 1.5, 'labour_hours_per_unit': 0.08,
-             'plant_cost': 0.0, 'total_cost': 3.99, 'notes': 'SPONs data gives us a baseline for a wood surface'},
-            {'id': 1224, 'type': 'flat_roof_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
-             'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None,
-             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
-             'thermal_conductivity_unit': None,
-             'link': 'SPONs', 'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
-             'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02,
-             'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None},
-            {'id': 1234, 'type': 'flat_roof_waterproofing',
-             'description': '20 mm thick two coat coverings; felt isolating membrane; to concrete (or '
-                            'timber) base; flat or to falls or slopes not exceeding 10° from horizontal',
-             'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': None,
-             'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
-             'thermal_conductivity_unit': None, 'link': 'SPONs',
-             'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
-             'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 0.0,
-             'labour_hours_per_unit': 0.5, 'plant_cost': 0.0, 'total_cost': 31.13, 'notes': None}
-        ]
-
-        flat_roof_floor_results = costs.flat_roof_insulation(
+        flat_roof_floor_results = costs.loft_and_flat_insulation(
             floor_area=33.5,
             material=flat_roof_material,
-            non_insulation_materials=flat_roof_non_insulation_materials
         )
 
-        assert flat_roof_floor_results == {'total': 5325.327767999999, 'subtotal': 4437.773139999999,
-                                           'vat': 887.5546279999999, 'contingency': 459.07998,
-                                           'preliminaries': 306.05332, 'material': 1830.775, 'profit': 612.10664,
-                                           'labour_hours': 24.79, 'labour_days': 1.549375, 'labour_cost': 186.9032}
+        assert flat_roof_floor_results == {
+            'total': 2063.935, 'subtotal': 1719.9458333333334, 'vat': 343.9891666666665, 'labour_hours': 8,
+            'labour_days': 1
+        }
 
         assert costs.labour_adjustment_factor == 0.88
 
-    # Mock property instance for regional tests
-    @pytest.fixture(params=[
-        ("Northamptonshire", "East Midlands", 7927.44),
-        ("Greater London Authority", "Inner London", 10475.0),
-        ("Adur", "South East England", 8333.32),
-        ("Bournemouth", "South West England", 8452),
-        ("Basildon", "East of England", 7895.44),
-        ("Birmingham", "West Midlands", 7706.2),
-        ("County Durham", "North East England", 8113.96),
-        ("Allerdale", "North West England", 6481.68),
-        ("York", "Yorkshire and the Humber", 8243.6),
-        ("Cardiff", "Wales", 7595.32),
-        ("Glasgow City", "Scotland", 7871.88),
-        ("Belfast", "Northern Ireland", 8504.36)
-    ])
-    def mock_property_with_region(self, request):
-        county, region, expected_cost = request.param
-        mock_property = Mock()
-        mock_property.data = {"county": county}
-        return mock_property, region, expected_cost
-
     # Test for different wattages
-    @pytest.mark.parametrize("wattage, expected_cost", [
-        (3000, 5945.58),
-        (4000, 7927.44),
-        (5000, 9909.3),
-        (6000, 11891.16),
+    @pytest.mark.parametrize("n_panels, expected_cost", [
+        (7, 4055.0),
+        (10, 4540.0),
+        (12, 4863.0),
+        (15, 5707.0),
     ])
-    def test_solar_pv_different_wattages(self, wattage, expected_cost):
+    def test_solar_pv_different_wattages(self, n_panels, expected_cost):
         mock_property = Mock()
         mock_property.data = {"county": "Mansfield"}
         costs = Costs(mock_property)
-        result = costs.solar_pv(wattage)
-        assert result['total'] == pytest.approx(expected_cost, rel=0.01)
-
-    def test_solar_pv_regional_variation(self, mock_property_with_region):
-        # Test for regional cost variations
-        property_instance, expected_region, expected_cost = mock_property_with_region
-        costs = Costs(property_instance)
-
-        assert costs.region == expected_region
-
-        result = costs.solar_pv(4000)  # Testing with a fixed wattage of 4000
+        result = costs.solar_pv(n_panels)
         assert result['total'] == pytest.approx(expected_cost, rel=0.01)

recommendations/tests/test_data/roof_uvalue_test_cases.py

@@ -0,0 +1,466 @@
+roof_uvalue_test_cases = [
+    # Pitched roof
+    {
+        'insulation_thickness': '0',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 2.3,
+    },
+    {
+        'insulation_thickness': '12',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 1.5
+    },
+    {
+        'insulation_thickness': '25',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 1
+    },
+    {
+        'insulation_thickness': '50',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': '75',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.5
+    },
+    {
+        'insulation_thickness': '100',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': '150',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.3
+    },
+    {
+        'insulation_thickness': '200',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.21
+    },
+    {
+        'insulation_thickness': '250',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.17
+    },
+    {
+        'insulation_thickness': '270',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.16
+    },
+    {
+        'insulation_thickness': '300',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.14
+    },
+    {
+        'insulation_thickness': '350',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.12
+    },
+    {
+        'insulation_thickness': '400+',
+        'is_loft': True,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': True,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.11
+    },
+    # Flat roofs - no insulation
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'B',
+        'uvalue': 2.3
+    },
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'D',
+        'uvalue': 2.3
+    },
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'E',
+        'uvalue': 1.5
+    },
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'F',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'G',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'H',
+        'uvalue': 0.35
+    },
+    # Flat roofs - 50mm insulation
+    {
+        'insulation_thickness': '50',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'B',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': '50',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'E',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': '50',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'G',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': '50',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'I',
+        'uvalue': 0.35
+    },
+    # Flat roofs - 100mm insulation
+    {
+        'insulation_thickness': '100',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'B',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': '100',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'F',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': '100',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'J',
+        'uvalue': 0.25
+    },
+    # Flat roofs - 150mm insulation
+    {
+        'insulation_thickness': '150',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'B',
+        'uvalue': 0.3
+    },
+    {
+        'insulation_thickness': '150',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'J',
+        'uvalue': 0.25
+    },
+    {
+        'insulation_thickness': '150',
+        'is_loft': False,
+        'is_roof_room': False,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': True,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'L',
+        'uvalue': 0.18
+    },
+    # Room roof - age band A
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 2.3
+    },
+    {
+        'insulation_thickness': 'below average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': 'average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.4
+    },
+    {
+        'insulation_thickness': 'above average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'A',
+        'uvalue': 0.3
+    },
+    # Room roof - age band E
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'E',
+        'uvalue': 1.5
+    },
+    {
+        'insulation_thickness': 'average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'E',
+        'uvalue': 0.4
+    },
+    # Room roof - age band H
+    {
+        'insulation_thickness': 'none',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'H',
+        'uvalue': 0.35
+    },
+    {
+        'insulation_thickness': 'below average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'H',
+        'uvalue': 0.68
+    },
+    {
+        'insulation_thickness': 'average',
+        'is_loft': False,
+        'is_roof_room': True,
+        'is_thatched': False,
+        'has_dwelling_above': False,
+        'is_flat': False,
+        'is_pitched': False,
+        'is_at_rafters': False,
+        'age_band': 'H',
+        'uvalue': 0.4
+    },
+]

recommendations/tests/test_fireplace_recommendations.py

@@ -40,7 +40,7 @@ class TestFirepaceRecommendations:
 
         assert recommender.recommendation
         assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
-        assert recommender.recommendation[0]["total"] == 300
+        assert recommender.recommendation[0]["total"] == 235
 
     def test_multiple_fireplaces(self):
         epc_record = EPCRecord()
@@ -59,4 +59,4 @@ class TestFirepaceRecommendations:
 
         assert recommender.recommendation
         assert recommender.recommendation[0]["type"] == "sealing_open_fireplace"
-        assert recommender.recommendation[0]["total"] == 900
+        assert recommender.recommendation[0]["total"] == 235 * 3

recommendations/tests/test_floor_recommendations.py

@@ -5,13 +5,18 @@ from unittest.mock import Mock
 from recommendations.FloorRecommendations import FloorRecommendations
 from recommendations.tests.test_data.materials import materials
 from backend.Property import Property
+from etl.epc.Record import EPCRecord
 
 
+# import inspect
+#
+# file_path = inspect.getfile(lambda: None)
 # with open(
-#     os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
+#     os.path.abspath(os.path.dirname(file_path)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
 # ) as f:
 #     input_properties = pickle.load(f)
 
+
 class TestFloorRecommendations:
 
     @pytest.fixture
@@ -59,6 +64,7 @@ class TestFloorRecommendations:
         input_properties[2].floor_type = "suspended"
         input_properties[2].number_of_floors = 1
         input_properties[2].floor_level = 0
+        input_properties[2].already_installed = []
 
         recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials)
         assert recommender.estimated_u_value is None
@@ -71,8 +77,8 @@ class TestFloorRecommendations:
 
         assert types == {"suspended_floor_insulation"}
 
-        assert len(recommender.recommendations) == 6
-        assert recommender.recommendations[0]["total"] == 4925.205
+        assert len(recommender.recommendations) == 1
+        assert recommender.recommendations[0]["total"] == 4687.5
         assert recommender.recommendations[0]["new_u_value"] == 0.21
 
     def test_uvalue_0_12(self, input_properties):
@@ -108,6 +114,7 @@ class TestFloorRecommendations:
         input_properties[4].floor_type = "solid"
         input_properties[4].number_of_floors = 1
         input_properties[4].floor_level = 0
+        input_properties[4].already_installed = []
 
         # In this case, we have no county, so in this case, it should yse the local-authority-label if possible
         input_properties[4].data["county"] = ""
@@ -146,123 +153,131 @@ class TestFloorRecommendations:
         assert recommender.estimated_u_value is None
         assert not recommender.recommendations
 
-    # def test_exposed_floor_no_insulation(self):
-    #     input_property = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
-    #     input_property.floor = {
-    #         'original_description': 'To unheated space, no insulation (assumed)',
-    #         'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
-    #         'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
-    #         'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
-    #         'insulation_thickness': 'none'
-    #     }
-    #     input_property.age_band = "L"
-    #     input_property.set_floor_type()
-    #     input_property.data = {"floor-level": 0, "property-type": "House"}
-    #     input_property.floor_area = 100
-    #     input_property.number_of_floors = 1
-    #
-    #     recommender = FloorRecommendations(
-    #         property_instance=input_property,
-    #         materials=materials
-    #     )
-    #
-    #     assert not recommender.recommendations
-    #
-    #     recommender.recommend()
-    #
-    #     # Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation
-    #     assert not len(recommender.recommendations)
-    #     assert recommender.estimated_u_value == 0.22
-    #
-    #     # Now with an older age band
-    #
-    #     input_property2 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
-    #     input_property2.floor = {
-    #         'original_description': 'To unheated space, no insulation (assumed)',
-    #         'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
-    #         'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
-    #         'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
-    #         'insulation_thickness': 'none'
-    #     }
-    #     input_property2.age_band = "D"
-    #     input_property2.set_floor_type()
-    #     input_property2.data = {"floor-level": 0, "property-type": "House"}
-    #     input_property2.floor_area = 100
-    #     input_property2.number_of_floors = 1
-    #
-    #     recommender2 = FloorRecommendations(
-    #         property_instance=input_property2,
-    #         materials=materials
-    #     )
-    #
-    #     assert not recommender2.recommendations
-    #
-    #     recommender2.recommend()
-    #
-    #     assert len(recommender2.recommendations) == 1
-    #
-    #     assert recommender2.recommendations[0]["new_u_value"] == 0.23
-    #     assert recommender2.recommendations[0]["starting_u_value"] == 1.2
-    #     assert recommender2.recommendations[0]["cost"] == 1500
-    #
-    # def test_exposed_floor_below_average_insulated(self):
-    #     input_property3 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
-    #     input_property3.floor = {
-    #         'original_description': 'To unheated space, below average insulation (assumed)',
-    #         'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None,
-    #         'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
-    #         'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
-    #         'insulation_thickness': 'below average'
-    #     }
-    #     input_property3.age_band = "C"
-    #     input_property3.set_floor_type()
-    #     input_property3.data = {"floor-level": 0, "property-type": "House"}
-    #     input_property3.floor_area = 100
-    #     input_property3.number_of_floors = 1
-    #
-    #     recommender3 = FloorRecommendations(
-    #         property_instance=input_property3,
-    #         materials=materials
-    #     )
-    #
-    #     assert not recommender3.recommendations
-    #
-    #     recommender3.recommend()
-    #
-    #     assert recommender3.estimated_u_value == 0.5
-    #
-    #     assert len(recommender3.recommendations) == 1
-    #
-    #     assert recommender3.recommendations[0]["new_u_value"] == 0.22
-    #     assert recommender3.recommendations[0]["starting_u_value"] == 0.5
-    #     assert recommender3.recommendations[0]["cost"] == 1100
-    #     assert recommender3.recommendations[0]["parts"][0]["depths"] == [100]
-    #
-    #     # With average insulation, no recommendations
-    #
-    #     input_property4 = Property(id=1, postcode="F4k3 2", address1="223 fake street", epc_client=Mock())
-    #     input_property4.floor = {
-    #         'original_description': 'To unheated space, insulated (assumed)',
-    #         'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None,
-    #         'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
-    #         'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
-    #         'insulation_thickness': 'average'
-    #     }
-    #     input_property4.age_band = "C"
-    #     input_property4.set_floor_type()
-    #     input_property4.data = {"floor-level": 0, "property-type": "House"}
-    #     input_property4.floor_area = 100
-    #     input_property4.number_of_floors = 1
-    #
-    #     recommender4 = FloorRecommendations(
-    #         property_instance=input_property4,
-    #         materials=materials
-    #     )
-    #
-    #     assert not recommender4.recommendations
-    #
-    #     recommender4.recommend()
-    #
-    #     assert recommender4.estimated_u_value is None
-    #
-    #     assert len(recommender4.recommendations) == 0
+    def test_exposed_floor_no_insulation(self):
+        epc_record = EPCRecord()
+        epc_record.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"}
+        epc_record.full_sap_epc = {}
+
+        input_property = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record)
+        input_property.floor = {
+            'original_description': 'To unheated space, no insulation (assumed)',
+            'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
+            'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
+            'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
+            'insulation_thickness': 'none'
+        }
+        input_property.age_band = "L"
+        input_property.set_floor_type()
+        input_property.floor_area = 100
+        input_property.number_of_floors = 1
+
+        recommender = FloorRecommendations(
+            property_instance=input_property,
+            materials=materials
+        )
+
+        assert not recommender.recommendations
+
+        recommender.recommend()
+
+        # Because of age band L, this should have a u-value of 0.22 to begin with and no recommendation
+        assert not len(recommender.recommendations)
+        assert recommender.estimated_u_value == 0.22
+
+        # Now with an older age band
+        epc_record2 = EPCRecord()
+        epc_record2.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"}
+        epc_record2.full_sap_epc = {}
+
+        input_property2 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record2)
+        input_property2.floor = {
+            'original_description': 'To unheated space, no insulation (assumed)',
+            'clean_description': 'To unheated space, no insulation', 'thermal_transmittance': None,
+            'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
+            'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
+            'insulation_thickness': 'none'
+        }
+        input_property2.age_band = "D"
+        input_property2.set_floor_type()
+        input_property2.insulation_floor_area = 100
+        input_property2.number_of_floors = 1
+
+        recommender2 = FloorRecommendations(
+            property_instance=input_property2,
+            materials=materials
+        )
+
+        assert not recommender2.recommendations
+
+        recommender2.recommend()
+
+        assert len(recommender2.recommendations) == 1
+
+        assert recommender2.recommendations[0]["new_u_value"] == 0.24
+        assert recommender2.recommendations[0]["starting_u_value"] == 1.2
+        assert recommender2.recommendations[0]["total"] == 9375
+
+    def test_exposed_floor_below_average_insulated(self):
+        epc_record3 = EPCRecord()
+        epc_record3.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"}
+        epc_record3.full_sap_epc = {}
+        input_property3 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record3)
+        input_property3.floor = {
+            'original_description': 'To unheated space, below average insulation (assumed)',
+            'clean_description': 'To unheated space, below average insulation', 'thermal_transmittance': None,
+            'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
+            'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
+            'insulation_thickness': 'below average'
+        }
+        input_property3.age_band = "C"
+        input_property3.set_floor_type()
+        input_property3.insulation_floor_area = 100
+        input_property3.number_of_floors = 1
+
+        recommender3 = FloorRecommendations(
+            property_instance=input_property3,
+            materials=materials
+        )
+
+        assert not recommender3.recommendations
+
+        recommender3.recommend()
+
+        assert recommender3.estimated_u_value == 0.5
+
+        assert len(recommender3.recommendations) == 1
+
+        assert recommender3.recommendations[0]["new_u_value"] == 0.24
+        assert recommender3.recommendations[0]["starting_u_value"] == 0.5
+        assert recommender3.recommendations[0]["total"] == 7500
+        assert recommender3.recommendations[0]["parts"][0]["depth"] == 50
+
+        # With average insulation, no recommendations
+        epc_record4 = EPCRecord()
+        epc_record4.prepared_epc = {"county": "Greater London", "floor-level": 0, "property-type": "House"}
+        epc_record4.full_sap_epc = {}
+        input_property4 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record4)
+        input_property4.floor = {
+            'original_description': 'To unheated space, insulated (assumed)',
+            'clean_description': 'To unheated space, insulated', 'thermal_transmittance': None,
+            'thermal_transmittance_unit': None, 'is_assumed': True, 'is_to_unheated_space': True,
+            'is_to_external_air': False, 'is_suspended': False, 'is_solid': False, 'another_property_below': False,
+            'insulation_thickness': 'average'
+        }
+        input_property4.age_band = "C"
+        input_property4.set_floor_type()
+        input_property4.insulation_floor_area = 100
+        input_property4.number_of_floors = 1
+
+        recommender4 = FloorRecommendations(
+            property_instance=input_property4,
+            materials=materials
+        )
+
+        assert not recommender4.recommendations
+
+        recommender4.recommend()
+
+        assert recommender4.estimated_u_value is None
+
+        assert len(recommender4.recommendations) == 0

recommendations/tests/test_heating_recommendations.py

@@ -54,16 +54,6 @@ class TestHeatingRecommendations:
         :return:
         """
 
-        if test_case["epc"]["uprn"] == 100090311351:
-            raise Exception(
-                "This test has electric storage heaters with automatic charge control - this case should be researched"
-                "and checked that a high heat retention storage recommendation is actually sensible. If it's not, "
-                "we should adjust accordingly or perhaps have just a control recommendation"
-            )
-
-        if test_case["epc"]["uprn"] == 100021560521:
-            raise Exception("Finish this test - could do so while on the train")
-
         epc_records = {"original_epc": test_case["epc"].copy(), "full_sap_epc": {}, "old_data": []}
 
         epc_record = EPCRecord(
@@ -106,19 +96,19 @@ class TestHeatingRecommendations:
 
         recommender.recommend(has_cavity_or_loft_recommendations=False)
 
-        assert len(recommender.heating_recommendations) == len(test_case["heating_recommendation_descriptions"])
+        assert len(recommender.heating_recommendations) == len(test_case["heating_measure_types"])
         assert (
             len(recommender.heating_control_recommendations) ==
-            len(test_case["heating_controls_recommendation_descriptions"])
+            len(test_case["heating_controls_measure_types"])
         )
 
-        # Check the exact descriptions
+        # Check the exact measure types
         assert (
-            {x["description"] for x in recommender.heating_recommendations} ==
-            set(test_case["heating_recommendation_descriptions"])
+            {x["measure_type"] for x in recommender.heating_recommendations} ==
+            set(test_case["heating_measure_types"])
         )
 
         assert (
-            {x["description"] for x in recommender.heating_control_recommendations} ==
-            set(test_case["heating_controls_recommendation_descriptions"])
+            {x["measure_type"] for x in recommender.heating_control_recommendations} ==
+            set(test_case["heating_controls_measure_types"])
         )

recommendations/tests/test_lighting_recommendations.py

@@ -41,8 +41,18 @@ class TestLightingRecommendations:
         assert len(lr.recommendation) == 1
 
         assert lr.recommendation == [
-            {'parts': [], 'type': 'low_energy_lighting', 'description': 'Install low energy lighting in 4 outlets',
-             'starting_u_value': None, 'new_u_value': None, 'sap_points': 0.4, 'total': 240.24,
-             'subtotal': 200.20000000000002, 'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3,
-             'material': 80.0, 'profit': 28.6, 'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0}
+            {
+                'phase': 0, 'parts': [], 'type': 'low_energy_lighting', 'measure_type': 'low_energy_lighting',
+                'description': 'Install low energy lighting in 4 outlets', 'starting_u_value': None,
+                'new_u_value': None,
+                'already_installed': False, 'sap_points': 0.4, 'kwh_savings': 219.0, 'co2_equivalent_savings': 0.035478,
+                'description_simulation': {
+                    'lighting-energy-eff': 'Very Good',
+                    'lighting-description': 'Low energy lighting in all fixed outlets',
+                    'low-energy-lighting': 100
+                },
+                'total': 240.24, 'subtotal': 200.20000000000002,
+                'vat': 40.040000000000006, 'contingency': 14.3, 'preliminaries': 14.3, 'material': 80.0, 'profit': 28.6,
+                'labour_hours': 3.2, 'labour_days': 0.4, 'labour_cost': 63.0, 'survey': False
+            }
         ]

recommendations/tests/test_recommendation_utils.py

@@ -6,6 +6,7 @@ from recommendations import recommendation_utils
 from datatypes.enums import QuantityUnits
 from recommendations.tests.test_data.wall_uvalue_test_cases import wall_uvalue_test_cases
 from recommendations.tests.test_data.floor_uvalue_test_cases import floor_uvalue_test_cases
+from recommendations.tests.test_data.roof_uvalue_test_cases import roof_uvalue_test_cases
 
 
 class TestRecommendationUtils:
@@ -88,8 +89,8 @@ class TestRecommendationUtils:
 
     def test_get_roof_u_value_case_3(self):
         inputs = {
-            'original_description': 'Room-in-roof, 200 mm insulation at rafters',
-            'clean_description': 'Room-in-roof, 200 mm insulation at rafters',
+            'original_description': 'Room-in-roof, insulated at rafters',
+            'clean_description': 'Room-in-roof, insulated at rafters',
             'thermal_transmittance': None,
             'thermal_transmittance_unit': None,
             'is_pitched': False,
@@ -101,12 +102,12 @@ class TestRecommendationUtils:
             'is_assumed': False,
             'has_dwelling_above': False,
             'is_valid': True,
-            'insulation_thickness': '200',
+            'insulation_thickness': 'average',
             'age_band': "J"
         }
 
         u_value = recommendation_utils.get_roof_u_value(**inputs)
-        assert u_value == 0.21, f"Expected 0.21, but got {u_value}"
+        assert u_value == 0.4, f"Expected 0.4, but got {u_value}"
 
     def test_get_roof_u_value_case_4(self):
         inputs = {
@@ -179,8 +180,8 @@ class TestRecommendationUtils:
     def test_get_roof_u_value_case_7(self):
         # Test case where the roof has a room in it
         inputs = {
-            'original_description': 'Pitched, room-in-roof, 100mm insulation',
-            'clean_description': 'Pitched, room-in-roof, 100mm insulation',
+            'original_description': 'Pitched, room-in-roof, above average insulation',
+            'clean_description': 'Pitched, room-in-roof, above average insulation',
             'thermal_transmittance': None,
             'thermal_transmittance_unit': None,
             'is_pitched': True,
@@ -192,12 +193,12 @@ class TestRecommendationUtils:
             'is_assumed': False,
             'has_dwelling_above': False,
             'is_valid': True,
-            'insulation_thickness': '100',
+            'insulation_thickness': 'above average',
             'age_band': "J"
         }
 
         u_value = recommendation_utils.get_roof_u_value(**inputs)
-        assert u_value == 0.40, f"Expected 0.40, but got {u_value}"
+        assert u_value == 0.3, f"Expected 0.3, but got {u_value}"
 
     def test_get_roof_u_value_case_8(self):
         # Test case where there is a dwelling above the roof, U-value should be 0
@@ -222,6 +223,26 @@ class TestRecommendationUtils:
         u_value = recommendation_utils.get_roof_u_value(**inputs)
         assert u_value == 0.0, f"Expected 0.0, but got {u_value}"
 
+    @pytest.mark.parametrize(
+        "test_case",
+        roof_uvalue_test_cases
+    )
+    def test_roof_uvalues(self, test_case):
+        expected_uvalue = test_case["uvalue"]
+        inputs = test_case.copy()
+        del inputs["uvalue"]
+        # insulation_thickness = inputs["insulation_thickness"]
+        # has_dwelling_above = inputs["has_dwelling_above"]
+        # is_loft = inputs["is_loft"]
+        # is_roof_room = inputs["is_roof_room"]
+        # is_thatched = inputs["is_thatched"]
+        # age_band = inputs["age_band"]
+        # is_flat = inputs["is_flat"]
+        # is_pitched = inputs["is_pitched"]
+        # is_at_rafters = inputs["is_at_rafters"]
+        uvalue = recommendation_utils.get_roof_u_value(**inputs)
+        assert expected_uvalue == uvalue, f"Expected u value {expected_uvalue}, recieved {uvalue}"
+
     @pytest.mark.parametrize(
         "test_case",
         wall_uvalue_test_cases
@@ -359,60 +380,36 @@ def test_park_home():
     ) == 0
 
 
-def test_esimtate_pitched_roof_area():
-    roof_area1 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=100, floor_height=2
+def test_estimate_pitched_roof_area():
+    roof_area0 = recommendation_utils.estimate_pitched_roof_area(
+        floor_area=80,
+    )
+    assert np.isclose(roof_area0, 97.65333333333334)
+
+    roof_area1 = recommendation_utils.estimate_pitched_roof_area(
+        floor_area=100,
     )
 
-    assert np.isclose(roof_area1, 107.70329614269008)
+    assert np.isclose(roof_area1, 122.06666666666666)
 
-    # As the floor height gets bigger, the area should get bigger
-    roof_area2 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=100, floor_height=3
+    roof_area2 = recommendation_utils.estimate_pitched_roof_area(
+        floor_area=45,
     )
 
-    assert np.isclose(roof_area2, 116.61903789690601)
+    assert np.isclose(roof_area2, 54.93)
 
-    # As the floor area gets smaller, the area should get smaller
-    roof_area3 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=100, floor_height=1
+    roof_area3 = recommendation_utils.estimate_pitched_roof_area(
+        floor_area=60,
     )
 
-    assert np.isclose(roof_area3, 101.9803902718557)
+    assert np.isclose(roof_area3, 73.24)
 
-    # As the floor area decreases, area should decrease
-    roof_area4 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=50, floor_height=2
-    )
-
-    assert np.isclose(roof_area4, 57.44562646538029)
-
-    # As the floor area increases, area should increase
-    roof_area5 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=150, floor_height=2
-    )
-
-    assert np.isclose(roof_area5, 157.797338380595)
-
-    zero_roof_area = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=0, floor_height=1000
+    zero_roof_area = recommendation_utils.estimate_pitched_roof_area(
+        floor_area=0,
     )
 
     assert zero_roof_area == 0
 
-    # If the floor height zero, we don't have a traingle, it's a flat roof
-    flat_roof_area = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=1000, floor_height=0
-    )
-
-    assert flat_roof_area == 1000
-
-    zero_roof_area2 = recommendation_utils.esimtate_pitched_roof_area(
-        floor_area=0, floor_height=0
-    )
-
-    assert zero_roof_area2 == 0
-
 
 def test_external_wall_area():
     # Arrange: Define the test cases
@@ -437,7 +434,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1976-1982",
         floor_area=37,
         number_habitable_rooms=2,
-        extension_count=0,
     )
 
     assert windows_case_1 == 4, f"Expected 4 windows, got {windows_case_1}"
@@ -450,7 +446,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1950-1966",
         floor_area=69,
         number_habitable_rooms=4,
-        extension_count=0,
     )
 
     assert windows_case_2 == 6, f"Expected 6 windows, got {windows_case_2}"
@@ -463,7 +458,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1967-1975",
         floor_area=56,
         number_habitable_rooms=3,
-        extension_count=0,
     )
 
     assert windows_case_3 == 5, f"Expected 5 windows, got {windows_case_3}"
@@ -476,7 +470,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1967-1975",
         floor_area=77.28,
         number_habitable_rooms=4,
-        extension_count=0,
     )
 
     assert windows_case_4 == 7, f"Expected 7 windows, got {windows_case_4}"
@@ -489,7 +482,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1950-1966",
         floor_area=88.4,
         number_habitable_rooms=5,
-        extension_count=0,
     )
 
     assert windows_case_5 == 12, f"Expected 12 windows, got {windows_case_5}"
@@ -502,7 +494,6 @@ def test_estimate_windows():
         construction_age_band="",
         floor_area=100,
         number_habitable_rooms=3,
-        extension_count=0,
     )
 
     assert windows_case_6 == 5, f"Expected 5 windows, got {windows_case_6}"
@@ -514,7 +505,6 @@ def test_estimate_windows():
         construction_age_band="England and Wales: 1967-1975",
         floor_area=85,
         number_habitable_rooms=4,
-        extension_count=0,
     )
 
     assert windows_case_7 == 10, f"Expected 10 windows, got {windows_case_7}"
@@ -526,7 +516,6 @@ def test_estimate_windows():
         construction_age_band="",
         floor_area=50,
         number_habitable_rooms=3,
-        extension_count=0,
     )
 
     assert windows_case_8 == 5, f"Expected 5 windows, got {windows_case_8}"

recommendations/tests/test_roof_recommendations.py

@@ -28,13 +28,14 @@ class TestRoofRecommendations:
 
         assert not roof_recommender.recommendations
 
-        roof_recommender.recommend()
+        roof_recommender.recommend(phase=0)
 
-        assert len(roof_recommender.recommendations)
+        assert len(roof_recommender.recommendations) == 1
+        assert roof_recommender.recommendations[0]["parts"][0]["depth"] == 300
 
     def test_loft_insulation_recommendation_50mm_insulation(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "Kent"}
+        epc_record.prepared_epc = {"county": "Kent", "roof-energy-eff": "Very Poor"}
         property_instance2 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
         property_instance2.age_band = "F"
         property_instance2.insulation_floor_area = 100
@@ -52,16 +53,17 @@ class TestRoofRecommendations:
 
         assert not roof_recommender2.recommendations
 
-        roof_recommender2.recommend()
+        roof_recommender2.recommend(phase=0)
 
         assert len(roof_recommender2.recommendations) == 1
 
-        assert roof_recommender2.recommendations[0]["total"] == 1936.9206000000004
-        assert roof_recommender2.recommendations[0]["new_u_value"] == 0.14
+        assert roof_recommender2.recommendations[0]["total"] == 1653
+        assert roof_recommender2.recommendations[0]["new_u_value"] == 0.13
         assert roof_recommender2.recommendations[0]["starting_u_value"] == 0.68
+        assert roof_recommender2.recommendations[0]["parts"][0]["depth"] == 300
 
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "Greater London Authority"}
+        epc_record.prepared_epc = {"county": "Greater London Authority", "roof-energy-eff": "Very Poor"}
         property_instance3 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
         property_instance3.age_band = "F"
         property_instance3.insulation_floor_area = 100
@@ -79,15 +81,15 @@ class TestRoofRecommendations:
 
         assert not roof_recommender3.recommendations
 
-        roof_recommender3.recommend()
+        roof_recommender3.recommend(phase=0)
 
         assert roof_recommender3.recommendations
         assert len(roof_recommender3.recommendations) == 1
-        assert roof_recommender3.recommendations[0]["parts"][0]["depth"] == 270
+        assert roof_recommender3.recommendations[0]["parts"][0]["depth"] == 300.0
 
     def test_loft_insulation_recommendation_150mm_insulation(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "North East Lincolnshire"}
+        epc_record.prepared_epc = {"county": "North East Lincolnshire", "roof-energy-eff": "Good"}
         property_instance4 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
         property_instance4.age_band = "F"
         property_instance4.insulation_floor_area = 100
@@ -105,17 +107,17 @@ class TestRoofRecommendations:
 
         assert not roof_recommender4.recommendations
 
-        roof_recommender4.recommend()
+        roof_recommender4.recommend(phase=0, default_u_values=True)
 
-        assert len(roof_recommender4.recommendations) == 4
+        assert len(roof_recommender4.recommendations) == 1
 
-        assert roof_recommender4.recommendations[0]["total"] == 1128.744
-        assert roof_recommender4.recommendations[0]["new_u_value"] == 0.15
+        assert roof_recommender4.recommendations[0]["total"] == 1653.0
+        assert roof_recommender4.recommendations[0]["new_u_value"] == 0.14
         assert roof_recommender4.recommendations[0]["starting_u_value"] == 0.3
-        assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 150
+        assert roof_recommender4.recommendations[0]["parts"][0]["depth"] == 300
 
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "Somerset"}
+        epc_record.prepared_epc = {"county": "Somerset", "roof-energy-eff": "Good"}
         property_instance5 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
         property_instance5.age_band = "F"
         property_instance5.insulation_floor_area = 100
@@ -133,12 +135,11 @@ class TestRoofRecommendations:
 
         assert not roof_recommender5.recommendations
 
-        roof_recommender5.recommend()
+        roof_recommender5.recommend(phase=0)
 
-        # The 150mm insulation should be selected, since there it already 150mm
         assert roof_recommender5.recommendations
-        assert len(roof_recommender5.recommendations) == 4
-        assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 150
+        assert len(roof_recommender5.recommendations) == 1
+        assert roof_recommender5.recommendations[0]["parts"][0]["depth"] == 300
 
     def test_loft_insulation_recommendation_270mm_insulation(self):
         # We shouldn't recommend anything in this case
@@ -161,127 +162,121 @@ class TestRoofRecommendations:
 
         assert not roof_recommender6.recommendations
 
-        roof_recommender6.recommend()
+        roof_recommender6.recommend(phase=0)
 
         assert len(roof_recommender6.recommendations) == 0
 
-    # def test_uninsulated_room_in_roof(self):
-    #     property_instance7 = Property(id=0, address1="fake", postcode="fake", epc_client=Mock())
-    #     property_instance7.age_band = "F"
-    #     property_instance7.insulation_floor_area = 100
-    #     property_instance7.roof = {
-    #         'original_description': 'Roof room(s), no insulation (assumed)',
-    #         'clean_description': 'Roof room(s), no insulation',
-    #         'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
-    #         'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
-    #         'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'
-    #     }
-    #
-    #     property_instance7.pitched_roof_area = 110
-    #     property_instance7.data = {"county": "Southampton"}
-    #
-    #     roof_recommender7 = RoofRecommendations(property_instance=property_instance7, materials=materials)
-    #
-    #     assert not roof_recommender7.recommendations
-    #
-    #     roof_recommender7.recommend()
-    #
-    #     # Even though we have 3 depths, we only end with 1 due to diminishin returns
-    #     assert len(roof_recommender7.recommendations) == 1
-    #
-    #     assert roof_recommender7.recommendations[0]["parts"][0]["depths"] == [270]
-    #
-    #     assert roof_recommender7.recommendations[0]["new_u_value"] == 0.14
-    #     assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8
-    #     assert roof_recommender7.recommendations[0]["description"] == \
-    #            "Insulate your room roof with 270mm of Example room roof insulation"
-    #
-    # def test_ceiling_insulated_room_in_roof(self):
-    #     property_instance8 = Property(id=8, address1="fake", postcode="fake", epc_client=Mock())
-    #     property_instance8.age_band = "F"
-    #     property_instance8.insulation_floor_area = 100
-    #     property_instance8.roof = {
-    #         'original_description': 'Roof room(s), ceiling insulated',
-    #         'clean_description': 'Roof room(s), ceiling insulated',
-    #         'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
-    #         'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
-    #         'is_at_rafters': False,
-    #         'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
-    #         'insulation_thickness': 'average'
-    #     }
-    #
-    #     property_instance8.pitched_roof_area = 110
-    #
-    #     roof_recommender8 = RoofRecommendations(property_instance=property_instance8, materials=materials)
-    #
-    #     assert not roof_recommender8.recommendations
-    #
-    #     roof_recommender8.recommend()
-    #
-    #     # No recommendations in this case
-    #     assert not roof_recommender8.recommendations
-    #
-    # def test_insulated_room_in_roof(self):
-    #     property_instance9 = Property(id=9, address1="fake", postcode="fake", epc_client=Mock())
-    #     property_instance9.age_band = "F"
-    #     property_instance9.insulation_floor_area = 100
-    #     property_instance9.roof = {
-    #         'original_description': 'Roof room(s), insulated (assumed)',
-    #         'clean_description': 'Roof room(s), insulated',
-    #         'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
-    #         'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
-    #         'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'
-    #     }
-    #
-    #     property_instance9.pitched_roof_area = 110
-    #     property_instance9.data = {"county": "Rutland"}
-    #
-    #     roof_recommender9 = RoofRecommendations(property_instance=property_instance9, materials=materials)
-    #
-    #     assert not roof_recommender9.recommendations
-    #
-    #     roof_recommender9.recommend()
-    #
-    #     # No recommendations in this case
-    #     assert not roof_recommender9.recommendations
-    #
-    # def test_limited_insulated_room_in_roof(self):
-    #     property_instance10 = Property(id=10, address1="fake", postcode="fake", epc_client=Mock())
-    #     property_instance10.age_band = "F"
-    #     property_instance10.insulation_floor_area = 100
-    #     property_instance10.roof = {
-    #         'original_description': 'Roof room(s), limited insulation (assumed)',
-    #         'clean_description': 'Roof room(s), limited insulation',
-    #         'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
-    #         'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
-    #         'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
-    #         'insulation_thickness': 'below average'
-    #     }
-    #
-    #     property_instance10.pitched_roof_area = 110
-    #     property_instance10.data = {"county": "Westmorland"}
-    #
-    #     roof_recommender10 = RoofRecommendations(property_instance=property_instance10, materials=materials)
-    #
-    #     assert not roof_recommender10.recommendations
-    #
-    #     roof_recommender10.recommend()
-    #
-    #     assert len(roof_recommender10.recommendations) == 2
-    #
-    #     assert roof_recommender10.recommendations[0]["parts"][0]["depths"] == [220]
-    #     assert roof_recommender10.recommendations[1]["parts"][0]["depths"] == [270]
-    #
-    #     assert roof_recommender10.recommendations[0]["new_u_value"] == 0.16
-    #     assert roof_recommender10.recommendations[1]["new_u_value"] == 0.14
-    #
-    #     assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.8
-    #     assert roof_recommender10.recommendations[1]["starting_u_value"] == 0.8
-    #
-    #     assert roof_recommender10.recommendations[0]["description"] == \
-    #            "Insulate your room roof with 220mm of Example room roof insulation"
-    #     assert roof_recommender10.recommendations[1]["description"] == \
-    #            "Insulate your room roof with 270mm of Example room roof insulation"
+    def test_uninsulated_room_in_roof(self):
+        epc_record = EPCRecord()
+        epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"}
+        property_instance7 = Property(id=0, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance7.age_band = "F"
+        property_instance7.insulation_floor_area = 100
+        property_instance7.roof = {
+            'original_description': 'Roof room(s), no insulation (assumed)',
+            'clean_description': 'Roof room(s), no insulation',
+            'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
+            'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
+            'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'none'
+        }
+
+        property_instance7.pitched_roof_area = 110
+
+        roof_recommender7 = RoofRecommendations(property_instance=property_instance7, materials=materials)
+
+        assert not roof_recommender7.recommendations
+
+        roof_recommender7.recommend(phase=0)
+
+        assert len(roof_recommender7.recommendations) == 1
+        assert roof_recommender7.recommendations[0]["new_u_value"] == 0.2
+        assert roof_recommender7.recommendations[0]["starting_u_value"] == 0.8
+        assert roof_recommender7.recommendations[0]["description"] == "Insulate room in roof at rafters and re-decorate"
+
+    def test_ceiling_insulated_room_in_roof(self):
+        epc_record = EPCRecord()
+        epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"}
+        property_instance8 = Property(id=8, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance8.age_band = "F"
+        property_instance8.insulation_floor_area = 100
+        property_instance8.roof = {
+            'original_description': 'Roof room(s), ceiling insulated',
+            'clean_description': 'Roof room(s), ceiling insulated',
+            'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
+            'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False,
+            'is_at_rafters': False,
+            'is_assumed': False, 'has_dwelling_above': False, 'is_valid': True,
+            'insulation_thickness': 'average'
+        }
+
+        property_instance8.pitched_roof_area = 110
+
+        roof_recommender8 = RoofRecommendations(property_instance=property_instance8, materials=materials)
+
+        assert not roof_recommender8.recommendations
+
+        roof_recommender8.recommend(phase=0)
+
+        # No recommendations in this case
+        assert not roof_recommender8.recommendations
+
+    def test_insulated_room_in_roof(self):
+        epc_record = EPCRecord()
+        epc_record.prepared_epc = {"county": "Southampton", "roof-energy-eff": "Very Poor"}
+        property_instance9 = Property(id=9, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance9.age_band = "F"
+        property_instance9.insulation_floor_area = 100
+        property_instance9.roof = {
+            'original_description': 'Roof room(s), insulated (assumed)',
+            'clean_description': 'Roof room(s), insulated',
+            'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
+            'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
+            'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True, 'insulation_thickness': 'average'
+        }
+
+        property_instance9.pitched_roof_area = 110
+        property_instance9.data = {"county": "Rutland"}
+
+        roof_recommender9 = RoofRecommendations(property_instance=property_instance9, materials=materials)
+
+        assert not roof_recommender9.recommendations
+
+        roof_recommender9.recommend(phase=0)
+
+        # No recommendations in this case
+        assert not roof_recommender9.recommendations
+
+    def test_limited_insulated_room_in_roof(self):
+        epc_record = EPCRecord()
+        epc_record.prepared_epc = {"county": "Westmorland", "roof-energy-eff": "Poor"}
+        property_instance10 = Property(id=10, address="fake", postcode="fake", epc_record=epc_record)
+        property_instance10.age_band = "F"
+        property_instance10.insulation_floor_area = 100
+        property_instance10.roof = {
+            'original_description': 'Roof room(s), limited insulation (assumed)',
+            'clean_description': 'Roof room(s), limited insulation',
+            'thermal_transmittance': None, 'thermal_transmittance_unit': None, 'is_pitched': False,
+            'is_roof_room': True, 'is_loft': False, 'is_flat': False, 'is_thatched': False, 'is_at_rafters': False,
+            'is_assumed': True, 'has_dwelling_above': False, 'is_valid': True,
+            'insulation_thickness': 'below average'
+        }
+
+        property_instance10.pitched_roof_area = 110
+
+        roof_recommender10 = RoofRecommendations(property_instance=property_instance10, materials=materials)
+
+        assert not roof_recommender10.recommendations
+
+        roof_recommender10.recommend(phase=0)
+
+        assert len(roof_recommender10.recommendations) == 1
+
+        assert roof_recommender10.recommendations[0]["new_u_value"] == 0.19
+
+        assert roof_recommender10.recommendations[0]["starting_u_value"] == 0.68
+
+        assert (roof_recommender10.recommendations[0]["description"] ==
+                'Insulate room in roof at rafters and re-decorate')
 
     def test_flat_no_insulation(self):
         epc_record = EPCRecord()
@@ -302,12 +297,12 @@ class TestRoofRecommendations:
 
         assert not roof_recommender11.recommendations
 
-        roof_recommender11.recommend()
+        roof_recommender11.recommend(phase=0)
 
         assert len(roof_recommender11.recommendations) == 1
 
         assert roof_recommender11.recommendations[0]["parts"][0]["depth"] == 150
-        assert roof_recommender11.recommendations[0]["total"] == 4380.84324
+        assert roof_recommender11.recommendations[0]["total"] == 6532.5
         assert roof_recommender11.recommendations[0]["new_u_value"] == 0.14
         assert roof_recommender11.recommendations[0]["starting_u_value"] == 2.3
         assert roof_recommender11.recommendations[0]["description"] == \
@@ -334,7 +329,7 @@ class TestRoofRecommendations:
 
         assert not roof_recommender12.recommendations
 
-        roof_recommender12.recommend()
+        roof_recommender12.recommend(phase=0)
 
         assert not roof_recommender12.recommendations
 
@@ -358,13 +353,13 @@ class TestRoofRecommendations:
 
         assert not roof_recommender13.recommendations
 
-        roof_recommender13.recommend()
+        roof_recommender13.recommend(phase=0)
 
         assert len(roof_recommender13.recommendations) == 1
 
         assert roof_recommender13.recommendations[0]["parts"][0]["depth"] == 150
 
-        assert roof_recommender13.recommendations[0]["total"] == 5199.969120000002
+        assert roof_recommender13.recommendations[0]["total"] == 7800
         assert roof_recommender13.recommendations[0]["new_u_value"] == 0.14
         assert roof_recommender13.recommendations[0]["starting_u_value"] == 2.3
 
@@ -390,6 +385,6 @@ class TestRoofRecommendations:
 
         assert not roof_recommender14.recommendations
 
-        roof_recommender14.recommend()
+        roof_recommender14.recommend(phase=0)
 
         assert not roof_recommender14.recommendations

recommendations/tests/test_solar_pv_recommendations.py

@@ -3,12 +3,6 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations
 from backend.Property import Property
 from etl.epc.Record import EPCRecord
 import pandas as pd
-from datetime import datetime
-from utils.s3 import read_dataframe_from_s3_parquet, read_from_s3
-from etl.solar.SolarPhotoSupply import SolarPhotoSupply
-from recommendations.Recommendations import Recommendations
-from backend.ml_models.api import ModelApi
-import msgpack
 
 
 class TestSolarPvRecommendations:
@@ -50,360 +44,61 @@ class TestSolarPvRecommendations:
         epc_record = EPCRecord()
         epc_record.prepared_epc = {"property-type": "House", "photo-supply": None, "county": "Huntingdonshire"}
         property_instance_valid_all = Property(id=1, address="", postcode="", epc_record=epc_record)
-        property_instance_valid_all.solar_pv_roof_area = 20
-        property_instance_valid_all.solar_pv_percentage = 40
+        property_instance_valid_all.roof_area = 40
+        property_instance_valid_all.number_of_floors = 2
         property_instance_valid_all.roof = {"is_flat": True}
+        property_instance_valid_all.solar_panel_configuration = {
+            "panel_performance": pd.DataFrame(
+                [
+                    {
+                        "panneled_roof_area": 20,
+                        "n_panels": 10,
+                        "array_wattage": 4000,
+                        "initial_ac_kwh_per_year": 3800
+                    }
+                ]
+            )
+        }
+
         return property_instance_valid_all
 
     def test_invalid_property_type(self, property_instance_invalid_type):
         solar_pv = SolarPvRecommendations(property_instance_invalid_type)
-        solar_pv.recommend()
+        solar_pv.recommend(phase=0)
         assert not solar_pv.recommendation
 
     def test_invalid_roof_type(self, property_instance_invalid_roof):
         solar_pv = SolarPvRecommendations(property_instance_invalid_roof)
-        solar_pv.recommend()
+        solar_pv.recommend(phase=0)
         assert not solar_pv.recommendation
 
     def test_existing_solar_pv(self, property_instance_has_solar_pv):
         solar_pv = SolarPvRecommendations(property_instance_has_solar_pv)
-        solar_pv.recommend()
+        solar_pv.recommend(phase=0)
         assert not solar_pv.recommendation
 
     def test_valid_all_conditions(self, property_instance_valid_all):
         solar_pv = SolarPvRecommendations(property_instance_valid_all)
-        solar_pv.recommend()
+        solar_pv.recommend(phase=0)
+        assert len(solar_pv.recommendation) == 2
         assert solar_pv.recommendation == [
             {
-                'parts': [],
-                'type': 'solar_pv',
-                'description': 'Install a 4 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on the roof',
-                'starting_u_value': None,
-                'new_u_value': None,
-                'sap_points': None,
-                'total': 8527.0752,
-                'subtotal': 7105.896,
-                'vat': 1421.1791999999996,
-                'labour_hours': 72,
-                'labour_days': 2,
-                'photo_supply': 4000
+                'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+                'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the '
+                               'roof.',
+                'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False,
+                'total': 4850.0, 'subtotal': 4041.666666666667, 'vat': 808.333333333333, 'labour_hours': 48,
+                'labour_days': 2, 'photo_supply': 50.0, 'has_battery': False, 'initial_ac_kwh_per_year': 3800,
+                'description_simulation': {'photo-supply': 50.0}
+            },
+            {
+                'phase': 0, 'parts': [], 'type': 'solar_pv', 'measure_type': 'solar_pv',
+                'description': 'Install a 4.0 kilowatt-peak (kWp) solar photovoltaic (PV) panel system on 50% the '
+                               'roof, '
+                               'with a battery storage system.',
+                'starting_u_value': None, 'new_u_value': None, 'sap_points': None, 'already_installed': False,
+                'total': 7550.0, 'subtotal': 6291.666666666667, 'vat': 1258.333333333333, 'labour_hours': 48,
+                'labour_days': 2, 'photo_supply': 50.0, 'has_battery': True, 'initial_ac_kwh_per_year': 3800,
+                'description_simulation': {'photo-supply': 50.0}
             }
         ]
-
-    def test_model(self):
-        """
-        This function tests the recommendation engine, in conjunction with the model
-        :return:
-        """
-
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900',
-            'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
-            'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '85',
-            'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '',
-            'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
-            'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79',
-            'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N',
-            'constituency': 'E14000707', 'co2-emissions-potential': '1.5', 'number-heated-rooms': '5',
-            'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '92',
-            'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-11-17',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '61', 'address1': '27 Cromwell Street',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough',
-            'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430',
-            'environment-impact-current': '47', 'co2-emissions-current': '5.4',
-            'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH',
-            'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor',
-            'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor',
-            'walls-energy-eff': 'Very Poor', 'photo-supply': '0.0', 'lighting-cost-potential': '72',
-            'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2021-12-01 10:12:23', 'flat-top-storey': '', 'current-energy-rating': 'E',
-            'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor',
-            'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '54',
-            'energy-consumption-current': '346', 'mainheat-description': 'Boiler and radiators, mains gas',
-            'lighting-cost-current': '144', 'lodgement-date': '2021-12-01', 'extension-count': '2',
-            'mainheatc-env-eff': 'Good', 'lmk-key': '3ec5533af02ec78361c1f9bea8dd2e878c2c6fa6cf59e5cc505c3eeb038e0f91',
-            'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
-            'potential-energy-efficiency': '86', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0',
-            'walls-description': 'Solid brick, as built, no insulation (assumed)',
-            'hotwater-description': 'From main system'
-        }
-
-        ending_epc = {
-            'low-energy-fixed-light-count': '', 'address': '27 Cromwell Street', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.5', 'heating-cost-potential': '443', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '53', 'construction-age-band': 'England and Wales: before 1900',
-            'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
-            'lighting-energy-eff': 'Very Poor', 'environment-impact-potential': '86',
-            'glazed-type': 'double glazing installed before 2002', 'heating-cost-current': '904', 'address3': '',
-            'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
-            'property-type': 'House', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '10',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '79',
-            'county': 'Lincolnshire', 'postcode': 'DN21 1DH', 'solar-water-heating-flag': 'N',
-            'constituency': 'E14000707', 'co2-emissions-potential': '1.4', 'number-heated-rooms': '5',
-            'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '84',
-            'local-authority': 'E07000142', 'built-form': 'Mid-Terrace', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Normal', 'inspection-date': '2021-12-21',
-            'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '49', 'address1': '27 Cromwell Street',
-            'heat-loss-corridor': '', 'flat-storey-count': '', 'constituency-label': 'Gainsborough',
-            'roof-energy-eff': 'Very Poor', 'total-floor-area': '89.0', 'building-reference-number': '10001989430',
-            'environment-impact-current': '55', 'co2-emissions-current': '4.4',
-            'roof-description': 'Pitched, no insulation (assumed)', 'floor-energy-eff': 'N/A',
-            'number-habitable-rooms': '5', 'address2': '', 'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH',
-            'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Poor',
-            'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'No low energy lighting', 'roof-env-eff': 'Very Poor',
-            'walls-energy-eff': 'Very Poor', 'photo-supply': '50.0', 'lighting-cost-potential': '72',
-            'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2021-12-21 17:33:09', 'flat-top-storey': '', 'current-energy-rating': 'D',
-            'secondheat-description': 'Room heaters, mains gas', 'walls-env-eff': 'Very Poor',
-            'transaction-type': 'ECO assessment', 'uprn': '100030949912', 'current-energy-efficiency': '65',
-            'energy-consumption-current': '277', 'mainheat-description': 'Boiler and radiators, mains gas',
-            'lighting-cost-current': '144', 'lodgement-date': '2021-12-21', 'extension-count': '2',
-            'mainheatc-env-eff': 'Good', 'lmk-key': 'b0b19583c59afbc69db12f4d6c98cd8837e80da3214d577c426eb3e672d424fc',
-            'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
-            'potential-energy-efficiency': '88', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '0',
-            'walls-description': 'Solid brick, as built, no insulation (assumed)',
-            'hotwater-description': 'From main system'
-        }
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = SolarPvRecommendations(property_instance=home)
-        recommender.recommend(phase=0)
-
-        coverage_50_percent = [x for x in recommender.recommendation if x["photo_supply"] == 50]
-        assert len(coverage_50_percent) == 2
-
-        property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_50_percent])
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-
-        assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [65.9, 65.9]
-        assert ending_epc["current-energy-efficiency"] == '65'
-
-    def test_model2(self):
-        data[["uprn", "sap_ending"]]
-        #
-
-        searcher = SearchEpc(
-            address1="",
-            postcode="",
-            auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
-            os_api_key="",
-            full_address="",
-            uprn=100030952942,
-        )
-        searcher.find_property(False)
-
-        ending_epc = {
-            'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent',
-            'uprn-source': 'Energy Assessor', 'floor-height': '2.49', 'heating-cost-potential': '464',
-            'unheated-corridor-length': '', 'hot-water-cost-potential': '46',
-            'construction-age-band': 'England and Wales: 1967-1975', 'potential-energy-rating': 'B',
-            'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average', 'lighting-energy-eff': 'Very Good',
-            'environment-impact-potential': '91', 'glazed-type': 'not defined', 'heating-cost-current': '535',
-            'address3': '', 'mainheatcont-description': 'Programmer, room thermostat and TRVs',
-            'sheating-energy-eff': 'N/A', 'property-type': 'Bungalow',
-            'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69',
-            'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N',
-            'constituency': 'E14000707', 'co2-emissions-potential': '0.7', 'number-heated-rooms': '3',
-            'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '56',
-            'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical',
-            'inspection-date': '2022-08-24', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '18',
-            'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '',
-            'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0',
-            'building-reference-number': '10002845316', 'environment-impact-current': '85',
-            'co2-emissions-current': '1.2', 'roof-description': 'Pitched, 300 mm loft insulation',
-            'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '',
-            'hot-water-env-eff': 'Good', 'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good',
-            'main-fuel': 'mains gas (not community)', 'lighting-env-eff': 'Very Good',
-            'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A', 'sheating-env-eff': 'N/A',
-            'lighting-description': 'Low energy lighting in all fixed outlets', 'roof-env-eff': 'Very Good',
-            'walls-energy-eff': 'Average', 'photo-supply': '40.0', 'lighting-cost-potential': '65',
-            'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100', 'main-heating-controls': '',
-            'lodgement-datetime': '2022-08-24 15:39:42', 'flat-top-storey': '', 'current-energy-rating': 'B',
-            'secondheat-description': 'Room heaters, electric', 'walls-env-eff': 'Average',
-            'transaction-type': 'ECO assessment', 'uprn': '100030952942', 'current-energy-efficiency': '87',
-            'energy-consumption-current': '100', 'mainheat-description': 'Boiler and radiators, mains gas',
-            'lighting-cost-current': '65', 'lodgement-date': '2022-08-24', 'extension-count': '0',
-            'mainheatc-env-eff': 'Good',
-            'lmk-key': 'e20be883431b1fed15db7fa1f52634fb7655d2b80c2fdad37df779f93ec4dafd',
-            'wind-turbine-count': '0', 'tenure': 'Owner-occupied', 'floor-level': '',
-            'potential-energy-efficiency': '91', 'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-        starting_epc = {
-            'low-energy-fixed-light-count': '', 'address': '6 Kenmare Crescent', 'uprn-source': 'Energy Assessor',
-            'floor-height': '2.49', 'heating-cost-potential': '464', 'unheated-corridor-length': '',
-            'hot-water-cost-potential': '46', 'construction-age-band': 'England and Wales: 1967-1975',
-            'potential-energy-rating': 'B', 'mainheat-energy-eff': 'Good', 'windows-env-eff': 'Average',
-            'lighting-energy-eff': 'Very Good', 'environment-impact-potential': '85', 'glazed-type': 'not defined',
-            'heating-cost-current': '535', 'address3': '',
-            'mainheatcont-description': 'Programmer, room thermostat and TRVs', 'sheating-energy-eff': 'N/A',
-            'property-type': 'Bungalow', 'local-authority-label': 'West Lindsey', 'fixed-lighting-outlets-count': '9',
-            'energy-tariff': 'Single', 'mechanical-ventilation': 'natural', 'hot-water-cost-current': '69',
-            'county': 'Lincolnshire', 'postcode': 'DN21 1PR', 'solar-water-heating-flag': 'N',
-            'constituency': 'E14000707', 'co2-emissions-potential': '1.2', 'number-heated-rooms': '3',
-            'floor-description': 'Suspended, no insulation (assumed)', 'energy-consumption-potential': '102',
-            'local-authority': 'E07000142', 'built-form': 'Semi-Detached', 'number-open-fireplaces': '0',
-            'windows-description': 'Fully double glazed', 'glazed-area': 'Much More Than Typical',
-            'inspection-date': '2022-05-31', 'mains-gas-flag': 'Y', 'co2-emiss-curr-per-floor-area': '40',
-            'address1': '6 Kenmare Crescent', 'heat-loss-corridor': '', 'flat-storey-count': '',
-            'constituency-label': 'Gainsborough', 'roof-energy-eff': 'Very Good', 'total-floor-area': '66.0',
-            'building-reference-number': '10002845316', 'environment-impact-current': '68',
-            'co2-emissions-current': '2.6', 'roof-description': 'Pitched, 300 mm loft insulation',
-            'floor-energy-eff': 'N/A', 'number-habitable-rooms': '3', 'address2': '', 'hot-water-env-eff': 'Good',
-            'posttown': 'GAINSBOROUGH', 'mainheatc-energy-eff': 'Good', 'main-fuel': 'mains gas (not community)',
-            'lighting-env-eff': 'Very Good', 'windows-energy-eff': 'Average', 'floor-env-eff': 'N/A',
-            'sheating-env-eff': 'N/A', 'lighting-description': 'Low energy lighting in all fixed outlets',
-            'roof-env-eff': 'Very Good', 'walls-energy-eff': 'Average', 'photo-supply': '0.0',
-            'lighting-cost-potential': '65', 'mainheat-env-eff': 'Good', 'multi-glaze-proportion': '100',
-            'main-heating-controls': '', 'lodgement-datetime': '2022-06-15 08:38:02', 'flat-top-storey': '',
-            'current-energy-rating': 'D', 'secondheat-description': 'Room heaters, electric',
-            'walls-env-eff': 'Average', 'transaction-type': 'ECO assessment', 'uprn': '100030952942',
-            'current-energy-efficiency': '68', 'energy-consumption-current': '227',
-            'mainheat-description': 'Boiler and radiators, mains gas', 'lighting-cost-current': '65',
-            'lodgement-date': '2022-06-15', 'extension-count': '0', 'mainheatc-env-eff': 'Good',
-            'lmk-key': 'ce181970b7077cb9b4626242bfb010b30a0e48541b5f22427e81f1adbeeec4f2', 'wind-turbine-count': '0',
-            'tenure': 'Owner-occupied', 'floor-level': '', 'potential-energy-efficiency': '85',
-            'hot-water-energy-eff': 'Good', 'low-energy-lighting': '100',
-            'walls-description': 'Cavity wall, filled cavity', 'hotwater-description': 'From main system'
-        }
-
-        cleaning_data = read_dataframe_from_s3_parquet(
-            bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
-        )
-
-        cleaned = read_from_s3(
-            s3_file_name="cleaned_epc_data/cleaned.bson",
-            bucket_name="retrofit-data-dev"
-        )
-        cleaned = msgpack.unpackb(cleaned, raw=False)
-
-        photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
-
-        epc = EPCRecord(
-            epc_records={
-                'original_epc': starting_epc,
-                'full_sap_epc': {},
-                'old_data': []
-            },
-            run_mode="newdata",
-            cleaning_data=cleaning_data
-        )
-
-        home = Property(
-            id=0,
-            address="",
-            postcode="",
-            epc_record=epc,
-            already_installed={},
-            non_invasive_recommendations={},
-        )
-        home.in_conservation_area = False
-        home.is_listed = False
-        home.is_heritage = False
-        home.restricted_measures = True
-        home.get_components(
-            cleaned=cleaned,
-            photo_supply_lookup=photo_supply_lookup,
-            floor_area_decile_thresholds=floor_area_decile_thresholds
-        )
-
-        recommender = SolarPvRecommendations(property_instance=home)
-        recommender.recommend(phase=0)
-
-        coverage_40_percent = [x for x in recommender.recommendation if x["photo_supply"] == 40]
-        assert len(coverage_40_percent) == 2
-
-        property_recommendations = Recommendations.insert_temp_recommendation_id([coverage_40_percent])
-
-        home.create_base_difference_epc_record(cleaned_lookup=cleaned)
-        home.adjust_difference_record_with_recommendations(
-            property_recommendations, []
-        )
-
-        scoring_data = pd.DataFrame(home.recommendations_scoring_data).drop(
-            columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
-                     "carbon_ending"]
-        )
-
-        model_api = ModelApi(portfolio_id="ashp-test", timestamp=datetime.now().isoformat())
-        model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-        predictions_dict = model_api.predict_all(
-            df=scoring_data,
-            bucket="retrofit-data-dev",
-            prediction_buckets={
-                "sap_change_predictions": "retrofit-sap-predictions-dev",
-            }
-        )
-
-        assert predictions_dict["sap_change_predictions"]["predictions"].tolist() == [87.1, 87.1]
-        assert ending_epc["current-energy-efficiency"] == '87'
-        assert starting_epc["current-energy-efficiency"] == '68'

recommendations/tests/test_ventilation_recommendations.py

@@ -22,7 +22,7 @@ class TestVentilationRecommendations:
 
         assert len(recommender.recommendation) == 1
 
-        assert recommender.recommendation[0]["total"] == 1000
+        assert recommender.recommendation[0]["total"] == 1071.0
         assert recommender.recommendation[0]["type"] == "mechanical_ventilation"
         assert len(recommender.recommendation[0]["parts"]) == 1
         assert recommender.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'
@@ -44,7 +44,7 @@ class TestVentilationRecommendations:
 
         assert len(recommender2.recommendation) == 1
 
-        assert recommender2.recommendation[0]["total"] == 1000
+        assert recommender2.recommendation[0]["total"] == 1071.0
         assert recommender2.recommendation[0]["type"] == "mechanical_ventilation"
         assert len(recommender2.recommendation[0]["parts"]) == 1
         assert recommender2.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'
@@ -66,7 +66,7 @@ class TestVentilationRecommendations:
 
         assert len(recommender3.recommendation) == 1
 
-        assert recommender3.recommendation[0]["total"] == 1000
+        assert recommender3.recommendation[0]["total"] == 1071.0
         assert recommender3.recommendation[0]["type"] == "mechanical_ventilation"
         assert len(recommender3.recommendation[0]["parts"]) == 1
         assert recommender3.recommendation[0]["parts"][0]["description"] == 'Mechanical Extract Ventilation'

recommendations/tests/test_wall_recommendations.py

@@ -10,8 +10,10 @@ from recommendations.tests.test_data.materials import materials
 from etl.epc.Record import EPCRecord
 
 
+# import inspect
+# file_path = inspect.getfile(lambda: None)
 # with open(
-#     os.path.abspath(os.path.dirname(__file__)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
+#     os.path.abspath(os.path.dirname(file_path)) + "/recommendations/tests/test_data/input_properties.pkl", "rb"
 # ) as f:
 #     input_properties = pickle.load(f)
 
@@ -86,17 +88,21 @@ class TestWallRecommendations:
         input_properties[1].walls["is_sandstone_or_limestone"] = False
         input_properties[1].age_band = "A"
         input_properties[1].restricted_measures = False
+        input_properties[1].already_installed = []
+        input_properties[1].walls["is_park_home"] = False
+        input_properties[1].construction_age_band = "England and Wales: 1930-1949"
+        input_properties[1].non_invasive_recommendations = []
 
         recommender = WallRecommendations(
             property_instance=input_properties[1],
             materials=materials
         )
         assert recommender.property.walls["original_description"] == "Solid brick, as built, no insulation (assumed)"
-        assert not recommender.ewi_valid
+        assert not recommender.ewi_valid()
         assert recommender.property.in_conservation_area == "not_in_conservation_area"
         assert recommender.property.data["property-type"] == "Flat"
 
-        recommender.recommend()
+        recommender.recommend(phase=0)
         # This should result in some recommendations, all of which should be internal insulation
         assert recommender.recommendations
 
@@ -131,7 +137,7 @@ class TestWallRecommendations:
         )
 
         assert recommender.property.walls["original_description"] == "Solid brick, as built, insulated (assumed)"
-        assert not recommender.ewi_valid
+        assert not recommender.ewi_valid()
         assert recommender.property.in_conservation_area == "not_in_conservation_area"
         assert recommender.property.data["property-type"] == "Flat"
         assert recommender.estimated_u_value is None
@@ -204,6 +210,11 @@ class TestWallRecommendationsBase:
         property_mock.restricted_measures = False
         property_mock.insulation_wall_area = 100
         property_mock.data = {"county": "Derbyshire"}
+        property_mock.walls = {
+            "is_cob": False,
+            "is_sandstone_or_limestone": False,
+            "is_cavity_wall": False
+        }
         return property_mock
 
     @pytest.fixture
@@ -216,24 +227,24 @@ class TestWallRecommendationsBase:
     def test_ewi_valid_in_conservation_area(self, wall_recommendations_instance):
         wall_recommendations_instance.property.in_conservation_area = "in_conversation_area"
         wall_recommendations_instance.property.restricted_measures = True
-        assert wall_recommendations_instance.ewi_valid is False
+        assert wall_recommendations_instance.ewi_valid() is False
 
     def test_ewi_valid_is_flat(self, wall_recommendations_instance):
         wall_recommendations_instance.property.data = {"property-type": "flat"}
-        assert wall_recommendations_instance.ewi_valid is False
+        assert wall_recommendations_instance.ewi_valid() is False
 
     def test_ewi_valid_not_in_conservation_area_and_not_flat(self, wall_recommendations_instance):
         wall_recommendations_instance.property.in_conservation_area = "not_in_conversation_area"
         wall_recommendations_instance.property.restricted_measures = False
         wall_recommendations_instance.property.data = {"property-type": "house"}
-        assert wall_recommendations_instance.ewi_valid is True
+        assert wall_recommendations_instance.ewi_valid() is True
 
 
 class TestCavityWallRecommensations:
 
     def test_fill_empty_cavity(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "Derbyshire"}
+        epc_record.prepared_epc = {"county": "Derbyshire", "walls-energy-eff": "Very Poor"}
         input_property = Property(id=1, postcode="F4k3", address="123 fake street", epc_record=epc_record)
         input_property.walls = {
             'original_description': 'Cavity wall, as built, no insulation (assumed)',
@@ -248,6 +259,7 @@ class TestCavityWallRecommensations:
         }
         input_property.age_band = "C"
         input_property.insulation_wall_area = 50
+        input_property.construction_age_band = "England and Wales: 1930-1949"
 
         recommender = WallRecommendations(
             property_instance=input_property,
@@ -261,14 +273,11 @@ class TestCavityWallRecommensations:
         assert recommender.recommendations
         assert recommender.estimated_u_value == 1.5
         assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.35)
-        assert np.isclose(recommender.recommendations[0]["total"], 1668.6600000000003)
-
-        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.35)
-        assert np.isclose(recommender.recommendations[1]["total"], 2004.6600000000003)
+        assert np.isclose(recommender.recommendations[0]["total"], 710.5)
 
     def test_fill_partial_filled_cavity(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"county": "County Durham"}
+        epc_record.prepared_epc = {"county": "County Durham", "walls-energy-eff": "Poor"}
         input_property = Property(id=1, postcode="F4k3", address="123 fake street", epc_record=epc_record)
         input_property.walls = {
             'original_description': 'Cavity wall, as built, partial insulation (assumed)',
@@ -283,6 +292,7 @@ class TestCavityWallRecommensations:
         }
         input_property.age_band = "C"
         input_property.insulation_wall_area = 50
+        input_property.construction_age_band = "England and Wales: 1930-1949"
 
         recommender = WallRecommendations(
             property_instance=input_property,
@@ -296,14 +306,13 @@ class TestCavityWallRecommensations:
         assert recommender.recommendations
         assert recommender.estimated_u_value == 1.3
         assert np.isclose(recommender.recommendations[0]["new_u_value"], 0.41)
-        assert np.isclose(recommender.recommendations[0]["total"], 1663.9350000000002)
-
-        assert np.isclose(recommender.recommendations[1]["new_u_value"], 0.41)
-        assert np.isclose(recommender.recommendations[1]["total"], 1999.9350000000002)
+        assert np.isclose(recommender.recommendations[0]["total"], 710.5)
 
     def test_system_built_wall(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Detached"}
+        epc_record.prepared_epc = {
+            "property-type": "House", "county": "Derbyshire", "built-form": "Detached", "walls-energy-eff": "Very Poor"
+        }
         input_property2 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record)
         input_property2.walls = {
             'original_description': 'System built, as built, no insulation (assumed)',
@@ -319,6 +328,7 @@ class TestCavityWallRecommensations:
         input_property2.age_band = "F"
         input_property2.insulation_wall_area = 120
         input_property2.restricted_measures = False
+        input_property2.construction_age_band = "England and Wales: 1976-1982"
 
         assert input_property2.walls["is_system_built"]
 
@@ -332,26 +342,24 @@ class TestCavityWallRecommensations:
         recommender2.recommend()
 
         assert recommender2.recommendations
-        assert len(recommender2.recommendations) == 9
+        assert len(recommender2.recommendations) == 2
         assert recommender2.estimated_u_value == 1
-        assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.19)
-        assert np.isclose(recommender2.recommendations[0]["total"], 16429.960320000002)
+        assert np.isclose(recommender2.recommendations[0]["new_u_value"], 0.21)
+        assert np.isclose(recommender2.recommendations[0]["total"], 35802.0)
         assert recommender2.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender2.recommendations[0]["parts"][0]["depth"] == 100
+        assert recommender2.recommendations[0]["parts"][0]["depth"] == 150
 
-        assert np.isclose(recommender2.recommendations[8]["new_u_value"], 0.23)
-        assert np.isclose(recommender2.recommendations[8]["total"], 11292.768)
-        assert recommender2.recommendations[8]["parts"][0]["type"] == "internal_wall_insulation"
-        assert recommender2.recommendations[8]["parts"][0]["depth"] == 72.5
-
-        assert np.isclose(recommender2.recommendations[6]["new_u_value"], 0.29)
-        assert np.isclose(recommender2.recommendations[6]["total"], 10988.208)
-        assert recommender2.recommendations[6]["parts"][0]["type"] == "internal_wall_insulation"
-        assert recommender2.recommendations[6]["parts"][0]["depth"] == 52.5
+        assert np.isclose(recommender2.recommendations[1]["new_u_value"], 0.26)
+        assert np.isclose(recommender2.recommendations[1]["total"], 29376)
+        assert recommender2.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation"
+        assert recommender2.recommendations[1]["parts"][0]["depth"] == 95
 
     def test_timber_frame_wall(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Semi-Detached"}
+        epc_record.prepared_epc = {
+            "property-type": "House", "county": "Derbyshire", "built-form": "Semi-Detached",
+            "walls-energy-eff": "Very Poor"
+        }
         input_property3 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record)
         input_property3.walls = {
             'original_description': 'Timber frame, as built, no insulation (assumed)',
@@ -367,6 +375,7 @@ class TestCavityWallRecommensations:
         input_property3.age_band = "B"
         input_property3.insulation_wall_area = 99
         input_property3.restricted_measures = False
+        input_property3.construction_age_band = "England and Wales: 1950-1966"
 
         assert input_property3.walls["is_timber_frame"]
 
@@ -380,21 +389,24 @@ class TestCavityWallRecommensations:
         recommender3.recommend()
 
         assert recommender3.recommendations
-        assert len(recommender3.recommendations) == 6
+        assert len(recommender3.recommendations) == 2
         assert recommender3.estimated_u_value == 1.9
-        assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.2)
-        assert np.isclose(recommender3.recommendations[0]["total"], 13554.717263999999)
+        assert np.isclose(recommender3.recommendations[0]["new_u_value"], 0.23)
+        assert np.isclose(recommender3.recommendations[0]["total"], 29536.65)
         assert recommender3.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender3.recommendations[0]["parts"][0]["depth"] == 100.0
+        assert recommender3.recommendations[0]["parts"][0]["depth"] == 150.0
 
-        assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.23)
-        assert np.isclose(recommender3.recommendations[1]["total"], 35206.19308800001)
-        assert recommender3.recommendations[1]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender3.recommendations[1]["parts"][0]["depth"] == 150.0
+        assert np.isclose(recommender3.recommendations[1]["new_u_value"], 0.29)
+        assert np.isclose(recommender3.recommendations[1]["total"], 24235.2)
+        assert recommender3.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation"
+        assert recommender3.recommendations[1]["parts"][0]["depth"] == 95.0
 
     def test_granite_or_whinstone_wall(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached"}
+        epc_record.prepared_epc = {
+            "property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached",
+            "walls-energy-eff": "Very Poor"
+        }
         input_property4 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record)
         input_property4.walls = {
             'original_description': 'Granite or whinstone, as built, no insulation (assumed)',
@@ -410,6 +422,7 @@ class TestCavityWallRecommensations:
         input_property4.age_band = "A"
         input_property4.insulation_wall_area = 223
         input_property4.restricted_measures = False
+        input_property4.construction_age_band = "England and Wales: before 1900"
 
         assert input_property4.walls["is_granite_or_whinstone"]
 
@@ -423,21 +436,24 @@ class TestCavityWallRecommensations:
         recommender4.recommend()
 
         assert recommender4.recommendations
-        assert len(recommender4.recommendations) == 6
+        assert len(recommender4.recommendations) == 2
         assert recommender4.estimated_u_value == 2.3
-        assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.21)
-        assert np.isclose(recommender4.recommendations[0]["total"], 29547.42864)
+        assert np.isclose(recommender4.recommendations[0]["new_u_value"], 0.23)
+        assert np.isclose(recommender4.recommendations[0]["total"], 66532.05)
         assert recommender4.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender4.recommendations[0]["parts"][0]["depth"] == 100
+        assert recommender4.recommendations[0]["parts"][0]["depth"] == 150
 
-        assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.23)
-        assert np.isclose(recommender4.recommendations[1]["total"], 76744.68288000001)
-        assert recommender4.recommendations[1]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender4.recommendations[1]["parts"][0]["depth"] == 150
+        assert np.isclose(recommender4.recommendations[1]["new_u_value"], 0.3)
+        assert np.isclose(recommender4.recommendations[1]["total"], 54590.4)
+        assert recommender4.recommendations[1]["parts"][0]["type"] == "internal_wall_insulation"
+        assert recommender4.recommendations[1]["parts"][0]["depth"] == 95
 
     def test_cob_wall(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached"}
+        epc_record.prepared_epc = {
+            "property-type": "Bungalow", "county": "Derbyshire", "built-form": "Detached",
+            "walls-energy-eff": "Very Poor"
+        }
         input_property5 = Property(id=1, postcode="F4k3 2", address="223 fake street", epc_record=epc_record)
         input_property5.walls = {
             'original_description': 'Cob, as built',
@@ -453,6 +469,7 @@ class TestCavityWallRecommensations:
         input_property5.age_band = "E"
         input_property5.insulation_wall_area = 77
         input_property5.restricted_measures = False
+        input_property5.construction_age_band = "England and Wales: 1967-1975"
 
         assert input_property5.walls["is_cob"]
 
@@ -465,22 +482,15 @@ class TestCavityWallRecommensations:
 
         recommender5.recommend()
 
-        assert recommender5.recommendations
-        assert len(recommender5.recommendations) == 5
-        assert recommender5.estimated_u_value == 0.8
-        assert np.isclose(recommender5.recommendations[0]["new_u_value"], 0.29)
-        assert np.isclose(recommender5.recommendations[0]["total"], 8963.834880000002)
-        assert recommender5.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender5.recommendations[0]["parts"][0]["depth"] == 50
-
-        assert np.isclose(recommender5.recommendations[3]["new_u_value"], 0.26)
-        assert np.isclose(recommender5.recommendations[3]["total"], 20771.11344)
-        assert recommender5.recommendations[3]["parts"][0]["type"] == "internal_wall_insulation"
-        assert recommender5.recommendations[3]["parts"][0]["depth"] == 100
+        # No insulation recommendations for cob walls
+        assert not recommender5.recommendations
 
     def test_sandstone_or_limestone_wall(self):
         epc_record = EPCRecord()
-        epc_record.prepared_epc = {"property-type": "House", "county": "Derbyshire", "built-form": "Mid-Terrace"}
+        epc_record.prepared_epc = {
+            "property-type": "House", "county": "Derbyshire", "built-form": "Mid-Terrace",
+            "walls-energy-eff": "Very Poor"
+        }
         input_property6 = Property(id=1, postcode="F4k3 6", address="623 fake street", epc_record=epc_record)
         input_property6.walls = {
             'original_description': 'Sandstone or limestone, as built, no insulation (assumed)',
@@ -496,6 +506,7 @@ class TestCavityWallRecommensations:
         input_property6.age_band = "F"
         input_property6.insulation_wall_area = 350
         input_property6.restricted_measures = False
+        input_property6.construction_age_band = "England and Wales: 1976-1982"
 
         assert input_property6.walls["is_sandstone_or_limestone"]
 
@@ -508,20 +519,11 @@ class TestCavityWallRecommensations:
 
         recommender6.recommend()
 
+        # For sandstone walls, we only recommend internal wall insulation
         assert recommender6.recommendations
-        assert len(recommender6.recommendations) == 9
+        assert len(recommender6.recommendations) == 1
         assert recommender6.estimated_u_value == 1
-        assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.19)
-        assert np.isclose(recommender6.recommendations[0]["total"], 46374.888000000006)
-        assert recommender6.recommendations[0]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender6.recommendations[0]["parts"][0]["depth"] == 100
-
-        assert np.isclose(recommender6.recommendations[2]["new_u_value"], 0.21)
-        assert np.isclose(recommender6.recommendations[2]["total"], 120451.29600000002)
-        assert recommender6.recommendations[2]["parts"][0]["type"] == "external_wall_insulation"
-        assert recommender6.recommendations[2]["parts"][0]["depth"] == 150
-
-        assert np.isclose(recommender6.recommendations[4]["new_u_value"], 0.28)
-        assert np.isclose(recommender6.recommendations[4]["total"], 94414.15199999999)
-        assert recommender6.recommendations[4]["parts"][0]["type"] == "internal_wall_insulation"
-        assert recommender6.recommendations[4]["parts"][0]["depth"] == 100
+        assert np.isclose(recommender6.recommendations[0]["new_u_value"], 0.26)
+        assert np.isclose(recommender6.recommendations[0]["total"], 85680.0)
+        assert recommender6.recommendations[0]["parts"][0]["type"] == "internal_wall_insulation"
+        assert recommender6.recommendations[0]["parts"][0]["depth"] == 95