still testing non null stuff

.idea/.gitignore

@@ -1,3 +1,5 @@
 # Default ignored files
 /shelf/
 /workspace.xml
+# GitHub Copilot persisted chat sessions
+/copilot/chatSessions

backend/Property.py

@@ -1,4 +1,5 @@
 import os
+import ast
 from itertools import groupby
 import pandas as pd
 
@@ -55,7 +56,14 @@ class Property:
 
     DATA_ANOMALY_MATCHES = DATA_ANOMALY_MATCHES
 
-    def __init__(self, id, postcode, address, epc_record):
+    # Surplus information, that can be provided as optional inputs, by a customer
+    n_bathrooms = None
+    n_bedrooms = None
+
+    def __init__(
+        self, id, postcode, address, epc_record, already_installed=None, non_invasive_recommendations=None,
+        **kwargs
+    ):
 
         self.epc_record = epc_record
 
@@ -68,6 +76,15 @@ class Property:
         }
         self.old_data = epc_record.get("old_data")
         self.property_dimensions = None
+        # This is a list of measures that have already been installed in the property, typically found as a result
+        # of the non-invasive surveys. We reflect that this has been installed in the recommendations, but remove the
+        # cost and instead, provide a message that the measure has already been installed
+
+        self.already_installed = ast.literal_eval(already_installed['already_installed']) if already_installed else []
+        self.non_invasive_recommendations = (
+            ast.literal_eval(non_invasive_recommendations['recommendations']) if
+            non_invasive_recommendations else []
+        )
 
         self.uprn = epc_record.get("uprn")
         self.full_sap_epc = epc_record.get("full_sap_epc")
@@ -130,9 +147,40 @@ class Property:
 
         self.current_adjusted_energy = None
         self.expected_adjusted_energy = None
+        self.current_energy_bill = None
+        self.expected_energy_bill = None
 
         self.recommendations_scoring_data = []
 
+        self.parse_kwargs(kwargs)
+
+    @classmethod
+    def extract_kwargs(cls, kwargs):
+        """
+        This method is to be used in the router, to extract the kwargs from the request and prevent any errors such as
+        non-integer values, or inputs that clash with the __init__ method of this class
+        :param kwargs:
+        :return:
+        """
+        n_bathrooms = kwargs.get("n_bathrooms", None)
+        if n_bathrooms is not None:
+            # We add on a small value to ensure that the number of bathrooms is rounded up, in case the value is 0.5
+            n_bathrooms = int(round(float(n_bathrooms) + 1e-5))
+
+        n_bedrooms = kwargs.get("n_bedrooms", None)
+        if n_bedrooms is not None:
+            n_bedrooms = int(round(float(n_bedrooms) + 1e-5))
+
+        return {
+            "n_bathrooms": n_bathrooms,
+            "n_bedrooms": n_bedrooms,
+        }
+
+    def parse_kwargs(self, kwargs):
+        # We extract the elements from kwargs that we recognise. Anything additional is ignored
+        self.n_bathrooms = kwargs.get("n_bathrooms", None)
+        self.n_bedrooms = kwargs.get("n_bedrooms", None)
+
     def create_base_difference_epc_record(self, cleaned_lookup: dict):
         """
         Creates a EPCDifferenceRecord object, which is used to store the difference between the current and
@@ -236,6 +284,7 @@ class Property:
                     recommendation_record=recommendation_record,
                     recommendations=previous_phase_representatives + [rec],
                     primary_recommendation_id=rec["recommendation_id"],
+                    non_invasive_recommendations=self.non_invasive_recommendations,
                 )
                 self.recommendations_scoring_data.append(scoring_dict)
 
@@ -245,6 +294,7 @@ class Property:
         recommendation_record,
         recommendations: list,
         primary_recommendation_id: int,
+        non_invasive_recommendations: list = None,
     ):
         """
         This function will iterate through a list of recommendations and apply a simulation for each recommendation
@@ -253,10 +303,12 @@ class Property:
         :param recommendation_record: The record of the property, which will be updated
         :param recommendations: The list of recommendations to apply
         :param primary_recommendation_id: The id of the primary recommendation, which is used to identify the record
+        :param non_invasive_recommendations: The list of non-invasive recommendations
         :return: The updated recommendation record
         """
 
         output = recommendation_record.copy()
+        non_invasive_recommendations = [] if non_invasive_recommendations is None else non_invasive_recommendations
 
         for col in [
             "walls_insulation_thickness",
@@ -275,6 +327,13 @@ class Property:
                 "external_wall_insulation",
                 "cavity_wall_insulation",
             ]:
+
+                # # If we have a non-incasive recommendation that the cavity wall is partially filled, we skip the
+                # # cavity wall insulation recommendation (since on the EPC, the property will look like how it did
+                # # before any works)
+                # if "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations:
+                #     continue
+
                 # The upgrade made here is to the u-value of the walls and the description of the
                 # insulation thickness
                 output["walls_thermal_transmittance_ending"] = recommendation[
@@ -282,7 +341,7 @@ class Property:
                 ]
                 # Setting the insulation thickness here to above average should be tested further because we
                 # don't see a high volume of instances for this
-                output["walls_insulation_thickness_ending"] = "above average"
+                output["walls_insulation_thickness_ending"] = "average"
                 output["walls_energy_eff_ending"] = "Good"
 
                 # Note: often when the wall is insulatied, the internal/external insulation is not noted so we should
@@ -298,11 +357,6 @@ class Property:
                 if recommendation["type"] == "cavity_wall_insulation":
                     output["is_filled_cavity_ending"] = True
 
-                # TODO: perhaps detrimental
-                # When making a recommendation for the wall, we will also update the ventilation
-                # if output["mechanical_ventilation_ending"] == 'natural':
-                #     output["mechanical_ventilation_ending"] = 'mechanical, extract only'
-
             else:
                 if output["walls_thermal_transmittance_ending"] is None:
                     raise ValueError("We should not have a None value for the u value")
@@ -426,7 +480,9 @@ class Property:
                         "double glazing installed during or after 2002"
                     )
 
-            if recommendation["type"] in ["heating", "hot_water_tank_insulation"]:
+            if recommendation["type"] in [
+                "heating", "hot_water_tank_insulation", "heating_control", "secondary_heating"
+            ]:
                 # We update the data, as defined in the recommendaton
 
                 simulation_config = recommendation["simulation_config"]
@@ -442,21 +498,12 @@ class Property:
                 output["photo_supply_ending"] = recommendation["photo_supply"]
 
             if recommendation["type"] not in [
-                "sealing_open_fireplace",
-                "low_energy_lighting",
-                "internal_wall_insulation",
-                "external_wall_insulation",
-                "cavity_wall_insulation",
-                "loft_insulation",
-                "room_roof_insulation",
-                "flat_roof_insulation",
-                "solid_floor_insulation",
-                "suspended_floor_insulation",
-                "exposed_floor_insulation",
-                "windows_glazing",
-                "solar_pv",
-                "heating",
-                "hot_water_tank_insulation",
+                "sealing_open_fireplace", "low_energy_lighting",
+                "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
+                "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
+                "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation",
+                "windows_glazing", "solar_pv", "heating", "hot_water_tank_insulation",
+                "heating_control", "secondary_heating"
             ]:
                 raise NotImplementedError(
                     "Implement me, given type %s" % recommendation["type"]
@@ -704,7 +751,7 @@ class Property:
         filtered_df = uprn_filenames[
             (uprn_filenames["lower"] <= self.uprn)
             & (uprn_filenames["upper"] >= self.uprn)
-        ]
+            ]
         if filtered_df.empty:
             logger.warning("Could not find file containing UPRNS")
             return None
@@ -787,7 +834,7 @@ class Property:
         self.floor_level = (
             FLOOR_LEVEL_MAP[self.data["floor-level"]]
             if self.data["floor-level"] not in self.DATA_ANOMALY_MATCHES
-            and self.data["floor-level"] is not None
+               and self.data["floor-level"] is not None
             else None
         )
 
@@ -863,12 +910,16 @@ class Property:
 
         return component_data
 
-    def set_adjusted_energy(self, current_adjusted_energy, expected_adjusted_energy):
+    def set_adjusted_energy(
+        self, current_adjusted_energy, expected_adjusted_energy, current_energy_bill, expected_energy_bill
+    ):
         """
         Stores these values for usage later
         """
         self.current_adjusted_energy = current_adjusted_energy
         self.expected_adjusted_energy = expected_adjusted_energy
+        self.current_energy_bill = current_energy_bill
+        self.expected_energy_bill = expected_energy_bill
 
     def set_windows_count(self):
         """

backend/SearchEpc.py

@@ -30,7 +30,7 @@ vartypes = {
     'environment-impact-potential': "Int64",
     'glazed-type': 'str',
     'heating-cost-current': 'float',
-    'address3': 'str',
+    # 'address3': 'str',
     'mainheatcont-description': 'str',
     'sheating-energy-eff': 'str',
     'property-type': 'str',
@@ -40,7 +40,7 @@ vartypes = {
     'mechanical-ventilation': 'str',
     'hot-water-cost-current': 'str',
     'county': 'str',
-    'postcode': 'str',
+    # 'postcode': 'str',
     'solar-water-heating-flag': 'str',
     'constituency': 'str',
     'co2-emissions-potential': 'float',
@@ -55,7 +55,7 @@ vartypes = {
     # 'inspection-date': str,
     'mains-gas-flag': 'str',
     'co2-emiss-curr-per-floor-area': 'float',
-    'address1': 'str',
+    # 'address1': 'str',
     'heat-loss-corridor': 'str',
     'flat-storey-count': "Int64",
     'constituency-label': 'str',
@@ -67,7 +67,7 @@ vartypes = {
     'roof-description': 'str',
     'floor-energy-eff': 'str',
     'number-habitable-rooms': 'float',
-    'address2': 'str',
+    # 'address2': 'str',
     'hot-water-env-eff': 'str',
     'posttown': 'str',
     'mainheatc-energy-eff': 'str',
@@ -98,7 +98,7 @@ vartypes = {
     # 'lodgement-date',
     'extension-count': "Int64",
     'mainheatc-env-eff': 'str',
-    'lmk-key': 'str',
+    # 'lmk-key': 'str',
     'wind-turbine-count': "Int64",
     'tenure': 'str',
     'floor-level': 'str',
@@ -147,6 +147,7 @@ class SearchEpc:
         uprn: [int, None] = None,
         size=None,
         property_type=None,
+        fast=False
     ):
         """
         Address lines 1 and postcode are mandatory fields. The other address lines are optional
@@ -187,6 +188,7 @@ class SearchEpc:
         self.size = size if size is not None else 25
 
         self.property_type = property_type
+        self.fast = fast
 
     @classmethod
     def get_house_number(cls, address: str) -> str | None:
@@ -365,9 +367,6 @@ class SearchEpc:
         # Finally, we identify the newest epc and the rest, and then return
         newest_epc, older_epcs = self.filter_newest_epc(list_of_epcs=rows)
 
-        # Retrieve postcode and address
-        address_epc, postcode_epc = self.format_address(newest_epc=newest_epc)
-
         # Ge the uprn from the newest record for this home
         uprns = {r["uprn"] for r in rows if r["uprn"]}
         # We can sometimes have no uprn for a property
@@ -384,6 +383,12 @@ class SearchEpc:
 
         uprn = uprns.pop() if uprns else None
 
+        if self.fast:
+            return newest_epc, [], {}, "", "", None
+
+        # Retrieve postcode and address
+        address_epc, postcode_epc = self.format_address(newest_epc=newest_epc)
+
         return newest_epc, older_epcs, full_sap_epc, address_epc, postcode_epc, uprn
 
     @staticmethod
@@ -575,6 +580,11 @@ class SearchEpc:
             property_type=property_type
         )
 
+        # If we have missing lodgment date, we fill it with inspection-date
+        epc_data["lodgement-datetime"] = epc_data["lodgement-datetime"].fillna(epc_data["inspection-date"])
+        # If we still have missing dates, we set it to the mean of the non NA dates
+        epc_data["lodgement-datetime"] = epc_data["lodgement-datetime"].fillna(epc_data["lodgement-datetime"].mean())
+
         # For each attribute, we need to determine the datatype and use an appropriate method
         # to estimate.
         estimated_epc = {}
@@ -609,7 +619,11 @@ class SearchEpc:
         # Insert an estimated lodgement datetime, with a weighted average
         estimated_epc["lodgement-datetime"] = self.calculate_weighted_lodgement_datetime(epc_data=epc_data)
         # Extract logement date
-        estimated_epc["lodgement-date"] = estimated_epc["lodgement-datetime"].strftime("%Y-%m-%d")
+        # It is possible that there is still no lodgement date, so we need to handle this
+        if pd.isnull(estimated_epc["lodgement-datetime"]):
+            estimated_epc["lodgement-date"] = None
+        else:
+            estimated_epc["lodgement-date"] = estimated_epc["lodgement-datetime"].strftime("%Y-%m-%d")
 
         estimated_epc["postcode"] = self.postcode
         estimated_epc["uprn"] = self.uprn
@@ -695,8 +709,13 @@ class SearchEpc:
                 self.full_sap_epc = {}
 
                 # Finally, set a standardised address 1 and postcode
-                self.address_clean = self.ordnance_survey_client.address_os
-                self.postcode_clean = self.ordnance_survey_client.postcode_os
+                self.address_clean = (
+                    self.ordnance_survey_client.address_os if self.ordnance_survey_client.address_os else self.address1
+                )
+                self.postcode_clean = (
+                    self.ordnance_survey_client.postcode_os if self.ordnance_survey_client.postcode_os else
+                    self.postcode
+                )
             return
 
         os_response = self.ordnance_survey_client.get_places_api()

backend/app/db/functions/non_intrusive_surveys.py

@@ -0,0 +1,50 @@
+from sqlalchemy.orm import Session
+from backend.app.db.models.non_intrusive_surveys import NonIntrusiveSurvey, NonIntrusiveSurveyNotes
+
+
+def upload_non_intrusive_survey_notes(session: Session, non_invasive_notes, batch_size=500):
+    """
+    Uploads a list of non-intrusive survey notes into the database in batches. Each dictionary in the list represents
+    one survey and its associated notes.
+
+    :param session: SQLAlchemy Session object through which all database transactions are handled.
+    :param non_invasive_notes: List of dictionaries where each dictionary contains survey details including 'uprn',
+                               'survey_date', 'surveyor', and other notes as key-value pairs.
+    :param batch_size: The size of each batch to be processed (default is 500).
+    :return: None
+    """
+
+    # Helper function to process each batch
+    def process_batch(batch):
+        surveys = []
+        notes = []
+
+        for note in batch:
+            survey = NonIntrusiveSurvey(
+                uprn=note['uprn'],
+                survey_date=note['survey_date'],
+                surveyor=note['surveyor']
+            )
+            surveys.append(survey)
+
+        session.add_all(surveys)
+        session.flush()  # Get IDs for surveys
+
+        for note, survey in zip(batch, surveys):
+            for key, value in note.items():
+                if key not in ['uprn', 'survey_date', 'surveyor']:
+                    notes.append(NonIntrusiveSurveyNotes(
+                        survey_id=survey.id,
+                        title=key,
+                        note=value
+                    ))
+
+        session.bulk_save_objects(notes)
+        session.commit()
+
+    # Split the data into batches and process each batch
+    total = len(non_invasive_notes)
+    for start in range(0, total, batch_size):
+        end = min(start + batch_size, total)
+        batch = non_invasive_notes[start:end]
+        process_batch(batch)

backend/app/db/functions/portfolio_functions.py

@@ -4,14 +4,14 @@ from backend.app.db.models.portfolio import Portfolio
 
 
 def aggregate_portfolio_recommendations(
-    session, portfolio_id: int, total_valuation_increase: float, labour_days: float
+    session, portfolio_id: int, total_valuation_increase: float, labour_days: float, aggregated_data: dict
 ):
     # Aggregate multiple fields
     aggregates = (
         session.query(
             func.sum(Recommendation.estimated_cost).label("cost"),
             func.sum(Recommendation.total_work_hours).label("total_work_hours"),
-            func.sum(Recommendation.heat_demand).label("energy_savings"),
+            func.sum(Recommendation.adjusted_heat_demand).label("energy_savings"),
             func.sum(Recommendation.co2_equivalent_savings).label("co2_equivalent_savings"),
             func.sum(Recommendation.energy_cost_savings).label("energy_cost_savings"),
         )
@@ -27,6 +27,7 @@ def aggregate_portfolio_recommendations(
         "energy_savings": aggregates.energy_savings or 0,
         "co2_equivalent_savings": aggregates.co2_equivalent_savings or 0,
         "energy_cost_savings": aggregates.energy_cost_savings or 0,
+        **aggregated_data
     }
 
     # Get the portfolio and update the fields

backend/app/db/functions/recommendations_functions.py

@@ -85,7 +85,8 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
             "co2_equivalent_savings": rec["co2_equivalent_savings"],
             "total_work_hours": rec["labour_hours"],
             "energy_cost_savings": rec["energy_cost_savings"],
-            "labour_days": rec["labour_days"]
+            "labour_days": rec["labour_days"],
+            "already_installed": rec["already_installed"],
         }
         for rec in recommendations_to_upload
     ]

backend/app/db/models/non_intrusive_surveys.py

@@ -0,0 +1,22 @@
+from sqlalchemy import Column, BigInteger, String, TIMESTAMP, ForeignKey, Integer
+from sqlalchemy.orm import declarative_base
+
+Base = declarative_base()
+
+
+class NonIntrusiveSurvey(Base):
+    __tablename__ = 'non_intrusive_survey'
+
+    id = Column(BigInteger, primary_key=True, autoincrement=True)
+    uprn = Column(Integer, nullable=False)
+    survey_date = Column(TIMESTAMP, nullable=False)
+    surveyor = Column(String, nullable=False)
+
+
+class NonIntrusiveSurveyNotes(Base):
+    __tablename__ = 'non_intrusive_survey_notes'
+
+    id = Column(BigInteger, primary_key=True, autoincrement=True)
+    survey_id = Column(BigInteger, ForeignKey('non_intrusive_survey.id'), nullable=False)
+    title = Column(String, nullable=False)
+    note = Column(String, nullable=False)

backend/app/db/models/portfolio.py

@@ -45,6 +45,21 @@ class Portfolio(Base):
     labour_days = Column(Float)
     created_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
     updated_at = Column(DateTime, nullable=False, default=datetime.datetime.now(pytz.utc))
+    # Aggregations for summary
+    epc_breakdown_pre_retrofit = Column(Text)
+    epc_breakdown_post_retrofit = Column(Text)
+    n_units_to_retrofit = Column(Integer)
+    co2_per_unit_pre_retrofit = Column(Text)
+    co2_per_unit_post_retrofit = Column(Text)
+    energy_bill_per_unit_pre_retrofit = Column(Text)
+    energy_bill_per_unit_post_retrofit = Column(Text)
+    energy_consumption_per_unit_pre_retrofit = Column(Text)
+    energy_consumption_per_unit_post_retrofit = Column(Text)
+    valuation_improvement_per_unit = Column(Text)
+    cost_per_unit = Column(Text)
+    cost_per_co2_saved = Column(Text)
+    cost_per_sap_point = Column(Text)
+    valuation_return_on_investment = Column(Text)
 
 
 class PropertyCreationStatus(enum.Enum):

backend/app/db/models/recommendations.py

@@ -30,6 +30,7 @@ class Recommendation(Base):
     rental_yield_increase = Column(Float)
     total_work_hours = Column(Float)
     labour_days = Column(Float)
+    already_installed = Column(Boolean, nullable=False, default=False)
 
 
 class RecommendationMaterials(Base):

backend/app/plan/router.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 from tqdm import tqdm
@@ -24,7 +25,7 @@ from backend.app.db.models.portfolio import rating_lookup
 from backend.app.dependencies import validate_token
 from backend.app.plan.schemas import PlanTriggerRequest
 from backend.app.plan.utils import get_cleaned
-from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, sap_to_epc
+from backend.app.utils import epc_to_sap_lower_bound, sap_to_epc
 
 from backend.ml_models.api import ModelApi
 from backend.Property import Property
@@ -35,32 +36,186 @@ from recommendations.optimiser.GainOptimiser import GainOptimiser
 from recommendations.optimiser.optimiser_functions import prepare_input_measures
 from recommendations.Recommendations import Recommendations
 from utils.logger import setup_logger
-from utils.s3 import read_dataframe_from_s3_parquet
+from utils.s3 import read_dataframe_from_s3_parquet, read_csv_from_s3
 from backend.ml_models.Valuation import PropertyValuation
 
 logger = setup_logger()
 
 BATCH_SIZE = 5
+SCORING_BATCH_SIZE = 400
 
 
-def patch_epc(config, epc_records):
+def patch_epc(patch, epc_records):
     """
     This utility function is useful to patch the epc data if we have data from the customer
     :return:
     """
 
-    number_habitable_rooms = config.get("number-habitable-rooms", None)
-    number_heated_rooms = config.get("number-heated-rooms", None)
+    for patch_variable, patch_value in patch.items():
 
-    if number_habitable_rooms is not None:
-        epc_records["original_epc"]["number-habitable-rooms"] = int(number_habitable_rooms)
+        if patch_variable in ["address", "postcode"]:
+            continue
 
-    if number_heated_rooms is not None:
-        epc_records["original_epc"]["number-heated-rooms"] = int(number_heated_rooms)
+        if patch_value == "":
+            continue
+        if patch_variable in epc_records["original_epc"]:
+            epc_records["original_epc"][patch_variable] = patch_value
 
     return epc_records
 
 
+def extract_portfolio_aggregation_data(
+    input_properties, total_valuation_increase, recommendations, new_epc_bands, property_value_increase_ranges
+):
+    # We aggregate a number of metrics for the portfolio:
+    # 1) A breakdown of the number of properties in each EPC band
+    #    a) before retrofit
+    #    b) after retrofit
+    # 2) Number of units
+    # 3) Co2/unit
+    #    a) before retrofit
+    #    b) after retrofit
+    # 4) Energy bill/unit
+    #    a) before retrofit
+    #    b) after retrofit
+    # 5) Average valuation improvement/unit
+    # 6) Total cost
+    # 7) Cost per unit
+    # 8) £ per CO2 saved
+    # 9) £ per SAP point
+
+    # We need to construct the underlyind data for this
+
+    # Helper function to reformat the EPC data
+    def reformat_epc_data(epc_counts):
+        # Define all possible EPC bands in the required order
+        epc_bands = ["G", "F", "E", "D", "C", "B", "A"]
+
+        # Create the formatted data list by checking each band in the order
+        formatted_data = []
+        for band in epc_bands:
+            # Get the count from the dictionary, defaulting to 0 if not present
+            count = epc_counts.get(band, 0)
+            # Append the formatted dictionary to the list
+            formatted_data.append({"name": band, band: count})
+
+        return formatted_data
+
+    n_units = len(input_properties)
+
+    agg_data = []
+    for p in input_properties:
+        # Get the recommendations for the property - we include all properties, even ones without recommendations
+        property_recommendations = recommendations.get(p.id, [])
+
+        # Get just the default recommendations
+        default_recommendations = [r for r in property_recommendations if r["default"]]
+
+        has_recommendations = len(default_recommendations) > 0
+
+        # We can now calculate multiple outputs based on default recommendations
+        carbon_savings = sum([r["co2_equivalent_savings"] for r in default_recommendations])
+
+        pre_retrofit_co2 = p.data["co2-emissions-current"]
+        post_retrofit_co2 = pre_retrofit_co2 - carbon_savings
+
+        pre_retrofit_energy_bill = p.current_energy_bill
+        post_retrofit_energy_bill = p.current_energy_bill - sum(
+            [r["energy_cost_savings"] for r in default_recommendations]
+        )
+
+        pre_retrofit_energy_consumption = p.current_adjusted_energy
+        post_retrofit_energy_consumption = p.current_adjusted_energy - sum(
+            [r["adjusted_heat_demand"] for r in default_recommendations]
+        )
+
+        # Add up energy savings
+        cost = sum([r["total"] for r in default_recommendations])
+        sap_point_improvement = sum([r["sap_points"] for r in default_recommendations])
+
+        lower_bound_valuation_uplift = (
+            property_value_increase_ranges[p.id]["lower_bound_increased_value"] -
+            property_value_increase_ranges[p.id]["current_value"]
+        )
+        upper_bound_valuation_uplift = (
+            property_value_increase_ranges[p.id]["upper_bound_increased_value"] -
+            property_value_increase_ranges[p.id]["current_value"]
+        )
+
+        agg_data.append({
+            "pre_retrofit_epc": p.data["current-energy-rating"],
+            "post_retrofit_epc": new_epc_bands[p.id],
+            "pre_retrofit_co2": pre_retrofit_co2,
+            "post_retrofit_co2": post_retrofit_co2,
+            "pre_retrofit_energy_bill": pre_retrofit_energy_bill,
+            "post_retrofit_energy_bill": post_retrofit_energy_bill,
+            "pre_retrofit_energy_consumption": pre_retrofit_energy_consumption,
+            "post_retrofit_energy_consumption": post_retrofit_energy_consumption,
+            "cost": cost,
+            "sap_point_improvement": sap_point_improvement,
+            "lower_bound_valuation_uplift": lower_bound_valuation_uplift,
+            "upper_bound_valuation_uplift": upper_bound_valuation_uplift,
+            "has_recommendations": has_recommendations
+        })
+
+    agg_data = pd.DataFrame(agg_data)
+
+    n_units_to_retrofit = agg_data["has_recommendations"].sum()
+
+    valuation_improvement_lower_bound_per_unit = (
+        agg_data["lower_bound_valuation_uplift"].mean()
+    )
+    valuation_improvement_upper_bound_per_unit = (
+        agg_data["upper_bound_valuation_uplift"].mean()
+    )
+
+    total_carbon_saved = agg_data["pre_retrofit_co2"].sum() - agg_data["post_retrofit_co2"].sum()
+    total_sap_points = agg_data["sap_point_improvement"].sum()
+
+    def format_money(amount):
+        return f"£{amount:,.0f}"
+
+    valuation_improvment_per_unit = str(
+        format_money(
+            total_valuation_increase / n_units) + (f" ({format_money(valuation_improvement_lower_bound_per_unit)} - "
+                                                   f"{format_money(valuation_improvement_upper_bound_per_unit)})")
+    )
+
+    valuation_return_on_investment = str(
+        str(round(total_valuation_increase / agg_data["cost"].sum(), 2)) +
+        f" ("
+        f"{agg_data['lower_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f} - "
+        f"{agg_data['upper_bound_valuation_uplift'].sum() / agg_data['cost'].sum():,.2f})"
+    )
+
+    aggregation_data = {
+        "epc_breakdown_pre_retrofit": json.dumps(
+            reformat_epc_data(agg_data["pre_retrofit_epc"].value_counts().to_dict())
+        ),
+        "epc_breakdown_post_retrofit": json.dumps(
+            reformat_epc_data(agg_data["post_retrofit_epc"].value_counts().to_dict())
+        ),
+        "number_of_properties": int(n_units),
+        "n_units_to_retrofit": int(n_units_to_retrofit),
+        "co2_per_unit_pre_retrofit": str(round(agg_data["pre_retrofit_co2"].mean(), 2)) + "t",
+        "co2_per_unit_post_retrofit": str(round(agg_data["post_retrofit_co2"].mean(), 2)) + "t",
+        "energy_bill_per_unit_pre_retrofit": format_money(agg_data["pre_retrofit_energy_bill"].mean()),
+        "energy_bill_per_unit_post_retrofit": format_money(agg_data["post_retrofit_energy_bill"].mean()),
+        "energy_consumption_per_unit_pre_retrofit": str(
+            round(agg_data["pre_retrofit_energy_consumption"].mean())) + "kWh",
+        "energy_consumption_per_unit_post_retrofit": str(
+            round(agg_data["post_retrofit_energy_consumption"].mean())) + "kWh",
+        "valuation_improvement_per_unit": valuation_improvment_per_unit,
+        "cost_per_unit": format_money(agg_data["cost"].mean()),
+        "cost_per_co2_saved": format_money(agg_data["cost"].sum() / total_carbon_saved),
+        "cost_per_sap_point": format_money(agg_data["cost"].sum() / total_sap_points),
+        "valuation_return_on_investment": valuation_return_on_investment,
+        # TODO: Could we add 10yr carbon credits value?
+    }
+
+    return aggregation_data
+
+
 router = APIRouter(
     prefix="/plan",
     tags=["plan"],
@@ -78,12 +233,29 @@ async def trigger_plan(body: PlanTriggerRequest):
     # TODO: We should store the trigger file path in the database with the plan so we can track the file that
     #       triggered the plan
 
-    # TODO: Create the ability to congigure/switch off certain measures
+    # TODO: if the measure is already installed, it should actually be the very first phase
 
     try:
         session.begin()
         logger.info("Getting the inputs")
         plan_input = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.trigger_file_path)
+        # If we have patches or overrides, we should read them in here
+        patches = []
+        if body.patches_file_path:
+            patches = read_csv_from_s3(bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.patches_file_path)
+
+        already_installed = []
+        if body.already_installed_file_path:
+            already_installed = read_csv_from_s3(
+                bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.already_installed_file_path
+            )
+
+        non_invasive_recommendations = []
+        if body.non_invasive_recommendations_file_path:
+            non_invasive_recommendations = read_csv_from_s3(
+                bucket_name=get_settings().PLAN_TRIGGER_BUCKET, filepath=body.non_invasive_recommendations_file_path
+            )
+
         cleaning_data = read_dataframe_from_s3_parquet(
             bucket_name=get_settings().DATA_BUCKET, file_key="sap_change_model/cleaning_dataset.parquet",
         )
@@ -91,36 +263,46 @@ async def trigger_plan(body: PlanTriggerRequest):
         input_properties = []
         for config in tqdm(plan_input):
             # We validate each record in the file. If the record is NOT valid, we need to handle this accordingly
+            uprn = config.get("uprn", None)
+            if uprn:
+                uprn = int(float(uprn))
 
             epc_searcher = SearchEpc(
                 address1=config["address"],
                 postcode=config["postcode"],
+                uprn=uprn,
                 auth_token=get_settings().EPC_AUTH_TOKEN,
-                os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY
+                os_api_key=get_settings().ORDNANCE_SURVEY_API_KEY,
             )
-            epc_searcher.find_property()
+            epc_searcher.ordnance_survey_client.built_form = config.get("built_form", None)
+            epc_searcher.ordnance_survey_client.property_type = config.get("property_type", None)
+            # For the moment, our OS API access is unavailable, so we skip and interpolate
+            epc_searcher.find_property(skip_os=True)
             # Create a record in db
             property_id, is_new = create_property(
                 session, body.portfolio_id, epc_searcher.address_clean, epc_searcher.postcode_clean, epc_searcher.uprn
             )
-            # if a new record was not created, we don't produduce recommendations
-            if not is_new:
-                continue
-
-            create_property_targets(
-                session,
-                property_id=property_id,
-                portfolio_id=body.portfolio_id,
-                epc_target=body.goal_value,
-                heat_demand_target=None
-            )
+            # if not is_new:
+            #     continue
+            #
+            # create_property_targets(
+            #     session,
+            #     property_id=property_id,
+            #     portfolio_id=body.portfolio_id,
+            #     epc_target=body.goal_value,
+            #     heat_demand_target=None
+            # )
 
             epc_records = {
                 'original_epc': epc_searcher.newest_epc.copy(),
                 'full_sap_epc': epc_searcher.full_sap_epc.copy(),
                 'old_data': epc_searcher.older_epcs.copy(),
             }
-            epc_records = patch_epc(config, epc_records)
+
+            patch = next((
+                x for x in patches if (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
+            ), {})
+            epc_records = patch_epc(patch, epc_records)
 
             prepared_epc = EPCRecord(
                 epc_records=epc_records,
@@ -128,12 +310,25 @@ async def trigger_plan(body: PlanTriggerRequest):
                 cleaning_data=cleaning_data
             )
 
+            property_already_installed = next((
+                x for x in already_installed if
+                (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
+            ), {})
+
+            property_non_invasive_recommendations = next((
+                x for x in non_invasive_recommendations if
+                (x["address"] == config["address"]) and (x["postcode"] == config["postcode"])
+            ), {})
+
             input_properties.append(
                 Property(
                     id=property_id,
                     address=epc_searcher.address_clean,
                     postcode=epc_searcher.postcode_clean,
                     epc_record=prepared_epc,
+                    already_installed=property_already_installed,
+                    non_invasive_recommendations=property_non_invasive_recommendations,
+                    **Property.extract_kwargs(config)
                 )
             )
 
@@ -160,15 +355,13 @@ async def trigger_plan(body: PlanTriggerRequest):
         recommendations = {}
         recommendations_scoring_data = []
         representative_recommendations = {}
-        for p in input_properties:
+        for p in tqdm(input_properties):
 
             # Property recommendations
             p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
 
-            recommender = Recommendations(property_instance=p, materials=materials)
-            # TODO: portfolio id as an input is temp
-            print("DELETE PORTFOLIO ID AS AN INPUT!!")
-            property_recommendations, property_representative_recommendations = recommender.recommend(body.portfolio_id)
+            recommender = Recommendations(property_instance=p, materials=materials, exclusions=body.exclusions)
+            property_recommendations, property_representative_recommendations = recommender.recommend()
 
             if not property_recommendations:
                 continue
@@ -187,6 +380,7 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         logger.info("Preparing data for scoring in sap change api")
         recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
+
         recommendations_scoring_data = recommendations_scoring_data.drop(
             columns=["rdsap_change", "heat_demand_change", "carbon_change", "sap_ending", "heat_demand_ending",
                      "carbon_ending"]
@@ -194,15 +388,26 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         model_api = ModelApi(portfolio_id=body.portfolio_id, timestamp=created_at)
 
-        all_predictions = model_api.predict_all(
-            df=recommendations_scoring_data,
-            bucket=get_settings().DATA_BUCKET,
-            prediction_buckets={
-                "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET,
-                "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET,
-                "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET
-            }
-        )
+        all_predictions = {
+            "sap_change_predictions": pd.DataFrame(),
+            "heat_demand_predictions": pd.DataFrame(),
+            "carbon_change_predictions": pd.DataFrame()
+        }
+        to_loop_over = range(0, recommendations_scoring_data.shape[0], SCORING_BATCH_SIZE)
+        for chunk in tqdm(to_loop_over, total=len(to_loop_over)):
+            predictions_dict = model_api.predict_all(
+                df=recommendations_scoring_data.iloc[chunk:chunk + SCORING_BATCH_SIZE],
+                bucket=get_settings().DATA_BUCKET,
+                prediction_buckets={
+                    "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET,
+                    "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET,
+                    "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET
+                }
+            )
+
+            # Append the predictions to the predictions dictionary
+            for key, scored in predictions_dict.items():
+                all_predictions[key] = pd.concat([all_predictions[key], scored])
 
         # Insert the predictions into the recommendations and run the optimiser
         # TODO: If a recommendation has a negative impact on SAP, we should remove it - this seems to have become a
@@ -214,7 +419,13 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             property_instance = [p for p in input_properties if p.id == property_id][0]
 
-            recommendations_with_impact, current_adjusted_energy, expected_adjusted_energy = (
+            (
+                recommendations_with_impact,
+                current_adjusted_energy,
+                expected_adjusted_energy,
+                current_energy_bill,
+                expected_energy_bill
+            ) = (
                 Recommendations.calculate_recommendation_impact(
                     property_instance=property_instance,
                     all_predictions=all_predictions,
@@ -225,10 +436,12 @@ async def trigger_plan(body: PlanTriggerRequest):
             # Store the resulting adjusted energy in the property instance
             property_instance.set_adjusted_energy(
                 current_adjusted_energy=current_adjusted_energy,
-                expected_adjusted_energy=expected_adjusted_energy
+                expected_adjusted_energy=expected_adjusted_energy,
+                current_energy_bill=current_energy_bill,
+                expected_energy_bill=expected_energy_bill
             )
 
-            input_measures = prepare_input_measures(recommendations_with_impact, body.goal, body.housing_type)
+            input_measures = prepare_input_measures(recommendations_with_impact, body.goal)
 
             current_sap_points = int(property_instance.data["current-energy-efficiency"])
             target_sap_points = epc_to_sap_lower_bound(body.goal_value)
@@ -256,16 +469,14 @@ async def trigger_plan(body: PlanTriggerRequest):
             if any(x in [r["type"] for r in solution] for x in [
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"
             ]):
-                ventilation_rec = [
-                    r for r in recommendations_with_impact if r[0]["type"] == "mechanical_ventilation"
-                ][0]
-
-                selected_recommendations = set(
-                    list(selected_recommendations) + [ventilation_rec[0]["recommendation_id"]]
+                ventilation_rec = next(
+                    (r[0] for r in recommendations_with_impact if r[0]["type"] == "mechanical_ventilation"),
+                    None
                 )
 
-            # We check if the selected recommendation is wall ventilation and if so, we make sure
-            # mechanical ventilation is selected
+                # If a matching recommendation was found, add its ID to the selected recommendations
+                if ventilation_rec:
+                    selected_recommendations.add(ventilation_rec["recommendation_id"])
 
             # We'll use the set of selected recommendations to filter the recommendations to upload
             final_recommendations = [
@@ -289,6 +500,8 @@ async def trigger_plan(body: PlanTriggerRequest):
         logger.info("Uploading recommendations to the database")
         property_valuation_increases = []
         session.commit()
+        new_epc_bands = {}
+        property_value_increase_ranges = {}
         for i in range(0, len(input_properties), BATCH_SIZE):
             try:
                 # Take a slice of the input_properties list to make a batch
@@ -300,8 +513,10 @@ async def trigger_plan(body: PlanTriggerRequest):
                     total_sap_points = sum([r["sap_points"] for r in default_recommendations])
                     new_sap_points = float(p.data["current-energy-efficiency"]) + total_sap_points
                     new_epc = sap_to_epc(new_sap_points)
+                    new_epc_bands[p.id] = new_epc
 
                     valuations = PropertyValuation.estimate(property_instance=p, target_epc=new_epc)
+                    property_value_increase_ranges[p.id] = valuations
 
                     # Your existing operations
                     property_details_epc = p.get_property_details_epc(
@@ -365,11 +580,20 @@ async def trigger_plan(body: PlanTriggerRequest):
             [sum(r["labour_days"] for r in rec_group if r["default"]) for p_id, rec_group in recommendations.items()]
         ))
 
+        aggregated_data = extract_portfolio_aggregation_data(
+            input_properties=input_properties,
+            total_valuation_increase=total_valuation_increase,
+            recommendations=recommendations,
+            new_epc_bands=new_epc_bands,
+            property_value_increase_ranges=property_value_increase_ranges
+        )
+
         aggregate_portfolio_recommendations(
             session,
             portfolio_id=body.portfolio_id,
             total_valuation_increase=total_valuation_increase,
-            labour_days=labour_days
+            labour_days=labour_days,
+            aggregated_data=aggregated_data
         )
 
         # Commit final changes

backend/app/plan/schemas.py

@@ -1,10 +1,54 @@
-from pydantic import BaseModel
+from pydantic import BaseModel, conlist, validator
+from typing import Optional
 
 
 class PlanTriggerRequest(BaseModel):
-    budget: float | None = None
+    budget: Optional[float] = None
     goal: str
     housing_type: str
     goal_value: str
     portfolio_id: int
     trigger_file_path: str
+    already_installed_file_path: Optional[str] = None
+    patches_file_path: Optional[str] = None
+    non_invasive_recommendations_file_path: Optional[str] = None
+    exclusions: Optional[conlist(str, min_items=1)] = None
+
+    # Pre-defined list of possibilities for exclusions
+    _allowed_exclusions = {
+        "wall_insulation",
+        "ventilation",
+        "roof_insulation",
+        "floor_insulation",
+        "windows",
+        "fireplace",
+        "heating",
+        "hot_water",
+        "lighting",
+        "solar_pv"
+    }
+
+    _allowed_goals = {"Increase EPC"}
+
+    _allowed_housing_types = {"Social", "Private"}
+
+    # Validator to ensure exclusions are within the pre-defined possibilities
+    @validator('exclusions', each_item=True)
+    def check_exclusions(cls, v):
+        if v not in cls._allowed_exclusions:
+            raise ValueError(f"{v} is not an allowed exclusion")
+        return v
+
+    # Validator to ensure that the goal is within the pre-defined possibilities
+    @validator('goal')
+    def check_goal(cls, v):
+        if v not in cls._allowed_goals:
+            raise ValueError(f"{v} is not a valid goal")
+        return v
+
+    # Validator to ensure that the housing type is within the pre-defined possibilities
+    @validator('housing_type')
+    def check_housing_type(cls, v):
+        if v not in cls._allowed_housing_types:
+            raise ValueError(f"{v} is not a valid housing type")
+        return v

backend/app/utils.py

@@ -1,6 +1,4 @@
 import boto3
-import csv
-from io import StringIO
 import string
 import secrets
 import logging
@@ -41,25 +39,6 @@ def setup_logger(log_file=None, level=logging.INFO, overwrite_handler=False):
     return logger
 
 
-def read_csv_from_s3(bucket_name, filepath):
-    s3 = boto3.client('s3')
-
-    # Get the object from s3
-    s3_object = s3.get_object(Bucket=bucket_name, Key=filepath)
-
-    # Read the CSV body from the s3 object
-    body = s3_object['Body'].read()
-
-    # Use StringIO to create a file-like object from the string
-    csv_data = StringIO(body.decode('utf-8'))
-
-    # Use csv library to read it into a list of dictionaries
-    reader = csv.DictReader(csv_data)
-    data = list(reader)
-
-    return data
-
-
 def generate_api_key():
     # Define the characters that will be used to generate the api key
     characters = string.ascii_letters + string.digits

backend/ml_models/AnnualBillSavings.py

@@ -10,13 +10,17 @@ class AnnualBillSavings:
     AVERAGE_ELECTRICITY_CONSUMPTION = 2700
     AVERAGE_GAS_CONSUMPTION = 11500
 
-    # Latest price cap figures from Ofgem are for January 2024
-    # https://www.ofgem.gov.uk/publications/changes-energy-price-cap-1-january-2024
-    ELECTRICITY_PRICE_CAP = 0.29
-    GAS_PRICE_CAP = 0.07
+    # Latest price cap figures from Ofgem are for April 2024
+    # https://www.ofgem.gov.uk/publications/new-energy-price-cap-level-april-june-2024-starts-today
+    ELECTRICITY_PRICE_CAP = 0.245
+    GAS_PRICE_CAP = 0.0604
 
     # This is a weighted mean of the price caps, using the consumption figures above as weights
-    PRICE_FACTOR = 0.11183098591549295
+    PRICE_FACTOR = 0.09549999999999999
+
+    # Daily standard charge, based on average across England, Scotland and Wales, and includes VAT
+    DAILY_STANDARD_CHARGE_GAS = 0.3143
+    DAILY_STANDARD_CHARGE_ELECTRICITY = 0.601
 
     EPC_BANDS = ["G", "F", "E", "D", "C", "B", "A"]
 
@@ -38,6 +42,17 @@ class AnnualBillSavings:
         """
         return cls.ELECTRICITY_PRICE_CAP * kwh
 
+    @classmethod
+    def calculate_annual_bill(cls, kwh):
+        """
+        This method will estimate the total annual bill for a property
+        It assumed gas & electricity are used
+        :param kwh: The total kwh consumption
+        :return: An estimate for annual bill
+        """
+
+        return cls.PRICE_FACTOR * kwh + (cls.DAILY_STANDARD_CHARGE_GAS + cls.DAILY_STANDARD_CHARGE_ELECTRICITY * 365)
+
     @classmethod
     def adjust_energy_to_metered(cls, epc_energy_consumption, current_epc_rating):
         """

backend/ml_models/Valuation.py

@@ -52,6 +52,25 @@ class PropertyValuation:
         10070056829: 76_000,
         10070056920: 76_000,
         10023345463: 76_000,
+        # IMMO Dudley Pilot - search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
+        90070461: 172_000,  # Based on Zoopla
+        90022227: 181_000,  # Based on Zoopla
+        90106884: 180_000,  # Based on Zoopla
+        90051858: 201_000,  # Based on Zoopla
+        90060989: 172_000,  # Based on Zoopla
+        90048026: 196_000,  # Based on Zoopla
+        90077535: 192_000,  # Based on Zoopla
+        90093693: 279_000,  # Based on Zoopla
+        90055152: 149_000,  # Based on Zoopla
+        90028499: 238_000,  # Based on Zoopla
+        # IMMO Dudley Pilot 2- search by going to https://www.zoopla.co.uk/property/uprn/{uprn}/
+        90039318: 177_000,  # Based on Zoopla
+        90038384: 170_000,  # Based on Zoopla
+        90105380: 185_000,  # Based on Zoopla
+        90124001: 165_000,  # Based on Zoopla
+        90013980: 148_000,  # Based on Zoopla
+        90087154: 184_000,  # Based on Zoopla
+        90046817: 167_000,  # Based on Zoopla
     }
 
     # We base our valuation uplifts on a number of sources

etl/air_source_heat_pump/AirSourceHeatPumpEfficiency.py

@@ -0,0 +1,78 @@
+import pandas as pd
+from tqdm import tqdm
+from utils.s3 import save_dataframe_to_s3_parquet, read_dataframe_from_s3_parquet
+from utils.logger import setup_logger
+from etl.epc.settings import EARLIEST_EPC_DATE
+
+logger = setup_logger()
+
+
+class AirSourceHeatPumpEfficiency:
+
+    def __init__(self, file_directories, cleaned_lookup):
+        """
+        :param file_directories: A list of directories where files are stored.
+        :param cleaned_lookup: A dictionary containing cleaned lookup data.
+        """
+        self.file_directories = file_directories
+        self.cleaned_lookup = cleaned_lookup
+
+        self.results = []
+
+    def create_dataset(self):
+        logger.info("Creating solar photo supply dataset")
+        for dir in tqdm(self.file_directories):
+            filepath = dir / "certificates.csv"
+            df = pd.read_csv(filepath, low_memory=False)
+            df = df[~pd.isnull(df["UPRN"])]
+            df["UPRN"] = df["UPRN"].astype(int).astype(str)
+            # Take entries after SAP12
+            df["LODGEMENT_DATE"] = pd.to_datetime(df["LODGEMENT_DATE"])
+            df = df[df["LODGEMENT_DATE"] > EARLIEST_EPC_DATE]
+
+            df = df[
+                ~df["TENURE"].isin(
+                    [
+                        "unknown",
+                        "Not defined - use in the case of a new dwelling for which the intended tenure in not known. "
+                        "It is not to be used for an existing dwelling"
+                    ]
+                )
+            ]
+
+            # Take entries that contain an air source heat pump
+            df = df[
+                df["MAINHEAT_DESCRIPTION"].str.contains("air source heat pump", case=False, na=False)
+            ]
+            # Get the columns we're interested in
+            df = df[
+                [
+                    "MAINHEAT_DESCRIPTION",
+                    "MAINHEAT_ENERGY_EFF",
+                    "MAINHEATCONT_DESCRIPTION",
+                    "MAINHEATC_ENERGY_EFF",
+                    "MAIN_FUEL",
+                    "HOTWATER_DESCRIPTION",
+                    "HOT_WATER_ENERGY_EFF",
+                    "MAINS_GAS_FLAG"
+                ]
+            ]
+
+            counts = df.groupby(
+                [
+                    "MAINHEAT_DESCRIPTION",
+                    "MAINHEAT_ENERGY_EFF",
+                    "MAINHEATCONT_DESCRIPTION",
+                    "MAINHEATC_ENERGY_EFF",
+                    "MAIN_FUEL",
+                    "HOTWATER_DESCRIPTION",
+                    "HOT_WATER_ENERGY_EFF",
+                    "MAINS_GAS_FLAG"
+                ]
+            ).size().reset_index(name="count")
+
+            # Drop rows that have a missing PROPERTY_TYPE, BUILT_FORM, CONSTRUCTION_AGE_BAND, TOTAL_FLOOR_AREA
+            for col in ["PROPERTY_TYPE", "BUILT_FORM", "CONSTRUCTION_AGE_BAND", "TOTAL_FLOOR_AREA"]:
+                df = df[~pd.isnull(df[col])]
+            # Take newest LODGEMENT_DATE per UPRN
+            df = df.sort_values(by="LODGEMENT_DATE", ascending=False).drop_duplicates(subset=["UPRN"])

etl/air_source_heat_pump/app.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+from backend.app.plan.utils import get_cleaned
+from etl.air_source_heat_pump.AirSourceHeatPumpEfficiency import AirSourceHeatPumpEfficiency
+
+DATA_DIRECTORY = Path(__file__).parent / "local_data" / "all-domestic-certificates"
+
+
+def app():
+    """
+    This code reads in the EPC dataset and looks at the efficiency values for heating systems that inclue air source
+    heat pumps. This dataset is then used to inform the recommendations for the air source heat pump, so we know
+    how to set the simulation
+    :return:
+    """
+
+    directories = [entry for entry in DATA_DIRECTORY.iterdir() if entry.is_dir()]
+    cleaned_lookup = get_cleaned()
+
+    ashp_data_client = AirSourceHeatPumpEfficiency(
+        file_directories=directories,
+        cleaned_lookup=cleaned_lookup
+    )
+
+    ashp_data_client.create_dataset()

etl/customers/gla_croydon_demo/asset_list.py

@@ -0,0 +1,212 @@
+import pandas as pd
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 67
+
+archetype_1_uprns = [100020604138, 200001188299, 100020578756, 200001187196, 200001192253, 100020581792, 200001188304,
+                     100020625813, 100020618060, 100020585305, 100020617489, 100020615039, 100020618076, 100020588913,
+                     200001187197, 100020671205, 100020576940, 100020619814, 100020576472, 100020618083]
+archetype_2_uprns = [100020698027, 10001007455, 100020653785, 10090383198, 100020665632, 100020620659, 100020615603,
+                     100020609610, 100020625597, 100020665656, 100020665640, 100020587905, 100020665630, 100020624351,
+                     100020625451, 100020624348, 100020666735, 100020653786, 100020576458, 100020657902, 100020624350,
+                     100020637405, 100020666734, 100020616325, 100020666716, 100020653783, 100020665645, 100020642337,
+                     100020665638, 100022904981, 100020688226, 100020630285, 100020626800, 100020665634, 100022907528,
+                     100020665652, 100020624347, 100020666721, 100020585002, 10014055968, 10001008257, 100020621438,
+                     100020576459, 100020665643, 100020665654, 100022917303]
+archetype_3_uprns = [100020577523, 100020616446, 100020605342, 100020594652, 100020585394, 100020601138, 100020597485,
+                     100020614883, 100020633162, 100020697787, 200001185785, 100020646842, 100020581449, 100020595611,
+                     100020641814, 100020575611, 100020652986, 100020654671, 100020647336, 100020610518, 100020607980,
+                     100020692380, 100020581690]
+archetype_4_uprns = [100020650603, 100020582907, 100020605116, 100020650607, 100020589325, 100020655500, 100020642537,
+                     200001187539, 100020631683, 100020610165, 100020596436, 100020598277, 100020660228]
+
+
+def app():
+    """
+    We shall define a small portfolio of properties, based in Croydon
+    :return:
+    """
+
+    # Firstly, read in the EPC data for Croydon
+    epc_data = pd.read_csv(
+        "local_data/all-domestic-certificates/domestic-E09000008-Croydon/certificates.csv",
+        low_memory=False
+    )
+
+    z = epc_data[epc_data["MAINHEAT_DESCRIPTION"] == "Boiler and radiators, mains gas"]
+    z["HOTWATER_DESCRIPTION"].value_counts()
+    z["MAIN_FUEL"].value_counts()
+
+    # Filter on entries where we have a UPRN
+    epc_data = epc_data[~pd.isnull(epc_data["UPRN"])]
+
+    # Get the newest EPC for each UPRN. We use LODGEMENT_DATE as a proxy for this
+    epc_data["LODGEMENT_DATE"] = pd.to_datetime(epc_data["LODGEMENT_DATE"])
+
+    epc_data = epc_data.sort_values("LODGEMENT_DATE", ascending=False).drop_duplicates("UPRN")
+
+    # Now filter on social properties
+    epc_data = epc_data[epc_data["TENURE"].isin(["rental (social)", "Rented (social)"])]
+    # There are 17337 properties with a registered EPC in Croydon
+    # Take below EPC C properties
+    epc_data = epc_data[epc_data["CURRENT_ENERGY_EFFICIENCY"].astype(int) < 69]
+    # 7994 properties are below EPC C (46%)
+
+    # 79% D, 19% E, 1% F, 0.2% G - it probably makes the most sense to focus on E and D properties
+    epc_data["CURRENT_ENERGY_RATING"].value_counts(normalize=True)
+
+    # For the purpose of the sample, take the properties have surveys done in the last 3 years
+    # This gives us 1351 remaining properties
+    three_years_ago = pd.Timestamp.now() - pd.DateOffset(days=int(3 * 365))
+    epc_data = epc_data[epc_data["LODGEMENT_DATE"] >= three_years_ago]
+
+    # Archetype 1: defined below:
+    # 1) House
+    # 2) Unfilled cavity
+    # 3) A roof that could be insulated (flat or pitched with no more than 50mm insulation)
+    # 4) EPC E or D
+    # 24 properties
+    archetype_1_sample = epc_data[
+        epc_data["PROPERTY_TYPE"].isin(["House"]) &
+        (epc_data["CURRENT_ENERGY_RATING"].isin(["D", "E"])) &
+        epc_data["WALLS_DESCRIPTION"].isin(["Cavity wall, as built, no insulation (assumed)"]) &
+        epc_data["ROOF_DESCRIPTION"].isin(
+            [
+                "Pitched, 12 mm loft insulation",
+                "Pitched, 0 mm loft insulation",
+                "Pitched, no insulation",
+                "Pitched, 50 mm loft insulation",
+                "Flat, no insulation (assumed)",
+                "Pitched, no insulation (assumed)"
+            ]
+        )
+        ]
+    archetype_1_sample_asset_list = archetype_1_sample[["UPRN", "ADDRESS1", "POSTCODE"]].copy()
+    archetype_1_sample_asset_list["ARCHETYPE"] = "Archetype 1"
+
+    # Archetype 2: defined below:
+    # 1) Flat
+    # 2) Unfilled cavity
+    # 3) Another property above
+    # 4) EPC E
+    # 57 properties here
+    archetype_2_sample = epc_data[
+        epc_data["PROPERTY_TYPE"].isin(["Flat"]) &
+        (epc_data["CURRENT_ENERGY_RATING"].isin(["E", "D"])) &
+        epc_data["WALLS_DESCRIPTION"].isin(["Cavity wall, as built, no insulation (assumed)"]) &
+        epc_data["ROOF_DESCRIPTION"].isin(
+            [
+                "(another dwelling above)"
+            ]
+        )
+        ]
+    archetype_2_sample_asset_list = archetype_2_sample[["UPRN", "ADDRESS1", "POSTCODE"]].copy()
+    archetype_2_sample_asset_list["ARCHETYPE"] = "Archetype 2"
+
+    # Archetype 3: defined below:
+    # 1) EPC E or below
+    # 2) Solid brick wall
+    # 3) House
+    # 4) Pitched roof with no insulation
+    # Just 7 properties (more expensive to retrofit)
+    archetype_3_sample = epc_data[
+        epc_data["PROPERTY_TYPE"].isin(["House"]) &
+        (epc_data["CURRENT_ENERGY_RATING"].isin(["E", "F", "G"])) &
+        epc_data["WALLS_DESCRIPTION"].isin(["Solid brick, as built, no insulation (assumed)"]) &
+        epc_data["ROOF_DESCRIPTION"].isin(
+            [
+                "Pitched, no insulation",
+                "Pitched, limited insulation (assumed)",
+                "Pitched, 100 mm loft insulation",
+                "Pitched, no insulation (assumed)",
+            ]
+        )
+        ]
+    archetype_3_sample_asset_list = archetype_3_sample[["UPRN", "ADDRESS1", "POSTCODE"]].copy()
+    archetype_3_sample_asset_list["ARCHETYPE"] = "Archetype 3"
+
+    # Archetype 4: defined below:
+    # 1) Maisonette
+    # 2) Empty cavity
+    # 3) EPC E
+    # 16 properties here
+    archetype_4_sample = epc_data[
+        epc_data["PROPERTY_TYPE"].isin(["Maisonette"]) &
+        epc_data["WALLS_DESCRIPTION"].isin(
+            ["Cavity wall, as built, no insulation (assumed)"]
+        )
+        ]
+
+    archetype_4_sample_asset_list = archetype_4_sample[["UPRN", "ADDRESS1", "POSTCODE"]].copy()
+    archetype_4_sample_asset_list["ARCHETYPE"] = "Archetype 4"
+
+    asset_list = pd.concat(
+        [
+            archetype_1_sample_asset_list,
+            archetype_2_sample_asset_list,
+            archetype_3_sample_asset_list,
+            archetype_4_sample_asset_list
+        ]
+    )
+
+    asset_list = asset_list.rename(
+        columns={
+            "UPRN": "uprn",
+            "ADDRESS1": "address",
+            "POSTCODE": "postcode",
+            "ARCHETYPE": "archetype"
+        }
+    )
+
+    asset_list["uprn"] = asset_list["uprn"].astype(int)
+
+    # We end up with some properties that are currently an EPC C, but we do not have this data in the download, so we
+    # manually remove
+    # 1) 3 Reid Close, CR5 3BL
+    # 2) Flat 6, Collier Court 2A, St. Peters Road CR0 1HD
+    asset_list = asset_list[
+        ~asset_list["uprn"].isin(
+            [
+                100020576460,
+                100020624352,
+            ]
+        )
+    ]
+    # We have slightly too many properties, so we take a random sample of each archetype
+    # achetype_1_size = 20
+    # achetype_2_size = 46
+    # achetype_3_size = 23
+    # achetype_4_size = 13
+    # archetype_1_uprns = asset_list[asset_list["archetype"] == "Archetype 1"]["uprn"].sample(
+    #     int(achetype_1_size)
+    # ).tolist()
+    # archetype_2_uprns = asset_list[asset_list["archetype"] == "Archetype 2"]["uprn"].sample(
+    #     int(achetype_2_size)
+    # ).tolist()
+    # archetype_3_uprns = asset_list[asset_list["archetype"] == "Archetype 3"]["uprn"].sample(
+    #     int(achetype_3_size)
+    # ).tolist()
+    # archetype_4_uprns = asset_list[asset_list["archetype"] == "Archetype 4"]["uprn"].sample(
+    #     int(achetype_4_size)
+    # ).tolist()
+    uprns_to_keep = archetype_1_uprns + archetype_2_uprns + archetype_3_uprns + archetype_4_uprns
+    asset_list = asset_list[asset_list["uprn"].isin(uprns_to_keep)]
+
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/inputs.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Social",
+        "goal": "Increase EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "budget": None,
+        "exclusions": ["floor_insulation"]
+    }
+    print(body)

etl/customers/gla_croydon_demo/slides.py

@@ -0,0 +1,760 @@
+"""
+This script contains the code to generate the data required to populate the slides
+We connect to the database amd extract the data for the portfolio needed so it is recommended to use
+a environment akin to the backend to run this script
+"""
+import pandas as pd
+import numpy as np
+from backend.app.db.connection import db_engine
+from sqlalchemy.orm import sessionmaker
+from utils.s3 import read_csv_from_s3
+from etl.customers.slide_utils import (
+    plot_epc_distribution,
+    get_property_details_by_portfolio_id,
+    get_plan_by_portfolio_id,
+    get_properties_with_default_recommendations,
+    create_powerpoint,
+    create_recommendations_summary
+)
+from backend.ml_models.AnnualBillSavings import AnnualBillSavings
+
+USER_ID = 8
+PORTFOLIO_ID_1 = 67
+PORTFOLIO_ID_2 = 68
+EPC_TARGET_1 = "C"
+EPC_TARGET_2 = "A"
+SAP_TARGET_1 = 69
+SAP_TARGET_2 = 100
+CUSTOMER_KEY = "gla-demo"
+
+# Sample UPRNS
+archetype_1_sample = ['100020604138', '200001192253', '100020581792', '100020576940', '200001187196', '100020618060',
+                      '100020625813', '100020578756', '100020618076', '200001187197', '100020619814', '100020617489',
+                      '100020588913']
+
+archetype_2_sample = ['100020585002', '100020615603', '100020665652', '100020626800', '100020624347', '100020624348',
+                      '100020576459', '10001007455', '100020666716', '100020609610', '100020625451', '100020625597',
+                      '100020624351', '100020665634', '100020624350', '100020665640', '100020665632', '100022917303',
+                      '100020665656', '10014055968', '100020630285', '100020665638', '100020616325', '100020637405',
+                      '100020698027', '100020657902', '100020688226', '100020653786', '100020642337', '100020665643']
+
+archetype_3_sample = ['100020594652', '100020697787', '100020577523', '100020633162', '100020601138', '100020595611',
+                      '100020597485', '100020614883', '100020605342', '100020654671', '100020575611', '100020607980',
+                      '200001185785', '100020616446', '100020692380']
+
+archetype_4_sample = ['100020596436', '100020610165', '200001187539', '100020655500', '100020582907', '100020598277',
+                      '100020650607', '100020605116', '100020650603']
+
+
+def scenario_1():
+    # Connect to database
+    session = sessionmaker(bind=db_engine)()
+
+    ########################################################################
+    # Get the data we need
+    ########################################################################
+
+    portfolio_id = PORTFOLIO_ID_1
+
+    # Get the asset list
+    asset_list = read_csv_from_s3(
+        "retrofit-plan-inputs-dev", f"{USER_ID}/67/inputs.csv"
+    )
+    asset_list = pd.DataFrame(asset_list)
+
+    # Get the properties for the portfolio
+    properties = get_properties_with_default_recommendations(session, portfolio_id)
+    properties_df = pd.DataFrame(properties)
+
+    # We now pull the data for the property details
+    property_details = get_property_details_by_portfolio_id(session, portfolio_id)
+    property_details_df = pd.DataFrame(property_details)
+    # We estimate bills based on the adjusted_energy_consumption
+    property_details_df["energy_bill"] = property_details_df["adjusted_energy_consumption"].apply(
+        lambda x: AnnualBillSavings.calculate_annual_bill(x)
+    )
+    # Merge on uprn
+    property_details_df = property_details_df.merge(
+        properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}),
+        on="property_id"
+    )
+
+    plans = get_plan_by_portfolio_id(session, portfolio_id)
+    plans_df = pd.DataFrame(plans)
+
+    # Unnest the recommendations. Each recommendation is a list of dictionaries
+    recommendations_exploded = properties_df["recommendations"].explode().tolist()
+    recommendations_df = pd.DataFrame([r for r in recommendations_exploded if not pd.isnull(r)])
+    # Add uprn on
+    recommendations_df = recommendations_df.merge(
+        properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}),
+        how="left",
+        on="property_id"
+    )
+
+    recommendations_summary = create_recommendations_summary(
+        recommendations_df,
+        properties_df,
+        property_details_df,
+        SAP_TARGET_1
+    )
+
+    # Calculate % changes of energ, co2 and abs
+    recommendations_summary["carbon_percent_change"] = (
+        recommendations_summary["total_carbon"] / recommendations_summary["current_co2"]
+    )
+
+    recommendations_summary["energy_percent_change"] = (
+        recommendations_summary["adjusted_heat_demand"] / recommendations_summary["current_energy"]
+    )
+
+    recommendations_summary["bills_percent_change"] = (
+        recommendations_summary["total_bill_savings"] / recommendations_summary["current_energy_bill"]
+    )
+
+    ########################
+    # Overview
+    ########################
+    overview_totals = recommendations_summary.sum()
+    overview_means = recommendations_summary.mean()
+
+    ########################
+    # Measures
+    ########################
+    measures_count = recommendations_df.groupby("type")["id"].count().reset_index()
+    wall_insulation_measures = measures_count[
+        measures_count["type"].isin(["cavity_wall_insulation", "external_wall_insulation", "internal_wall_insulation"])
+    ]["id"].sum()
+    ventilation_measures = measures_count[
+        measures_count["type"].isin(["mechanical_ventilation"])
+    ]["id"].sum()
+    roof_insulation_measures = measures_count[
+        measures_count["type"].isin(["loft_insulation", "flat_roof_insulation"])
+    ]["id"].sum()
+    floor_insulation_measures = measures_count[
+        measures_count["type"].isin(["solid_floor_insulation", "suspended_floor_insulation"])
+    ]["id"].sum()
+    windows = measures_count[
+        measures_count["type"].isin(["windows_glazing"])
+    ]["id"].sum()
+    heating = measures_count[
+        measures_count["type"].isin(["heating"])
+    ]["id"].sum()
+    heating_controls = measures_count[
+        measures_count["type"].isin(["heating_control"])
+    ]["id"].sum()
+    solar = measures_count[
+        measures_count["type"].isin(["solar_pv"])
+    ]["id"].sum()
+    other = measures_count[
+        ~measures_count["type"].isin([
+            "cavity_wall_insulation", "external_wall_insulation", "internal_wall_insulation",
+            "loft_insulation", "flat_roof_insulation", "solid_floor_insulation",
+            "suspended_floor_insulation", "windows_glazing", "heating", "heating_control", "solar_pv",
+            "mechanical_ventilation"
+        ])
+    ]["id"].sum()
+
+    # Summary information by each archetype
+    ########################
+    # Archetype 1
+    ########################
+    archetype_1 = asset_list[asset_list["archetype"] == "Archetype 1"]
+    recommendations_arch_1_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_1["uprn"].values)
+    ]
+
+    arch_1_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_1["uprn"].values)
+    ]
+    arch_1_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    cols_to_keep = ["total_cost", "total_carbon", "total_bill_savings", "total_sap_points", "adjusted_heat_demand",
+                    "energy_percent_change", "carbon_percent_change", "bills_percent_change"]
+    arch_1_recommendation_min = recommendations_arch_1_summary.min()[cols_to_keep]
+    arch_1_recommendation_max = recommendations_arch_1_summary.max()[cols_to_keep]
+    arch_1_recommendation_means = recommendations_arch_1_summary.mean()[cols_to_keep]
+    arch_1_totals = recommendations_arch_1_summary.sum()[cols_to_keep]
+
+    annual_total_co2 = recommendations_arch_1_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_1_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_1_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_1["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_1_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_1_recommendation_min['total_cost']} - {arch_1_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_1_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_1_recommendation_min['total_sap_points']} - {arch_1_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_1_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_1_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_1_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_1_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_1_recommendation_min['energy_percent_change']} - "
+                           f"{arch_1_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_1_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_1_recommendation_min['total_carbon']} - {arch_1_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_1_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_1_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_1_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_1_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_1_recommendation_min['total_bill_savings']} - "
+                 f"{arch_1_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_1_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_1_recommendation_min['bills_percent_change']} - "
+                         f"{arch_1_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 2
+    ########################
+    archetype_2 = asset_list[asset_list["archetype"] == "Archetype 2"]
+    recommendations_arch_2_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_2["uprn"].values)
+    ]
+
+    arch_2_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_2["uprn"].values)
+    ]
+    arch_2_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_2_recommendation_min = recommendations_arch_2_summary.min()
+    arch_2_recommendation_max = recommendations_arch_2_summary.max()
+    arch_2_recommendation_means = recommendations_arch_2_summary.mean().round(2)
+
+    total_cost = recommendations_arch_2_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_2_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_2_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_2_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_2["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_2_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_2_recommendation_min['total_cost']} - {arch_2_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_2_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_2_recommendation_min['total_sap_points']} - {arch_2_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_2_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_2_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_2_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_2_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_2_recommendation_min['energy_percent_change']} - "
+                           f"{arch_2_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_2_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_2_recommendation_min['total_carbon']} - {arch_2_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_2_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_2_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_2_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_2_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_2_recommendation_min['total_bill_savings']} - "
+                 f"{arch_2_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_2_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_2_recommendation_min['bills_percent_change']} - "
+                         f"{arch_2_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 3
+    ########################
+    archetype_3 = asset_list[asset_list["archetype"] == "Archetype 3"]
+    recommendations_arch_3_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_3["uprn"].values)
+    ]
+
+    arch_3_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_3["uprn"].values)
+    ]
+    arch_3_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_3_recommendation_min = recommendations_arch_3_summary.min()
+    arch_3_recommendation_max = recommendations_arch_3_summary.max()
+    arch_3_recommendation_means = recommendations_arch_3_summary.mean()
+
+    total_cost = recommendations_arch_3_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_3_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_3_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_3_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_3_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_3_recommendation_min['total_cost']} - {arch_3_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_3_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_3_recommendation_min['total_sap_points']} - {arch_3_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_3_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_3_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_3_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_3_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_3_recommendation_min['energy_percent_change']} - "
+                           f"{arch_3_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_3_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_3_recommendation_min['total_carbon']} - {arch_3_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_3_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_3_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_3_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_3_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_3_recommendation_min['total_bill_savings']} - "
+                 f"{arch_3_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_3_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_3_recommendation_min['bills_percent_change']} - "
+                         f"{arch_3_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 4
+    ########################
+    archetype_4 = asset_list[asset_list["archetype"] == "Archetype 4"]
+    recommendations_arch_4_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_4["uprn"].values)
+    ]
+
+    arch_4_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_4["uprn"].values)
+    ]
+    arch_4_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_4_recommendation_min = recommendations_arch_4_summary.min()
+    arch_4_recommendation_max = recommendations_arch_4_summary.max()
+    arch_4_recommendation_means = recommendations_arch_4_summary.mean()
+
+    total_cost = recommendations_arch_4_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_4_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_4_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_4_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_4["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_4_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_4_recommendation_min['total_cost']} - {arch_4_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_4_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_4_recommendation_min['total_sap_points']} - {arch_4_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_4_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_4_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_4_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_4_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_4_recommendation_min['energy_percent_change']} - "
+                           f"{arch_4_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_4_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_4_recommendation_min['total_carbon']} - {arch_4_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_4_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_4_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_4_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_4_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_4_recommendation_min['total_bill_savings']} - "
+                 f"{arch_4_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_4_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_4_recommendation_min['bills_percent_change']} - "
+                         f"{arch_4_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Overview
+    ########################
+    overview_totals = recommendations_summary.sum()
+
+
+def make_sample():
+    # sample_proportion = 67 / 102
+    # Get the asset list
+    asset_list = read_csv_from_s3(
+        "retrofit-plan-inputs-dev", f"{USER_ID}/67/inputs.csv"
+    )
+    asset_list = pd.DataFrame(asset_list)
+
+    # From the asset list, we deduce how many properties we need
+    # Need to figure out the sizes
+    archetype_1_sample_size = 13
+    archetype_2_sample_size = 30
+    archetype_3_sample_size = 15
+    archetype_4_sample_size = 9
+
+    # We take the sample and we'll keep the uprns static
+    archetype_1_sample = asset_list[
+        asset_list["archetype"] == "Archetype 1"
+        ].sample(archetype_1_sample_size)["uprn"].to_list()
+
+    archetype_2_sample = asset_list[
+        asset_list["archetype"] == "Archetype 2"
+        ].sample(archetype_2_sample_size)["uprn"].to_list()
+
+    archetype_3_sample = asset_list[
+        asset_list["archetype"] == "Archetype 3"
+        ].sample(archetype_3_sample_size)["uprn"].to_list()
+
+    archetype_4_sample = asset_list[
+        asset_list["archetype"] == "Archetype 4"
+        ].sample(archetype_4_sample_size)["uprn"].to_list()
+
+
+def scenario_2():
+    # Connect to database
+    session = sessionmaker(bind=db_engine)()
+
+    ########################################################################
+    # Get the data we need
+    ########################################################################
+
+    portfolio_id = PORTFOLIO_ID_2
+
+    # Get the asset list
+    asset_list = read_csv_from_s3(
+        "retrofit-plan-inputs-dev", f"{USER_ID}/67/inputs.csv"
+    )
+    asset_list = pd.DataFrame(asset_list)
+
+    sample_uprns = archetype_1_sample + archetype_2_sample + archetype_3_sample + archetype_4_sample
+
+    # Filter on sample uprns
+    asset_list = asset_list[asset_list["uprn"].astype(str).isin(sample_uprns)]
+
+    # Get the properties for the portfolio
+    properties = get_properties_with_default_recommendations(session, portfolio_id)
+    properties_df = pd.DataFrame(properties)
+    properties_df = properties_df[properties_df["uprn"].astype(str).isin(sample_uprns)]
+
+    # We now pull the data for the property details
+    property_details = get_property_details_by_portfolio_id(session, portfolio_id)
+    property_details_df = pd.DataFrame(property_details)
+    property_details_df = property_details_df[property_details_df["property_id"].isin(properties_df["id"].values)]
+    # We estimate bills based on the adjusted_energy_consumption
+    property_details_df["energy_bill"] = property_details_df["adjusted_energy_consumption"].apply(
+        lambda x: AnnualBillSavings.calculate_annual_bill(x)
+    )
+    # Merge on uprn
+    property_details_df = property_details_df.merge(
+        properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}),
+        on="property_id"
+    )
+
+    plans = get_plan_by_portfolio_id(session, portfolio_id)
+    plans_df = pd.DataFrame(plans)
+
+    # Unnest the recommendations. Each recommendation is a list of dictionaries
+    recommendations_exploded = properties_df["recommendations"].explode().tolist()
+    recommendations_df = pd.DataFrame([r for r in recommendations_exploded if not pd.isnull(r)])
+    # Add uprn on
+    recommendations_df = recommendations_df.merge(
+        properties_df[["uprn", "id"]].rename(columns={"id": "property_id"}),
+        how="left",
+        on="property_id"
+    )
+
+    recommendations_summary = create_recommendations_summary(
+        recommendations_df,
+        properties_df,
+        property_details_df,
+        SAP_TARGET_1
+    )
+
+    # Calculate % changes of energ, co2 and abs
+    recommendations_summary["carbon_percent_change"] = (
+        recommendations_summary["total_carbon"] / recommendations_summary["current_co2"]
+    )
+
+    recommendations_summary["energy_percent_change"] = (
+        recommendations_summary["adjusted_heat_demand"] / recommendations_summary["current_energy"]
+    )
+
+    recommendations_summary["bills_percent_change"] = (
+        recommendations_summary["total_bill_savings"] / recommendations_summary["current_energy_bill"]
+    )
+
+    ########################
+    # Overview
+    ########################
+    overview_totals = recommendations_summary.sum()
+    overview_means = recommendations_summary.mean()
+
+    ########################
+    # Measures
+    ########################
+    measures_count = recommendations_df.groupby("type")["id"].count().reset_index()
+    wall_insulation_measures = measures_count[
+        measures_count["type"].isin(["cavity_wall_insulation", "external_wall_insulation", "internal_wall_insulation"])
+    ]["id"].sum()
+    ventilation_measures = measures_count[
+        measures_count["type"].isin(["mechanical_ventilation"])
+    ]["id"].sum()
+    roof_insulation_measures = measures_count[
+        measures_count["type"].isin(["loft_insulation", "flat_roof_insulation"])
+    ]["id"].sum()
+    floor_insulation_measures = measures_count[
+        measures_count["type"].isin(["solid_floor_insulation", "suspended_floor_insulation"])
+    ]["id"].sum()
+    windows = measures_count[
+        measures_count["type"].isin(["windows_glazing"])
+    ]["id"].sum()
+    heating = measures_count[
+        measures_count["type"].isin(["heating"])
+    ]["id"].sum()
+    heating_controls = measures_count[
+        measures_count["type"].isin(["heating_control"])
+    ]["id"].sum()
+    solar = measures_count[
+        measures_count["type"].isin(["solar_pv"])
+    ]["id"].sum()
+    other = measures_count[
+        ~measures_count["type"].isin([
+            "cavity_wall_insulation", "external_wall_insulation", "internal_wall_insulation",
+            "loft_insulation", "flat_roof_insulation", "solid_floor_insulation",
+            "suspended_floor_insulation", "windows_glazing", "heating", "heating_control", "solar_pv",
+            "mechanical_ventilation"
+        ])
+    ]["id"].sum()
+
+    z = recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3_sample)]
+
+    recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3_sample)]["type"].value_counts()
+
+    # Summary information by each archetype
+    ########################
+    # Archetype 1
+    ########################
+    archetype_1 = asset_list[asset_list["archetype"] == "Archetype 1"]
+    recommendations_arch_1_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_1["uprn"].values)
+    ]
+
+    arch_1_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_1["uprn"].values)
+    ]
+    arch_1_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_1_recommendation_min = recommendations_arch_1_summary.min()
+    arch_1_recommendation_max = recommendations_arch_1_summary.max()
+    arch_1_recommendation_means = recommendations_arch_1_summary.mean()
+
+    arch_1_totals = recommendations_arch_1_summary.sum()
+
+    annual_total_co2 = recommendations_arch_1_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_1_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_1_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_1["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_1_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_1_recommendation_min['total_cost']} - {arch_1_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_1_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_1_recommendation_min['total_sap_points']} - {arch_1_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_1_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_1_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_1_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_1_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_1_recommendation_min['energy_percent_change']} - "
+                           f"{arch_1_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_1_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_1_recommendation_min['total_carbon']} - {arch_1_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_1_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_1_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_1_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_1_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_1_recommendation_min['total_bill_savings']} - "
+                 f"{arch_1_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_1_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_1_recommendation_min['bills_percent_change']} - "
+                         f"{arch_1_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 2
+    ########################
+    archetype_2 = asset_list[asset_list["archetype"] == "Archetype 2"]
+    recommendations_arch_2_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_2["uprn"].values)
+    ]
+
+    arch_2_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_2["uprn"].values)
+    ]
+    arch_2_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_2_recommendation_min = recommendations_arch_2_summary.min()
+    arch_2_recommendation_max = recommendations_arch_2_summary.max()
+    arch_2_recommendation_means = recommendations_arch_2_summary.mean().round(2)
+
+    total_cost = recommendations_arch_2_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_2_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_2_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_2_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_2["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_2_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_2_recommendation_min['total_cost']} - {arch_2_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_2_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_2_recommendation_min['total_sap_points']} - {arch_2_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_2_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_2_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_2_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_2_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_2_recommendation_min['energy_percent_change']} - "
+                           f"{arch_2_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_2_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_2_recommendation_min['total_carbon']} - {arch_2_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_2_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_2_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_2_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_2_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_2_recommendation_min['total_bill_savings']} - "
+                 f"{arch_2_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_2_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_2_recommendation_min['bills_percent_change']} - "
+                         f"{arch_2_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 3
+    ########################
+    archetype_3 = asset_list[asset_list["archetype"] == "Archetype 3"]
+    recommendations_arch_3_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_3["uprn"].values)
+    ]
+
+    arch_3_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_3["uprn"].values)
+    ]
+    arch_3_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_3_recommendation_min = recommendations_arch_3_summary.min()
+    arch_3_recommendation_max = recommendations_arch_3_summary.max()
+    arch_3_recommendation_means = recommendations_arch_3_summary.mean()
+
+    total_cost = recommendations_arch_3_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_3_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_3_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_3_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_3["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_3_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_3_recommendation_min['total_cost']} - {arch_3_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_3_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_3_recommendation_min['total_sap_points']} - {arch_3_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_3_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_3_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_3_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_3_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_3_recommendation_min['energy_percent_change']} - "
+                           f"{arch_3_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_3_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_3_recommendation_min['total_carbon']} - {arch_3_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_3_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_3_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_3_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_3_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_3_recommendation_min['total_bill_savings']} - "
+                 f"{arch_3_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_3_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_3_recommendation_min['bills_percent_change']} - "
+                         f"{arch_3_recommendation_max['bills_percent_change']}")
+
+    ########################
+    # Archetype 4
+    ########################
+    archetype_4 = asset_list[asset_list["archetype"] == "Archetype 4"]
+    recommendations_arch_4_summary = recommendations_summary[
+        recommendations_summary["uprn"].astype(str).isin(archetype_4["uprn"].values)
+    ]
+
+    arch_4_property_details = property_details_df[
+        property_details_df["uprn"].astype(str).isin(archetype_4["uprn"].values)
+    ]
+    arch_4_property_details["co2_emissions"].sum() / property_details_df["co2_emissions"].sum()
+
+    # Take the mean, median and maximum of each value
+    arch_4_recommendation_min = recommendations_arch_4_summary.min()
+    arch_4_recommendation_max = recommendations_arch_4_summary.max()
+    arch_4_recommendation_means = recommendations_arch_4_summary.mean()
+
+    total_cost = recommendations_arch_4_summary["total_cost"].sum()
+    annual_total_co2 = recommendations_arch_4_summary["total_carbon"].sum()
+    annual_total_bills = recommendations_arch_4_summary["total_bill_savings"].sum()
+    annual_total_energy_savings = recommendations_arch_4_summary["adjusted_heat_demand"].sum()
+    archetype_measures = \
+        recommendations_df[recommendations_df["uprn"].astype(str).isin(archetype_4["uprn"].values)].groupby("type")[
+            "id"].count().reset_index()
+
+    cost_text = (f"{round(arch_4_recommendation_means['total_cost'], 2)}: "
+                 f"{arch_4_recommendation_min['total_cost']} - {arch_4_recommendation_max['total_cost']}")
+
+    sap_text = (f"{round(arch_4_recommendation_means['total_sap_points'], 2)}: "
+                f"{arch_4_recommendation_min['total_sap_points']} - {arch_4_recommendation_max['total_sap_points']}")
+
+    energy_text = (f"{round(arch_4_recommendation_means['adjusted_heat_demand'], 2)}: "
+                   f"{arch_4_recommendation_min['adjusted_heat_demand']} - "
+                   f"{arch_4_recommendation_max['adjusted_heat_demand']}")
+
+    energy_percent_text = (f"{round(arch_4_recommendation_means['energy_percent_change'], 2)}: "
+                           f"{arch_4_recommendation_min['energy_percent_change']} - "
+                           f"{arch_4_recommendation_max['energy_percent_change']}")
+
+    carbon_text = (f"{round(arch_4_recommendation_means['total_carbon'], 2)}: "
+                   f"{arch_4_recommendation_min['total_carbon']} - {arch_4_recommendation_max['total_carbon']}")
+
+    carbon_percent_text = (f"{round(arch_4_recommendation_means['carbon_percent_change'], 2)}: "
+                           f"{arch_4_recommendation_min['carbon_percent_change']} - "
+                           f"{arch_4_recommendation_max['carbon_percent_change']}")
+
+    bill_text = (f"{round(arch_4_recommendation_means['total_bill_savings'], 2)}: "
+                 f"{arch_4_recommendation_min['total_bill_savings']} - "
+                 f"{arch_4_recommendation_max['total_bill_savings']}")
+
+    bill_percent_text = (f"{round(arch_4_recommendation_means['bills_percent_change'], 2)}: "
+                         f"{arch_4_recommendation_min['bills_percent_change']} - "
+                         f"{arch_4_recommendation_max['bills_percent_change']}")

etl/customers/immo/pilot/asset_list.py

@@ -0,0 +1,157 @@
+import pandas as pd
+from utils.s3 import read_excel_from_s3
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 70
+
+council_tax_bands = [
+    {'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'band': 'A'},
+    {'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'band': 'A'},
+    {'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'band': 'A'},
+    {'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'band': 'A'},
+    {'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'band': 'A'},
+    {'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'band': 'B'},
+    {'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'band': 'B'},
+    {'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'band': 'C'},
+    {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'band': 'A'},
+    {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'band': 'A'},
+]
+council_tax_bands = pd.DataFrame(council_tax_bands)
+
+# This is information we need to override on the EPC itself, for instance if a new survey has been conducted and
+# that has not reached the API
+# For 53 Bromley, the non-invasives found the walls to be partially filled
+patches = [
+    {
+        'address': '6 Beech Road', 'postcode': 'DY1 4BP',
+        'walls-description': 'Cavity wall, filled cavity',
+        'walls-energy-eff': 'Good',
+        'roof-description': 'Pitched, 12 mm loft insulation',
+        'roof-energy-eff': 'Very Poor',
+        'windows-description': 'Fully double glazed',
+        'windows-energy-eff': 'Good',
+        'mainheat-description': 'Room heaters, electric',
+        'mainheat-energy-eff': 'Very Poor',
+        'mainheatcont-description': 'Appliance thermostats',
+        'mainheatc-energy-eff': 'Good',
+        'lighting-description': 'Low energy lighting in 25% of fixed outlets',
+        'lighting-energy-eff': 'Good',
+        'floor-description': 'Solid, no insulation (assumed)',
+        'secondheat-description': 'None',
+        'current-energy-efficiency': '32',
+        'energy-consumption-current': '491',
+        'co2-emissions-current': '5.0',
+        'potential-energy-efficiency': '87'
+    },
+    {
+        'address': '53 Bromley', 'postcode': 'DY5 4PJ',
+        'walls-description': 'Cavity wall, partial insulation (assumed)',
+    },
+]
+
+# This is information that is found as a result of the non-invasives, that mean that certain measures
+# have been installed already. To reflect this in the front end, it is included in the recommendation, however
+# the cost is removed and instead, a message is presented saying that the measure is already installed.
+already_installed = [
+    {
+        'address': '5 Oaklands',
+        'postcode': 'B62 0JA',
+        "already_installed": ["windows_glazing"]
+    }
+]
+
+non_invasive_recommendations = [
+    {'address': '8 Corporation Road', 'postcode': 'DY2 7PX', 'recommendations': []},
+    {'address': '21 Wells Road', 'postcode': 'DY5 3TB', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '27 Milton Road', 'postcode': 'WV14 8HZ', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '195 Ashenhurst Road', 'postcode': 'DY1 2JB', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '53 Bromley', 'postcode': 'DY5 4PJ', 'recommendations': ['cavity_surveyed_as_filled_is_partial']},
+    {'address': '91 Osprey Drive', 'postcode': 'DY1 2JS', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '47 Fairfield Road', 'postcode': 'DY8 5UJ', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '150 Huntingtree Road', 'postcode': 'B63 4HP', 'recommendations': ['cavity_extract_and_refill']},
+    {'address': '6 Beech Road', 'postcode': 'DY1 4BP', 'recommendations': []},
+    {'address': '5 Oaklands', 'postcode': 'B62 0JA', 'recommendations': ['cavity_extract_and_refill']},
+]
+
+
+def app():
+    raw_asset_list = read_excel_from_s3(
+        bucket_name="retrofit-datalake-dev",
+        file_key="customers/Immo/IMMO Sample Assets_Dudley.xlsx",
+        header_row=0
+    )
+    raw_asset_list = raw_asset_list.drop(columns=["Unnamed: 0"])
+    # Extract address and postcode
+    raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0]
+    raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip()
+
+    asset_list = raw_asset_list.merge(council_tax_bands, how="left", on=["address", "postcode"])
+
+    # We're provided with number of bathrooms and number of bedrooms.
+    asset_list = asset_list.rename(
+        columns={
+            "No. of Beds": "n_bedrooms",
+            "No. of WC's": "n_bathrooms"
+        }
+    )
+
+    # Store the asset list in s3
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    # Store overrides in s3
+    already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(already_installed),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=already_installed_filename
+    )
+
+    # Store patches in s3
+    patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(patches),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=patches_filename
+    )
+
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    # EPC C portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)
+
+    # EPC B portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID + 1),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "B",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)

etl/customers/immo/pilot/asset_list_2.py

@@ -0,0 +1,152 @@
+import pandas as pd
+from utils.s3 import read_excel_from_s3
+from utils.s3 import save_csv_to_s3
+
+USER_ID = 8
+PORTFOLIO_ID = 72
+
+# For
+patches = [
+    {
+        'address': '116 Parkes Hall Road',
+        'postcode': 'DY1 3RJ',
+        'uprn': '90046817',
+        'walls-description': 'Cavity wall, filled cavity',
+        'walls-energy-eff': 'Average',
+        'roof-description': 'Pitched, 270 mm loft insulation',
+        'roof-energy-eff': 'Good',
+        'windows-description': 'Fully double glazed',
+        'windows-energy-eff': 'Good',
+        'mainheat-description': 'Boiler and radiators, mains gas',
+        'mainheat-energy-eff': 'Good',
+        'mainheatcont-description': 'Programmer, room thermostat and TRVs',
+        'mainheatc-energy-eff': 'Good',
+        'lighting-description': 'Low energy lighting in 27% of fixed outlets',
+        'lighting-energy-eff': 'Average',
+        'floor-description': 'Solid, no insulation (assumed)',
+        'secondheat-description': 'None',
+        'current-energy-efficiency': '73',
+        'current-energy-rating': 'C',
+        'energy-consumption-current': '184',
+        'co2-emissions-current': '2.4',
+        'potential-energy-efficiency': '88',
+        'total-floor-area': '73',
+        'construction-age-band': 'England and Wales: 1930-1949',
+        'property-type': 'House',
+        'built-form': 'Mid-Terrace',
+    }
+]
+
+# This is information that is found as a result of the non-invasives, that mean that certain measures
+# have been installed already. To reflect this in the front end, it is included in the recommendation, however
+# the cost is removed and instead, a message is presented saying that the measure is already installed.
+already_installed = [
+    {
+        'address': '28 Sangwin Road', 'postcode': 'WV14 9EQ', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '51 Hillwood Road', 'postcode': 'B62 8NQ', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '47 Watsons Close', 'postcode': 'DY2 7HL', "already_installed": ["loft_insulation"]
+    },
+    {
+        'address': '44 Hatfield Road',
+        'postcode': 'DY9 7LW',
+        "already_installed": ["loft_insulation", "cavity_wall_insulation"]
+    }
+]
+
+non_invasive_recommendations = []
+
+
+def app():
+    raw_asset_list = read_excel_from_s3(
+        bucket_name="retrofit-datalake-dev",
+        file_key="customers/Immo/Dudley Asset List - Hestia - pilot2.xlsx",
+        header_row=0
+    )
+
+    raw_asset_list = raw_asset_list[raw_asset_list["in_pilot"]].copy()
+
+    # Extract address and postcode
+    raw_asset_list["address"] = raw_asset_list["Full Address"].str.split(",").str[0]
+    raw_asset_list["postcode"] = raw_asset_list["Full Address"].str.split(",").str[-1].str.strip()
+
+    # We're provided with number of bathrooms and number of bedrooms.
+    # THe UPRNs are not the official ones
+    asset_list = raw_asset_list.rename(
+        columns={
+            "No. of Beds": "n_bedrooms",
+            "No. of WC's": "n_bathrooms",
+            'Property Type': 'property_type',
+            'Architype': 'built_form'
+        }
+    )
+
+    # Remap the values
+    asset_list["built_form"] = asset_list["built_form"].map({
+        "SEMI DETACHED": "Semi-Detached",
+        "MID TERRACE": "Mid-Terrace",
+        "END TERRACE": "End-Terrace",
+    })
+
+    # Store the asset list in s3
+    filename = f"{USER_ID}/{PORTFOLIO_ID}/pilot.csv"
+    save_csv_to_s3(
+        dataframe=asset_list,
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=filename
+    )
+
+    # Store overrides in s3
+    already_installed_filename = f"{USER_ID}/{PORTFOLIO_ID}/already_installed.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(already_installed),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=already_installed_filename
+    )
+
+    # Store patches in s3
+    patches_filename = f"{USER_ID}/{PORTFOLIO_ID}/patches.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(patches),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=patches_filename
+    )
+
+    # Store non-invasive recommendations in S3
+    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.json"
+    save_csv_to_s3(
+        dataframe=pd.DataFrame(non_invasive_recommendations),
+        bucket_name="retrofit-plan-inputs-dev",
+        file_name=non_invasive_recommendations_filename
+    )
+
+    # EPC C portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "C",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)
+
+    # EPC B portoflio
+    body = {
+        "portfolio_id": str(PORTFOLIO_ID + 1),
+        "housing_type": "Private",
+        "goal": "Increase EPC",
+        "goal_value": "B",
+        "trigger_file_path": filename,
+        "already_installed_file_path": already_installed_filename,
+        "patches_file_path": patches_filename,
+        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
+        "budget": None,
+    }
+    print(body)

etl/customers/immo/pilot/non_invasive.py

@@ -0,0 +1,210 @@
+# import extract_msg
+from datetime import datetime
+from sqlalchemy.orm import sessionmaker
+from backend.app.db.connection import db_engine
+from backend.app.db.functions.non_intrusive_surveys import upload_non_intrusive_survey_notes
+
+
+def parse_msg_body(text):
+    # Split the text into lines
+    lines = text.split('\r\n')
+
+    # Dictionary to hold the parsed data
+    data = {}
+
+    # Process each line
+    for line in lines:
+        # Remove all asterisks and extra whitespace
+        clean_line = line.replace('*', '').strip()
+
+        if clean_line:  # Ensure the line is not empty after cleaning
+            # Attempt to split clean '=' if present
+            if '=' in clean_line:
+                clean_line = clean_line.replace(' = ', ': ')
+
+            # Use line content as a key with a default value indicating presence
+            # Generate a unique key for lines without '='
+            data[f"Info{len(data) + 1}"] = clean_line
+
+    return data
+
+
+def app():
+    """
+    This code retrieves the results of the non-invasive surveys, to be stored in S3
+    :return:
+    """
+
+    # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/5 Oaklands B62 "
+    #             "0JA/Immo - 5 Oaklands Halesowen B62 0JA.msg")
+    # filepath = ("/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/6 Beech Rd DY1 "
+    #             "4BP/IMMO - 6 Beech Road Dudley DY1 4BP.msg")
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/8 Corporation Rd DY2 "
+    #     "7PX/IMMO - 8 Corporation Road Dudley DY2 7PX.msg"
+    # )
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/21 Wells Rd DY5 3TB/"
+    #     "IMMO - 21 Wells Road Brierley Hill DY5 3TB.msg"
+    # )
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/47 Fairfield Rd DY8 "
+    #     "5UJ/IMMO - 47 Fairfield Road Wordsley Stourbridge DY8 5UJ.msg"
+    # )
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/91 Osprey Drive DY1 "
+    #     "2JS/IMMO - 91 Osprey Drive Dudley DY1 2JS.msg"
+    # )
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/195 Ashenhurst Rd DY1 "
+    #     "2JB/IMMO - 195 Ashenhurst Road Dudley DY1 2JB.msg"
+    # )
+    # filepath = (
+    #     "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data/27 Milton Rd DY1 2JB/IMMO "
+    #     "- 27 Milton Road Coseley Bilston WV14 8HZ.msg"
+    # )
+    #
+    # with extract_msg.Message(filepath) as msg:
+    #     body = msg.body
+    #
+    # from pprint import pprint
+    # pprint(parse_msg_body(body))
+
+    # We manually create the non-invasive notes for the pilot
+    non_invasive_notes = [
+        {
+            'uprn': 90028499,
+            # 'address': '5 Oaklands',
+            # 'postcode': 'B62 0JA',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation. '
+                               'There is a shared alleyway with the neighbour, that is a solid brick wall.',
+            'Wall Render': 'Partial render between top of ground floor window and bottom of 1st floor window',
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: North East, Back house direction: South West',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90055152,
+            # 'address': '6 Beech Road',
+            # 'postcode': 'DY1 4BP',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': '1st floor is solid brick with external wall insulation. 2nd floor is cavity, '
+                               'retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': None,
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Side house direction: North East',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90070461,
+            # 'address': '8 Corporation Road',
+            # 'postcode': 'DY2 7PX',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': "External wall insulation",
+            'Wall Render': "Render finish throughout",
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: North East, Back house direction: South West',
+            'Access to mains?': None,
+        },
+        {
+            'uprn': 90022227,
+            # 'address': '21 Wells Road',
+            # 'postcode': 'DY5 3TB',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': None,
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: East, Back house direction: West',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90077535,
+            # 'address': '47 Fairfield Road',
+            # 'postcode': 'DY8 5UJ',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': None,
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: East, Back house direction: West',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90060989,
+            # 'address': '53 Bromley',
+            # 'postcode': 'DY5 4PJ',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': "Filled at build, partially filled - celotex/king board, 50mm cavity remaining - "
+                               "recommends a cavity wall fill",
+            "Roof": "Hipped roof",
+            'Existing solar PV': 'No existing solar',
+            'Orientation': "Front house direction: North, Back house direction: South, Side house direction: West",
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90048026,
+            # 'address': '91 Osprey Drive',
+            # 'postcode': 'DY1 2JS',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': 'Tile hung front and rear of property',
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Side house direction: East',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90093693,
+            # 'address': '150 Huntingtree Road',
+            # 'postcode': 'B63 4HP',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Heating': 'Electric (storage heaters)',
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            "Roof": "Hipped roof",
+            'Existing solar PV': 'No existing solar',
+            'Orientation': "Front house direction: North West, Back house direction: South East, Side house direction: "
+                           "North East",
+        },
+        {
+            'uprn': 90051858,
+            # 'address': '195 Ashenhurst Road',
+            # 'postcode': 'DY1 2JB',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': "Solid render front and rear of property",
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: South, Back house direction: North',
+            'Access to mains?': 'Property has access to the mains',
+        },
+        {
+            'uprn': 90106884,
+            # 'address': '27 Milton Road',
+            # 'postcode': 'WV14 8HZ',
+            'surveyor': 'Carl Fitzgerald - The Warmfront Team',
+            'survey_date': datetime.strptime('2024-04-11', '%Y-%m-%d'),
+            'Wall Insulation': 'Cavity wall, retro drilled, containing loose fibre insulation. Consider getting a '
+                               'CIGA check and extracting the cavity, replacing with bead insulation.',
+            'Wall Render': "Solid render front and rear of property",
+            'Existing solar PV': 'No existing solar',
+            'Orientation': 'Front house direction: South East, Back house direction: North West',
+            'Access to mains?': 'Property has access to the mains',
+        },
+    ]
+
+    session = sessionmaker(bind=db_engine)()
+    upload_non_intrusive_survey_notes(session=session, non_invasive_notes=non_invasive_notes, batch_size=500)

etl/customers/immo/pilot/requirements.txt

@@ -0,0 +1 @@
+extract-msg

etl/customers/slide_utils.py

@@ -246,7 +246,7 @@ def create_powerpoint(data, save_location):
     prs.save(save_location)
 
 
-def create_recommendations_summary(recommendations_df, properties_df, sap_target):
+def create_recommendations_summary(recommendations_df, properties_df, property_details_df, sap_target):
     # Aggregate the impact of the recommendations
     # We want:
     # Total number of sap points
@@ -259,13 +259,15 @@ def create_recommendations_summary(recommendations_df, properties_df, sap_target
         total_valuation_impact=("property_valuation_increase", "sum"),
         total_bill_savings=("energy_cost_savings", "sum"),
         total_cost=("estimated_cost", "sum"),
-        total_carbon=("co2_equivalent_savings", "sum")
+        total_carbon=("co2_equivalent_savings", "sum"),
+        adjusted_heat_demand=("adjusted_heat_demand", "sum")
     ).reset_index()
-    # Merge on current sap points
+    # Merge on current sap points, current CO2, current adjusted_heat_demand, current annual bill
     recommendations_summary = recommendations_summary.merge(
         properties_df[["id", "uprn", "current_sap_points"]].rename(columns={"id": "property_id"}), on="property_id",
         how="left"
     )
+
     recommendations_summary["expected_sap_points"] = (
         recommendations_summary["current_sap_points"] + recommendations_summary["total_sap_points"]
     )
@@ -274,4 +276,18 @@ def create_recommendations_summary(recommendations_df, properties_df, sap_target
     )
     recommendations_summary["sap_difference"] = sap_target - recommendations_summary["expected_sap_points"]
 
+    if property_details_df is not None:
+        recommendations_summary = recommendations_summary.merge(
+            property_details_df[["uprn", "co2_emissions", "adjusted_energy_consumption", "energy_bill"]].rename(
+                columns={
+                    "id": "property_id",
+                    "co2_emissions": "current_co2",
+                    "adjusted_energy_consumption": "current_energy",
+                    "energy_bill": "current_energy_bill"
+                }
+            ),
+            on="uprn",
+            how="left"
+        )
+
     return recommendations_summary

etl/eligibility/Eligibility.py

@@ -145,6 +145,7 @@ class Eligibility:
                 "reason": None,
                 "thickness_classification": thickness_classification
             }
+            return
 
         # Insulation is already thick enough
         self.loft = {
@@ -164,8 +165,10 @@ class Eligibility:
         """
 
         is_cavity = self.walls["is_cavity_wall"]
-        is_empty = (not self.walls["is_filled_cavity"]) or (
+        is_empty = (not self.walls["is_filled_cavity"])
+        is_as_built = (
             self.walls["is_as_built"] and self.walls["insulation_thickness"] not in ["average", "above average"]
+            and self.walls["is_assumed"]
         )
         is_partial_filled = "partial" in self.walls["clean_description"].lower()
         # We look for potentially under performing cavities - anything that is assumed, as built and insulated
@@ -175,6 +178,7 @@ class Eligibility:
 
         is_unfilled_cavity = is_cavity and (is_empty and not is_partial_filled)
         is_partial_filled_cavity = is_cavity and is_partial_filled
+        is_assumed_filled_cavity = is_cavity and is_as_built
         is_underperforming_cavity = is_cavity and is_underperforming
 
         # Check if it has internal or external wall insulation
@@ -195,6 +199,13 @@ class Eligibility:
             }
             return
 
+        if is_assumed_filled_cavity:
+            self.cavity = {
+                "suitability": True,
+                "type": "as built assumed",
+            }
+            return
+
         if is_partial_filled_cavity:
             self.cavity = {
                 "suitability": True,
@@ -340,13 +351,35 @@ class Eligibility:
 
         # Check if the property is suitable for cavity wall
         self.cavity_insulation()
-        self.loft_insulation()
 
-        self.gbis_warmfront = (self.cavity["suitability"]) and (
-            int(self.epc["current-energy-efficiency"]) <= 68
-        )
+        current_sap = int(self.epc["current-energy-efficiency"])
+        # We have a strict suitability check and a non-strict check
 
-    def check_eco4_warmfront(self, post_retrofit_sap=None):
+        # Perfect strictness
+        if (self.cavity["type"] == "empty") and (current_sap < 69):
+            self.gbis_warmfront = {
+                "eligible": True,
+                "strict": True,
+                "message": "Perfect suitability",
+            }
+            return
+
+        # Near perfect
+        if self.cavity["suitability"] and (current_sap < 69):
+            self.gbis_warmfront = {
+                "eligible": True,
+                "strict": True,
+                "message": "Near perfect suitability",
+            }
+            return
+
+        self.gbis_warmfront = {
+            "eligible": False,
+            "strict": False,
+            "message": "All conditions fail",
+        }
+
+    def check_eco4_warmfront(self):
         """
         This funciton will check if the property is eligible for funding under the ECO4 scheme
 
@@ -378,49 +411,121 @@ class Eligibility:
         self.cavity_insulation()
         self.loft_insulation()
 
-        # make sure conditions 2 and 3 are true
-        is_eligible = self.cavity["suitability"] & self.loft["suitability"]
+        # We put in a placeholder when the roof is not a loft
+        if self.loft["reason"] == "roof not loft":
+            self.loft["thickness"] = 999
 
-        if current_sap >= 69:
+        # Case 1: No conditions meet
+        if not self.cavity["suitability"] and (self.loft["thickness"] > 100) and current_sap >= 55:
             self.eco4_warmfront = {
                 "eligible": False,
-                "message": "sap too high",
+                "strict": False,
+                "message": "All conditions fail",
                 "cavity_type": self.cavity["type"],
                 "loft_type": self.loft["thickness_classification"]
             }
             return
 
-        if post_retrofit_sap is None:
-
-            if current_sap >= 55:
-                message = "Possibly eligible but property currently EPC D"
-            else:
-                message = "subject to post retrofit sap" if is_eligible else "not eligible"
-
-            # Update the message to flag properties that failed just because of a full cavity.
-            # We need to double check that the wall is a cavity, that the loft is suitable and that the
-            # sap is within reason
-            # We can then estimate the age of the cavity fill
-            if not is_eligible and (current_sap < 69) and self.loft["suitability"] and self.walls["is_cavity_wall"]:
-                message = "Failed due to full cavity - check cavity age"
-
+        # Case 2 - perfect match
+        if (self.cavity["type"] == "empty") and (self.loft["thickness"] <= 100) and (current_sap < 55):
             self.eco4_warmfront = {
-                "eligible": is_eligible,
-                "message": message,
+                "eligible": True,
+                "strict": True,
+                "message": "Perfect suitability",
                 "cavity_type": self.cavity["type"],
                 "loft_type": self.loft["thickness_classification"]
             }
             return
 
-        is_eligible = is_eligible & (post_retrofit_sap >= 69)
+        # Case 2.5 - near perfect match - but we would not recommend this using the model
+        if self.cavity["suitability"] and (self.loft["thickness"] <= 100) and (current_sap < 55):
+            self.eco4_warmfront = {
+                "eligible": True,
+                "strict": True,
+                "message": "Near perfect suitability",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
 
-        self.eco4_warmfront = {
-            "eligible": is_eligible,
-            "message": None,
-            "cavity_type": self.cavity["type"],
-            "loft_type": self.loft["thickness_classification"]
-        }
-        return
+        # Case 3 - cavity is suitable, loft is within 150mm, sap is good
+        if self.cavity["suitability"] and (self.loft["thickness"] <= 150) and (current_sap < 55):
+            self.eco4_warmfront = {
+                "eligible": True,
+                "strict": False,
+                "message": "Meets cavity, loft borderline, meets sap",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 3 - cavity is suitable, loft is not, sap is good
+        if self.cavity["suitability"] and (self.loft["thickness"] > 150) and (current_sap < 55):
+            self.eco4_warmfront = {
+                "eligible": True,
+                "strict": False,
+                "message": "Meets cavity and sap",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 4 - cavity is not suitable, loft is, sap is not - we say this is not elifible
+        if not self.cavity["suitability"] and (self.loft["thickness"] <= 100) and (current_sap < 55):
+            self.eco4_warmfront = {
+                "eligible": False,
+                "strict": False,
+                "message": "failed fabric check",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 5 - cavity and loft suitable, sap too high
+        if self.cavity["suitability"] and (self.loft["thickness"] <= 150) and (current_sap >= 55):
+            self.eco4_warmfront = {
+                "eligible": True,
+                "strict": False,
+                "message": "Meets fabric, fails SAP check",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 6 - meets just cavity
+        if self.cavity["suitability"] and (self.loft["thickness"] > 100) and (current_sap >= 55):
+            self.eco4_warmfront = {
+                "eligible": True,
+                "strict": False,
+                "message": "Meets just cavity",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 7 - fails cavity, loft but meets sap
+        if not self.cavity["suitability"] and (self.loft["thickness"] > 100) and (current_sap < 55):
+            self.eco4_warmfront = {
+                "eligible": False,
+                "strict": False,
+                "message": "Fails cavity and loft, meets SAP",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        # Case 8 - fails cavity, meets loft, fails sap
+        if not self.cavity["suitability"] and (self.loft["thickness"] <= 100) and (current_sap >= 55):
+            self.eco4_warmfront = {
+                "eligible": False,
+                "strict": False,
+                "message": "Fails cavity, meets loft, fails SAP",
+                "cavity_type": self.cavity["type"],
+                "loft_type": self.loft["thickness_classification"]
+            }
+            return
+
+        raise ValueError("Implement me")
 
     def check_gbis(self):
 

etl/eligibility/ha_15_32/app.py

@@ -387,17 +387,19 @@ def prepare_model_data_row(
     }
 
     simulations = [
-        [cavity_simulation],
-        [loft_simulation]
+        cavity_simulation,
+        loft_simulation
     ]
 
-    p.adjust_difference_record_with_recommendations(simulations)
+    recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
+    scoring_dict = p.create_recommendation_scoring_data(
+        property_id=p.id,
+        recommendation_record=recommendation_record,
+        recommendations=simulations,
+        primary_recommendation_id=cavity_simulation["recommendation_id"]
+    )
 
-    # Make sure we definitely have the correct data
-    cavity_scoring = [x for x in p.recommendations_scoring_data if "cavity" in x["id"]][0]
-    loft_scoring = [x for x in p.recommendations_scoring_data if "loft" in x["id"]][0]
-
-    return [cavity_scoring, loft_scoring]
+    return [scoring_dict]
 
 
 def get_ha_32data(ha_data, cleaned, cleaning_data, created_at):

etl/epc/Dataset.py

@@ -203,11 +203,11 @@ class TrainingDataset(BaseDataset):
         common_cols = [[col + "_starting", col + "_ending"] for col in common_cols]
 
         self.df = self.df.loc[
-            :,
-            no_suffix_cols
-            + only_ending_cols
-            + [col for cols in common_cols for col in cols],
-        ]
+                  :,
+                  no_suffix_cols
+                  + only_ending_cols
+                  + [col for cols in common_cols for col in cols],
+                  ]
 
     def _remove_abnormal_change_in_floor_area(self):
         """
@@ -509,7 +509,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["is_sandstone_or_limestone"]
                     == expanded_df["is_sandstone_or_limestone_ending"]
                 )
-            ]
+                ]
         elif component == "floor":
             expanded_df = expanded_df[
                 (expanded_df["is_suspended"] == expanded_df["is_suspended_ending"])
@@ -526,7 +526,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["is_to_external_air"]
                     == expanded_df["is_to_external_air_ending"]
                 )
-            ]
+                ]
         elif component == "roof":
             expanded_df = expanded_df[
                 (expanded_df["is_pitched"] == expanded_df["is_pitched_ending"])
@@ -539,7 +539,7 @@ class TrainingDataset(BaseDataset):
                     expanded_df["has_dwelling_above"]
                     == expanded_df["has_dwelling_above_ending"]
                 )
-            ]
+                ]
 
         return expanded_df
 

etl/epc/Record.py

@@ -191,7 +191,7 @@ class EPCRecord:
         This method will clean the records using the data processor
         """
         epc_data_processor = EPCDataProcessor(
-            data=self.epc_record_as_dataframe("prepared_epc"),
+            data=self.epc_record_as_dataframe("prepared_epc").copy(),
             run_mode="newdata",
             cleaning_averages=self.cleaning_data,
         )
@@ -725,26 +725,26 @@ class EPCRecord:
         if self.prepared_epc["construction-age-band"] in DATA_ANOMALY_MATCHES:
             if self.old_data:
                 # Take the most recent
-                max_datetime = max(
-                    [
-                        old_record["lodgement-datetime"]
-                        for old_record in self.old_data
-                        if old_record["construction-age-band"]
-                           not in DATA_ANOMALY_MATCHES
-                    ]
-                )
-
-                most_recent = [
-                    old_record
+                old_age_bands = [
+                    old_record["lodgement-datetime"]
                     for old_record in self.old_data
-                    if old_record["lodgement-datetime"] == max_datetime
+                    if old_record["construction-age-band"] not in DATA_ANOMALY_MATCHES
                 ]
 
-                self.prepared_epc["construction-age-band"] = (
-                    EPCDataProcessor.clean_construction_age_band(
-                        most_recent[0]["construction-age-band"]
+                if old_age_bands:
+                    max_datetime = max(old_age_bands)
+
+                    most_recent = [
+                        old_record
+                        for old_record in self.old_data
+                        if old_record["lodgement-datetime"] == max_datetime
+                    ]
+
+                    self.prepared_epc["construction-age-band"] = (
+                        EPCDataProcessor.clean_construction_age_band(
+                            most_recent[0]["construction-age-band"]
+                        )
                     )
-                )
 
         self.construction_age_band = self.prepared_epc["construction-age-band"]
         self.age_band = england_wales_age_band_lookup.get(self.construction_age_band)

etl/epc/generate_scenarios_data.py

@@ -66,7 +66,11 @@ scenario_properties = [
                 {"walls_insulation_thickness_ending": "average"},
                 [0],
             ],
+<<<<<<< HEAD
             [["solar", "windows"], "12-15", {"photo_supply_ending": 50}, [0, 1]],
+=======
+            [["solar", "windows"], "15", {"photo_supply_ending": 50}, [0, 1]],
+>>>>>>> 2f45ed895530a6dfb7ebe7bd40b3463fe54993bb
         ],
     },
     {
@@ -123,6 +127,7 @@ scenario_properties = [
     },
 ]
 
+
 recommendations_scoring_data = []
 
 for scenario_property in scenario_properties:
@@ -247,6 +252,9 @@ for scenario_property in scenario_properties:
     recommendations_scoring_data.extend(scoring_list)
 
 recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
+recommendations_scoring_data["impact"] = recommendations_scoring_data["impact"].astype(
+    int
+)
 recommendations_scoring_data = recommendations_scoring_data.drop(
     columns=[
         "rdsap_change",

etl/epc_clean/app.py

@@ -36,8 +36,11 @@ def app():
     cleaned_data = {}
     epc_directories = [entry for entry in EPC_DIRECTORY.iterdir() if entry.is_dir()]
 
+    WALLS = []
     for directory in tqdm(epc_directories):
         data = pd.read_csv(directory / "certificates.csv", low_memory=False)
+        z = data["WALLS_DESCRIPTION"].unique().tolist()
+        WALLS.extend(z)
         # Rename the columns to the same format as the api returns
         data.columns = [c.replace("_", "-").lower() for c in data.columns]
         # Take just date before the date threshold

etl/epc_clean/epc_attributes/RoofAttributes.py

@@ -122,6 +122,13 @@ class RoofAttributes(Definitions):
         result["is_valid"] = "invalid" not in description
         description = description.replace("invalid", "")
 
+        # We handle an edge case where the description is "pitched, 150  loft insulation" and is missing the mm
+        if result["is_pitched"] or result["is_loft"]:
+            # Search for a regular expression that matches 150   insulation
+            match = re.search(r"(\d+\+?)\s*insulation", description)
+            if match:
+                result['insulation_thickness'] = match.group(1)
+
         # insulation thickness
         thickness_map = {
             "ceiling insulated": "average",
@@ -137,11 +144,11 @@ class RoofAttributes(Definitions):
                 # Remove the match from the description
                 # description = description.replace(key, "")
                 break
-        else:
-            # Extract insulation thickness in mm, if present
-            match = re.search(r'(\d+\+?)\s*mm', description)
-            if match:
-                result['insulation_thickness'] = match.group(1)
+
+        # Extract insulation thickness in mm, if present
+        match = re.search(r'(\d+\+?)\s*mm', description)
+        if match:
+            result['insulation_thickness'] = match.group(1)
 
         if "insulation_thickness" not in result:
             result['insulation_thickness'] = None

etl/non_intrusive_surveys/photos/README.md

@@ -0,0 +1,19 @@
+# Non Intrusive Surveys - photo upload
+
+This folder contains photos taken during non-intrusive surveys. Photos are stored in folders named after the survey ID.
+
+## Getting started
+
+Install the required packages by running the following command:
+
+```bash
+pip install -r requirements.txt
+```
+
+## Usage
+
+The main application is found in the app.py file. To run the application, use the following command:
+
+```bash
+python app.py
+```

etl/non_intrusive_surveys/photos/app.py

@@ -0,0 +1,149 @@
+import boto3
+import os
+from PIL import Image
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Inputs
+ENV_FILEPATH = "etl/non_intrusive_surveys/photos/.env"
+PHOTO_DIRECTORY = "/Users/khalimconn-kowlessar/Downloads/IMMO - Dudley Pilot - non-invasive raw data"
+FOLDER_UPRN_LOOKUP = {
+    "91 Osprey Drive DY1 2JS": 90048026,
+    "195 Ashenhurst Rd DY1 2JB": 90051858,
+    "6 Beech Rd DY1 4BP": 90055152,
+    "53 Bromley DY5 4PJ": 90060989,
+    "5 Oaklands B62 0JA": 90028499,
+    "47 Fairfield Rd DY8 5UJ": 90077535,
+    "150 Huntingtree Rd B63 4HP": 90093693,
+    "27 Milton Rd DY1 2JB": 90106884,
+    "21 Wells Rd DY5 3TB": 90022227,
+    "8 Corporation Rd DY2 7PX": 90070461
+}
+
+load_dotenv(ENV_FILEPATH)
+CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME = os.getenv("CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME", None)
+CDN_BUCKET_NAME = os.getenv("CDN_BUCKET_NAME", None)
+
+
+def list_subdirectories(directory_path):
+    """
+    List all subdirectories within a given directory.
+
+    :param directory_path: Path to the directory.
+    :return: A list of paths to the subdirectories.
+    """
+    directory = Path(directory_path)
+    subdirectories = [subdir for subdir in directory.iterdir() if subdir.is_dir()]
+    return subdirectories
+
+
+def list_files_in_directory(directory_path, file_extension=".jpg"):
+    """
+    List all files with a specific extension within a given directory and its subdirectories.
+
+    :param directory_path: Path to the directory to scan.
+    :param file_extension: File extension to filter by.
+    :return: A list of paths to the files.
+    """
+    # Convert the directory path to a Path object if it's not already one
+    directory = Path(directory_path) if not isinstance(directory_path, Path) else directory_path
+
+    # List all files of the specified type in the directory and subdirectories
+    file_list = [file for file in directory.rglob(f'*{file_extension}')]
+
+    return file_list
+
+
+def create_images(input_path, uprn):
+    # Define the base directory path
+    base_directory = f"non_intrusive_photos/{uprn}"
+    print(f"Creating directory: {base_directory}")  # Debug: print the directory to be created
+
+    # Need to create local directory if it doesn't exist
+    os.makedirs(base_directory, exist_ok=True)
+
+    # Define output paths
+    thumbnail_path = os.path.join(base_directory, "thumbnail.jpg")
+    full_hd_path = os.path.join(base_directory, "1080p.jpg")
+    webp_path = os.path.join(base_directory, "webp.webp")  # Save as WebP format
+
+    # Load the image
+    with Image.open(input_path) as img:
+        # Create a thumbnail
+        thumbnail = img.copy()
+        thumbnail.thumbnail((128, 128), Image.Resampling.LANCZOS)
+        thumbnail.save(thumbnail_path, 'JPEG', quality=85)
+
+        # Create a 1080p version
+        full_hd = img.copy()
+        full_hd.thumbnail((1920, 1080), Image.Resampling.LANCZOS)
+        full_hd.save(full_hd_path, 'JPEG', quality=90)
+
+        # Convert to WebP for better compression
+        webp = img.copy()
+        webp.save(webp_path, 'WEBP', quality=90)
+
+    # Return paths to the processed images
+    return thumbnail_path, full_hd_path, webp_path
+
+
+def upload_to_s3(bucket_name, file_path, object_name):
+    s3_client = boto3.client('s3')
+    s3_client.upload_file(file_path, bucket_name, object_name)
+    print(f"Uploaded {object_name} to S3 bucket {bucket_name}")
+
+
+def upload_photos_to_s3(bucket_name, photo_paths):
+    # Upload each photo
+    for path in photo_paths:
+        object_name = path.split('/')[-1]  # Assuming the path format is folder/filename
+        upload_to_s3(bucket_name, path, object_name)
+
+
+def generate_cdn_url(distribution_domain, object_name):
+    return f"https://{distribution_domain}/{object_name}"
+
+
+def process_and_upload_images(uprn, input_image_path, bucket_name, distribution_domain):
+    # Create images
+    thumbnail, full_hd, original = create_images(input_image_path, uprn=str(uprn))
+
+    # Upload images
+    upload_photos_to_s3(bucket_name, photo_paths=[thumbnail, full_hd, original])
+
+    # Generate CDN links
+    cdn_links = [generate_cdn_url(distribution_domain, path.split('/')[-1]) for path in [thumbnail, full_hd, original]]
+
+    # Delete local files
+    for path in [thumbnail, full_hd, original]:
+        os.remove(path)
+
+    return cdn_links
+
+
+def app():
+    """
+    This application is tasked with uploading the photos, recorded during the non-invasive surveys, to s3 and the
+    database.
+    To begin with, this app will simply read the files from the local machine, however we will come up with a more
+    efficient way to do this in the future.
+
+    :return:
+    """
+
+    # List all files in the directory using pathlib
+    property_directories = list_subdirectories(PHOTO_DIRECTORY)
+
+    # For each property, we want to list all of the photos in the directory
+    for property_dir in property_directories:
+        photo_files = list_files_in_directory(property_dir)
+        uprn = FOLDER_UPRN_LOOKUP[property_dir.name]
+
+        # We now want to convert each file, and upload it to s3
+        for photo_filepath in photo_files:
+            process_and_upload_images(
+                uprn=uprn,
+                input_image_path=photo_filepath,
+                bucket_name=CDN_BUCKET_NAME,
+                distribution_domain=CLOUDFRONT_DISTRIBUTION_DOMAIN_NAME
+            )

etl/non_intrusive_surveys/photos/requirements.txt

@@ -0,0 +1,3 @@
+Pillow
+boto3
+python-dotenv

infrastructure/terraform/main.tf

@@ -66,7 +66,7 @@ resource "aws_security_group" "allow_db" {
 resource "aws_db_instance" "default" {
   allocated_storage      = var.allocated_storage
   engine                 = "postgres"
-  engine_version         = "14.7"
+  engine_version         = "14.10"
   instance_class         = var.instance_class
   db_name                = var.database_name
   username               = jsondecode(data.aws_secretsmanager_secret_version.db_credentials.secret_string)["db_assessment_model_username"]
@@ -181,4 +181,16 @@ module "lambda_carbon_prediction_ecr" {
 module "lambda_heat_prediction_ecr" {
   ecr_name = "lambda-heat-prediction-${var.stage}"
   source   = "./modules/ecr"
+}
+
+##############################################
+# CDN - Cloudfront
+##############################################
+module "cloudfront_distribution" {
+  source             = "./modules/cloudfront"
+  bucket_name        = module.s3.bucket_name
+  bucket_id          = module.s3.bucket_id
+  bucket_arn         = module.s3.bucket_arn
+  bucket_domain_name = module.s3.bucket_domain_name
+  stage              = var.stage
 }

infrastructure/terraform/modules/cloudfront/main.tf

@@ -0,0 +1,65 @@
+resource "aws_cloudfront_distribution" "s3_distribution" {
+  origin {
+    domain_name = var.bucket_domain_name
+    origin_id   = "S3-${var.bucket_name}"
+
+    s3_origin_config {
+      origin_access_identity = aws_cloudfront_origin_access_identity.oai.cloudfront_access_identity_path
+    }
+  }
+
+  enabled = true
+
+  default_cache_behavior {
+    allowed_methods        = ["GET", "HEAD"]
+    cached_methods         = ["GET", "HEAD"]
+    target_origin_id       = "S3-${var.bucket_name}"
+    viewer_protocol_policy = "redirect-to-https"
+    compress               = true
+
+    forwarded_values {
+      query_string = false
+      cookies {
+        forward = "none"
+      }
+    }
+
+    min_ttl     = 0
+    default_ttl = 86400
+    max_ttl     = 31536000
+  }
+
+  price_class = "PriceClass_All"
+
+  restrictions {
+    geo_restriction {
+      restriction_type = "none"
+    }
+  }
+
+  viewer_certificate {
+    cloudfront_default_certificate = true
+  }
+}
+
+resource "aws_cloudfront_origin_access_identity" "oai" {
+  comment = "OAI for ${var.bucket_name}"
+}
+
+resource "aws_s3_bucket_policy" "bucket_policy" {
+  bucket = var.bucket_id
+
+  policy = jsonencode({
+    Version   = "2012-10-17"
+    Statement = [
+      {
+        Effect    = "Allow"
+        Principal = {
+          AWS = "arn:aws:iam::cloudfront:user/CloudFront Origin Access Identity ${aws_cloudfront_origin_access_identity.oai.id}"
+        }
+        Action   = "s3:GetObject"
+        Resource = "${var.bucket_arn}/*"
+      },
+    ]
+  })
+}

infrastructure/terraform/modules/cloudfront/variables.tf

@@ -0,0 +1,24 @@
+variable "bucket_name" {
+  description = "The name of the bucket"
+  type        = string
+}
+
+variable "stage" {
+  description = "The deployment stage"
+  type        = string
+}
+
+variable "bucket_id" {
+  description = "The ID of the S3 bucket"
+  type        = string
+}
+
+variable "bucket_arn" {
+  description = "The ARN of the S3 bucket"
+  type        = string
+}
+
+variable "bucket_domain_name" {
+  description = "The regional domain name of the S3 bucket"
+  type        = string
+}

infrastructure/terraform/modules/s3/outputs.tf

@@ -2,3 +2,15 @@ output "bucket_name" {
   description = "The name of the S3 bucket"
   value       = aws_s3_bucket.bucket.bucket
 }
+
+output "bucket_id" {
+  value = aws_s3_bucket.bucket.id
+}
+
+output "bucket_arn" {
+  value = aws_s3_bucket.bucket.arn
+}
+
+output "bucket_domain_name" {
+  value = aws_s3_bucket.bucket.bucket_regional_domain_name
+}

recommendations/Costs.py

@@ -42,7 +42,52 @@ BATTERY_COST = 3500
 
 # This is based on https://www.checkatrade.com/blog/cost-guides/cost-smart-thermostat/
 SMART_APPLIANCE_THERMOSTAT_COST = 400
-PROGRAMMER_COST = 200
+PROGRAMMER_COST = 120
+ROOM_THERMOSTAT_COST = 150
+TRVS_COST = 35
+
+# Cost for TTZC
+# Smart thermostat based on checkatrade https://www.checkatrade.com/blog/cost-guides/cost-smart-thermostat/
+# Based on the Nest system
+TTZC_SMART_THERMOSTAT_COST = 205
+TTZC_SMART_THERMOSTAT_LABOUR_HOURS = 2
+TTZC_ELECTRICIAN_HOURLY_RATE = 45
+# Based on cost of a Nest temperature sensor
+TTZC_ROOM_TEMPERATURE_SENSOR_COST = 50
+TTZC_ROOM_TEMPERATURE_SENSOR_LABOUR_HOURS = 0.17  # (Assume ~ 10 mins install per sensor)
+# Basedon an average cost of smart radiator values
+TTZC_SMART_RADIATOR_VALUES = 50
+TTZC_SMART_RADIATOR_VALUES_LABOUR_HOURS = 0.37  # (Assume ~ 15-30 mins install per valve)
+
+# Low carbon combi boiler - median value based on £2200 - £3000 range
+LOW_CARBON_COMBI_BOILER = 2200
+
+# boiler prices based on
+# https://www.greenmatch.co.uk/boilers/30kw-boiler
+# https://www.greenmatch.co.uk/boilers/35kw-boiler
+# https://www.greenmatch.co.uk/boilers/40kw-boiler
+# These are exclusive of installation costs
+CONDENSING_BOILER_COSTS = {
+    "30kw": 1550,
+    "35kw": 1610,
+    "40kw": 1625
+}
+
+# Assumes 3 hours to remove each heater (including re-decorating)
+ROOM_HEATER_REMOVAL_COST = 120
+ROOM_HEATER_REMOVAL_LABOUR_HOURS = 3
+
+# This is a cost quoted by Jim for a system flush - existig system will run more efficiently
+SYSTEM_FLUSH_COST = 250
+
+SINGLE_RADIATOR_COST = 150
+DOUBLE_RADIATOR_COST = 300
+FLUE_COST = 600
+PIPEWORK_COST = 750  # Min cost is £500
+
+# This is the cost per meter squared for cavity extraction
+# https://www.checkatrade.com/blog/cost-guides/cavity-wall-insulation-removal-cost/
+CAVITY_EXTRACTION_COST = 21.5
 
 
 class Costs:
@@ -126,7 +171,7 @@ class Costs:
         if not self.labour_adjustment_factor:
             raise ValueError("Labour adjustment factor not found")
 
-    def cavity_wall_insulation(self, wall_area, material):
+    def cavity_wall_insulation(self, wall_area, material, is_extraction_and_refill=False):
         """
         Calculates the total cost for cavity wall insulation based on material and labor costs,
         including contingency, preliminaries, profit, and VAT.
@@ -161,6 +206,13 @@ class Costs:
         # Assume a team of 2
         labour_days = (labour_hours / 8) / 2
 
+        if is_extraction_and_refill:
+            # bump up the cost of the work
+            total_cost = total_cost + CAVITY_EXTRACTION_COST * wall_area
+            # Additional 2 days work
+            labour_hours = labour_hours + (2 * 8)
+            labour_days = labour_days + 2
+
         return {
             "total": total_cost,
             "subtotal": subtotal_before_vat,
@@ -998,3 +1050,193 @@ class Costs:
             "labour_hours": 0,
             "labour_days": 0,
         }
+
+    def roomstat_programmer_trvs(
+        self, number_heated_rooms, has_programmer, has_trvs, has_room_thermostat
+    ):
+        """
+
+        :return:
+        """
+
+        total_cost = 0
+        labour_hours = 0
+
+        if not has_programmer:
+            total_cost += PROGRAMMER_COST
+            labour_hours += 1
+
+        if not has_trvs:
+            total_cost += TRVS_COST * number_heated_rooms
+            labour_hours += 0.25 * number_heated_rooms
+
+        if not has_room_thermostat:
+            total_cost += ROOM_THERMOSTAT_COST
+            labour_hours += 0.5
+
+        subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
+        vat = total_cost - subtotal_before_vat
+
+        return {
+            "total": total_cost,
+            "subtotal": subtotal_before_vat,
+            "vat": vat,
+            "labour_hours": labour_hours,
+            "labour_days": 1,
+        }
+
+    def time_and_temperature_zone_control(self, number_heated_rooms):
+
+        # The product costs are inclusive of VAT
+        product_costs = (
+            TTZC_SMART_THERMOSTAT_COST +
+            TTZC_ROOM_TEMPERATURE_SENSOR_COST * number_heated_rooms +
+            TTZC_SMART_RADIATOR_VALUES * number_heated_rooms
+        )
+        labour_hours = (
+            TTZC_SMART_THERMOSTAT_LABOUR_HOURS +
+            TTZC_ROOM_TEMPERATURE_SENSOR_LABOUR_HOURS * number_heated_rooms +
+            TTZC_SMART_RADIATOR_VALUES_LABOUR_HOURS * number_heated_rooms
+        )
+        labour_costs = TTZC_ELECTRICIAN_HOURLY_RATE * labour_hours
+        # Add continency and preliminaries to the labour to account for the complexity of the job
+        labour_costs = labour_costs * (1 + self.CONTINGENCY + self.PRELIMINARIES)
+
+        vat = labour_costs * self.VAT_RATE
+
+        subtotal_before_vat = product_costs + labour_costs
+        total_cost = subtotal_before_vat + vat
+
+        labour_days = np.ceil(labour_hours / 8)
+
+        return {
+            "total": total_cost,
+            "subtotal": subtotal_before_vat,
+            "vat": vat,
+            "labour_hours": labour_hours,
+            "labour_days": labour_days,
+        }
+
+    def heater_removal(self, n_rooms):
+        """
+        Estimates the costs of removal of heaters, including the redecoration costs of the space behind the heater
+        :return:
+        """
+
+        removal_cost = ROOM_HEATER_REMOVAL_COST * n_rooms
+        removal_labour_hours = ROOM_HEATER_REMOVAL_LABOUR_HOURS * n_rooms
+
+        vat = removal_cost * self.VAT_RATE
+
+        subtotal_before_vat = removal_cost
+        total_cost = subtotal_before_vat + vat
+
+        return {
+            "total": total_cost,
+            "subtotal": subtotal_before_vat,
+            "vat": vat,
+            "labour_hours": removal_labour_hours,
+            "labour_days": np.ceil(removal_labour_hours / 8),
+        }
+
+    @staticmethod
+    def _estimate_n_radiators(number_habitable_rooms, total_floor_area, property_type, built_form):
+        # Base number of radiators: one per habitable room
+        base_radiators = number_habitable_rooms
+
+        # Additional radiators for non-habitable essential areas (e.g., kitchens, hallways)
+        additional_radiators = 3  # Initial assumption
+
+        # Adjust additional radiators based on property type
+        if property_type == 'Flat':
+            additional_radiators -= 1  # Flats may need fewer radiators due to less exposure
+        elif property_type in ['House', 'Bungalow', 'Maisonette']:
+            # Multiple floors in Maisonette may require additional heating points
+            additional_radiators += 2  # Houses and bungalows might need more due to greater exposure
+        else:
+            raise Exception("Invalid property type")
+
+        # Adjust total radiator needs based on built form
+        form_factor = {
+            'Mid-Terrace': 0.95,
+            'Semi-Detached': 1.05,
+            'Detached': 1.25,
+            'End-Terrace': 1.05
+        }
+
+        # Calculate total heating power needed and number of radiators based on standard output
+        total_heating_power_required = total_floor_area * 80  # Watts per square meter
+        radiator_output = 1000  # Average wattage per radiator
+        total_radiators_based_on_power = (total_heating_power_required / radiator_output) * form_factor[built_form]
+
+        # Final estimation taking the higher of calculated needs or base room count
+        estimated_radiators = max(total_radiators_based_on_power, base_radiators + additional_radiators)
+        return round(estimated_radiators)
+
+    def boiler(self, size, exising_room_heaters, system_change, n_heated_rooms, n_rooms):
+        """
+        Based on a basic estimate of median value £2600 to install a low carbon combi boiler
+        First time central heating vosts can als be found here:
+        https://www.checkatrade.com/blog/cost-guides/central-heating-installation-cost/
+        :return:
+        """
+
+        unit_cost = CONDENSING_BOILER_COSTS[size]
+        # The unit cost is the cost without VAT
+        # We now need to estimate the cost of the works
+        labour_days = 2
+        labour_hours = labour_days * 8
+        labour_rate = 300
+
+        # Average cost of installation is 1 (maybe 2days) at £300 per day
+        # https://www.checkatrade.com/blog/cost-guides/new-boiler-cost/
+        # To be pessimistic, assume 2 days work
+        labour_cost = labour_rate * self.labour_adjustment_factor * labour_days
+        # Add contingency and preliminaries
+        labour_cost = labour_cost * (1 + self.CONTINGENCY + self.PRELIMINARIES)
+
+        # labour_days = labour_days + (removal_labour_hours / 8)
+
+        vat = labour_cost * self.VAT_RATE
+
+        subtotal_before_vat = unit_cost + labour_cost
+        total_cost = subtotal_before_vat + vat
+
+        # if there are existing room heaters, we need to add the cost of removing them
+        if exising_room_heaters:
+            removal_costing = self.heater_removal(n_rooms=n_heated_rooms)
+            # Add the totals to the existing totals
+            total_cost += removal_costing["total"]
+            subtotal_before_vat += removal_costing["subtotal"]
+            labour_hours += removal_costing["labour_hours"]
+            labour_days += removal_costing["labour_days"]
+            vat += removal_costing["vat"]
+
+        if system_change:
+            # We need the cost of radiators
+            n_radiators = self._estimate_n_radiators(
+                number_habitable_rooms=n_rooms,
+                total_floor_area=self.property.floor_area,
+                property_type=self.property.data["property-type"],
+                built_form=self.property.data["built-form"]
+            )
+
+            additionals_labour_cost = labour_rate * self.labour_adjustment_factor
+            radiator_cost = DOUBLE_RADIATOR_COST * n_radiators
+            system_change_cost = radiator_cost + FLUE_COST + PIPEWORK_COST + additionals_labour_cost
+            system_change_cost_before_vat = system_change_cost / (1 + self.VAT_RATE)
+            system_change_vat = system_change_cost - system_change_cost_before_vat
+            # We add an extra labour day for the system change
+            labour_days += 1
+            labour_hours += 8
+            total_cost += system_change_cost
+            subtotal_before_vat += system_change_cost_before_vat
+            vat += system_change_vat
+
+        return {
+            "total": total_cost,
+            "subtotal": subtotal_before_vat,
+            "vat": vat,
+            "labour_hours": labour_hours,
+            "labour_days": labour_days,
+        }

recommendations/FireplaceRecommendations.py

@@ -32,7 +32,8 @@ class FireplaceRecommendations(Definitions):
         if number_open_fireplaces == 0:
             return
 
-        estimated_cost = number_open_fireplaces * self.COST_OF_WORK
+        already_installed = "sealing_open_fireplace" in self.property.already_installed
+        estimated_cost = number_open_fireplaces * self.COST_OF_WORK if not already_installed else 0
 
         # We recommend installing two mechanical ventilation systems
         self.recommendation = [
@@ -44,6 +45,7 @@ class FireplaceRecommendations(Definitions):
                 "starting_u_value": None,
                 "new_u_value": None,
                 "sap_points": None,
+                "already_installed": already_installed,
                 "total": estimated_cost,
                 # Take a very basic estimate of 6 hours, multipled by the number of open fireplaces to seal
                 "labour_hours": 6 * number_open_fireplaces,

recommendations/FloorRecommendations.py

@@ -8,7 +8,7 @@ from datatypes.enums import QuantityUnits
 from backend.Property import Property
 from recommendations.recommendation_utils import (
     r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
-    get_recommended_part, get_floor_u_value
+    get_recommended_part, get_floor_u_value, override_costs
 )
 from recommendations.Costs import Costs
 
@@ -192,12 +192,21 @@ class FloorRecommendations(Definitions):
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
+
+                        already_installed = "suspended_floor_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
+
                     elif material["type"] == "solid_floor_insulation":
                         cost_result = self.costs.solid_floor_insulation(
                             insulation_floor_area=self.property.insulation_floor_area,
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
+
+                        already_installed = "solid_floor_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
                     else:
                         raise NotImplementedError("Implement me!")
 
@@ -217,6 +226,7 @@ class FloorRecommendations(Definitions):
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": None,
+                            "already_installed": already_installed,
                             **cost_result
                         }
                     )

recommendations/HeatingControlRecommender.py

@@ -1,5 +1,5 @@
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import check_simulation_difference
+from recommendations.recommendation_utils import check_simulation_difference, override_costs
 from backend.Property import Property
 from etl.epc_clean.epc_attributes.MainheatControlAttributes import MainheatControlAttributes
 
@@ -27,6 +27,14 @@ class HeatingControlRecommender:
             self.recommend_high_heat_retention_controls()
             return
 
+        if heating_description in ["Boiler and radiators, mains gas"]:
+            # We can recommend roomstat programmer trvs
+            self.recommend_roomstat_programmer_trvs()
+            # We can also recommend time and temperature zone controls
+            self.recommend_time_temperature_zone_controls()
+
+            return
+
     def recommend_room_heaters_electric_controls(self):
         """
         If the home has Room heaters, electric, we start by identifying potential heating controls that could
@@ -105,3 +113,136 @@ class HeatingControlRecommender:
 
         # We don't implement any other recommendations right now
         return
+
+    def recommend_roomstat_programmer_trvs(self):
+        """
+        If the home has a boiler and radiators, mains gas, we start by identifying potential heating controls that could
+        be upgraded, that would provide a practical impact.
+
+        The criteria for recommending an upgrade to heating controls are (one of these must be true)
+        1) There are no controls
+        2) No programmer
+        3) No room thermostat
+        4) No TRVs
+
+
+        :return:
+        """
+
+        # We check if we have the conditions to recommend this upgrade
+
+        needs_programmer = self.property.main_heating_controls["switch_system"] is None
+        needs_room_thermostat = self.property.main_heating_controls["thermostatic_control"] is None
+        needs_trvs = self.property.main_heating_controls["trvs"] is None
+
+        can_recommend = (
+            (self.property.main_heating_controls["no_control"] is not None) or
+            needs_programmer or
+            needs_room_thermostat or
+            needs_trvs
+        )
+
+        if not can_recommend:
+            return
+
+        ending_config = MainheatControlAttributes("Programmer, room thermostat and TRVS").process()
+        # We use this to determine how we should be updating the config
+        simulation_config = check_simulation_difference(
+            new_config=ending_config, old_config=self.property.main_heating_controls
+        )
+        # This upgrade will only take the heating system to average energy efficiency
+        # If the current system is below good, we make it good
+        if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average"]:
+            simulation_config["mainheatc_energy_eff_ending"] = "Good"
+
+        has_programmer = not needs_programmer
+        has_room_thermostat = not needs_room_thermostat
+        has_trvs = not needs_trvs
+
+        cost_result = self.costs.roomstat_programmer_trvs(
+            number_heated_rooms=int(self.property.data["number-heated-rooms"]),
+            has_programmer=has_programmer,
+            has_room_thermostat=has_room_thermostat,
+            has_trvs=has_trvs
+        )
+
+        description = "upgrade heating controls to Room thermostat, programmer and TRVs"
+
+        already_installed = "heating_control" in self.property.already_installed
+        if already_installed:
+            cost_result = override_costs(cost_result)
+            description = "Heating controls have already been upgraded, no further action needed."
+
+        self.recommendation.append(
+            {
+                "type": "heating_control",
+                "parts": [],
+                "description": description,
+                **cost_result,
+                "starting_u_value": None,
+                "new_u_value": None,
+                "sap_points": None,
+                "already_installed": already_installed,
+                "simulation_config": simulation_config
+            }
+        )
+
+        return
+
+    def recommend_time_temperature_zone_controls(self):
+        """
+        If the home has a boiler, we can recommend time and temperature zone controls. This is a more advanced
+        and more efficient control system than the standard controls that come with a boiler. However, it may come
+        with a higher cost and more involved usage
+        :return:
+        """
+
+        # We check if the efficiency of the current heating controls is good or below, and
+
+        # Conditions for installation are as follows:
+        # 1) The current heating controls are not time and temperature zone controls
+        # 2) The current heating controls are not already at 'Very Good' or above
+
+        if (
+            (self.property.main_heating_controls["thermostatic_control"] == "time and temperature zone control") or
+            (self.property.data["mainheatc-energy-eff"] in ["Very Good"])
+        ):
+            # No recommendation needed
+            return
+
+        ending_config = MainheatControlAttributes("Time and temperature zone control").process()
+
+        # We use this to determine how we should be updating the config
+        simulation_config = check_simulation_difference(
+            new_config=ending_config, old_config=self.property.main_heating_controls
+        )
+
+        # If the current system is below very good, we make it very good
+        if self.property.data["mainheatc-energy-eff"] in ["Poor", "Very Poor", "Average", "Good"]:
+            simulation_config["mainheatc_energy_eff_ending"] = "Very Good"
+
+        cost_result = self.costs.time_and_temperature_zone_control(
+            number_heated_rooms=int(self.property.data["number-heated-rooms"])
+        )
+
+        description = ("Upgrade heating controls to Smart Thermostats, room sensors and smart radiator valves (time & "
+                       "temperature zone control)")
+
+        already_installed = "heating_control" in self.property.already_installed
+        if already_installed:
+            cost_result = override_costs(cost_result)
+            description = "Heating controls have already been upgraded, no further action needed."
+
+        self.recommendation.append(
+            {
+                "type": "heating_control",
+                "parts": [],
+                "description": description,
+                **cost_result,
+                "starting_u_value": None,
+                "new_u_value": None,
+                "sap_points": None,
+                "already_installed": already_installed,
+                "simulation_config": simulation_config
+            }
+        )

recommendations/HeatingRecommender.py

@@ -1,9 +1,11 @@
 import pandas as pd
 
 from recommendations.Costs import Costs
-from recommendations.recommendation_utils import check_simulation_difference
+from recommendations.recommendation_utils import check_simulation_difference, override_costs
 from backend.Property import Property
 from etl.epc_clean.epc_attributes.MainheatAttributes import MainHeatAttributes
+from etl.epc_clean.epc_attributes.HotWaterAttributes import HotWaterAttributes
+from etl.epc_clean.epc_attributes.MainFuelAttributes import MainFuelAttributes
 from recommendations.HeatingControlRecommender import HeatingControlRecommender
 
 
@@ -13,18 +15,73 @@ class HeatingRecommender:
         self.property = property_instance
         self.costs = Costs(self.property)
 
-        self.recommendations = []
+        self.heating_recommendations = []
+        self.heating_control_recommendations = []
 
     def recommend(self, phase=0):
-        self.recommendations = []
+
+        # TODO: We could have a system flush recommendation for an existing boiler, where there is no need to replace
+        #       the boiler, but instead flushing the system will make it run more efficiently. There is a cost for this
+        #       in the Costs class, stored as SYSTEM_FLUSH_COST
+
+        self.heating_recommendations = []
+        self.heating_control_recommendations = []
         # This first iteration of the recommender will provide very basic recommendation
         # We recommend heating controls based on the main heating system
-        if self.property.main_heating["clean_description"] in [
+
+        has_electric_heating_description = self.property.main_heating["clean_description"] in [
             "Room heaters, electric", "Electric storage heaters", "Electric storage heaters, radiators"
-        ]:
+        ]
+
+        no_heating_no_mains = (
+            self.property.main_heating["clean_description"] in ["No system present, electric heaters assumed"] and
+            not self.property.data["mains-gas-flag"]
+        )
+
+        if has_electric_heating_description or no_heating_no_mains:
             # Recommend high heat retention storage heaters
-            self.recommend_electric_storage_heaters(phase=phase, system_change=True, heating_controls_only=False)
-            return
+            self.recommend_hhr_storage_heaters(phase=phase, system_change=True, heating_controls_only=False)
+
+        # if the property has mains heating with boiler and radiators, we recommend optimal heating controls
+        has_boiler = self.property.main_heating["clean_description"] in ["Boiler and radiators, mains gas"]
+
+        # We also check that the property doesn't have a heating system, but it has access to the mains gas
+        no_heating_has_mains = self.property.main_heating["clean_description"] in [
+            'No system present, electric heaters assumed'
+        ] and self.property.data["mains-gas-flag"]
+
+        has_gas_heaters = (
+            self.property.main_heating["clean_description"] in ["Room heaters, mains gas"] and
+            self.property.data["mains-gas-flag"]
+        )
+
+        # We also check if the property has electric heating, but it has access to the mains gas
+        electic_heating_has_mains = has_electric_heating_description and self.property.data["mains-gas-flag"]
+
+        portable_heaters_has_mains = (
+            self.property.main_heating["clean_description"] in ["Portable electric heaters assumed for most rooms"] and
+            self.property.data["mains-gas-flag"]
+        )
+
+        if (
+            has_boiler or
+            no_heating_has_mains or
+            electic_heating_has_mains or
+            has_gas_heaters or
+            portable_heaters_has_mains
+        ):
+            # This indicates that the home previously did not have a boiler in place and so would require
+            # an overhaul to the system - right now, this is all reasons, apart from if there is an existing boiler
+            system_change = not has_boiler
+            exising_room_heaters = self.property.main_heating["clean_description"] in [
+                "Room heaters, electric", "Room heaters, mains gas"
+            ]
+
+            self.recommend_boiler_upgrades(
+                phase=phase, system_change=system_change, exising_room_heaters=exising_room_heaters
+            )
+
+        return
 
     @staticmethod
     def check_simulation_difference(old_config, new_config):
@@ -39,9 +96,8 @@ class HeatingRecommender:
 
         return differences
 
-    @staticmethod
     def combine_heating_and_controls(
-        controls_recommendations, heating_simulation_config, costs, description, phase, heating_controls_only,
+        self, controls_recommendations, heating_simulation_config, costs, description, phase, heating_controls_only,
         system_change
     ):
         """
@@ -82,8 +138,18 @@ class HeatingRecommender:
                     **recommendation_simulation_config,
                     **controls_recommendations[0]["simulation_config"]
                 }
+                controls_description = controls_recommendations[0]['description']
+                # Make the first letter of the description lowercase
+                controls_description = (
+                    controls_description[0].lower() + controls_description[1:]
+                )
 
-                recommendation_description = f"{description} and {controls_recommendations[0]['description']}"
+                recommendation_description = f"{description} and {controls_description}"
+
+            already_installed = "cavity_wall_insulation" in self.property.already_installed
+            if already_installed:
+                total_costs = override_costs(total_costs)
+                recommendation_description = "Heating system has already been upgraded, no further action needed."
 
             recommendation = {
                 "phase": phase,
@@ -95,6 +161,7 @@ class HeatingRecommender:
                 "starting_u_value": None,
                 "new_u_value": None,
                 "sap_points": None,
+                "already_installed": already_installed,
                 **total_costs,
                 "simulation_config": recommendation_simulation_config
             }
@@ -126,9 +193,8 @@ class HeatingRecommender:
 
         return output
 
-    def recommend_electric_storage_heaters(self, phase, system_change, heating_controls_only):
+    def recommend_hhr_storage_heaters(self, phase, system_change, heating_controls_only):
         """
-        We recommend electric storage heaters as an upgrade to the heating system.
         We will recommend upgrading to a high heat retention storage system, if the current system is not already
         high heat retention storage
 
@@ -165,9 +231,18 @@ class HeatingRecommender:
         # This upgrade will only take the heating system to average energy efficiency
         heating_simulation_config["mainheat_energy_eff_ending"] = "Average"
 
+        # If the property is off-gas and has no heating system in place, the number of heated rooms will actually
+        # be 0, so we use the number of rooms as the figure
+        number_heated_rooms = (
+            self.property.data["number-heated-rooms"] if self.property.data["number-heated-rooms"] > 0
+            else (
+                self.property.number_of_rooms - 1 if self.property.number_of_rooms > 1 else
+                self.property.number_of_rooms
+            )
+        )
         # Upgrade to electric storage heaters
         costs = self.costs.high_heat_electric_storage_heaters(
-            number_heated_rooms=self.property.data["number-heated-rooms"]
+            number_heated_rooms=number_heated_rooms
         )
         description = "Install high heat retention electric storage heaters"
 
@@ -181,4 +256,183 @@ class HeatingRecommender:
             system_change=system_change
         )
 
-        self.recommendations.extend(recommendations)
+        self.heating_recommendations.extend(recommendations)
+
+    @staticmethod
+    def estimate_boiler_size(property_type, built_form, floor_area, floor_height, num_heated_rooms):
+        # Step 1: Base size estimation based on property type (as a starting point)
+        base_size = {
+            'Flat': 25,
+            'House': 30,
+            'Maisonette': 28,
+            'Bungalow': 27
+        }
+
+        # Step 2: Calculate the volume of the property
+        volume = floor_area * floor_height
+
+        # Step 3: Adjust base size for built form (to account for heat retention)
+        form_adjustment = {
+            'Mid-Terrace': 0,
+            'End-Terrace': 2,
+            'Semi-Detached': 4,
+            'Detached': 6
+        }
+
+        # Step 4: Further adjust for the total volume and number of heated rooms
+        volume_adjustment = (volume / 100)  # Simplified adjustment factor for volume
+        rooms_adjustment = (num_heated_rooms - 5) * 0.5  # Assuming base case of 5 rooms
+
+        # Calculate the estimated boiler size
+        estimated_size = base_size[property_type] + form_adjustment[built_form] + volume_adjustment + rooms_adjustment
+
+        # Step 5: Align with available boiler sizes and ensure it does not exceed 35kW, as it's rare to need more
+        available_sizes = [30, 35, 40, 45, 50]
+        estimated_size = min(max(estimated_size, 30), 40)  # Ensure within 30kW to 35kW range
+
+        # Find the closest available size (in this case, either rounding up or down to align with 30 or 35)
+        closest_size = min(available_sizes, key=lambda x: abs(x - estimated_size))
+
+        return closest_size
+
+    def recommend_boiler_upgrades(self, phase, system_change, exising_room_heaters):
+        """
+        This boiler recommendation will only recommend a like-for-like upgrade, since changing the system
+        is generally more expensive
+        :param phase:
+        :param system_change: Indicates if the property would be undergoing a heating system change. This could be true
+                              if the home didn't have a heating system in place, or if the home had electric heating
+                              previously
+        :param exising_room_heaters: Indicates if the property had room heaters previously - if so, a boiler
+                                     recommendation will need to be accompanied by removal of the room heaters
+        :return:
+        """
+
+        recommendation_phase = phase
+
+        # We now recommend boiler upgrades, if applicable
+        simulation_config = {}
+        boiler_costs = {}
+        boiler_recommendation = {}
+
+        has_inefficient_space_heating = self.property.data["mainheat-energy-eff"] in ["Very Poor", "Poor", "Average"]
+
+        has_inefficient_mains_water = (
+            self.property.hotwater["clean_description"] in ["From main system"] and
+            self.property.data["hot-water-energy-eff"] in ["Very Poor", "Poor", "Average"]
+        )
+
+        if has_inefficient_space_heating or has_inefficient_mains_water:
+            boiler_size = self.estimate_boiler_size(
+                property_type=self.property.data["property-type"],
+                built_form=self.property.data["built-form"],
+                floor_area=self.property.floor_area,
+                floor_height=self.property.floor_height,
+                num_heated_rooms=self.property.data["number-heated-rooms"],
+            )
+
+            description = "Upgrade to a new condensing boiler"
+
+            simulation_config = {
+                "mainheat_energy_eff_ending": "Good",
+                "hot_water_energy_eff_ending": "Good"
+            }
+            if system_change:
+                # Installation of a boiler improves the hot water system so we need to reflect this in
+                # the outcome of the recommendation
+                heating_ending_config = MainHeatAttributes("Boiler and radiators, mains gas").process()
+                hotwater_ending_config = HotWaterAttributes("From main system").process()
+                fuel_ending_config = MainFuelAttributes("mains gas (not community)").process()
+
+                heating_simulation_config = check_simulation_difference(
+                    new_config=heating_ending_config, old_config=self.property.main_heating
+                )
+                hotwater_simulation_config = check_simulation_difference(
+                    new_config=hotwater_ending_config, old_config=self.property.hotwater
+                )
+                fuel_simulation_config = check_simulation_difference(
+                    new_config=fuel_ending_config, old_config=self.property.main_fuel
+                )
+
+                simulation_config = {
+                    **simulation_config,
+                    **heating_simulation_config,
+                    **hotwater_simulation_config,
+                    **fuel_simulation_config,
+                    "hot_water_energy_eff_ending": "Good"
+                }
+
+            boiler_costs = self.costs.boiler(
+                size=f"{boiler_size}kw",
+                exising_room_heaters=exising_room_heaters,
+                system_change=system_change,
+                n_heated_rooms=self.property.data["number-heated-rooms"],
+                n_rooms=self.property.number_of_rooms
+            )
+
+            already_installed = "heating" in self.property.already_installed
+            if already_installed:
+                boiler_costs = override_costs(boiler_costs)
+                description = "Heating system has already been upgraded, no further action needed."
+
+            boiler_recommendation = {
+                "phase": recommendation_phase,
+                "parts": [
+                    # TODO
+                ],
+                "type": "heating",
+                "description": description,
+                "starting_u_value": None,
+                "new_u_value": None,
+                "sap_points": None,
+                "already_installed": already_installed,
+                "simulation_config": simulation_config,
+                **boiler_costs
+            }
+
+        # We recommend the heating controls
+        # If the property did not previously have a boiler, we combine
+        controls_recommender = HeatingControlRecommender(self.property)
+        controls_recommender.recommend(heating_description="Boiler and radiators, mains gas")
+        # We may have 2 recommendations from the heating controls
+
+        if not controls_recommender.recommendation and not boiler_recommendation:
+            return
+
+        if not system_change and len(boiler_recommendation):
+            # If there is not a system change, we add the boiler recommendation at point.
+            self.heating_recommendations.extend([boiler_recommendation])
+
+        if system_change:
+            # We combine the heating and controls recommendations, in the case of a system change
+            combined_recommendations = []
+            for controls_recommendation in controls_recommender.recommendation:
+                combined_recommendation = self.combine_heating_and_controls(
+                    controls_recommendations=[controls_recommendation],
+                    heating_simulation_config=simulation_config,
+                    costs=boiler_costs,
+                    description=boiler_recommendation["description"],
+                    phase=recommendation_phase,
+                    heating_controls_only=False,
+                    system_change=True
+                )
+                combined_recommendations.extend(combined_recommendation)
+
+            # Overwrite the existing boiler recommendation
+            self.heating_recommendations.extend(combined_recommendations)
+        else:
+            # We increment the recommendation phase, since the heating controls are separate from the boiler upgrade
+            # but we'll only upgrade if we have a heating recommendation
+            has_heating_recommendation = any(
+                rec["type"] == "heating" for rec in self.heating_recommendations
+            )
+            if has_heating_recommendation:
+                recommendation_phase += 1
+            # The heating controls recommendation is distrinct from the boiler upgrade recommendation
+            # We insert phase into the recommendations for heating controls
+            for recommendation in controls_recommender.recommendation:
+                recommendation["phase"] = recommendation_phase
+
+            self.heating_control_recommendations.extend(controls_recommender.recommendation)
+
+        return

recommendations/HotwaterRecommendations.py

@@ -1,5 +1,6 @@
 from backend.Property import Property
 from recommendations.Costs import Costs
+from recommendations.recommendation_utils import override_costs
 
 
 class HotwaterRecommendations:
@@ -22,8 +23,14 @@ class HotwaterRecommendations:
 
         # This first iteration of the recommender will provide very basic recommendation
         # We recommend heating controls based on the main heating system
-        if (self.property.hotwater["heater_type"] in ["electric immersion"]) & \
-            (self.property.data["hot-water-energy-eff"] == "Very Poor"):
+
+        # If there is no system present, but access to the mains, we
+
+        if (
+            (self.property.hotwater["heater_type"] in ["electric immersion"]) &
+            (self.property.data["hot-water-energy-eff"] == "Very Poor") &
+            (self.property.hotwater["no_system_present"] is None)
+        ):
             self.recommend_tank_insulation(phase=phase)
             return
 
@@ -35,6 +42,13 @@ class HotwaterRecommendations:
 
         recommendation_cost = self.costs.hot_water_tank_insulation()
 
+        already_installed = "hot_water_tank_insulation" in self.property.already_installed
+        if already_installed:
+            recommendation_cost = override_costs(recommendation_cost)
+            description = "Insulation tank has already been insulated, no further action required"
+        else:
+            description = "Insulate hot water tank"
+
         self.recommendations.append(
             {
                 "phase": phase,
@@ -42,10 +56,11 @@ class HotwaterRecommendations:
                     # TODO
                 ],
                 "type": "hot_water_tank_insulation",
-                "description": "Insulate the hot water tank with an insulation jacket",
+                "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
                 "sap_points": None,
+                "already_installed": already_installed,
                 **recommendation_cost,
                 "simulation_config": {"hot_water_energy_eff_ending": "Average"}
             }

recommendations/LightingRecommendations.py

@@ -1,6 +1,7 @@
 from backend.Property import Property
 from typing import List
 from recommendations.Costs import Costs
+from recommendations.recommendation_utils import override_costs
 
 
 class LightingRecommendations:
@@ -91,6 +92,11 @@ class LightingRecommendations:
 
         heat_demand_change, carbon_change = self.estimate_lighting_impact(number_non_lel_outlets)
 
+        already_installed = "low_energy_lighting" in self.property.already_installed
+        if already_installed:
+            cost_result = override_costs(cost_result)
+            description = "Low energy lighting has already been installed, no further action required"
+
         self.recommendation = [
             {
                 "phase": phase,
@@ -99,6 +105,7 @@ class LightingRecommendations:
                 "description": description,
                 "starting_u_value": None,
                 "new_u_value": None,
+                "already_installed": already_installed,
                 # For SAP points, we use the fact that lighting is usually worth 2 points and we scale this to
                 # the proportion of lights that will be set to low energy
                 "sap_points": round(2 * (number_non_lel_outlets / number_lighting_outlets), 2),

recommendations/Recommendations.py

@@ -11,6 +11,7 @@ from recommendations.SolarPvRecommendations import SolarPvRecommendations
 from recommendations.WindowsRecommendations import WindowsRecommendations
 from recommendations.HeatingRecommender import HeatingRecommender
 from recommendations.HotwaterRecommendations import HotwaterRecommendations
+from recommendations.SecondaryHeating import SecondaryHeating
 from backend.ml_models.AnnualBillSavings import AnnualBillSavings
 
 
@@ -22,7 +23,8 @@ class Recommendations:
     def __init__(
         self,
         property_instance: Property,
-        materials: List
+        materials: List,
+        exclusions: List[str] = None,
     ):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
@@ -31,6 +33,7 @@ class Recommendations:
 
         self.property_instance = property_instance
         self.materials = materials
+        self.exclusions = exclusions if exclusions else []
 
         self.floor_recommender = FloorRecommendations(property_instance=property_instance, materials=materials)
         self.wall_recomender = WallRecommendations(property_instance=property_instance, materials=materials)
@@ -44,8 +47,9 @@ class Recommendations:
         self.solar_recommender = SolarPvRecommendations(property_instance=property_instance)
         self.heating_recommender = HeatingRecommender(property_instance=property_instance)
         self.hotwater_recommender = HotwaterRecommendations(property_instance=property_instance)
+        self.secondary_heating_recommender = SecondaryHeating(property_instance=property_instance)
 
-    def recommend(self, portfolio_id):
+    def recommend(self):
 
         """
         This method runs the recommendations for the individual measures and then appends them to a list for output
@@ -58,78 +62,113 @@ class Recommendations:
         property_recommendations = []
         phase = 0
 
-        print("WALL RECOMMENDATIONS HAVE BEEN COMMENTED OUT TEMPORARILY - ADD ME BACK IN")
-        if portfolio_id != 66:
-            # Building Fabric
+        # Building Fabric
+        if "wall_insulation" not in self.exclusions:
             self.wall_recomender.recommend(phase=phase)
             if self.wall_recomender.recommendations:
                 property_recommendations.append(self.wall_recomender.recommendations)
                 phase += 1
 
-            # Ventilation recommendations
-            # We only produce a ventilation recommendation if the property is recommended to have wall or roof
-            # insulation
-            # We will not attribute a SAP impact to the ventilation recommendation, since we've seen that this has no
-            # real impact on the SAP score. Therefore, we don't need to include phasing for ventilation. If we have any
-            # wall or roof recommendations, we will ensure that ventilation is included in the simulation
+        if "roof_insulation" not in self.exclusions:
+            self.roof_recommender.recommend(phase=phase)
+            if self.roof_recommender.recommendations:
+                property_recommendations.append(self.roof_recommender.recommendations)
+                phase += 1
+
+        # Ventilation recommendations
+        # We only produce a ventilation recommendation if the property is recommended to have wall or roof
+        # insulation
+        # We will not attribute a SAP impact to the ventilation recommendation, since we've seen that this
+        # has no
+        # real impact on the SAP score. Therefore, we don't need to include phasing for ventilation. If we
+        # have any
+        # wall or roof recommendations, we will ensure that ventilation is included in the simulation
+        if "ventilation" not in self.exclusions:
             if self.wall_recomender.recommendations or self.roof_recommender.recommendations:
                 self.ventilation_recomender.recommend()
                 if self.ventilation_recomender.recommendation:
                     property_recommendations.append(self.ventilation_recomender.recommendation)
 
-        self.roof_recommender.recommend(phase=phase)
-        if self.roof_recommender.recommendations:
-            property_recommendations.append(self.roof_recommender.recommendations)
-            phase += 1
+        if "floor_insulation" not in self.exclusions:
+            self.floor_recommender.recommend(phase=phase)
+            if self.floor_recommender.recommendations:
+                property_recommendations.append(self.floor_recommender.recommendations)
+                phase += 1
 
-        self.floor_recommender.recommend(phase=phase)
-        if self.floor_recommender.recommendations:
-            property_recommendations.append(self.floor_recommender.recommendations)
-            phase += 1
+        if "windows" not in self.exclusions:
+            self.windows_recommender.recommend(phase=phase)
+            if self.windows_recommender.recommendation:
+                property_recommendations.append(self.windows_recommender.recommendation)
+                phase += 1
 
-        self.windows_recommender.recommend(phase=phase)
-        if self.windows_recommender.recommendation:
-            property_recommendations.append(self.windows_recommender.recommendation)
-            phase += 1
-
-        self.fireplace_recommender.recommend(phase=phase)
-        if self.fireplace_recommender.recommendation:
-            property_recommendations.append(self.fireplace_recommender.recommendation)
-            phase += 1
+        if "fireplace" not in self.exclusions:
+            self.fireplace_recommender.recommend(phase=phase)
+            if self.fireplace_recommender.recommendation:
+                property_recommendations.append(self.fireplace_recommender.recommendation)
+                phase += 1
 
         # Heating and Electical systems
-        self.heating_recommender.recommend(phase=phase)
-        if self.heating_recommender.recommendations:
-            property_recommendations.append(self.heating_recommender.recommendations)
-            phase += 1
+        if "heating" not in self.exclusions:
+            self.heating_recommender.recommend(phase=phase)
+            if (
+                self.heating_recommender.heating_recommendations or
+                self.heating_recommender.heating_control_recommendations
+            ):
+                if self.heating_recommender.heating_recommendations:
+                    property_recommendations.append(self.heating_recommender.heating_recommendations)
+
+                if self.heating_recommender.heating_control_recommendations:
+                    property_recommendations.append(self.heating_recommender.heating_control_recommendations)
+
+                # We check if we have distinct heating and heating controls recommendations
+                # If so, we increment by 2 (one of the heating system, one for the heating controls)
+                # otherwise we incremenet by 1
+                max_used_phase = max(
+                    [rec["phase"] for rec in
+                     self.heating_recommender.heating_recommendations +
+                     self.heating_recommender.heating_control_recommendations]
+                )
+                amount_to_increment = max_used_phase - phase + 1
+                phase += amount_to_increment
 
         # Hot water
-        self.hotwater_recommender.recommend(phase=phase)
-        if self.hotwater_recommender.recommendations:
-            property_recommendations.append(self.hotwater_recommender.recommendations)
-            phase += 1
+        if "hot_water" not in self.exclusions:
+            self.hotwater_recommender.recommend(phase=phase)
+            if self.hotwater_recommender.recommendations:
+                property_recommendations.append(self.hotwater_recommender.recommendations)
+                phase += 1
 
-        self.lighting_recommender.recommend(phase=phase)
-        if self.lighting_recommender.recommendation:
-            property_recommendations.append(self.lighting_recommender.recommendation)
-            phase += 1
+        if "lighting" not in self.exclusions:
+            self.lighting_recommender.recommend(phase=phase)
+            if self.lighting_recommender.recommendation:
+                property_recommendations.append(self.lighting_recommender.recommendation)
+                phase += 1
+
+        if "secondary_heating" not in self.exclusions:
+            self.secondary_heating_recommender.recommend(phase=phase)
+            if self.secondary_heating_recommender.recommendation:
+                property_recommendations.append(self.secondary_heating_recommender.recommendation)
+                phase += 1
 
         # Renewables
-        self.solar_recommender.recommend(phase=phase)
-        if self.solar_recommender.recommendation:
-            property_recommendations.append(self.solar_recommender.recommendation)
-            phase += 1
+        if "solar_pv" not in self.exclusions:
+            self.solar_recommender.recommend(phase=phase)
+            if self.solar_recommender.recommendation:
+                property_recommendations.append(self.solar_recommender.recommendation)
+                phase += 1
 
         # We insert temporary ids into the recommendations which is important for the optimiser later
         property_recommendations = self.insert_temp_recommendation_id(property_recommendations)
 
         # We also need to create the representative recommendations for each recommendation type
-        property_representative_recommendations = self.create_representative_recommendations(property_recommendations)
+        property_representative_recommendations = self.create_representative_recommendations(
+            property_recommendations, non_invasive_recommendations=self.property_instance.non_invasive_recommendations
+        )
 
         return property_recommendations, property_representative_recommendations
 
     @staticmethod
-    def create_representative_recommendations(property_recommendations):
+    def create_representative_recommendations(property_recommendations, non_invasive_recommendations):
         """
         This method will create a representative recommendation for each recommendation type
         In order to create a representative recommendation, we choose the recommendation that has:
@@ -144,6 +183,13 @@ class Recommendations:
 
         for recommendations_by_type in property_recommendations:
 
+            # If the property was initially surveyed as filled, but the cavity was only partially filled, we don't
+            # want to include the cavity wall insulation recommendation in the defaults
+            # if (recommendations_by_type[0].get("type") == "cavity_wall_insulation") and (
+            #     "cavity_surveyed_as_filled_is_partial" in non_invasive_recommendations
+            # ):
+            #     continue
+
             if recommendations_by_type[0].get("type") == "mechanical_ventilation":
                 continue
 
@@ -213,13 +259,13 @@ class Recommendations:
 
         property_sap_predictions = all_predictions["sap_change_predictions"][
             all_predictions["sap_change_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
         property_heat_predictions = all_predictions["heat_demand_predictions"][
             all_predictions["heat_demand_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
         property_carbon_predictions = all_predictions["carbon_change_predictions"][
             all_predictions["carbon_change_predictions"]["property_id"] == str(property_instance.id)
-            ]
+            ].copy()
 
         property_recommendations = recommendations[property_instance.id].copy()
 
@@ -247,6 +293,8 @@ class Recommendations:
             current_epc_rating=property_instance.data["current-energy-rating"],
         )
 
+        # TODO: This isn't quite right as this is based on EVERY possible measure, not just the ones that are
+        #       actually implemented
         expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
             epc_energy_consumption=expected_heat_demand,
             current_epc_rating=property_instance.data["current-energy-rating"],
@@ -256,6 +304,10 @@ class Recommendations:
             current_adjusted_energy - expected_adjusted_energy
         )
 
+        # TODO: We should determine if the home is gas & electricity or just electricity
+        current_energy_bill = AnnualBillSavings.calculate_annual_bill(current_adjusted_energy)
+        expected_energy_bill = AnnualBillSavings.calculate_annual_bill(expected_adjusted_energy)
+
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
 
@@ -330,4 +382,10 @@ class Recommendations:
                     rec["heat_demand"] is None) or (rec["energy_cost_savings"] is None):
                     raise ValueError("sap points, co2 or heat demand is missing")
 
-        return property_recommendations, current_adjusted_energy, expected_adjusted_energy
+        return (
+            property_recommendations,
+            current_adjusted_energy,
+            expected_adjusted_energy,
+            current_energy_bill,
+            expected_energy_bill
+        )

recommendations/RoofRecommendations.py

@@ -5,7 +5,7 @@ from typing import List
 from datatypes.enums import QuantityUnits
 from recommendations.recommendation_utils import (
     get_roof_u_value, r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns,
-    update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric
+    update_lowest_selected_u_value, get_recommended_part, convert_thickness_to_numeric, override_costs
 )
 from recommendations.Costs import Costs
 
@@ -20,8 +20,9 @@ class RoofRecommendations:
 
     DIMINISHING_RETURNS_U_VALUE = 0.14
 
-    # It is recommended that lofts should have at least 270mm of insulation
-    MINIMUM_LOFT_ISULATION_MM = 270
+    # It is recommended that lofts should have at least 270mm of insulation. If the property has more than 200mm of
+    # loft insulation in place already, we do not recommend anything for the moment
+    MINIMUM_LOFT_ISULATION_MM = 200
     # Flat roof should have at least 100mm of insulation
     MINIMUM_FLAT_ROOF_ISULATION_MM = 100
 
@@ -71,7 +72,7 @@ class RoofRecommendations:
         # Building regulations part L recommend installing at least 270mm of insulation, however generally we
         # experience diminishing returns in terms of SAP once we go beyond around 150mm of insulation
         # This only holds true for pitched roofs.
-        if (insulation_thickness >= self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]:
+        if (insulation_thickness > self.MINIMUM_LOFT_ISULATION_MM) and self.property.roof["is_pitched"]:
             return
 
         if (insulation_thickness >= self.MINIMUM_FLAT_ROOF_ISULATION_MM) and self.property.roof["is_flat"]:
@@ -206,12 +207,18 @@ class RoofRecommendations:
                             floor_area=self.property.insulation_floor_area,
                             material=material
                         )
+                        already_installed = "loft_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
                     elif material["type"] == "flat_roof_insulation":
                         cost_result = self.costs.flat_roof_insulation(
                             floor_area=self.property.insulation_floor_area,
                             material=material,
                             non_insulation_materials=non_insulation_materials
                         )
+                        already_installed = "flat_roof_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
                     else:
                         raise ValueError("Invalid material type")
 
@@ -231,6 +238,7 @@ class RoofRecommendations:
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": None,
+                            "already_installed": already_installed,
                             **cost_result
                         }
                     )

recommendations/SecondaryHeating.py

@@ -0,0 +1,65 @@
+from recommendations.Costs import Costs
+from recommendations.recommendation_utils import override_costs
+from backend.Property import Property
+
+
+class SecondaryHeating:
+    """
+    This class recommends the removal of the secondary heating system for properties that have a primary heating
+    system.
+    """
+
+    # The list of existing heating systems that are accepted
+    ACCEPTED_MAINHEAT_DESCRIPTIONS = ["Boiler and radiators, mains gas"]
+    ACCEPTED_SECONDHEAT_DESCRIPTIONS = ["Room heaters, electric"]
+    # These are the heaters where works are required to remove them
+    FIXED_HEATER_DESCRIPTIONS = ["Room heaters, electric"]
+
+    def __init__(self, property_instance: Property):
+        self.property = property_instance
+        self.costs = Costs(self.property)
+
+        self.recommendation = []
+
+    def recommend(self, phase: int):
+        # Reset
+        self.recommendation = []
+
+        if self.property.main_heating["clean_description"] not in self.ACCEPTED_MAINHEAT_DESCRIPTIONS:
+            return
+
+        # TODO: We need to clean secondary data
+        if self.property.data['secondheat-description'] not in self.ACCEPTED_SECONDHEAT_DESCRIPTIONS:
+            return
+
+        if self.property.data['secondheat-description'] in self.FIXED_HEATER_DESCRIPTIONS:
+            # We have an associated cost otherwise, there is no cost
+            n_rooms = self.property.data['number-heated-rooms']
+        else:
+            n_rooms = 0
+
+        costs = self.costs.heater_removal(n_rooms=n_rooms)
+
+        already_installed = "secondary_heating" in self.property.already_installed
+        if already_installed:
+            costs = override_costs(costs)
+            description = "Secondary heating system has already been removed, no further action required"
+        else:
+            description = "Remove the secondary heating system"
+
+        self.recommendation.append(
+            {
+                "phase": phase,
+                "parts": [],
+                "type": "secondary_heating",
+                "description": description,
+                "starting_u_value": None,
+                "new_u_value": None,
+                "sap_points": None,
+                "already_installed": already_installed,
+                **costs,
+                "simulation_config": {
+                    "secondheat_description_ending": "None"
+                }
+            }
+        )

recommendations/SolarPvRecommendations.py

@@ -1,5 +1,6 @@
 import numpy as np
 from recommendations.Costs import Costs
+from recommendations.recommendation_utils import override_costs
 
 
 class SolarPvRecommendations:
@@ -8,6 +9,9 @@ class SolarPvRecommendations:
     # Wattage per panel - this is based on the average wattage of a solar panel being between 250w and 420w
     SOLAR_PANEL_WATTAGE = 250
 
+    MAX_SYSTEM_WATTAGE = 6000
+    MIN_SYSTEM_WATTAGE = 1000
+
     def __init__(self, property_instance):
         """
         :param property_instance: Instance of the Property class, for the home associated to property_id
@@ -18,6 +22,19 @@ class SolarPvRecommendations:
 
         self.recommendation = []
 
+    @staticmethod
+    def trim_solar_wattage_options(scenarios_with_wattage):
+        # Initialize the list with the first element, assuming the list is not empty
+        trimmed_list = [scenarios_with_wattage[0]]
+
+        # Iterate over the list starting from the second element
+        for scenario in scenarios_with_wattage[1:]:
+            # Compare the second element (index 1) of the current tuple with the last tuple in the trimmed list
+            if scenario[1] > trimmed_list[-1][1]:
+                trimmed_list.append(scenario)
+
+        return trimmed_list
+
     def recommend(self, phase):
         """
         We check if a property is potentially suitable for solar PV based on the following criteria:
@@ -39,33 +56,56 @@ class SolarPvRecommendations:
         if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv:
             return
 
+        solar_pv_percentage = self.property.solar_pv_percentage
+        # We round up to the neaest 10%
+        solar_pv_percentage = np.ceil(solar_pv_percentage * 10) / 10
+
         # For the solar recommendations, we produce the following scenarios:
         # 1) Solar panels only, we present a high, medium and low coverage
         # 2) With and without battery
         roof_coverage_scenarios = [
-            self.property.solar_pv_percentage - 0.1, self.property.solar_pv_percentage,
-            self.property.solar_pv_percentage + 0.1
+            solar_pv_percentage - 0.1, solar_pv_percentage,
         ]
-        # We make sure we haven't gone too low or high
-        roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 1]
+        if solar_pv_percentage <= 0.4:
+            roof_coverage_scenarios.append(solar_pv_percentage + 0.1)
+        # We make sure we haven't gone too low or high - we allow no more than 60% coverage
+        roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 0.6]
+        # If we only have two scenarios, we add a coverage scenario 10% less than the smallest
+        if len(roof_coverage_scenarios) == 2:
+            roof_coverage_scenarios.insert(0, roof_coverage_scenarios[0] - 0.1)
         battery_scenarios = [False, True]
 
-        # I now produce the cross product of the scenarios
-        scenarios = [(roof, battery) for roof in roof_coverage_scenarios for battery in battery_scenarios]
-
-        for roof_coverage, has_battery in scenarios:
+        scenarios_with_wattage = []
+        for roof_coverage in roof_coverage_scenarios:
             # We now have a property which is potentially suitable for solar PV
             solar_pv_roof_area = self.property.get_solar_pv_roof_area(roof_coverage)
 
             number_solar_panels = np.floor(solar_pv_roof_area / self.SOLAR_PANEL_AREA)
             solar_panel_wattage = number_solar_panels * self.SOLAR_PANEL_WATTAGE
 
-            roof_coverage_percent = round(roof_coverage * 100)
+            if solar_panel_wattage < self.MIN_SYSTEM_WATTAGE:
+                continue
 
+            solar_panel_wattage = np.clip(
+                a=solar_panel_wattage, a_min=self.MIN_SYSTEM_WATTAGE, a_max=self.MAX_SYSTEM_WATTAGE
+            )
+            scenarios_with_wattage.append((roof_coverage, solar_panel_wattage))
+
+        # We trim the scenarios, so that we don't have duplicate wattages
+        scenarios_with_wattage = self.trim_solar_wattage_options(scenarios_with_wattage)
+
+        # Produce the cross product of the scenarios
+        scenarios = [
+            (roof, wattage, battery) for roof, wattage in scenarios_with_wattage for battery in battery_scenarios
+        ]
+        # We deduce the wattage of the solar panels based on the roof coverage
+
+        for roof_coverage, solar_panel_wattage, has_battery in scenarios:
+            # We now have a property which is potentially suitable for solar PV
+            roof_coverage_percent = round(roof_coverage * 100)
             # Given the wattage, we estimate the cost of the solar PV system. This is based on the MCS database
             # of solar PV installations
             cost_result = self.costs.solar_pv(wattage=solar_panel_wattage, has_battery=has_battery)
-
             kw = np.floor(solar_panel_wattage / 100) / 10
 
             if has_battery:
@@ -75,6 +115,10 @@ class SolarPvRecommendations:
                 description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p"
                                f"anel system on {round(roof_coverage_percent)}% the roof.")
 
+            already_installed = "solar_pv" in self.property.already_installed
+            if already_installed:
+                cost_result = override_costs(cost_result)
+
             self.recommendation.append(
                 {
                     "phase": phase,
@@ -84,9 +128,11 @@ class SolarPvRecommendations:
                     "starting_u_value": None,
                     "new_u_value": None,
                     "sap_points": None,
+                    "already_installed": already_installed,
                     **cost_result,
                     # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale
                     # back up here
-                    "photo_supply": 100 * roof_coverage
+                    "photo_supply": 100 * roof_coverage,
+                    "has_battery": has_battery
                 }
             )

recommendations/VentilationRecommendations.py

@@ -50,7 +50,11 @@ class VentilationRecommendations(Definitions):
 
         part = self.materials.copy()
 
-        estimated_cost = n_units * part[0]["cost"]
+        already_installed = "cavity_wall_insulation" in self.property.already_installed
+
+        estimated_cost = n_units * part[0]["cost"] if not already_installed else 0
+        labour_hours = 4 * n_units if not already_installed else 0
+        labour_days = 4 * n_units / 8.0 if not already_installed else 0
 
         part[0]["total"] = estimated_cost
         part[0]["quantity"] = n_units
@@ -65,6 +69,7 @@ class VentilationRecommendations(Definitions):
                 "description": f"Install {n_units} {part[0]['description']} units",
                 "starting_u_value": None,
                 "new_u_value": None,
+                "already_installed": already_installed,
                 "sap_points": 0,
                 "heat_demand": 0,
                 "adjusted_heat_demand": 0,
@@ -72,7 +77,7 @@ class VentilationRecommendations(Definitions):
                 "energy_cost_savings": 0,
                 "total": estimated_cost,
                 # We use a very simple and rough estimate of 4 hours per unit
-                "labour_hours": 4 * n_units,
-                "labour_days": 4 * n_units / 8.0  # Assume 8 hour day
+                "labour_hours": labour_hours,
+                "labour_days": labour_days  # Assume 8 hour day
             }
         ]

recommendations/WallRecommendations.py

@@ -8,7 +8,7 @@ from backend.Property import Property
 from BaseUtility import Definitions
 from recommendations.recommendation_utils import (
     r_value_per_mm_to_u_value, calculate_u_value_uplift, is_diminishing_returns, update_lowest_selected_u_value,
-    get_recommended_part, get_wall_u_value
+    get_recommended_part, get_wall_u_value, override_costs
 )
 from recommendations.config import PARTIALLY_FILLED_PERCENTAGE_ASSUMPTION
 from recommendations.Costs import Costs
@@ -113,7 +113,9 @@ class WallRecommendations(Definitions):
         insulation_thickness = self.property.walls["insulation_thickness"]
 
         # We check if the wall is already insulated and if so, we exit
-        if (insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]:
+        if ((insulation_thickness in ["average", "above average"]) or self.property.walls["is_filled_cavity"]) and (
+            "cavity_extract_and_refill" not in self.property.non_invasive_recommendations
+        ):
             return
 
         if u_value:
@@ -216,11 +218,26 @@ class WallRecommendations(Definitions):
             if new_u_value <= self.BUILDING_REGULATIONS_PART_L_CAVITY_WALL_MAX_U_VALUE:
                 lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
 
+                is_extraction_and_refill = "cavity_extract_and_refill" in self.property.non_invasive_recommendations
+
                 cost_result = self.costs.cavity_wall_insulation(
                     wall_area=self.property.insulation_wall_area,
                     material=material.to_dict(),
+                    is_extraction_and_refill=is_extraction_and_refill
                 )
 
+                already_installed = "cavity_wall_insulation" in self.property.already_installed
+                if already_installed:
+                    cost_result = override_costs(cost_result)
+
+                if is_extraction_and_refill:
+                    description = f"Extract and refill cavity wall insulation with {material['description']}"
+                else:
+                    description = self._make_description(material)
+
+                # updated the new u-value with the best possible our installers have
+                new_u_value = max(0.31, new_u_value)
+
                 recommendations.append(
                     {
                         "phase": phase,
@@ -233,10 +250,11 @@ class WallRecommendations(Definitions):
                             )
                         ],
                         "type": "cavity_wall_insulation",
-                        "description": self._make_description(material),
+                        "description": description,
                         "starting_u_value": u_value,
                         "new_u_value": new_u_value,
                         "sap_points": None,
+                        "already_installed": already_installed,
                         **cost_result
                     }
                 )
@@ -277,12 +295,19 @@ class WallRecommendations(Definitions):
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
+                        already_installed = "internal_wall_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
+
                     elif material["type"] == "external_wall_insulation":
                         cost_result = self.costs.external_wall_insulation(
                             wall_area=self.property.insulation_wall_area,
                             material=material.to_dict(),
                             non_insulation_materials=non_insulation_materials
                         )
+                        already_installed = "external_wall_insulation" in self.property.already_installed
+                        if already_installed:
+                            cost_result = override_costs(cost_result)
                     else:
                         raise ValueError("Invalid material type")
 
@@ -301,6 +326,7 @@ class WallRecommendations(Definitions):
                             "description": self._make_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
+                            "already_installed": already_installed,
                             "sap_points": None,
                             **cost_result
                         }

recommendations/WindowsRecommendations.py

@@ -4,6 +4,7 @@ import numpy as np
 
 from backend.Property import Property
 from recommendations.Costs import Costs
+from recommendation_utils import override_costs
 
 
 class WindowsRecommendations:
@@ -70,18 +71,23 @@ class WindowsRecommendations:
             is_secondary_glazing=is_secondary_glazing
         )
 
-        glazing_type = "secondary glazing" if is_secondary_glazing else "double glazing"
-        if self.property.windows["glazing_coverage"] in ["partial", "most"]:
-            description = f"Install {glazing_type} to the remaining windows"
+        already_installed = "windows_glazing" in self.property.already_installed
+        if already_installed:
+            cost_result = override_costs(cost_result)
+            description = "The property already has double glazing installed. No further action is required."
         else:
-            description = f"Install {glazing_type} to all windows"
+            glazing_type = "secondary glazing" if is_secondary_glazing else "double glazing"
+            if self.property.windows["glazing_coverage"] in ["partial", "most"]:
+                description = f"Install {glazing_type} to the remaining windows"
+            else:
+                description = f"Install {glazing_type} to all windows"
 
-        if self.property.is_listed:
-            description += ". Secondary glazing recommended due to listed building status"
-        elif self.property.is_heritage:
-            description += ". Secondary glazing recommended due to herigate building status"
-        elif self.property.in_conservation_area:
-            description += ". Secondary glazing recommended due to conservation area status"
+            if self.property.is_listed:
+                description += ". Secondary glazing recommended due to listed building status"
+            elif self.property.is_heritage:
+                description += ". Secondary glazing recommended due to herigate building status"
+            elif self.property.in_conservation_area:
+                description += ". Secondary glazing recommended due to conservation area status"
 
         self.recommendation = [
             {
@@ -92,6 +98,7 @@ class WindowsRecommendations:
                 "starting_u_value": None,
                 "new_u_value": None,
                 "sap_points": None,
+                "already_installed": already_installed,
                 **cost_result,
                 "is_secondary_glazing": is_secondary_glazing
             }

recommendations/optimiser/optimiser_functions.py

@@ -1,17 +1,13 @@
-def prepare_input_measures(property_recommendations, goal, housing_type):
+def prepare_input_measures(property_recommendations, goal):
     """
     Basic function to convert recommendations_to_upload to a format that is
     suitable for the optimiser - large
     :param property_recommendations:   object containing the recommendations, created in the plan trigger api
     :param goal:    goal to be optimised for, should be one of the keys in gain_map. E.g. if the gain is SAP points,
                     the goal should reflect that desired gain
-    :param housing_type:    type of housing the recommendations are for - should be one of "Social" or "Private"
     :return:    Nested list of input measures
     """
 
-    if housing_type not in ["Social", "Private"]:
-        raise ValueError("Invalid housing type - investigate me")
-
     goal_map = {
         "Increase EPC": "sap_points"
     }
@@ -20,12 +16,14 @@ def prepare_input_measures(property_recommendations, goal, housing_type):
     if not goal_key:
         raise NotImplementedError("Not implemented this gain type - investigate me")
 
-    # We don't include suspended and solid floor insulation as possible measures in private housing, because
-    # of the need to decant the tenant
-    ignored_measures = ["suspended_floor_insulation", "solid_floor_insulation"] if housing_type == "Private" else []
-
     input_measures = []
     for recs in property_recommendations:
+        if recs[0]["type"] == "solar_pv":
+            # if the recommendation is a solar recommendation without a battery, we exclude it from the optimisation.
+            # That will ensure that the optimiser only considers solar recommendations with batteries, so we don't
+            # under-report the potential cost
+            recs = [r for r in recs if r["has_battery"]]
+
         input_measures.append(
             [
                 {
@@ -34,7 +32,7 @@ def prepare_input_measures(property_recommendations, goal, housing_type):
                     "gain": rec[goal_key],
                     "type": rec["type"]
                 }
-                for rec in recs if rec["type"] not in ignored_measures
+                for rec in recs
             ]
         )
 

recommendations/recommendation_utils.py

@@ -767,3 +767,15 @@ def check_simulation_difference(old_config, new_config):
     differences = {key + "_ending": new_config[key] for key in new_config if old_config[key] != new_config[key]}
 
     return differences
+
+
+def override_costs(costs):
+    """
+    If the method is overridden, we want to make sure that the costs are zero. This function sets the costs to zero
+    :param costs: Dictionary of costing, as returned by the Costs class
+    :return:
+    """
+    for k in costs:
+        costs[k] = 0
+
+    return costs

utils/s3.py

@@ -1,9 +1,10 @@
 import pickle
 import boto3
-from io import BytesIO, StringIO
-from botocore.exceptions import NoCredentialsError, PartialCredentialsError
+import csv
 import pandas as pd
+from io import BytesIO, StringIO
 from utils.logger import setup_logger
+from botocore.exceptions import NoCredentialsError, PartialCredentialsError
 
 logger = setup_logger()
 
@@ -184,7 +185,7 @@ def read_pickle_from_s3(bucket_name, s3_file_name):
         logger.errpr("Incomplete credentials provided.")
         return None
     except Exception as e:
-        logger.errpr(f'Failed to download data from {bucket_name}/{s3_file_name}: {str(e)}')
+        logger.error(f'Failed to download data from {bucket_name}/{s3_file_name}: {str(e)}')
         return None
 
     # Deserialize data from pickle format
@@ -224,3 +225,22 @@ def read_excel_from_s3(bucket_name, file_key, header_row):
     df.reset_index(drop=True, inplace=True)
 
     return df
+
+
+def read_csv_from_s3(bucket_name, filepath):
+    s3 = boto3.client('s3')
+
+    # Get the object from s3
+    s3_object = s3.get_object(Bucket=bucket_name, Key=filepath)
+
+    # Read the CSV body from the s3 object
+    body = s3_object['Body'].read()
+
+    # Use StringIO to create a file-like object from the string
+    csv_data = StringIO(body.decode('utf-8'))
+
+    # Use csv library to read it into a list of dictionaries
+    reader = csv.DictReader(csv_data)
+    data = list(reader)
+
+    return data