From a0bbdadd1f621401e9ba6465e69f03a97e4c3446 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 15:13:04 +0000
Subject: [PATCH 01/16] Adding in pipline to test model simulations

---
 backend/Property.py                      |  14 +
 etl/testing_data/sap_model_simulation.py | 385 +++++++++++++++++++++++
 2 files changed, 399 insertions(+)
 create mode 100644 etl/testing_data/sap_model_simulation.py

diff --git a/backend/Property.py b/backend/Property.py
index de87099b..d1f5a1e2 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -184,6 +184,15 @@ class Property:
             recommendation_record["walls_thermal_transmittance_ending"] = recommendation["new_u_value"]
             recommendation_record["walls_insulation_thickness_ending"] = "above average"
             recommendation_record["walls_energy_eff_ending"] = "Good"
+
+            if recommendation["type"] == "external_wall_insulation":
+                recommendation_record["external_insulation"] = True
+                recommendation_record["internal_insulation"] = False
+
+            if recommendation["type"] == "internal_wall_insulation":
+                recommendation_record["external_insulation"] = False
+                recommendation_record["internal_insulation"] = True
+
         else:
             if recommendation_record["walls_thermal_transmittance_ending"] is None:
                 raise ValueError("We should not have a None value for the u value")
@@ -269,6 +278,11 @@ class Property:
             else:
                 raise ValueError("Invalid glazing type - implement me")
 
+            if is_secondary_glazing:
+                recommendation_record["glazed_type_ending"] = "secondary glazing"
+            else:
+                recommendation_record["glazed_type_ending"] = "double glazing installed during or after 2002 "
+
         if recommendation["type"] == "solar_pv":
             recommendation_record["photo_supply_ending"] = recommendation["photo_supply"]
 
diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
new file mode 100644
index 00000000..31cc21e7
--- /dev/null
+++ b/etl/testing_data/sap_model_simulation.py
@@ -0,0 +1,385 @@
+import pandas as pd
+from utils.s3 import read_dataframe_from_s3_parquet
+from backend.Property import Property
+
+
+def app():
+    dataset = read_dataframe_from_s3_parquet(
+        bucket_name="retrofit-datalake-dev",
+        file_key="dataset_with0perm_all.parquet"
+    )
+
+    thresholds = dataset["total_floor_area_starting"].quantile(
+        [0.3, 0.6, 0.9]
+    ).values
+
+    dataset["floor_area_quantile"] = pd.cut(
+        dataset["total_floor_area_starting"],
+        bins=[0] + list(thresholds) + [float('inf')],
+        labels=False,
+        include_lowest=True
+    )
+
+    # We want to set up some tests to deduce the following:
+    # For different property types, of various sizes, what is the impact of the various measures that we recommend
+    # 1) Insulating the loft. We test the impact of bringing the loft to 270mm insulation and 300mm insulation
+    property_types = dataset[
+        ["property_type", "built_form", "floor_area_quantile", "construction_age_band"]
+    ].drop_duplicates()
+
+    property_types = property_types.sort_values(
+        ["property_type", "built_form", "floor_area_quantile", "construction_age_band"]
+    )
+
+    # For each property type congifuration, we take an example property with different starting loft thresholds. We take
+    # the value with the lowest U-value, since when simulating, we often work with particularly low u-values
+
+    # TODOS
+    # 1) When simulating with loft insulation, make sure is_loft is definitely true, because the roof could start as
+    #    pitched, but is_loft false
+
+    # TODO: We have a description: "Pitched, loft insulation", which seems to have its insulation thickness set to
+    #  "none"
+    #       Example UPRN: 100021359753, 10001204228
+
+    # TODO: For windows, we have glazing_type and glazed_type. When simulating, we don't set glazed_type_ending which
+    #       could be set to "double glazing installed during or after 2002" (THIS HAS BEEN ADDED!)
+
+    # TODO: When simulating external wall insulation vs internal wall insulation, I need to set the external_insulation
+    #       or internal_insulation boolean values to true (THIS HAS BEEN ADDED!)
+
+    # TODO: We could probably re-map some of the values of glazed_type_ending
+
+    # For simulating
+    # 1) loft insulation - we take the lowest u-value when loft insulation is 270mm and 300mm, the values we most
+    # commonly simulate to - For loft insulation, these values are in-line with
+    best_270mm_uvalue = dataset[dataset["roof_insulation_thickness"] == "270"]["roof_thermal_transmittance"].min()
+    best_300mm_uvalue = dataset[dataset["roof_insulation_thickness"] == "300"]["roof_thermal_transmittance"].min()
+
+    # 2) Intenal wall insulation - we take the lowest u-value when simulating internal wall insulation
+    best_internal_wall_uvalue = dataset[
+        dataset["internal_insulation"] & dataset["is_solid_brick"]
+        ]["walls_thermal_transmittance"].min()
+
+    # 3) External wall insulation - we take the lowest u-value when simulating external wall insulation
+    best_external_wall_uvalue = dataset[
+        dataset["external_insulation"] & dataset["is_solid_brick"]
+        ]["walls_thermal_transmittance"].min()
+
+    # 4) Cavity wall insulation - we take the lowest u-value when simulating cavity wall insulation
+    # This is 0.28, which is a sufficiently low value
+    best_cavity_wall_uvalue = dataset[
+        dataset["is_cavity_wall"] & dataset["is_filled_cavity"] & (~dataset["external_insulation"]) & (
+            ~dataset["internal_insulation"])
+        ]["walls_thermal_transmittance"].min()
+
+    loft_insulation_testing_data = []
+    solid_wall_testing_data = []
+    cavity_wall_testing_data = []
+    solid_floor_testing_data = []
+    suspended_floor_testing_data = []
+    single_glazed_testing_data = []
+    partial_double_glazed_testing_data = []
+    partial_secondary_glazed_testing_data = []
+    for property_config in property_types.itertuples():
+        # Take a sample row
+        population = dataset[
+            (dataset["property_type"] == property_config.property_type) &
+            (dataset["built_form"] == property_config.built_form) &
+            (dataset["floor_area_quantile"] == property_config.floor_area_quantile) &
+            (dataset["construction_age_band"] == property_config.construction_age_band)
+            ]
+
+        # 1) Loft insulation
+
+        # For loft insulation, there are two scenarios we test.
+        # 1) Loft insulation to 270mm
+        # 2) Lost insulation to 300mm
+
+        for insulation_thickness in ["none", "12", "50", "75", "100", "150", "200", "250"]:
+            if insulation_thickness == "none":
+                row = population[
+                    (population["roof_insulation_thickness"] == "none") &
+                    (population["is_pitched"])
+                    ].sample(1)
+
+            else:
+                row = population[
+                    (population["roof_insulation_thickness"] == insulation_thickness) &
+                    (population["is_pitched"])
+                    ].sample(1)
+
+            loft_insulation_270mm_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "loft_insulation_270mm",
+                    "type": "loft_insulation",
+                    "new_u_value": best_270mm_uvalue,
+                    "parts": [
+                        {"depth": 270}
+                    ]
+                }
+            )
+
+            loft_insulation_300mm_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "loft_insulation_300mm",
+                    "type": "loft_insulation",
+                    "new_u_value": best_300mm_uvalue,
+                    "parts": [
+                        {"depth": 300}
+                    ]
+                }
+            )
+
+            # Insert simulation specific configuration details
+            loft_insulation_270mm_simulation = {
+                "simulation_ending_insulation_thickness": "270",
+                "simulation_starting_insulation_thickness": insulation_thickness,
+                **loft_insulation_270mm_simulation
+            }
+
+            loft_insulation_300mm_simulation = {
+                "simulation_ending_insulation_thickness": "300",
+                "simulation_starting_insulation_thickness": insulation_thickness,
+                **loft_insulation_300mm_simulation
+            }
+
+            loft_insulation_testing_data.append(loft_insulation_270mm_simulation)
+            loft_insulation_testing_data.append(loft_insulation_300mm_simulation)
+
+        # 2) Solid wall insulation
+        solid_wall_sample = population[
+            population["is_solid_brick"] & (population["walls_insulation_thickness"] == "none")
+            ]
+
+        # We take 1 sample for each value of walls_thermal_transmittance
+        for uvalue in solid_wall_sample["walls_thermal_transmittance"].unique():
+            row = solid_wall_sample[
+                solid_wall_sample["walls_thermal_transmittance"] == uvalue
+                ].sample(1)
+
+            # Simulated IWI
+            internal_wall_insulation_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0].to_dict("records")[0],
+                recommendation_record=row.copy(),
+                recommendation={
+                    "recommendation_id": "internal_wall_insulation",
+                    "type": "internal_wall_insulation",
+                    "new_u_value": best_internal_wall_uvalue,
+                    "parts": []
+                }
+            )
+
+            # Simulated EWI
+            best_external_wall_uvalue_wall_insulation_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy(),
+                recommendation={
+                    "recommendation_id": "external_wall_insulation",
+                    "type": "external_wall_insulation",
+                    "new_u_value": best_external_wall_uvalue,
+                    "parts": []
+                }
+            )
+
+            # The iww/ewi simulations will be next to each other, so we can see how they differ for the same property
+            solid_wall_testing_data.append(internal_wall_insulation_simulation)
+            solid_wall_testing_data.append(best_external_wall_uvalue_wall_insulation_simulation)
+
+        # 3) Cavity wall insulation
+        cavity_wall_sample = population[
+            population["is_cavity_wall"] & (~population["is_filled_cavity"]) & (
+                ~population["external_insulation"]
+            ) & (~population["internal_insulation"])
+            ]
+
+        # We take 1 sample for each value of walls_thermal_transmittance
+        for uvalue in cavity_wall_sample["walls_thermal_transmittance"].unique():
+            row = cavity_wall_sample[
+                cavity_wall_sample["walls_thermal_transmittance"] == uvalue
+                ].sample(1)
+
+            # Simulated filled cavity
+            filled_cavity_wall_insulation_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "cavity_wall_insulation",
+                    "type": "cavity_wall_insulation",
+                    "new_u_value": best_cavity_wall_uvalue,
+                    "parts": []
+                }
+            )
+
+            cavity_wall_testing_data.append(filled_cavity_wall_insulation_simulation)
+
+        # 4) Solid floor insulation
+        solid_floor_sample = population[
+            population["is_solid"] & (population["floor_insulation_thickness"] == "none")
+            ]
+
+        # We have many different values of u-value for solid floors, we we'll take a sample at the 25%, 50% and 75%
+        # values
+        # We must take a value that is in one of the unique values for floor_thermal_transmittance
+        for uvalue in solid_floor_sample["floor_thermal_transmittance"].quantile([0.25, 0.5, 0.75]).values:
+            nearest_value = solid_floor_sample['floor_thermal_transmittance'].sub(uvalue).abs().idxmin()
+            nearest_row = solid_floor_sample.loc[[nearest_value]].sample(1)
+
+            # Simulated solid floor insulation
+            solid_floor_insulation_simulation = Property.create_recommendation_scoring_data(
+                property_id=nearest_row["uprn"].values[0],
+                recommendation_record=nearest_row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "solid_floor_insulation",
+                    "type": "solid_floor_insulation",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": []
+                }
+            )
+
+            solid_floor_testing_data.append(solid_floor_insulation_simulation)
+
+        # 5) Suspended floor insulation
+        suspended_floor_sample = population[
+            population["is_suspended"] & (population["floor_insulation_thickness"] == "none")
+            ]
+
+        # We take the same approach as for solid floors
+        for uvalue in suspended_floor_sample["floor_thermal_transmittance"].quantile([0.25, 0.5, 0.75]).values:
+            nearest_value = suspended_floor_sample['floor_thermal_transmittance'].sub(uvalue).abs().idxmin()
+            nearest_row = suspended_floor_sample.loc[[nearest_value]].sample(1)
+
+            # Simulated suspended floor insulation
+            suspended_floor_insulation_simulation = Property.create_recommendation_scoring_data(
+                property_id=nearest_row["uprn"].values[0],
+                recommendation_record=nearest_row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "suspended_floor_insulation",
+                    "type": "suspended_floor_insulation",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": []
+                }
+            )
+
+            suspended_floor_testing_data.append(suspended_floor_insulation_simulation)
+
+        # 6) Windows - single glazing
+        single_glazing_sample = population[
+            (population["glazing_type"] == "single")
+        ]
+
+        # We take multiple values for multi_glaze_proportion_starting. We definitely need zero, but then we also
+        # take the 25%, 50% and 75% values
+        multi_glaze_values = [0] + list(
+            single_glazing_sample["multi_glaze_proportion_starting"].quantile([0.25, 0.5, 0.75]).values
+        )
+        multi_glaze_values = set(multi_glaze_values)
+
+        for value in multi_glaze_values:
+            row = single_glazing_sample[
+                single_glazing_sample["multi_glaze_proportion_starting"] == value
+                ].sample(1)
+
+            # For single glazed windows, we can recommend double glazing or secondary glazing
+
+            # Simulated double glazing
+            double_glazing_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "windows_glazing",
+                    "type": "windows_glazing",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "is_secondary_glazing": False
+                }
+            )
+
+            # Simulated secondary glazing
+            secondary_glazing_simulation = Property.create_recommendation_scoring_data(
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "windows_glazing",
+                    "type": "windows_glazing",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "is_secondary_glazing": True
+                }
+            )
+
+            # Add in simulation specific details
+            # Add to the beginning of the dictionary
+            double_glazing_simulation = {
+                "simulation_ending_window_finish": "double",
+                **double_glazing_simulation
+            }
+            secondary_glazing_simulation = {
+                "simulation_ending_window_finish": "secondary",
+                **secondary_glazing_simulation
+            }
+
+            single_glazed_testing_data.append(double_glazing_simulation)
+            single_glazed_testing_data.append(secondary_glazing_simulation)
+
+        # 7) Windows - partial double glazed
+        partial_double_glazing_sample = population[
+            (population["glazing_type"] == "double") & (population["multi_glaze_proportion_starting"] > 0) & (
+                population["multi_glaze_proportion_starting"] < 100
+            )
+            ]
+
+        for value in partial_double_glazing_sample["multi_glaze_proportion_starting"].quantile(
+            [0.25, 0.5, 0.75]
+        ).values:
+            nearest_value = partial_double_glazing_sample['multi_glaze_proportion_starting'].sub(value).abs().idxmin()
+            nearest_row = partial_double_glazing_sample.loc[[nearest_value]].sample(1)
+            # If we start with partial double glazing, we recommend completing the job
+            # Simulated double glazing
+            double_glazing_simulation = Property.create_recommendation_scoring_data(
+                property_id=nearest_row["uprn"].values[0],
+                recommendation_record=nearest_row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "windows_glazing",
+                    "type": "windows_glazing",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "is_secondary_glazing": False
+                }
+            )
+
+            partial_double_glazed_testing_data.append(double_glazing_simulation)
+
+        # 8) Windows - partial secondary glazed
+        partial_secondary_glazing_sample = population[
+            (population["glazing_type"] == "secondary") & (population["multi_glaze_proportion_starting"] > 0) & (
+                population["multi_glaze_proportion_starting"] < 100
+            )
+            ]
+
+        for value in partial_secondary_glazing_sample["multi_glaze_proportion_starting"].quantile(
+            [0.25, 0.5, 0.75]
+        ).values:
+            nearest_value = partial_secondary_glazing_sample['multi_glaze_proportion_starting'].sub(
+                value).abs().idxmin()
+            nearest_row = partial_secondary_glazing_sample.loc[[nearest_value]].sample(1)
+
+            # If we start with partial secondary glazing, we recommend completing the job
+            # Simulated secondary glazing
+            secondary_glazing_simulation = Property.create_recommendation_scoring_data(
+                property_id=nearest_row["uprn"].values[0],
+                recommendation_record=nearest_row.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "windows_glazing",
+                    "type": "windows_glazing",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "is_secondary_glazing": True
+                }
+            )
+
+            partial_secondary_glazed_testing_data.append(secondary_glazing_simulation)

From 61488b1aea337de171b43103ab2c1dbf0b38189e Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 15:53:27 +0000
Subject: [PATCH 02/16] debugging simulation testing pipeline

---
 etl/testing_data/sap_model_simulation.py | 104 ++++++++++++++++++++---
 1 file changed, 92 insertions(+), 12 deletions(-)

diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 31cc21e7..e8b97464 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -1,12 +1,15 @@
+import json
+
 import pandas as pd
-from utils.s3 import read_dataframe_from_s3_parquet
+from tqdm import tqdm
+from utils.s3 import read_dataframe_from_s3_parquet, save_data_to_s3
 from backend.Property import Property
 
 
 def app():
     dataset = read_dataframe_from_s3_parquet(
-        bucket_name="retrofit-datalake-dev",
-        file_key="dataset_with0perm_all.parquet"
+        bucket_name="retrofit-data-dev",
+        file_key="sap_change_model/dataset.parquet"
     )
 
     thresholds = dataset["total_floor_area_starting"].quantile(
@@ -81,7 +84,9 @@ def app():
     single_glazed_testing_data = []
     partial_double_glazed_testing_data = []
     partial_secondary_glazed_testing_data = []
-    for property_config in property_types.itertuples():
+    pitched_roof_solar = []
+    flat_roof_solar = []
+    for property_config in tqdm(property_types.itertuples(), total=property_types.shape[0]):
         # Take a sample row
         population = dataset[
             (dataset["property_type"] == property_config.property_type) &
@@ -101,13 +106,18 @@ def app():
                 row = population[
                     (population["roof_insulation_thickness"] == "none") &
                     (population["is_pitched"])
-                    ].sample(1)
+                    ]
 
             else:
                 row = population[
                     (population["roof_insulation_thickness"] == insulation_thickness) &
                     (population["is_pitched"])
-                    ].sample(1)
+                    ]
+
+            if row.empty:
+                continue
+
+            row = row.sample(1)
 
             loft_insulation_270mm_simulation = Property.create_recommendation_scoring_data(
                 property_id=row["uprn"].values[0],
@@ -164,8 +174,8 @@ def app():
 
             # Simulated IWI
             internal_wall_insulation_simulation = Property.create_recommendation_scoring_data(
-                property_id=row["uprn"].values[0].to_dict("records")[0],
-                recommendation_record=row.copy(),
+                property_id=row["uprn"].values[0],
+                recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
                     "recommendation_id": "internal_wall_insulation",
                     "type": "internal_wall_insulation",
@@ -333,9 +343,14 @@ def app():
             )
             ]
 
-        for value in partial_double_glazing_sample["multi_glaze_proportion_starting"].quantile(
+        partial_double_glazed_values = partial_double_glazing_sample["multi_glaze_proportion_starting"].quantile(
             [0.25, 0.5, 0.75]
-        ).values:
+        ).values
+        # Take non-null values
+        partial_double_glazed_values = [v for v in partial_double_glazed_values if not pd.isnull(v)]
+        partial_double_glazed_values = set(partial_double_glazed_values)
+
+        for value in partial_double_glazed_values:
             nearest_value = partial_double_glazing_sample['multi_glaze_proportion_starting'].sub(value).abs().idxmin()
             nearest_row = partial_double_glazing_sample.loc[[nearest_value]].sample(1)
             # If we start with partial double glazing, we recommend completing the job
@@ -361,9 +376,14 @@ def app():
             )
             ]
 
-        for value in partial_secondary_glazing_sample["multi_glaze_proportion_starting"].quantile(
+        partial_secondary_glazed_values = partial_secondary_glazing_sample["multi_glaze_proportion_starting"].quantile(
             [0.25, 0.5, 0.75]
-        ).values:
+        ).values
+        # Take non-null values
+        partial_secondary_glazed_values = [v for v in partial_secondary_glazed_values if not pd.isnull(v)]
+        partial_secondary_glazed_values = set(partial_secondary_glazed_values)
+
+        for value in partial_secondary_glazed_values:
             nearest_value = partial_secondary_glazing_sample['multi_glaze_proportion_starting'].sub(
                 value).abs().idxmin()
             nearest_row = partial_secondary_glazing_sample.loc[[nearest_value]].sample(1)
@@ -383,3 +403,63 @@ def app():
             )
 
             partial_secondary_glazed_testing_data.append(secondary_glazing_simulation)
+
+        # 9) Solar PV
+
+        # We only recommend solar for properties that have flat or pitched roofs, and no existing solar
+        pitched_roof_no_solar = population[
+            (population["is_pitched"]) & (population["photo_supply_starting"] == 0)
+            ].sample(1)
+
+        flat_roof_no_solar = population[
+            (population["is_flat"]) & (population["photo_supply_starting"] == 0)
+            ].sample(1)
+
+        # We simulate 30%, 40% and 50% coverage
+        for coverage in [30, 40, 50]:
+            solar_simulation_pitched = Property.create_recommendation_scoring_data(
+                property_id=pitched_roof_no_solar["uprn"].values[0],
+                recommendation_record=pitched_roof_no_solar.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "solar_pv",
+                    "type": "solar_pv",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "photo_supply": coverage
+                }
+            )
+
+            solar_simulation_flat = Property.create_recommendation_scoring_data(
+                property_id=flat_roof_no_solar["uprn"].values[0],
+                recommendation_record=flat_roof_no_solar.copy().to_dict("records")[0],
+                recommendation={
+                    "recommendation_id": "solar_pv",
+                    "type": "solar_pv",
+                    "new_u_value": None,  # This doesn't matter at the moment
+                    "parts": [],
+                    "photo_supply": coverage
+                }
+            )
+
+            pitched_roof_solar.append(solar_simulation_pitched)
+            flat_roof_solar.append(solar_simulation_flat)
+
+    # We store all of this data in s3, as it is
+    save_data_to_s3(
+        bucket_name="retrofit-datalake-dev",
+        s3_file_name="sap_change_model/loft_insulation_testing_data.parquet",
+        data=json.dumps(
+            {
+                "loft_insulation_testing_data": loft_insulation_testing_data,
+                "solid_wall_testing_data": solid_wall_testing_data,
+                "cavity_wall_testing_data": cavity_wall_testing_data,
+                "solid_floor_testing_data": solid_floor_testing_data,
+                "suspended_floor_testing_data": suspended_floor_testing_data,
+                "single_glazed_testing_data": single_glazed_testing_data,
+                "partial_double_glazed_testing_data": partial_double_glazed_testing_data,
+                "partial_secondary_glazed_testing_data": partial_secondary_glazed_testing_data,
+                "pitched_roof_solar": pitched_roof_solar,
+                "flat_roof_solar": flat_roof_solar
+            }
+        )
+    )

From 7c109ebf5d2ad3004e5c1615edb4337e99b45d51 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 16:03:51 +0000
Subject: [PATCH 03/16] handling empty data cases in testing piipeline

---
 etl/testing_data/sap_model_simulation.py | 81 +++++++++++++-----------
 1 file changed, 45 insertions(+), 36 deletions(-)

diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index e8b97464..6ff89691 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -232,10 +232,13 @@ def app():
             population["is_solid"] & (population["floor_insulation_thickness"] == "none")
             ]
 
+        solid_floor_uvalues = solid_floor_sample["floor_thermal_transmittance"].quantile([0.25, 0.5, 0.75]).values
+        solid_floor_uvalues = {v for v in solid_floor_uvalues if not pd.isnull(v)}
+
         # We have many different values of u-value for solid floors, we we'll take a sample at the 25%, 50% and 75%
         # values
         # We must take a value that is in one of the unique values for floor_thermal_transmittance
-        for uvalue in solid_floor_sample["floor_thermal_transmittance"].quantile([0.25, 0.5, 0.75]).values:
+        for uvalue in solid_floor_uvalues:
             nearest_value = solid_floor_sample['floor_thermal_transmittance'].sub(uvalue).abs().idxmin()
             nearest_row = solid_floor_sample.loc[[nearest_value]].sample(1)
 
@@ -258,8 +261,13 @@ def app():
             population["is_suspended"] & (population["floor_insulation_thickness"] == "none")
             ]
 
+        suspended_floor_uvalues = suspended_floor_sample["floor_thermal_transmittance"].quantile(
+            [0.25, 0.5, 0.75]
+        ).values
+        suspended_floor_uvalues = {v for v in suspended_floor_uvalues if not pd.isnull(v)}
+
         # We take the same approach as for solid floors
-        for uvalue in suspended_floor_sample["floor_thermal_transmittance"].quantile([0.25, 0.5, 0.75]).values:
+        for uvalue in suspended_floor_uvalues:
             nearest_value = suspended_floor_sample['floor_thermal_transmittance'].sub(uvalue).abs().idxmin()
             nearest_row = suspended_floor_sample.loc[[nearest_value]].sample(1)
 
@@ -282,14 +290,7 @@ def app():
             (population["glazing_type"] == "single")
         ]
 
-        # We take multiple values for multi_glaze_proportion_starting. We definitely need zero, but then we also
-        # take the 25%, 50% and 75% values
-        multi_glaze_values = [0] + list(
-            single_glazing_sample["multi_glaze_proportion_starting"].quantile([0.25, 0.5, 0.75]).values
-        )
-        multi_glaze_values = set(multi_glaze_values)
-
-        for value in multi_glaze_values:
+        if not single_glazing_sample.empty:
             row = single_glazing_sample[
                 single_glazing_sample["multi_glaze_proportion_starting"] == value
                 ].sample(1)
@@ -409,40 +410,48 @@ def app():
         # We only recommend solar for properties that have flat or pitched roofs, and no existing solar
         pitched_roof_no_solar = population[
             (population["is_pitched"]) & (population["photo_supply_starting"] == 0)
-            ].sample(1)
+            ]
+
+        if not pitched_roof_no_solar.empty:
+            pitched_roof_no_solar = pitched_roof_no_solar.sample(1)
 
         flat_roof_no_solar = population[
             (population["is_flat"]) & (population["photo_supply_starting"] == 0)
-            ].sample(1)
+            ]
+
+        if not flat_roof_no_solar.empty:
+            flat_roof_no_solar = flat_roof_no_solar.sample(1)
 
         # We simulate 30%, 40% and 50% coverage
         for coverage in [30, 40, 50]:
-            solar_simulation_pitched = Property.create_recommendation_scoring_data(
-                property_id=pitched_roof_no_solar["uprn"].values[0],
-                recommendation_record=pitched_roof_no_solar.copy().to_dict("records")[0],
-                recommendation={
-                    "recommendation_id": "solar_pv",
-                    "type": "solar_pv",
-                    "new_u_value": None,  # This doesn't matter at the moment
-                    "parts": [],
-                    "photo_supply": coverage
-                }
-            )
 
-            solar_simulation_flat = Property.create_recommendation_scoring_data(
-                property_id=flat_roof_no_solar["uprn"].values[0],
-                recommendation_record=flat_roof_no_solar.copy().to_dict("records")[0],
-                recommendation={
-                    "recommendation_id": "solar_pv",
-                    "type": "solar_pv",
-                    "new_u_value": None,  # This doesn't matter at the moment
-                    "parts": [],
-                    "photo_supply": coverage
-                }
-            )
+            if not pitched_roof_no_solar.empty:
+                solar_simulation_pitched = Property.create_recommendation_scoring_data(
+                    property_id=pitched_roof_no_solar["uprn"].values[0],
+                    recommendation_record=pitched_roof_no_solar.copy().to_dict("records")[0],
+                    recommendation={
+                        "recommendation_id": "solar_pv",
+                        "type": "solar_pv",
+                        "new_u_value": None,  # This doesn't matter at the moment
+                        "parts": [],
+                        "photo_supply": coverage
+                    }
+                )
+                pitched_roof_solar.append(solar_simulation_pitched)
 
-            pitched_roof_solar.append(solar_simulation_pitched)
-            flat_roof_solar.append(solar_simulation_flat)
+            if not flat_roof_no_solar.empty:
+                solar_simulation_flat = Property.create_recommendation_scoring_data(
+                    property_id=flat_roof_no_solar["uprn"].values[0],
+                    recommendation_record=flat_roof_no_solar.copy().to_dict("records")[0],
+                    recommendation={
+                        "recommendation_id": "solar_pv",
+                        "type": "solar_pv",
+                        "new_u_value": None,  # This doesn't matter at the moment
+                        "parts": [],
+                        "photo_supply": coverage
+                    }
+                )
+                flat_roof_solar.append(solar_simulation_flat)
 
     # We store all of this data in s3, as it is
     save_data_to_s3(

From 1028861a1b0f8e9bb4a117335150923d6101f222 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 16:54:16 +0000
Subject: [PATCH 04/16] implementing the scoring process into testing pipeline

---
 etl/testing_data/sap_model_simulation.py | 82 +++++++++++++++++++++---
 1 file changed, 73 insertions(+), 9 deletions(-)

diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 6ff89691..0434ffd5 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -2,9 +2,12 @@ import json
 
 import pandas as pd
 from tqdm import tqdm
-from utils.s3 import read_dataframe_from_s3_parquet, save_data_to_s3
+from utils.s3 import read_dataframe_from_s3_parquet, save_data_to_s3, save_dataframe_to_s3_parquet
 from backend.Property import Property
 
+# This is the github pr number
+MODEL_VERSION = "100"
+
 
 def app():
     dataset = read_dataframe_from_s3_parquet(
@@ -76,6 +79,20 @@ def app():
             ~dataset["internal_insulation"])
         ]["walls_thermal_transmittance"].min()
 
+    ending_colums = [col for col in dataset.columns if col.endswith("_ending")]
+    # For the purpose of scoring, we want to simulate JUST the impact of the measure we're testing. We therefore
+    # need to make sure that every "_ending" column is equal to its starting value
+    column_config = {}
+    for ending_col in ending_colums:
+        base_col = ending_col.replace("_ending", "")
+        # We check if the starting column ends with _starting or is just the base col
+        if base_col + "_starting" in dataset.columns:
+            column_config[ending_col] = base_col + "_starting"
+        elif base_col in dataset.columns:
+            column_config[ending_col] = base_col
+        else:
+            raise ValueError("something went wrong")
+
     loft_insulation_testing_data = []
     solid_wall_testing_data = []
     cavity_wall_testing_data = []
@@ -93,7 +110,11 @@ def app():
             (dataset["built_form"] == property_config.built_form) &
             (dataset["floor_area_quantile"] == property_config.floor_area_quantile) &
             (dataset["construction_age_band"] == property_config.construction_age_band)
-            ]
+            ].copy()
+
+        # Re-set all of the ending columns
+        for col in ending_colums:
+            population[col] = population[column_config[col]]
 
         # 1) Loft insulation
 
@@ -185,9 +206,9 @@ def app():
             )
 
             # Simulated EWI
-            best_external_wall_uvalue_wall_insulation_simulation = Property.create_recommendation_scoring_data(
+            external_wall_insulation_simulation = Property.create_recommendation_scoring_data(
                 property_id=row["uprn"].values[0],
-                recommendation_record=row.copy(),
+                recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
                     "recommendation_id": "external_wall_insulation",
                     "type": "external_wall_insulation",
@@ -198,7 +219,7 @@ def app():
 
             # The iww/ewi simulations will be next to each other, so we can see how they differ for the same property
             solid_wall_testing_data.append(internal_wall_insulation_simulation)
-            solid_wall_testing_data.append(best_external_wall_uvalue_wall_insulation_simulation)
+            solid_wall_testing_data.append(external_wall_insulation_simulation)
 
         # 3) Cavity wall insulation
         cavity_wall_sample = population[
@@ -291,9 +312,7 @@ def app():
         ]
 
         if not single_glazing_sample.empty:
-            row = single_glazing_sample[
-                single_glazing_sample["multi_glaze_proportion_starting"] == value
-                ].sample(1)
+            row = single_glazing_sample.sample(1)
 
             # For single glazed windows, we can recommend double glazing or secondary glazing
 
@@ -456,7 +475,7 @@ def app():
     # We store all of this data in s3, as it is
     save_data_to_s3(
         bucket_name="retrofit-datalake-dev",
-        s3_file_name="sap_change_model/loft_insulation_testing_data.parquet",
+        s3_file_name="sap_change_model/simulation-pipeline-data.json",
         data=json.dumps(
             {
                 "loft_insulation_testing_data": loft_insulation_testing_data,
@@ -472,3 +491,48 @@ def app():
             }
         )
     )
+
+    # For each simulation type, we score against the model
+    from backend.ml_models.api import ModelApi
+    from datetime import datetime
+
+    created_at = datetime.now().isoformat()
+    model_api = ModelApi(portfolio_id="simulation-testing-pipeline", timestamp=created_at)
+    model_api.MODEL_PREFIXES = ["sap_change_predictions"]
+
+    # 1) Loft insulation
+    # We chunk up the data into 200 rows
+    loft_insulation_testing_df = pd.DataFrame(loft_insulation_testing_data)
+
+    loft_insulation_predictions = []
+    loft_to_loop_over = range(0, loft_insulation_testing_df.shape[0], 200)
+    for chunk in tqdm(loft_to_loop_over, total=len(loft_to_loop_over)):
+        loft_insulation_predictions_dict = model_api.predict_all(
+            df=loft_insulation_testing_df.iloc[chunk:chunk + 200],
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        )
+
+        loft_insulation_predictions.append(loft_insulation_predictions_dict["sap_change_predictions"])
+
+    loft_insulation_predictions = pd.concat(loft_insulation_predictions)
+    # Store final parquet in s3
+    save_dataframe_to_s3_parquet(
+        df=loft_insulation_predictions,
+        bucket_name="retrofit-data-dev",
+        file_key=f"sap_change_model/simulation-pipeline-loft-insulation-predictions_{MODEL_VERSION}.parquet"
+    )
+
+    # We now merge the loft insulation predictions onto the scoring data and calculate exactly how much the insulation
+    # is worth
+
+    loft_insulation_comparison_df = loft_insulation_testing_df[
+        ["simulation_ending_insulation_thickness", "simulation_starting_insulation_thickness", "uprn", "id", ""]
+    ].merge(
+        loft_insulation_predictions,
+        left_on="id",
+        right_on="id",
+        how="left"
+    )

From bd0fb8c2c17aa4b0572bf38d1c328bf156ed63cc Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 18:11:31 +0000
Subject: [PATCH 05/16] adding unique ids and config hash to prevent
 duplications

---
 etl/testing_data/sap_model_simulation.py | 62 +++++++++++++++++-------
 1 file changed, 45 insertions(+), 17 deletions(-)

diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 0434ffd5..0a044201 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -104,6 +104,9 @@ def app():
     pitched_roof_solar = []
     flat_roof_solar = []
     for property_config in tqdm(property_types.itertuples(), total=property_types.shape[0]):
+
+        config_hash = hash(str(property_config))
+
         # Take a sample row
         population = dataset[
             (dataset["property_type"] == property_config.property_type) &
@@ -144,7 +147,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "loft_insulation_270mm",
+                    "recommendation_id": f"loft_insulation_{insulation_thickness}_270mm_{config_hash}",
                     "type": "loft_insulation",
                     "new_u_value": best_270mm_uvalue,
                     "parts": [
@@ -157,7 +160,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "loft_insulation_300mm",
+                    "recommendation_id": f"loft_insulation_{insulation_thickness}_300mm_{config_hash}",
                     "type": "loft_insulation",
                     "new_u_value": best_300mm_uvalue,
                     "parts": [
@@ -198,7 +201,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "internal_wall_insulation",
+                    "recommendation_id": f"internal_wall_insulation_uvalue_{uvalue}_{config_hash}",
                     "type": "internal_wall_insulation",
                     "new_u_value": best_internal_wall_uvalue,
                     "parts": []
@@ -210,7 +213,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "external_wall_insulation",
+                    "recommendation_id": f"external_wall_insulation_uvalue_{uvalue}_{config_hash}",
                     "type": "external_wall_insulation",
                     "new_u_value": best_external_wall_uvalue,
                     "parts": []
@@ -239,7 +242,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "cavity_wall_insulation",
+                    "recommendation_id": f"cavity_wall_insulation_uvalue_{uvalue}_{config_hash}",
                     "type": "cavity_wall_insulation",
                     "new_u_value": best_cavity_wall_uvalue,
                     "parts": []
@@ -268,7 +271,7 @@ def app():
                 property_id=nearest_row["uprn"].values[0],
                 recommendation_record=nearest_row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "solid_floor_insulation",
+                    "recommendation_id": f"solid_floor_insulation_uvalue_{uvalue}_{config_hash}",
                     "type": "solid_floor_insulation",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": []
@@ -297,7 +300,7 @@ def app():
                 property_id=nearest_row["uprn"].values[0],
                 recommendation_record=nearest_row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "suspended_floor_insulation",
+                    "recommendation_id": f"suspended_floor_insulation_uvalue_{uvalue}_{config_hash}",
                     "type": "suspended_floor_insulation",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": []
@@ -321,7 +324,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "windows_glazing",
+                    "recommendation_id": f"windows_glazing_single_to_double_{config_hash}",
                     "type": "windows_glazing",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": [],
@@ -334,7 +337,7 @@ def app():
                 property_id=row["uprn"].values[0],
                 recommendation_record=row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "windows_glazing",
+                    "recommendation_id": f"windows_glazing_single_to_secondary_{config_hash}",
                     "type": "windows_glazing",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": [],
@@ -379,7 +382,7 @@ def app():
                 property_id=nearest_row["uprn"].values[0],
                 recommendation_record=nearest_row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "windows_glazing",
+                    "recommendation_id": f"windows_glazing_partial_double_to_double_{value}_{config_hash}",
                     "type": "windows_glazing",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": [],
@@ -414,7 +417,7 @@ def app():
                 property_id=nearest_row["uprn"].values[0],
                 recommendation_record=nearest_row.copy().to_dict("records")[0],
                 recommendation={
-                    "recommendation_id": "windows_glazing",
+                    "recommendation_id": f"windows_glazing_partial_secondary_to_secondary_{value}_{config_hash}",
                     "type": "windows_glazing",
                     "new_u_value": None,  # This doesn't matter at the moment
                     "parts": [],
@@ -449,7 +452,7 @@ def app():
                     property_id=pitched_roof_no_solar["uprn"].values[0],
                     recommendation_record=pitched_roof_no_solar.copy().to_dict("records")[0],
                     recommendation={
-                        "recommendation_id": "solar_pv",
+                        "recommendation_id": f"pitched_solar_pv_coverage_{coverage}_percent_{config_hash}",
                         "type": "solar_pv",
                         "new_u_value": None,  # This doesn't matter at the moment
                         "parts": [],
@@ -463,7 +466,7 @@ def app():
                     property_id=flat_roof_no_solar["uprn"].values[0],
                     recommendation_record=flat_roof_no_solar.copy().to_dict("records")[0],
                     recommendation={
-                        "recommendation_id": "solar_pv",
+                        "recommendation_id": f"flat_solar_pv_coverage_{coverage}_percent_{config_hash}",
                         "type": "solar_pv",
                         "new_u_value": None,  # This doesn't matter at the moment
                         "parts": [],
@@ -521,18 +524,43 @@ def app():
     # Store final parquet in s3
     save_dataframe_to_s3_parquet(
         df=loft_insulation_predictions,
-        bucket_name="retrofit-data-dev",
+        bucket_name="retrofit-datalake-dev",
         file_key=f"sap_change_model/simulation-pipeline-loft-insulation-predictions_{MODEL_VERSION}.parquet"
     )
 
     # We now merge the loft insulation predictions onto the scoring data and calculate exactly how much the insulation
     # is worth
 
-    loft_insulation_comparison_df = loft_insulation_testing_df[
-        ["simulation_ending_insulation_thickness", "simulation_starting_insulation_thickness", "uprn", "id", ""]
+    loft_insulation_comparison_matrix = loft_insulation_testing_df[
+        ["simulation_starting_insulation_thickness", "simulation_ending_insulation_thickness", "uprn", "id",
+         "sap_starting"]
     ].merge(
-        loft_insulation_predictions,
+        loft_insulation_predictions.drop(columns=["recommendation_id"]),
         left_on="id",
         right_on="id",
         how="left"
     )
+
+    loft_insulation_comparison_matrix["measure_impact"] = loft_insulation_comparison_matrix["predictions"] - \
+                                                          loft_insulation_comparison_matrix["sap_starting"]
+    # Perform a group by describe
+    loft_insulation_describe = loft_insulation_comparison_matrix.groupby(
+        ["simulation_starting_insulation_thickness", "simulation_ending_insulation_thickness"]
+    )[["measure_impact"]].describe().reset_index()
+
+    z = loft_insulation_comparison_matrix[loft_insulation_comparison_matrix["measure_impact"] < 0]
+    z.head(1)[["uprn", "id"]]
+    error_row = loft_insulation_testing_df[
+        (loft_insulation_testing_df["id"] == "100090292333+loft_insulation_150_270mm")
+    ]
+
+    error_dataset = dataset[
+        (dataset["uprn"] == "10070401239") & (dataset["roof_insulation_thickness"] == "250")
+        ]
+
+    changed_from_dataset = []
+    for c in column_config:
+        ending_value = error_row[column_config[c]].values[0]
+        starting_value = error_row[column_config[c]].values[0]
+    error_dataset["roof_insulation_thickness"]
+    error_dataset["roof_insulation_thickness_ending"]

From 7a219285fc192145aedb337c54803fa4f442a142 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Mon, 12 Feb 2024 19:16:10 +0000
Subject: [PATCH 06/16] looking into loft insulation data

---
 etl/testing_data/sap_model_simulation.py | 116 ++++++++++++++++++++---
 1 file changed, 102 insertions(+), 14 deletions(-)

diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 0a044201..467b50db 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -543,24 +543,112 @@ def app():
 
     loft_insulation_comparison_matrix["measure_impact"] = loft_insulation_comparison_matrix["predictions"] - \
                                                           loft_insulation_comparison_matrix["sap_starting"]
+
+    # We create a sap band grouping, for every 10 points of sap. So 1-10, 11-20, 21-30 etc
+    loft_insulation_comparison_matrix["sap_band"] = pd.cut(
+        loft_insulation_comparison_matrix["sap_starting"],
+        bins=range(0, 101, 10),
+        labels=range(1, 11)
+    )
+
     # Perform a group by describe
     loft_insulation_describe = loft_insulation_comparison_matrix.groupby(
-        ["simulation_starting_insulation_thickness", "simulation_ending_insulation_thickness"]
+        ["sap_band", "simulation_starting_insulation_thickness", "simulation_ending_insulation_thickness"]
     )[["measure_impact"]].describe().reset_index()
 
-    z = loft_insulation_comparison_matrix[loft_insulation_comparison_matrix["measure_impact"] < 0]
-    z.head(1)[["uprn", "id"]]
-    error_row = loft_insulation_testing_df[
-        (loft_insulation_testing_df["id"] == "100090292333+loft_insulation_150_270mm")
-    ]
+    for col in ["simulation_starting_insulation_thickness", "simulation_ending_insulation_thickness"]:
+        loft_insulation_describe[col] = loft_insulation_describe[col].str.replace('none', "0")
+        loft_insulation_describe[col] = loft_insulation_describe[col].astype(int)
 
-    error_dataset = dataset[
-        (dataset["uprn"] == "10070401239") & (dataset["roof_insulation_thickness"] == "250")
+    loft_insulation_describe = loft_insulation_describe.sort_values(
+        ["simulation_ending_insulation_thickness", "simulation_starting_insulation_thickness"], ascending=True
+    )
+
+    # In the training data, try and get just the rows that are loft insulation only
+    # Things that change:
+    # 1) roof_insulation_thickness
+    # 3) roof_thermal_transmittance
+    # 4) roof_energy_eff_ending
+    loft_insulation_training_data = dataset.copy()
+    loft_insulation_columns_we_need_the_same = [c for c in column_config.keys() if c not in [
+        "roof_insulation_thickness_ending", "roof_thermal_transmittance_ending", "roof_energy_eff_ending",
+        "transaction_type_ending", "days_to_ending", "sap_ending", "heat_demand_ending", "carbon_ending",
+        "total_floor_area_ending", "floor_height_ending", "estimated_perimeter_ending"
+    ]]
+
+    for ending_col in tqdm(loft_insulation_columns_we_need_the_same):
+        starting_col = column_config[ending_col]
+        loft_insulation_training_data = loft_insulation_training_data[
+            loft_insulation_training_data[ending_col] == loft_insulation_training_data[starting_col]
+            ]
+
+    # We get rows where the insulation starts at 200mm
+    insulation_200mm_starting = loft_insulation_training_data[
+        (loft_insulation_training_data["roof_insulation_thickness"] == "200") &
+        (loft_insulation_training_data["roof_insulation_thickness_ending"] == "300")
         ]
 
-    changed_from_dataset = []
-    for c in column_config:
-        ending_value = error_row[column_config[c]].values[0]
-        starting_value = error_row[column_config[c]].values[0]
-    error_dataset["roof_insulation_thickness"]
-    error_dataset["roof_insulation_thickness_ending"]
+    # Let's use the API to find exactly the record
+    from backend.SearchEpc import SearchEpc
+    searcher = SearchEpc(
+        address1="2 Darkfield Way",
+        postcode="TA7 8HY",
+        auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
+        os_api_key=""
+    )
+    searcher.uprn = "10009320092"
+    searcher.find_property(skip_os=True)
+
+    newest_epc = searcher.newest_epc
+    older_epc = [epc for epc in searcher.older_epcs if
+                 epc["lmk-key"] == "5ae2f073004839510f9eeb1886160776a05697f8518b8b3b63d45f65686c4757"][0]
+    # Iterate through the keys in the newest_epc and find the values in older epc that are different to the newest epc
+
+    differences = {}
+    for k, v in newest_epc.items():
+        if v != older_epc[k]:
+            differences[k] = (v, older_epc[k])
+
+    testing_model_api = ModelApi(portfolio_id="simulation-testing-loft-example", timestamp=created_at)
+    testing_model_api.MODEL_PREFIXES = ["sap_change_predictions"]
+
+    ############################################################################################################
+    # TODO:!
+    # Findings: 1) For uprn 10009320092, the number of rooms and number of heated rooms has changed and can change from
+    # epc to epc. We should therefore include a starting and ending value for this
+
+    # Investigation 1)
+    testing_row = insulation_200mm_starting[insulation_200mm_starting["uprn"] == "10009320092"].copy()
+    testing_row["id"] = "testing-200mm-loft-insulation-starting-baseline+recommendation_id_baseline"
+    testing_row["recommendation_id"] = "recommendation_id_baseline"
+    # The testing row has 4 rooms
+    # Score in the model to see what we get
+
+    baseline_prediction = testing_model_api.predict_all(
+        df=testing_row,
+        bucket="retrofit-data-dev",
+        prediction_buckets={
+            "sap_change_predictions": "retrofit-sap-predictions-dev",
+        }
+    )
+
+    baseline_pred_df = baseline_prediction["sap_change_predictions"]
+    impact = baseline_pred_df["predictions"].values[0] - testing_row["sap_starting"].values[0]
+
+    # Changing this from 4 rooms to 5 rooms has NO impact!!
+    testing_row_5_rooms = testing_row.copy()
+    testing_row_5_rooms["id"] = "testing-200mm-loft-insulation-starting-baseline+recommendation_id_5_rooms"
+    testing_row_5_rooms["recommendation_id"] = "recommendation_id_5_rooms"
+    testing_row_5_rooms["number_habitable_rooms"] = float(5)
+    testing_row_5_rooms["number_heated_rooms"] = float(5)
+
+    prediction_5_rooms = testing_model_api.predict_all(
+        df=testing_row_5_rooms,
+        bucket="retrofit-data-dev",
+        prediction_buckets={
+            "sap_change_predictions": "retrofit-sap-predictions-dev",
+        }
+    )
+
+    pred_df_5_rooms = prediction_5_rooms["sap_change_predictions"]
+    impact_5_rooms = pred_df_5_rooms["predictions"].values[0] - testing_row_5_rooms["sap_starting"].values[0]

From ec80473f3e4af72bbab1621e58fc112bb018fe6f Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Wed, 14 Feb 2024 11:06:10 +0000
Subject: [PATCH 07/16] implemented further scenarios into solar
 recommendations

---
 backend/Property.py                       |   21 +-
 backend/ml_models/Valuation.py            |    9 +-
 etl/testing_data/sap_model_simulation.py  | 1514 ++++++++++++++++++++-
 recommendations/Costs.py                  |   14 +-
 recommendations/Recommendations.py        |    3 +
 recommendations/SolarPvRecommendations.py |   55 +-
 6 files changed, 1586 insertions(+), 30 deletions(-)

diff --git a/backend/Property.py b/backend/Property.py
index d1f5a1e2..e57d8326 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -118,7 +118,6 @@ class Property:
         self.number_lighting_outlets = epc_record.prepared_epc.get("fixed_lighting_outlets_count")
         self.floor_level = None
         self.number_of_windows = None
-        self.solar_pv_roof_area = None
         self.solar_pv_percentage = None
 
         self.current_adjusted_energy = None
@@ -185,6 +184,8 @@ class Property:
             recommendation_record["walls_insulation_thickness_ending"] = "above average"
             recommendation_record["walls_energy_eff_ending"] = "Good"
 
+            # Note: often when the wall is insulatied, the internal/external insulation is not noted so we should
+            #       test the impact of using these booleans
             if recommendation["type"] == "external_wall_insulation":
                 recommendation_record["external_insulation"] = True
                 recommendation_record["internal_insulation"] = False
@@ -238,7 +239,10 @@ class Property:
 
             recommendation_record["roof_insulation_thickness_ending"] = str(proposed_depth)
             if recommendation["type"] == "loft_insulation":
-                recommendation_record["roof_energy_eff_ending"] = "Good"
+                if proposed_depth >= 270:
+                    recommendation_record["roof_energy_eff_ending"] = "Very Good"
+                else:
+                    recommendation_record["roof_energy_eff_ending"] = "Good"
             else:
                 recommendation_record["roof_energy_eff_ending"] = "Very Good"
         else:
@@ -682,9 +686,16 @@ class Property:
         percentage_of_roof = photo_supply_matched["photo_supply_median"].mean()
         percentage_of_roof = percentage_of_roof / 100
 
-        self.solar_pv_roof_area = (
+        self.solar_pv_percentage = percentage_of_roof
+
+    def get_solar_pv_roof_area(self, percentage_of_roof):
+        """
+        Given a percentage of the roof, this method will return the estimated area of the solar panels
+        :param percentage_of_roof:
+        :return:
+        """
+
+        return (
             self.insulation_floor_area * percentage_of_roof if self.roof["is_flat"] else
             self.pitched_roof_area * percentage_of_roof
         )
-
-        self.solar_pv_percentage = percentage_of_roof
diff --git a/backend/ml_models/Valuation.py b/backend/ml_models/Valuation.py
index ff771252..83c20bb4 100644
--- a/backend/ml_models/Valuation.py
+++ b/backend/ml_models/Valuation.py
@@ -51,13 +51,14 @@ class PropertyValuation:
     KNIGHT_FRANK_MAPPING = [
         {"start": "D", "end": "C", "increase_percentage": 0.03},
         {"start": "D", "end": "B", "increase_percentage": 0.088},
+        {"start": "D", "end": "A", "increase_percentage": 0.088},
     ]
 
     NATIONWIDE_MAPPING = [
-        {"start": "G", "end": "D", "increase_percentage": 0.035},
-        {"start": "F", "end": "D", "increase_percentage": 0.035},
-        {"start": "D", "end": "B", "increase_percentage": 0.017},
-        {"start": "D", "end": "A", "increase_percentage": 0.017},
+        # {"start": "G", "end": "D", "increase_percentage": 0.035},
+        # {"start": "F", "end": "D", "increase_percentage": 0.035},
+        # {"start": "D", "end": "B", "increase_percentage": 0.017},
+        # {"start": "D", "end": "A", "increase_percentage": 0.017},
     ]
 
     EPC_BANDS = ["G", "F", "E", "D", "C", "B", "A"]
diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 467b50db..726c2428 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -6,7 +6,7 @@ from utils.s3 import read_dataframe_from_s3_parquet, save_data_to_s3, save_dataf
 from backend.Property import Property
 
 # This is the github pr number
-MODEL_VERSION = "100"
+MODEL_VERSION = "101"
 
 
 def app():
@@ -590,6 +590,22 @@ def app():
 
     # Let's use the API to find exactly the record
     from backend.SearchEpc import SearchEpc
+
+    testing_model_api = ModelApi(portfolio_id="simulation-testing-loft-example", timestamp=created_at)
+    testing_model_api.MODEL_PREFIXES = ["sap_change_predictions"]
+
+    ############################################################################################################
+    # TODO:!
+    # Findings:
+    # 1) For uprn 10009320092, the number of rooms and number of heated rooms has changed and can change from
+    # epc to epc. We should therefore include a starting and ending value for this
+    # 2) Maybe we should include tenure??? Owner occupied homes are going to be a lot more unusual. Both investigation
+    # 2 and 3 were owner occupied
+    # 3) Maybe we should treat photo_supply missing differently than 0?
+
+    ################################################################################################
+    # Investigation 1)
+
     searcher = SearchEpc(
         address1="2 Darkfield Way",
         postcode="TA7 8HY",
@@ -609,15 +625,6 @@ def app():
         if v != older_epc[k]:
             differences[k] = (v, older_epc[k])
 
-    testing_model_api = ModelApi(portfolio_id="simulation-testing-loft-example", timestamp=created_at)
-    testing_model_api.MODEL_PREFIXES = ["sap_change_predictions"]
-
-    ############################################################################################################
-    # TODO:!
-    # Findings: 1) For uprn 10009320092, the number of rooms and number of heated rooms has changed and can change from
-    # epc to epc. We should therefore include a starting and ending value for this
-
-    # Investigation 1)
     testing_row = insulation_200mm_starting[insulation_200mm_starting["uprn"] == "10009320092"].copy()
     testing_row["id"] = "testing-200mm-loft-insulation-starting-baseline+recommendation_id_baseline"
     testing_row["recommendation_id"] = "recommendation_id_baseline"
@@ -652,3 +659,1490 @@ def app():
 
     pred_df_5_rooms = prediction_5_rooms["sap_change_predictions"]
     impact_5_rooms = pred_df_5_rooms["predictions"].values[0] - testing_row_5_rooms["sap_starting"].values[0]
+
+    ################################################################################################
+    # Investigation 2
+
+    searcher = SearchEpc(
+        address1="19 Rossal Place",
+        postcode="MK12 6JE",
+        auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
+        os_api_key=""
+    )
+    searcher.uprn = "25006966"
+    searcher.find_property(skip_os=True)
+
+    newest_epc = searcher.newest_epc
+    older_epc = [epc for epc in searcher.older_epcs if
+                 epc["lmk-key"] == "fe23917ac59fbf3a608c76431941011ab4c7938546a432fc6212182caab31d73"][0]
+    # Iterate through the keys in the newest_epc and find the values in older epc that are different to the newest epc
+
+    differences = {}
+    for k, v in newest_epc.items():
+        if v != older_epc[k]:
+            differences[k] = (v, older_epc[k])
+
+    testing_row2 = insulation_200mm_starting[insulation_200mm_starting["uprn"] == "25006966"].copy()
+    # THERE IS NOTHING CLEAR THAT IS CHANGING IN THIS RECORD THAT INDICATES SUGGESTS WE'RE MISSING INFORMATION
+
+    ################################################################################################
+    # Investigation 3
+
+    insulation_200mm_starting[
+        insulation_200mm_starting["rdsap_change"] == insulation_200mm_starting["rdsap_change"].max()
+        ]["uprn"].values[0]
+    # This UPRN: 100060350521
+
+    searcher = SearchEpc(
+        address1="138 Nicholas Crescent",
+        postcode="PO15 5AN",
+        auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
+        os_api_key=""
+    )
+    searcher.uprn = "100060350521"
+    searcher.find_property(skip_os=True)
+
+    newest_epc = searcher.newest_epc
+    older_epc = [epc for epc in searcher.older_epcs if
+                 epc["lmk-key"] == "9c4059762189b451191c98d2ef980a5364b8b7e0be3f064f681abcd4a0da681b"][0]
+    # Iterate through the keys in the newest_epc and find the values in older epc that are different to the newest epc
+
+    differences = {}
+    for k, v in newest_epc.items():
+        if v != older_epc[k]:
+            differences[k] = (v, older_epc[k])
+
+    # Nothing looks amiss about this record - let's score it in the model
+    testing_row_3 = insulation_200mm_starting[insulation_200mm_starting["uprn"] == "100060350521"].copy()
+
+    testing_row_3["id"] = "testing-200mm-loft-insulation-starting-baseline+recommendation_id_baseline"
+    testing_row_3["recommendation_id"] = "recommendation_id_baseline"
+    # The testing row has 4 rooms
+    # Score in the model to see what we get
+
+    baseline_prediction3 = testing_model_api.predict_all(
+        df=testing_row_3,
+        bucket="retrofit-data-dev",
+        prediction_buckets={
+            "sap_change_predictions": "retrofit-sap-predictions-dev",
+        }
+    )
+
+    baseline_pred_df3 = baseline_prediction3["sap_change_predictions"]
+    impact3 = baseline_pred_df3["predictions"].values[0] - testing_row_3["sap_starting"].values[0]
+
+    # TODO: Look at some of the example properties we have, and test using the model to score the impact of the
+    #       different measures when multiple measures are scored together e.g. cavity + loft together vs individually
+
+    # Look at performance on loft only rows
+    loft_insulation_training_data["id"] = (loft_insulation_training_data["uprn"] +
+                                           "_loft_insulation + recommendation_id_placeholder")
+
+    loft_only_predictions = []
+    loft_to_loop_over = range(0, loft_insulation_training_data.shape[0], 400)
+    for chunk in tqdm(loft_to_loop_over, total=len(loft_to_loop_over)):
+        loft_insulation_predictions_dict = model_api.predict_all(
+            df=loft_insulation_training_data.iloc[chunk:chunk + 400],
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        )
+
+        loft_only_predictions.append(loft_insulation_predictions_dict["sap_change_predictions"])
+
+    loft_only_predictions = pd.concat(loft_only_predictions)
+
+    def calculate_mape(actuals, predictions):
+        """
+        Calculate the Mean Absolute Percentage Error (MAPE).
+
+        Parameters:
+        - actuals: A list or array of actual values.
+        - predictions: A list or array of predicted values, corresponding to actuals.
+
+        Returns:
+        - mape: The MAPE score as a float.
+
+        Note: This function assumes actuals and predictions are of the same length and
+        does not contain zeros in the actuals (to avoid division by zero).
+        """
+        # Convert inputs to numpy arrays for vectorized operations
+        import numpy as np
+        actuals = np.array(actuals)
+        predictions = np.array(predictions)
+
+        # Calculate the absolute percentage errors
+        ape = np.abs((actuals - predictions) / actuals) * 100
+
+        # Calculate the mean of these percentage errors
+        mape = np.mean(ape)
+
+        return mape
+
+    calculate_mape(actuals=loft_insulation_training_data["sap_ending"],
+                   predictions=loft_only_predictions["predictions"])
+
+    comparison_df = pd.DataFrame(
+        {
+            "sap_starting": loft_insulation_training_data["sap_starting"].values,
+            "actual_sap_ending": loft_insulation_training_data["sap_ending"].values,
+            "predicted_sap_ending": loft_only_predictions["predictions"].values
+        }
+    )
+
+    comparison_df["residual"] = abs(comparison_df["actual_sap_ending"] - comparison_df["predicted_sap_ending"])
+    comparison_df["residual"].describe()
+
+    comparison_df[comparison_df["residual"] == comparison_df["residual"].max()]
+
+    # TODO: Test scoring separately vs combined
+    from etl.epc.Record import EPCRecord
+    from etl.solar.SolarPhotoSupply import SolarPhotoSupply
+    from utils.s3 import read_from_s3
+    import msgpack
+    from recommendations.Recommendations import Recommendations
+    import datetime
+    from numpy import nan
+    cleaning_data = read_dataframe_from_s3_parquet(
+        bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
+    )
+    uprn_filenames = read_dataframe_from_s3_parquet(
+        bucket_name="retrofit-data-dev", file_key="spatial/filename_meta.parquet"
+    )
+    photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")
+    cleaned = read_from_s3(
+        s3_file_name="cleaned_epc_data/cleaned.bson",
+        bucket_name="retrofit-data-dev".format(environment="retrofit-data-dev")
+    )
+    cleaned = msgpack.unpackb(cleaned, raw=False)
+    materials = [
+        {'id': 17, 'type': 'mechanical_ventilation', 'description': 'Mechanical Extract Ventilation', 'depth': None,
+         'depth_unit': None, 'cost': 500, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None, 'r_value_unit': None,
+         'thermal_conductivity': None, 'thermal_conductivity_unit': None, 'link': None,
+         'created_at': datetime.datetime(2023, 10, 18, 16, 39, 9, 827188), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': None, 'labour_cost': None, 'labour_hours_per_unit': None,
+         'plant_cost': None, 'total_cost': None, 'notes': None},
+        {'id': 1221, 'type': 'flat_roof_preparation', 'description': 'clean surface to receive new damp-proof membrane',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14,
+         'plant_cost': 0.0, 'total_cost': 4.36,
+         'notes': 'This data is based on concrete however forms a decent baseline for a Bituminous Felt flat roof'},
+        {'id': 1223, 'type': 'flat_roof_preparation',
+         'description': 'One coat primer; on wood surfaces before fixing; General surfaces; over 300 mm girth',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 2.49, 'labour_cost': 1.5, 'labour_hours_per_unit': 0.08,
+         'plant_cost': 0.0, 'total_cost': 3.99, 'notes': 'SPONs data gives us a baseline for a wood surface'},
+        {'id': 1224, 'type': 'flat_roof_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02,
+         'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None},
+        {'id': 1226, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 22.14, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
+         'plant_cost': 0.0, 'total_cost': 32.8, 'notes': None},
+        {'id': 1227, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 120.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://www.panelsystems.co.uk/product/floormate-ravatherm-sb?attribute_pa_group=floormate-500a'
+                 '&attribute_pa_product-name=ravatherm-xps-x-500-sl&attribute_pa_length=1250&attribute_pa_width=600'
+                 '&attribute_pa_thickness=120&attribute_pa_unit-of-sale=pack-3-brds&attribute_pa_min-order-qty=10'
+                 '&gclid=CjwKCAiAjrarBhAWEiwA2qWdCKJK2iqlzUZ-mBFOfCLy2f5TldAbOj7G3LrvYw5JLaigplJAajzYpRoCtB8QAvD_BwE',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 26.187656, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
+         'plant_cost': 0.0, 'total_cost': 36.847656,
+         'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, "
+                  "the 120mm thickness is 18% more expensive per board than the 100mm thickness"},
+        {'id': 1228, 'type': 'flat_roof_insulation', 'description': 'Ravatherm XPS × 500 SL', 'depth': 140.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://www.panelsystems.co.uk/product/floormate-ravatherm-sb?attribute_pa_group=floormate-500a'
+                 '&attribute_pa_product-name=ravatherm-xps-x-500-sl&attribute_pa_length=1250&attribute_pa_width=600'
+                 '&attribute_pa_thickness=120&attribute_pa_unit-of-sale=pack-3-brds&attribute_pa_min-order-qty=10'
+                 '&gclid=CjwKCAiAjrarBhAWEiwA2qWdCKJK2iqlzUZ-mBFOfCLy2f5TldAbOj7G3LrvYw5JLaigplJAajzYpRoCtB8QAvD_BwE',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 31.114737, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
+         'plant_cost': 0.0, 'total_cost': 41.77474,
+         'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, "
+                  "the 140mm thickness is 40% more expensive per board than the 100mm thickness"},
+        {'id': 1229, 'type': 'flat_roof_insulation', 'description': 'Foamglas T3+ Flat Roof Insulation', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.027777778,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.036,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': 95.83, 'material_cost': 109.09, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 139.79, 'notes': None},
+        {'id': 1230, 'type': 'flat_roof_insulation', 'description': 'Foamglas T4+ Flat Roof Insulation', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.024390243,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.041,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': 63.89, 'material_cost': 76.19, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 104.53, 'notes': None}, {'id': 1231, 'type': 'flat_roof_insulation',
+                                                                   'description': 'Ecotherm Eco-Versal General '
+                                                                                  'Purpose Insulation Board',
+                                                                   'depth': 100.0, 'depth_unit': 'mm', 'cost': None,
+                                                                   'cost_unit': 'gbp_per_m2',
+                                                                   'r_value_per_mm': 0.045454547,
+                                                                   'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                   'thermal_conductivity': 0.022,
+                                                                   'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                   'link': 'SPONs',
+                                                                   'created_at': datetime.datetime(2023, 12, 4, 20, 1,
+                                                                                                   49, 298076),
+                                                                   'is_active': True, 'prime_material_cost': 15.12,
+                                                                   'material_cost': 25.96, 'labour_cost': 30.7,
+                                                                   'labour_hours_per_unit': 1.3, 'plant_cost': 0.0,
+                                                                   'total_cost': 56.66, 'notes': None},
+        {'id': 1232, 'type': 'flat_roof_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 120.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': 20.16, 'material_cost': 34.613335, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 65.31333,
+         'notes': "SPONs didn't have this thickness, so the material price is based on the fact that on the link, "
+                  "the 120mm thickness is 33% more expensive than the 100mm thickness"},
+        {'id': 1233, 'type': 'flat_roof_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 150.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': 23.53, 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4,
+         'plant_cost': 0.0, 'total_cost': 67.68, 'notes': None}, {'id': 1234, 'type': 'flat_roof_waterproofing',
+                                                                  'description': '20 mm thick two coat coverings; '
+                                                                                 'felt isolating membrane; to '
+                                                                                 'concrete (or timber) base; flat or '
+                                                                                 'to falls or slopes not exceeding '
+                                                                                 '10° from horizontal',
+                                                                  'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': None,
+                                                                  'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 12, 4, 20, 1,
+                                                                                                  49, 298076),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 0.0, 'labour_cost': 0.0,
+                                                                  'labour_hours_per_unit': 0.5, 'plant_cost': 0.0,
+                                                                  'total_cost': 31.13, 'notes': None},
+        {'id': 1225, 'type': 'flat_roof_insulation',
+         'description': 'Kingspan Thermaroof TR21 zero OPD urethene insulation board', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.025,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 4, 20, 1, 49, 298076), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 50.95, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.48,
+         'plant_cost': 0.0, 'total_cost': 61.61,
+         'notes': "SPONs didn't have a labour hours so we use 0.48 which is similar to other materials"},
+        {'id': 1235, 'type': 'windows_glazing',
+         'description': 'uPVC windows; Profile 22 or other equal and approved; reinforced where appropriate with '
+                        'aluminium alloy; in refurbishment work, including standard ironmongery; sills and factory '
+                        'glazed with low-e 24 mm double glazing; removing existing windows and fixing new in '
+                        'position; including lugs plugged and screwed to brickwork or blockwork; Casement/fixed '
+                        'light; including vents; e.p.d.m. glazing gaskets and weather seals; 1770 mm × 1200 mm; ref '
+                        'P312WW',
+         'depth': 0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': None,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 12, 20, 14, 37, 51, 728866), 'is_active': True,
+         'prime_material_cost': 176.55, 'material_cost': 182.25, 'labour_cost': 163.36, 'labour_hours_per_unit': 6.5,
+         'plant_cost': 0.0, 'total_cost': 345.61,
+         'notes': 'This is the cost of removal of existing windows and installation of new windows. This is a '
+                  'casement style window, which is the most common but also the cheapest style. In the cost '
+                  'estimation framework, we can inflate prices for different finishes, to be conservative on price. '},
+        {'id': 1109, 'type': 'cavity_wall_insulation',
+         'description': 'Expanded Polystyrene Beads cavity wall insulation', 'depth': 75.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://www.styrene.co.uk/downloads/Datasheets'
+                 '/Stylite_Cavity_Loose_Fill_Insulation_Datasheet_v20211.pdf',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 18.875, 'labour_cost': 1.125, 'labour_hours_per_unit': 0.065,
+         'plant_cost': 0.0, 'total_cost': 20.0,
+         'notes': "It is hard to find materials online. To price this, we've used this article: "
+                  "https://www.greenmatch.co.uk/blog/cavity-wall-insulation-cost It puts EPS beads at around £22 per "
+                  "meter squared, blowing wool insulation at £18 per meter squared and Polyurethane Foam at £26 per "
+                  "meter squared, when taking the most pessimistic prices. These rates have been used to adjust the "
+                  "price of the mineral wool insulation to give us the other forms of insulation"},
+        {'id': 1110, 'type': 'cavity_wall_insulation',
+         'description': 'Injected Polyurthane Foam cavity wall insulation', 'depth': 75.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://www.foaminstall.co.uk/wp-content/uploads/2017/04/Lapolla-Cavity-Fill-BBA-certificate-sheet1'
+                 '.pdf',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 22.875, 'labour_cost': 1.125, 'labour_hours_per_unit': 0.065,
+         'plant_cost': 0.0, 'total_cost': 24.0, 'notes': None},
+        {'id': 1111, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 2.03, 'material_cost': 2.1, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 3.66, 'notes': None},
+        {'id': 1112, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 150.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 3.06, 'material_cost': 3.16, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 4.94, 'notes': None},
+        {'id': 1113, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 170.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://insulation4less.co.uk/products/knauf-170mm-combi-cut?variant=31671561257013&dfw_tracker'
+                 '=77750-31671561257013&utm_source=google&utm_medium=shopping&utm_campaign=shoptimised&gad_source=1'
+                 '&gclid=CjwKCAiAx_GqBhBQEiwAlDNAZi1LiTWKVn0W1vktOYAPPQU3hss5Tq2qNn6GNhodCQoRD_tvqCLdxhoCKnIQAvD_BwE',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 3.81938, 'labour_cost': 1.71304, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 5.53242,
+         'notes': "We don't have a 170mm in SPONs so the material cost is based on the fact that the 170mm insulation "
+                  "is 87.4% of the cost of the 200mm insulation"},
+        {'id': 1114, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 200.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 4.25, 'material_cost': 4.37, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 6.33, 'notes': None},
+        {'id': 1115, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 270.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 5.91938, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 7.87938, 'notes': 'This is the 100mm product + the 170mm product'},
+        {'id': 1116, 'type': 'loft_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll', 'depth': 300.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.47, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 8.43, 'notes': 'This is the 100mm product + the 200mm product'},
+        {'id': 1117, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 1.99, 'material_cost': 2.05, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 3.65, 'notes': None},
+        {'id': 1118, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 150.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 2.96, 'material_cost': 3.05, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 4.83, 'notes': None},
+        {'id': 1119, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 170.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://flooringwarehousedirect.co.uk/product/isover-spacesaver-roll-170mm-x-1160mm-x-7-03m-8-15m2/',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 3.8706238, 'labour_cost': 2.281361,
+         'labour_hours_per_unit': 0.12816635, 'plant_cost': 0.0, 'total_cost': 6.1519847,
+         'notes': "We don't have a 170mm in SPONs so the material cost is based on the fact that the 170mm insulation "
+                  "is 85.4% of the cost of the 200mm insulation"},
+        {'id': 1120, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 200.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 4.4, 'material_cost': 4.53, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 7.2, 'notes': None},
+        {'id': 1121, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 270.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 5.920624, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 8.590624, 'notes': 'This is the 100mm product + the 170mm product'},
+        {'id': 1122, 'type': 'loft_insulation', 'description': 'Isover Mineral Wool Modular Roll', 'depth': 300.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.58, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 9.25, 'notes': 'This is the 100mm product + the 200mm product'},
+        {'id': 1123, 'type': 'loft_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 5.93, 'material_cost': 6.4, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 9.07, 'notes': 'This provides acoustic insulation as well'},
+        {'id': 1124, 'type': 'loft_insulation', 'description': 'Isover Acoustic Partition Roll', 'depth': 300.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 17.79, 'material_cost': 19.2, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 21.87, 'notes': 'This provides acoustic insulation as well'},
+        {'id': 1125, 'type': 'loft_insulation', 'description': 'Thermafleece EcoRoll Insulation', 'depth': 300.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 24.78, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 27.45,
+         'notes': 'This material is based on installing 3 layers of the 100mm product'},
+        {'id': 1126, 'type': 'loft_insulation', 'description': 'Thermafleece EcoRoll Insulation', 'depth': 280.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 23.36, 'labour_cost': 3.12, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 26.48,
+         'notes': 'This material is based on installed 2 layers of the 140mm product'},
+        {'id': 1127, 'type': 'iwi_wall_demolition',
+         'description': 'Solid & Dry Lined walls: Hack of wall finishes with chipping hammer; plaster to walls.',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 10.27, 'labour_hours_per_unit': 0.33,
+         'plant_cost': 1.28, 'total_cost': 11.55, 'notes': None}, {'id': 1128, 'type': 'iwi_wall_demolition',
+                                                                   'description': 'Stud walls: Remove wall linings '
+                                                                                  'including battening behind; '
+                                                                                  'plasterboard and skim',
+                                                                   'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                   'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+                                                                   'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                   'thermal_conductivity': None,
+                                                                   'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                   'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                   12, 244907),
+                                                                   'is_active': True, 'prime_material_cost': None,
+                                                                   'material_cost': 0.0, 'labour_cost': 6.23,
+                                                                   'labour_hours_per_unit': 0.2, 'plant_cost': 1.25,
+                                                                   'total_cost': 7.48, 'notes': None},
+        {'id': 1129, 'type': 'iwi_wall_demolition',
+         'description': 'Lathe and Plaster walls: Remove wall linings including battening behind; wood lath and '
+                        'plaster',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.85, 'labour_hours_per_unit': 0.22,
+         'plant_cost': 2.09, 'total_cost': 8.94, 'notes': None},
+        {'id': 1130, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 60.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 41.69, 'material_cost': 53.33, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 82.85, 'notes': None},
+        {'id': 1131, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 86.86, 'material_cost': 99.85, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 129.37, 'notes': None},
+        {'id': 1132, 'type': 'internal_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 130.29, 'material_cost': 144.58, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 174.1, 'notes': None},
+        {'id': 1133, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 30.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 6.16, 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 45.07, 'notes': None},
+        {'id': 1134, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 8.46, 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 47.44, 'notes': None},
+        {'id': 1135, 'type': 'internal_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 15.12, 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 56.66, 'notes': None},
+        {'id': 1136, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard',
+         'depth': 37.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 26.86, 'labour_cost': 5.21, 'labour_hours_per_unit': 0.23,
+         'plant_cost': 0.0, 'total_cost': 32.07, 'notes': None},
+        {'id': 1137, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard',
+         'depth': 42.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 17.37, 'labour_cost': 5.21, 'labour_hours_per_unit': 0.23,
+         'plant_cost': 0.0, 'total_cost': 22.58, 'notes': None},
+        {'id': 1138, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard',
+         'depth': 52.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 21.74, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25,
+         'plant_cost': 0.0, 'total_cost': 27.53, 'notes': None},
+        {'id': 1139, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard',
+         'depth': 62.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 19.3, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25,
+         'plant_cost': 0.0, 'total_cost': 25.09, 'notes': None},
+        {'id': 1140, 'type': 'internal_wall_insulation', 'description': 'Kingspan Kooltherm K18 insulated plasterboard',
+         'depth': 72.5, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.04761905,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.021,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 23.15, 'labour_cost': 5.79, 'labour_hours_per_unit': 0.25,
+         'plant_cost': 0.0, 'total_cost': 28.94, 'notes': None},
+        {'id': 1141, 'type': 'iwi_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02,
+         'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None}, {'id': 1142, 'type': 'iwi_redecoration',
+                                                                 'description': 'Plaster; one coat Thistle board '
+                                                                                'finish or other equal; steel '
+                                                                                'trowelled; 3 mm thick work to walls '
+                                                                                'or ceilings; one coat; to '
+                                                                                'plasterboard base; over 600mm wide',
+                                                                 'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                 'cost_unit': None, 'r_value_per_mm': nan,
+                                                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                 'thermal_conductivity': None,
+                                                                 'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                 'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                 12, 244907),
+                                                                 'is_active': True, 'prime_material_cost': None,
+                                                                 'material_cost': 0.06, 'labour_cost': 6.58,
+                                                                 'labour_hours_per_unit': 0.25, 'plant_cost': 0.0,
+                                                                 'total_cost': 6.64, 'notes': None},
+        {'id': 1143, 'type': 'iwi_redecoration',
+         'description': 'Two coats emulsion paint on plaster, over 40mm girth; 3.5m - 5m high', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.41, 'labour_cost': 3.93, 'labour_hours_per_unit': 0.21,
+         'plant_cost': 0.0, 'total_cost': 4.34, 'notes': None}, {'id': 1144, 'type': 'iwi_redecoration',
+                                                                 'description': 'Fitting existing softwood skirting '
+                                                                                'or architrave to new frames; 150mm '
+                                                                                'high',
+                                                                 'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                 'cost_unit': None, 'r_value_per_mm': nan,
+                                                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                 'thermal_conductivity': None,
+                                                                 'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                 'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                 12, 244907),
+                                                                 'is_active': True, 'prime_material_cost': None,
+                                                                 'material_cost': 0.01, 'labour_cost': 4.87,
+                                                                 'labour_hours_per_unit': 0.12, 'plant_cost': 0.0,
+                                                                 'total_cost': 4.88, 'notes': None},
+        {'id': 1145, 'type': 'suspended_floor_demolition', 'description': 'Removal of carpet and underfelt',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 3.32,
+         'notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and '
+                  'therefore there is no need for a skip'},
+        {'id': 1146, 'type': 'suspended_floor_demolition',
+         'description': 'Remove boarding; withdraw nails; set aside for reuse; ground level', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 9.34, 'labour_hours_per_unit': 0.3,
+         'plant_cost': 0.0, 'total_cost': 9.34, 'notes': None}, {'id': 1147, 'type': 'suspended_floor_vapour_barrier',
+                                                                 'description': 'Visqueen High Performance Vapour '
+                                                                                'Barrier',
+                                                                 'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                 'cost_unit': None, 'r_value_per_mm': nan,
+                                                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                 'thermal_conductivity': None,
+                                                                 'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                 'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                 12, 244907),
+                                                                 'is_active': True, 'prime_material_cost': 0.58,
+                                                                 'material_cost': 1.21, 'labour_cost': 0.48,
+                                                                 'labour_hours_per_unit': 0.02, 'plant_cost': 0.0,
+                                                                 'total_cost': 1.69, 'notes': None},
+        {'id': 1148, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 50.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 4.24, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 5.8,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1149, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 75.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.31, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 7.87,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1150, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 100.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 8.26, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 9.82,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1151, 'type': 'suspended_floor_insulation', 'description': 'Thermafleece CosyWool Roll', 'depth': 140.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 11.68, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 13.46,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1152, 'type': 'suspended_floor_insulation',
+         'description': 'Thermafleece TF35 high density wool insulating batts', 'depth': 50.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.028571429,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.035,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.63, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 8.19,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1153, 'type': 'suspended_floor_insulation',
+         'description': 'Thermafleece TF35 high density wool insulating batts', 'depth': 75.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.028571429,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.035,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 10.31, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 11.87,
+         'notes': 'Spons did not contain labour costs so we use values for similar insulations. We use the same '
+                  'values as in Crown loft roll 44, since it is also an insulation roll'},
+        {'id': 1154, 'type': 'suspended_floor_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 30.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 6.16, 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 45.07, 'notes': None}, {'id': 1155, 'type': 'suspended_floor_insulation',
+                                                                  'description': 'Ecotherm Eco-Versal General Purpose '
+                                                                                 'Insulation Board',
+                                                                  'depth': 50.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': 8.46,
+                                                                  'material_cost': 19.1, 'labour_cost': 28.34,
+                                                                  'labour_hours_per_unit': 1.2, 'plant_cost': 0.0,
+                                                                  'total_cost': 47.44, 'notes': None},
+        {'id': 1156, 'type': 'suspended_floor_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 100.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 15.12, 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 56.66, 'notes': None}, {'id': 1157, 'type': 'suspended_floor_insulation',
+                                                                  'description': 'Ecotherm Eco-Versal General Purpose '
+                                                                                 'Insulation Board',
+                                                                  'depth': 150.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': 23.53,
+                                                                  'material_cost': 34.62, 'labour_cost': 33.06,
+                                                                  'labour_hours_per_unit': 1.4, 'plant_cost': 0.0,
+                                                                  'total_cost': 67.68, 'notes': None},
+        {'id': 1158, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 2.03, 'material_cost': 2.1, 'labour_cost': 1.56, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 3.66, 'notes': None},
+        {'id': 1159, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll',
+         'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 3.06, 'material_cost': 3.16, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 4.94, 'notes': None},
+        {'id': 1160, 'type': 'suspended_floor_insulation', 'description': 'Crown Loft Roll 44 glass fibre roll',
+         'depth': 200.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.022727273,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.044,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 4.25, 'material_cost': 4.37, 'labour_cost': 1.96, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 6.33, 'notes': None},
+        {'id': 1161, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 1.99, 'material_cost': 2.05, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 3.65, 'notes': None},
+        {'id': 1162, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll',
+         'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 2.96, 'material_cost': 3.05, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 4.83, 'notes': None},
+        {'id': 1163, 'type': 'suspended_floor_insulation', 'description': 'Isover Mineral Wool Modular Roll',
+         'depth': 200.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 4.4, 'material_cost': 4.53, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 7.2, 'notes': None},
+        {'id': 1164, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll',
+         'depth': 25.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 1.67, 'material_cost': 2.01, 'labour_cost': 1.43, 'labour_hours_per_unit': 0.08,
+         'plant_cost': 0.0, 'total_cost': 3.44, 'notes': None},
+        {'id': 1165, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.025641026,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.039,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 2.74, 'material_cost': 3.11, 'labour_cost': 1.6, 'labour_hours_per_unit': 0.09,
+         'plant_cost': 0.0, 'total_cost': 4.71, 'notes': None},
+        {'id': 1166, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll',
+         'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 4.57, 'material_cost': 5.01, 'labour_cost': 1.78, 'labour_hours_per_unit': 0.1,
+         'plant_cost': 0.0, 'total_cost': 6.79, 'notes': None},
+        {'id': 1167, 'type': 'suspended_floor_insulation', 'description': 'Isover Acoustic Partition Roll',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_unit', 'r_value_per_mm': 0.023255814,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.043,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 5.93, 'material_cost': 6.4, 'labour_cost': 2.67, 'labour_hours_per_unit': 0.15,
+         'plant_cost': 0.0, 'total_cost': 9.07, 'notes': None},
+        {'id': 1168, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 25.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 3.88, 'labour_cost': 3.24, 'labour_hours_per_unit': 0.14,
+         'plant_cost': 0.0, 'total_cost': 7.12, 'notes': None},
+        {'id': 1169, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.62, 'labour_cost': 3.71, 'labour_hours_per_unit': 0.16,
+         'plant_cost': 0.0, 'total_cost': 10.33, 'notes': None},
+        {'id': 1170, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 9.3, 'labour_cost': 4.17, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 13.47, 'notes': None},
+        {'id': 1171, 'type': 'suspended_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 12.02, 'labour_cost': 4.4, 'labour_hours_per_unit': 0.19,
+         'plant_cost': 0.0, 'total_cost': 16.42, 'notes': None}, {'id': 1172, 'type': 'suspended_floor_insulation',
+                                                                  'description': 'Kingspan Thermafloor TF70 High '
+                                                                                 'Performance Rigid Floor Insulation',
+                                                                  'depth': 50.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 10.36, 'labour_cost': 4.06,
+                                                                  'labour_hours_per_unit': 0.18, 'plant_cost': 0.0,
+                                                                  'total_cost': 14.42, 'notes': None},
+        {'id': 1173, 'type': 'suspended_floor_insulation',
+         'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 75.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 15.35, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 19.41, 'notes': None}, {'id': 1174, 'type': 'suspended_floor_insulation',
+                                                                  'description': 'Kingspan Thermafloor TF70 High '
+                                                                                 'Performance Rigid Floor Insulation',
+                                                                  'depth': 100.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 19.17, 'labour_cost': 4.06,
+                                                                  'labour_hours_per_unit': 0.18, 'plant_cost': 0.0,
+                                                                  'total_cost': 23.23, 'notes': None},
+        {'id': 1175, 'type': 'suspended_floor_insulation',
+         'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 125.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 26.59, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 30.65, 'notes': None}, {'id': 1176, 'type': 'suspended_floor_insulation',
+                                                                  'description': 'Kingspan Thermafloor TF70 High '
+                                                                                 'Performance Rigid Floor Insulation',
+                                                                  'depth': 150.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 31.13, 'labour_cost': 4.64,
+                                                                  'labour_hours_per_unit': 0.2, 'plant_cost': 0.0,
+                                                                  'total_cost': 35.77, 'notes': None},
+        {'id': 1177, 'type': 'suspended_floor_redecoration', 'description': 'refix floorboards previously set aside',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 1.54, 'labour_cost': 24.98, 'labour_hours_per_unit': 0.74,
+         'plant_cost': 0.0, 'total_cost': 26.52, 'notes': None},
+        {'id': 1178, 'type': 'suspended_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37,
+         'plant_cost': 0.0, 'total_cost': 6.59,
+         'notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; Gradus '
+                  'woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, therefore we need '
+                  'just labour rates'},
+        {'id': 1179, 'type': 'solid_floor_demolition', 'description': 'Removal of carpet and underfelt', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 3.32, 'labour_hours_per_unit': 0.11,
+         'plant_cost': 0.0, 'total_cost': 3.32,
+         'notes': 'We ignore the plant cost that is in SPONs because we assume the carpet is not scrapped and '
+                  'therefore there is no need for a skip'},
+        {'id': 1180, 'type': 'solid_floor_preparation',
+         'description': 'clean surface of concrete to receive new damp-proof membrane', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': None,
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 4.36, 'labour_hours_per_unit': 0.14,
+         'plant_cost': 0.0, 'total_cost': 4.36, 'notes': None}, {'id': 1181, 'type': 'solid_floor_preparation',
+                                                                 'description': 'Clean out crack to form a 20mm×20mm '
+                                                                                'groove and fill with cement: mortar '
+                                                                                'mixed with bonding agent',
+                                                                 'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                 'cost_unit': None, 'r_value_per_mm': nan,
+                                                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                 'thermal_conductivity': None,
+                                                                 'thermal_conductivity_unit': None, 'link': None,
+                                                                 'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                 12, 244907),
+                                                                 'is_active': True, 'prime_material_cost': None,
+                                                                 'material_cost': 6.91, 'labour_cost': 18.99,
+                                                                 'labour_hours_per_unit': 0.61, 'plant_cost': 0.16,
+                                                                 'total_cost': 26.06,
+                                                                 'notes': 'This step is the assessment and repair of '
+                                                                          'any damage to the concrete floor such as '
+                                                                          'filling cracks or levelling uneven areas'},
+        {'id': 1182, 'type': 'solid_floor_vapour_barrier', 'description': 'Visqueen High Performance Vapour Barrier',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 0.58, 'material_cost': 1.21, 'labour_cost': 0.48, 'labour_hours_per_unit': 0.02,
+         'plant_cost': 0.0, 'total_cost': 1.69, 'notes': None},
+        {'id': 1183, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 25.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 3.88, 'labour_cost': 3.24, 'labour_hours_per_unit': 0.14,
+         'plant_cost': 0.0, 'total_cost': 7.12, 'notes': None},
+        {'id': 1184, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 6.62, 'labour_cost': 3.71, 'labour_hours_per_unit': 0.16,
+         'plant_cost': 0.0, 'total_cost': 10.33, 'notes': None},
+        {'id': 1185, 'type': 'solid_floor_insulation', 'description': 'Kay-Cel Expanded Polystyrene Board',
+         'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.030303031,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.033,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 9.3, 'labour_cost': 4.17, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 13.47, 'notes': None}, {'id': 1186, 'type': 'solid_floor_insulation',
+                                                                  'description': 'Kingspan Thermafloor TF70 High '
+                                                                                 'Performance Rigid Floor Insulation',
+                                                                  'depth': 50.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 10.36, 'labour_cost': 4.06,
+                                                                  'labour_hours_per_unit': 0.18, 'plant_cost': 0.0,
+                                                                  'total_cost': 14.42, 'notes': None},
+        {'id': 1187, 'type': 'solid_floor_insulation',
+         'description': 'Kingspan Thermafloor TF70 High Performance Rigid Floor Insulation', 'depth': 75.0,
+         'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 15.35, 'labour_cost': 4.06, 'labour_hours_per_unit': 0.18,
+         'plant_cost': 0.0, 'total_cost': 19.41, 'notes': None}, {'id': 1188, 'type': 'solid_floor_insulation',
+                                                                  'description': 'Ecotherm Eco-Versal General Purpose '
+                                                                                 'Insulation Board',
+                                                                  'depth': 30.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': 6.16,
+                                                                  'material_cost': 16.73, 'labour_cost': 28.34,
+                                                                  'labour_hours_per_unit': 1.2, 'plant_cost': 0.0,
+                                                                  'total_cost': 45.07, 'notes': None},
+        {'id': 1189, 'type': 'solid_floor_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 50.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 8.46, 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 47.44, 'notes': None}, {'id': 1190, 'type': 'solid_floor_insulation',
+                                                                  'description': 'Ecotherm Eco-Versal General Purpose '
+                                                                                 'Insulation Board',
+                                                                  'depth': 60.0, 'depth_unit': 'mm', 'cost': None,
+                                                                  'cost_unit': 'gbp_per_m2',
+                                                                  'r_value_per_mm': 0.045454547,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': 0.022,
+                                                                  'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+                                                                  'link': 'https://londonbuildingsupplies.co.uk/products/60mm--ecotherm-eco-versal-general-purpose-pir-insulation-board---2.4m-x-1.2m-x-60mm.html',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 24.081198, 'labour_cost': 28.34,
+                                                                  'labour_hours_per_unit': 1.2, 'plant_cost': 0.0,
+                                                                  'total_cost': 52.421196,
+                                                                  'notes': "This material isn't in SPONs but checking"
+                                                                           " online, is around 92% of the cost of the"
+                                                                           " 100mm"},
+        {'id': 1191, 'type': 'solid_floor_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 70.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin',
+         'link': 'https://londonbuildingsupplies.co.uk/products/70mm--ecotherm-eco-versal-general-purpose-pir'
+                 '-insulation-board---2.4m-x-1.2m-x-70mm.html',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 27.089088, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 55.42909,
+         'notes': "This material isn't in SPONs but checking online, is around 104% of the cost of the 100mm (more "
+                  "expensive than 100mm)"},
+        {'id': 1192, 'type': 'solid_floor_insulation',
+         'description': 'Ecotherm Eco-Versal General Purpose Insulation Board', 'depth': 100.0, 'depth_unit': 'mm',
+         'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 15.12, 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 56.66, 'notes': None},
+        {'id': 1193, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.032258064,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.031,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 11.07, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.46,
+         'plant_cost': 0.0, 'total_cost': 21.73,
+         'notes': "In Spons, the thermal conductivity is 0.033 however the datasheet indicates it's 0.32: "
+                  "https://ravagobuildingsolutions.com/uk/wp-content/uploads/sites/30/2022/08/ravatherm-xps-x-500-sl"
+                  "-tds-version-1-20210901.pdf"},
+        {'id': 1194, 'type': 'solid_floor_insulation', 'description': 'Ravatherm XPS X 500 SL Polystyrene Foam',
+         'depth': 75.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.03125,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.032,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 16.28, 'labour_cost': 10.66, 'labour_hours_per_unit': 0.46,
+         'plant_cost': 0.0, 'total_cost': 26.94, 'notes': None}, {'id': 1195, 'type': 'solid_floor_redecoration',
+                                                                  'description': 'Screeded beds; protection to '
+                                                                                 'compressible formwork exceeding '
+                                                                                 '600mm wide',
+                                                                  'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                  'cost_unit': None, 'r_value_per_mm': nan,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': None,
+                                                                  'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': 9.6,
+                                                                  'material_cost': 9.89, 'labour_cost': 2.67,
+                                                                  'labour_hours_per_unit': 0.15, 'plant_cost': 0.0,
+                                                                  'total_cost': 12.56,
+                                                                  'notes': 'This is the screed layer, placed on top '
+                                                                           'of the insulation'},
+        {'id': 1196, 'type': 'solid_floor_redecoration', 'description': 'Fitting carpet', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.59, 'labour_hours_per_unit': 0.37,
+         'plant_cost': 0.0, 'total_cost': 6.59,
+         'notes': 'SPONs does not have data on re-fitting the carpet so we use the data in Fitted carpeting; Gradus '
+                  'woven polypropylene tufted loop\n\n as a baseline. We assume re-use of carpets, therefore we need '
+                  'just labour rates'},
+        {'id': 1197, 'type': 'solid_floor_redecoration',
+         'description': 'Fitting existing softwood skirting or architrave to new frames; 150mm high', 'depth': 0.0,
+         'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.01, 'labour_cost': 4.87, 'labour_hours_per_unit': 0.12,
+         'plant_cost': 0.0, 'total_cost': 4.88, 'notes': None}, {'id': 1198, 'type': 'ewi_wall_demolition',
+                                                                 'description': 'Solid & Dry Lined walls: Hack of '
+                                                                                'wall finishes with chipping hammer; '
+                                                                                'plaster to walls.',
+                                                                 'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                 'cost_unit': None, 'r_value_per_mm': nan,
+                                                                 'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                 'thermal_conductivity': None,
+                                                                 'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                 'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                 12, 244907),
+                                                                 'is_active': True, 'prime_material_cost': None,
+                                                                 'material_cost': 0.0, 'labour_cost': 10.27,
+                                                                 'labour_hours_per_unit': 0.33, 'plant_cost': 1.28,
+                                                                 'total_cost': 11.55, 'notes': None},
+        {'id': 1199, 'type': 'ewi_wall_demolition',
+         'description': 'Stud walls: Remove wall linings including battening behind; plasterboard and skim',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 0.0, 'labour_cost': 6.23, 'labour_hours_per_unit': 0.2,
+         'plant_cost': 1.25, 'total_cost': 7.48, 'notes': None}, {'id': 1200, 'type': 'ewi_wall_demolition',
+                                                                  'description': 'Lathe and Plaster walls: Remove '
+                                                                                 'wall linings including battening '
+                                                                                 'behind; wood lath and plaster',
+                                                                  'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                  'cost_unit': None, 'r_value_per_mm': nan,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': None,
+                                                                  'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 0.0, 'labour_cost': 6.85,
+                                                                  'labour_hours_per_unit': 0.22, 'plant_cost': 2.09,
+                                                                  'total_cost': 8.94, 'notes': None},
+        {'id': 1201, 'type': 'ewi_wall_preparation',
+         'description': 'Clean and prepare surfaces, one coat Keim dilution, one coat primer and two coats of Keim '
+                        'Ecosil paint; Brick or block walls; over 300 mm girth',
+         'depth': 0.0, 'depth_unit': None, 'cost': None, 'cost_unit': None, 'r_value_per_mm': nan,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': None,
+         'thermal_conductivity_unit': None, 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 7.3, 'labour_cost': 5.62, 'labour_hours_per_unit': 0.3,
+         'plant_cost': 0.0, 'total_cost': 12.92,
+         'notes': 'This work covers the preparation and priming of the wall before insulating'},
+        {'id': 1202, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 30.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 6.16, 'material_cost': 16.73, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 45.07, 'notes': None},
+        {'id': 1203, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 50.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 8.46, 'material_cost': 19.1, 'labour_cost': 28.34, 'labour_hours_per_unit': 1.2,
+         'plant_cost': 0.0, 'total_cost': 47.44, 'notes': None},
+        {'id': 1204, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 15.12, 'material_cost': 25.96, 'labour_cost': 30.7, 'labour_hours_per_unit': 1.3,
+         'plant_cost': 0.0, 'total_cost': 56.66, 'notes': None},
+        {'id': 1205, 'type': 'external_wall_insulation', 'description': 'Ecotherm Eco-Versal PIR Insulation Board',
+         'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.045454547,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.022,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 23.53, 'material_cost': 34.62, 'labour_cost': 33.06, 'labour_hours_per_unit': 1.4,
+         'plant_cost': 0.0, 'total_cost': 67.68, 'notes': None},
+        {'id': 1206, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 60.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 41.69, 'material_cost': 53.33, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 82.85, 'notes': None},
+        {'id': 1207, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 100.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 86.86, 'material_cost': 99.85, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 129.37, 'notes': None},
+        {'id': 1208, 'type': 'external_wall_insulation', 'description': 'Foamglas Grade F Wall Insulation Slabs',
+         'depth': 150.0, 'depth_unit': 'mm', 'cost': None, 'cost_unit': 'gbp_per_m2', 'r_value_per_mm': 0.02631579,
+         'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
+         'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': 'SPONs',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': 130.29, 'material_cost': 144.58, 'labour_cost': 29.52, 'labour_hours_per_unit': 1.25,
+         'plant_cost': 0.0, 'total_cost': 174.1, 'notes': None}, {'id': 1209, 'type': 'ewi_wall_redecoration',
+                                                                  'description': 'EPS insulation fixed with adhesive '
+                                                                                 'to SFS structure (measured '
+                                                                                 'separately) with horizontal PVC '
+                                                                                 'intermediate track and vertical '
+                                                                                 'T-spines; with glassfibre mesh '
+                                                                                 'reinforcement embedded in Sto Armat '
+                                                                                 'Classic Basecoat Render and Stolit '
+                                                                                 'K 1.5 Decorative Topcoat Render ('
+                                                                                 'white)',
+                                                                  'depth': 0.0, 'depth_unit': None, 'cost': None,
+                                                                  'cost_unit': None, 'r_value_per_mm': nan,
+                                                                  'r_value_unit': 'square_meter_kelvin_per_watt',
+                                                                  'thermal_conductivity': None,
+                                                                  'thermal_conductivity_unit': None, 'link': 'SPONs',
+                                                                  'created_at': datetime.datetime(2023, 11, 28, 22, 49,
+                                                                                                  12, 244907),
+                                                                  'is_active': True, 'prime_material_cost': None,
+                                                                  'material_cost': 0.0, 'labour_cost': 0.0,
+                                                                  'labour_hours_per_unit': 0.0, 'plant_cost': 0.0,
+                                                                  'total_cost': 69.94,
+                                                                  'notes': 'This material in SPONs is for 70mm EPS '
+                                                                           'insulation, which comes in at a cost of '
+                                                                           '99.17 per meter square. This includes the '
+                                                                           'cost of insulation. To get the costing '
+                                                                           'for just the works and not the '
+                                                                           'insulation, we subtract the cost of EPS '
+                                                                           'insulation, using Ravathem 75mm '
+                                                                           'insulation as an example, which costs '
+                                                                           '£29.23 per meter square, giving us the '
+                                                                           'cost of the remaining works without '
+                                                                           'insulation. This material gives us a cost '
+                                                                           'for basecoat, mesh application and a '
+                                                                           'render finish'},
+        {'id': 1210, 'type': 'low_energy_lighting_installation',
+         'description': 'Installation of fittings and cost of bub', 'depth': 0.0, 'depth_unit': None, 'cost': None,
+         'cost_unit': 'gbp_per_unit', 'r_value_per_mm': nan, 'r_value_unit': 'square_meter_kelvin_per_watt',
+         'thermal_conductivity': None, 'thermal_conductivity_unit': None,
+         'link': 'https://www.checkatrade.com/blog/cost-guides/cost-install-downlights/ '
+                 'https://www.hamuch.com/cost/led-spot-light#:~:text=It%20costs%20an%20average%20of,'
+                 'will%20drive%20up%20the%20cost.',
+         'created_at': datetime.datetime(2023, 11, 28, 22, 49, 12, 244907), 'is_active': True,
+         'prime_material_cost': None, 'material_cost': 20.0, 'labour_cost': 15.0, 'labour_hours_per_unit': 0.8,
+         'plant_cost': 0.0, 'total_cost': 66.0,
+         'notes': 'We estimate the unit economics from the checkatrade article. We assume that the average job '
+                  'consists of installing 6 lights based on the hamuch article. We use the median value of 400 for a '
+                  'job of 6 lights'}]
+
+    testing_properties = [
+        {
+            "address": "2 South Terrace",
+            "postcode": "NN1 5JY"
+        },
+        {
+            "address": "8 Lindlings",
+            "postcode": "HP1 2HA",
+        },
+        {
+            "address": "44 Lindlings",
+            "postcode": "HP1 2HE",
+        },
+        {
+            "address": "46 Chaulden Terrace",
+            "postcode": "HP1 2AN",
+        },
+        {
+            "address": "73 Long Chaulden",
+            "postcode": "HP1 2HX",
+        },
+        {
+            "address": "77 Simmons Drive",
+            "postcode": "B32 1SL",
+        },
+        {
+            "address": "139 School Road",
+            "postcode": "B28 8JF",
+        },
+    ]
+
+    testing_properties_results = []
+    for testing_property_config in testing_properties:
+
+        searcher = SearchEpc(
+            address1=testing_property_config["address"],
+            postcode=testing_property_config["postcode"],
+            auth_token="a2Nvbm5rb3dsZXNzYXJAZ21haWwuY29tOjY5MGJiMWM0NmIyOGI5ZDUxYzAxMzQzYzNiZGNlZGJjZDNmODQwMzA=",
+            os_api_key=""
+        )
+        searcher.find_property(skip_os=True)
+        epc_records = {
+            'original_epc': searcher.newest_epc.copy(),
+            'full_sap_epc': searcher.full_sap_epc.copy(),
+            'old_data': searcher.older_epcs.copy(),
+        }
+
+        prepared_epc = EPCRecord(
+            epc_records=epc_records,
+            run_mode="newdata",
+            cleaning_data=cleaning_data
+        )
+
+        p = Property(
+            id=prepared_epc.uprn,
+            address=searcher.address_clean,
+            postcode=searcher.postcode_clean,
+            epc_record=prepared_epc,
+        )
+        p.get_spatial_data(uprn_filenames)
+        p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
+
+        recommender = Recommendations(property_instance=p, materials=materials)
+        recommender.recommend()
+
+        wall_recommendations = recommender.wall_recomender.recommendations
+        loft_recommendations = recommender.roof_recommender.recommendations
+        floor_recommendations = recommender.floor_recommender.recommendations
+        solar_recommendations = recommender.solar_recommender.recommendation
+
+        # TODO: TEMP!
+        solar_recommendations[0]["photo_supply"] = 50
+
+        # Take just the cavity wall and loft recommendations
+        p.create_base_difference_epc_record(cleaned_lookup=cleaned)
+
+        wall_scoring_data = []
+        for wall_rec in wall_recommendations:
+            recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
+            scoring_dict = p.create_recommendation_scoring_data(
+                property_id=p.id, recommendation_record=recommendation_record, recommendation=wall_rec,
+            )
+            wall_scoring_data.append(scoring_dict)
+
+        roof_scoring_data = []
+        for roof_rec in loft_recommendations:
+            recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
+            scoring_dict = p.create_recommendation_scoring_data(
+                property_id=p.id, recommendation_record=recommendation_record, recommendation=roof_rec,
+            )
+            roof_scoring_data.append(scoring_dict)
+
+        floor_scoring_data = []
+        for floor_rec in floor_recommendations:
+            recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
+            scoring_dict = p.create_recommendation_scoring_data(
+                property_id=p.id, recommendation_record=recommendation_record, recommendation=floor_rec,
+            )
+            floor_scoring_data.append(scoring_dict)
+
+        solar_scoring_data = []
+        for solar_rec in solar_recommendations:
+            recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
+            scoring_dict = p.create_recommendation_scoring_data(
+                property_id=p.id, recommendation_record=recommendation_record, recommendation=solar_rec,
+            )
+            solar_scoring_data.append(scoring_dict)
+
+        # We now produce a combined, applying just the first roof recommendation to the first wall recommendation
+
+        # Firstly apply the wall
+        starting_record = p.base_difference_record.df.to_dict("records")[0].copy()
+        scoring_dict_with_wall = p.create_recommendation_scoring_data(
+            property_id=p.id,
+            recommendation_record=starting_record.copy(),
+            recommendation=wall_recommendations[0],
+        ) if wall_recommendations else starting_record.copy()
+
+        scoring_dict_with_wall_and_roof = p.create_recommendation_scoring_data(
+            property_id=p.id,
+            recommendation_record=scoring_dict_with_wall.copy(),
+            recommendation=loft_recommendations[0],
+        ) if loft_recommendations else scoring_dict_with_wall.copy()
+
+        scoring_dict_with_wall_roof_floor = p.create_recommendation_scoring_data(
+            property_id=p.id,
+            recommendation_record=scoring_dict_with_wall_and_roof.copy(),
+            recommendation=floor_recommendations[0],
+        ) if floor_recommendations else scoring_dict_with_wall_and_roof.copy()
+
+        scoring_dict_with_wall_roof_floor_solar = p.create_recommendation_scoring_data(
+            property_id=p.id,
+            recommendation_record=scoring_dict_with_wall_roof_floor.copy(),
+            recommendation=solar_recommendations[0],
+        ) if solar_recommendations else scoring_dict_with_wall_roof_floor.copy()
+
+        # We score each dataset with the model
+        wall_only_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame(wall_scoring_data),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        ) if wall_scoring_data else {"sap_change_predictions": pd.DataFrame()}
+
+        roof_only_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame(roof_scoring_data),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        ) if roof_scoring_data else {"sap_change_predictions": pd.DataFrame()}
+
+        floor_only_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame(floor_scoring_data),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        ) if floor_scoring_data else {"sap_change_predictions": pd.DataFrame()}
+
+        solar_only_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame(solar_scoring_data),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        ) if solar_scoring_data else {"sap_change_predictions": pd.DataFrame()}
+
+        wall_roof_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame([scoring_dict_with_wall_and_roof]),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        )
+
+        wall_roof_floor_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame([scoring_dict_with_wall_roof_floor]),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        )
+
+        wall_roof_floor_solar_predictions_dict = model_api.predict_all(
+            df=pd.DataFrame([scoring_dict_with_wall_roof_floor_solar]),
+            bucket="retrofit-data-dev",
+            prediction_buckets={
+                "sap_change_predictions": "retrofit-sap-predictions-dev",
+            }
+        )
+
+        wall_only_predictions = wall_only_predictions_dict["sap_change_predictions"].copy()
+        wall_only_predictions["type"] = "wall_only"
+
+        roof_only_predictions = roof_only_predictions_dict["sap_change_predictions"].copy()
+        roof_only_predictions["type"] = "roof_only"
+
+        floor_only_predictions = floor_only_predictions_dict["sap_change_predictions"].copy()
+        floor_only_predictions["type"] = "floor_only"
+
+        solar_only_predictions = solar_only_predictions_dict["sap_change_predictions"].copy()
+        solar_only_predictions["type"] = "solar_only"
+
+        wall_and_roof_predictions = wall_roof_predictions_dict["sap_change_predictions"].copy()
+        wall_and_roof_predictions["type"] = "wall_and_roof"
+
+        wall_roof_floor_predictions = wall_roof_floor_predictions_dict["sap_change_predictions"].copy()
+        wall_roof_floor_predictions["type"] = "wall_roof_floor"
+
+        wall_roof_floor_solar_predictions = wall_roof_floor_solar_predictions_dict["sap_change_predictions"].copy()
+        wall_roof_floor_solar_predictions["type"] = "wall_roof_floor_solar"
+
+        # We collate the results
+        results = pd.concat(
+            [
+                wall_only_predictions,
+                roof_only_predictions,
+                floor_only_predictions,
+                solar_only_predictions,
+                wall_and_roof_predictions,
+                wall_roof_floor_predictions,
+                wall_roof_floor_solar_predictions
+            ]
+        )
+        results["gain"] = results["predictions"] - int(p.data["current-energy-efficiency"])
+        results["address"] = p.address
+
+        testing_properties_results.append(results)
+
+    testing_properties_results = pd.concat(testing_properties_results)
diff --git a/recommendations/Costs.py b/recommendations/Costs.py
index 106f4453..dd1f1a21 100644
--- a/recommendations/Costs.py
+++ b/recommendations/Costs.py
@@ -37,6 +37,9 @@ MCS_SOLAR_PV_COST_DATA = {
     "average_cost_per_kwh-Northern Ireland": 2126.09,
 }
 
+# This is based on quotes from installers
+BATTERY_COST = 3500
+
 
 class Costs:
     """
@@ -835,7 +838,7 @@ class Costs:
             "labour_days": labour_days
         }
 
-    def solar_pv(self, wattage: float):
+    def solar_pv(self, wattage: float, has_battery: bool = False):
 
         """
         Calculates the total cost for solar PV based data provided by the MCS dashboard, which contains
@@ -847,8 +850,8 @@ class Costs:
 
         Price can also be benchmarked against this checkatrade article:
         https://www.checkatrade.com/blog/cost-guides/cost-of-solar-panel-installation/
-        :param wattage: Peak wattage of the solar PV system
-        :return:
+        :param wattage: Peak wattage of the solar PV system]
+        :param has_battery: Bool, whether the system includes a battery
         """
 
         # Get the cost data relevant to the region
@@ -858,6 +861,11 @@ class Costs:
         total_cost = kw * regional_cost
 
         subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
+
+        if has_battery:
+            # The battery cost is based on the £3500 quote, recieved from installers
+            subtotal_before_vat += BATTERY_COST
+            
         vat = total_cost - subtotal_before_vat
 
         # Labour hours are based on estimates from online research but an average team seems to consist of 3 people
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index 0444006d..6ca7badb 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -44,6 +44,9 @@ class Recommendations:
 
         """
         This method runs the recommendations for the individual measures and then appends them to a list for output
+
+        The recommendations are implemented in order of suggested phase, from fabric first to heating systems, to
+        renewables.
         :return:
         """
 
diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py
index 8a773570..9995bbe1 100644
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@@ -39,17 +39,56 @@ class SolarPvRecommendations:
         if not is_valid_property_type or not is_valid_roof_type or not has_no_existing_solar_pv:
             return
 
-        # We now have a property which is potentially suitable for solar PV
-        number_solar_panels = np.floor(self.property.solar_pv_roof_area / self.SOLAR_PANEL_AREA)
-        solar_panel_wattage = number_solar_panels * self.SOLAR_PANEL_WATTAGE
+        # For the solar recommendations, we produce the following scenarios:
+        # 1) Solar panels only, we present a high, medium and low coverage
+        # 2) With and without battery
+        roof_coverage_scenarios = [
+            self.property.solar_pv_percentage - 0.1, self.property.solar_pv_percentage,
+            self.property.solar_pv_percentage + 0.1
+        ]
+        # We make sure we haven't gone too low or high
+        roof_coverage_scenarios = [v for v in roof_coverage_scenarios if 0 <= v <= 1]
+        battery_scenarios = [False, True]
 
-        roof_coverage_percent = round(self.property.solar_pv_percentage * 100)
+        # I now produce the cross product of the scenarios
+        scenarios = [(roof, battery) for roof in roof_coverage_scenarios for battery in battery_scenarios]
 
-        # Given the wattage, we estimate the cost of the solar PV system. This is based on the MCS database
-        # of solar PV installations
-        cost_result = self.costs.solar_pv(wattage=solar_panel_wattage)
+        for roof_coverage, has_battery in scenarios:
+            # We now have a property which is potentially suitable for solar PV
+            solar_pv_roof_area = self.property.get_solar_pv_roof_area(roof_coverage)
 
-        kw = np.floor(solar_panel_wattage / 100) / 10
+            number_solar_panels = np.floor(solar_pv_roof_area / self.SOLAR_PANEL_AREA)
+            solar_panel_wattage = number_solar_panels * self.SOLAR_PANEL_WATTAGE
+
+            roof_coverage_percent = round(roof_coverage * 100)
+
+            # Given the wattage, we estimate the cost of the solar PV system. This is based on the MCS database
+            # of solar PV installations
+            cost_result = self.costs.solar_pv(wattage=solar_panel_wattage, has_battery=has_battery)
+
+            kw = np.floor(solar_panel_wattage / 100) / 10
+
+            if has_battery:
+                description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) panel system on "
+                               f"{round(roof_coverage_percent * 100)}% the roof, with a battery storage system.")
+            else:
+                description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p"
+                               f"anel system on {round(roof_coverage_percent * 100)}% the roof.")
+
+            self.recommendation.append(
+                {
+                    "parts": [],
+                    "type": "solar_pv",
+                    "description": description,
+                    "starting_u_value": None,
+                    "new_u_value": None,
+                    "sap_points": None,
+                    **cost_result,
+                    # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale
+                    # back up here
+                    "photo_supply": 100 * scenario
+                }
+            )
 
         self.recommendation = [
             {

From 925a6c1887cb11c16e231167923a286fdc552f7b Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Wed, 14 Feb 2024 12:00:49 +0000
Subject: [PATCH 08/16] Added phasing figures into recommender

---
 recommendations/FireplaceRecommendations.py   |  3 +-
 recommendations/FloorRecommendations.py       |  8 +--
 recommendations/LightingRecommendations.py    |  3 +-
 recommendations/Recommendations.py            | 54 +++++++++++--------
 recommendations/RoofRecommendations.py        | 12 +++--
 recommendations/SolarPvRecommendations.py     | 21 ++------
 recommendations/VentilationRecommendations.py |  1 +
 recommendations/WallRecommendations.py        | 20 ++++---
 recommendations/WindowsRecommendations.py     |  3 +-
 9 files changed, 66 insertions(+), 59 deletions(-)

diff --git a/recommendations/FireplaceRecommendations.py b/recommendations/FireplaceRecommendations.py
index c193b7ce..5d620d49 100644
--- a/recommendations/FireplaceRecommendations.py
+++ b/recommendations/FireplaceRecommendations.py
@@ -20,7 +20,7 @@ class FireplaceRecommendations(Definitions):
         self.has_ventilaion = None
         self.recommendation = None
 
-    def recommend(self):
+    def recommend(self, phase=0):
         """
         Based on the number of open fireplcaes found, we recommend sealing each one at a cost of
         around £500
@@ -37,6 +37,7 @@ class FireplaceRecommendations(Definitions):
         # We recommend installing two mechanical ventilation systems
         self.recommendation = [
             {
+                "phase": phase,
                 "parts": [],
                 "type": "sealing_open_fireplace",
                 "description": "Seal %s open fireplaces" % str(number_open_fireplaces),
diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py
index 2f568264..a19ee025 100644
--- a/recommendations/FloorRecommendations.py
+++ b/recommendations/FloorRecommendations.py
@@ -69,7 +69,7 @@ class FloorRecommendations(Definitions):
         # TODO: To be completed
         self.exposed_floor_non_insulation_materials = []
 
-    def recommend(self):
+    def recommend(self, phase=0):
         u_value = self.property.floor["thermal_transmittance"]
 
         property_type = self.property.data["property-type"]
@@ -129,7 +129,8 @@ class FloorRecommendations(Definitions):
             self.recommend_floor_insulation(
                 u_value=u_value,
                 insulation_materials=self.solid_floor_insulation_materials,
-                non_insulation_materials=self.solid_floor_non_insulation_materials
+                non_insulation_materials=self.solid_floor_non_insulation_materials,
+                phase=phase
             )
             return
 
@@ -156,7 +157,7 @@ class FloorRecommendations(Definitions):
 
         raise ValueError("Invalid material type - implement me!")
 
-    def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials):
+    def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials, phase):
         """
         This method is tasked with estimating the impact of performing suspended floor insulation
         :return:
@@ -198,6 +199,7 @@ class FloorRecommendations(Definitions):
 
                     self.recommendations.append(
                         {
+                            "phase": phase,
                             "parts": [
                                 get_recommended_part(
                                     part=material.to_dict(),
diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py
index 788d1ad1..6d50f0a2 100644
--- a/recommendations/LightingRecommendations.py
+++ b/recommendations/LightingRecommendations.py
@@ -51,7 +51,7 @@ class LightingRecommendations:
 
         return total_energy_savings_per_year, carbon_reduction_tonnes
 
-    def recommend(self):
+    def recommend(self, phase=0):
         """
         This method will check if there are any lighting fittings that aren't low energy.
 
@@ -90,6 +90,7 @@ class LightingRecommendations:
 
         self.recommendation = [
             {
+                "phase": phase,
                 "parts": [],
                 "type": "low_energy_lighting",
                 "description": description,
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index 6ca7badb..b4b764c2 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -51,48 +51,58 @@ class Recommendations:
         """
 
         property_recommendations = []
-
-        # Floor recommendations
-        self.floor_recommender.recommend()
-        if self.floor_recommender.recommendations:
-            property_recommendations.append(self.floor_recommender.recommendations)
-
+        phase = 0
         # Wall recommendations
-        self.wall_recomender.recommend()
+        self.wall_recomender.recommend(phase=phase)
         if self.wall_recomender.recommendations:
             property_recommendations.append(self.wall_recomender.recommendations)
-
-        # Roof recommendations
-        self.roof_recommender.recommend()
-        if self.roof_recommender.recommendations:
-            property_recommendations.append(self.roof_recommender.recommendations)
+            phase += 1
 
         # Ventilation recommendations
         # We only produce a ventilation recommendation if the property is recommended to have wall or roof insulation
+        # We will not attribute a SAP impact to the ventilation recommendation, since we've seen that this has no
+        # real impact on the SAP score. Therefore, we don't need to include phasing for ventilation. If we have any
+        # wall or roof recommendations, we will ensure that ventilation is included in the simulation
         if self.wall_recomender.recommendations or self.roof_recommender.recommendations:
             self.ventilation_recomender.recommend()
             if self.ventilation_recomender.recommendation:
                 property_recommendations.append(self.ventilation_recomender.recommendation)
 
-        # Fireplace sealing recommendations
-        self.fireplace_recommender.recommend()
-        if self.fireplace_recommender.recommendation:
-            property_recommendations.append(self.fireplace_recommender.recommendation)
+        # Roof recommendations
+        self.roof_recommender.recommend(phase=phase)
+        if self.roof_recommender.recommendations:
+            property_recommendations.append(self.roof_recommender.recommendations)
+            phase += 1
 
-        # Lighting recommendations
-        self.lighting_recommender.recommend()
-        if self.lighting_recommender.recommendation:
-            property_recommendations.append(self.lighting_recommender.recommendation)
+        # Floor recommendations
+        self.floor_recommender.recommend(phase=phase)
+        if self.floor_recommender.recommendations:
+            property_recommendations.append(self.floor_recommender.recommendations)
+            phase += 1
 
         # Windows recommendations
-        self.windows_recommender.recommend()
+        self.windows_recommender.recommend(phase=phase)
         if self.windows_recommender.recommendation:
             property_recommendations.append(self.windows_recommender.recommendation)
+            phase += 1
+
+        # Fireplace sealing recommendations
+        self.fireplace_recommender.recommend(phase=phase)
+        if self.fireplace_recommender.recommendation:
+            property_recommendations.append(self.fireplace_recommender.recommendation)
+            phase += 1
+
+        # Lighting recommendations
+        self.lighting_recommender.recommend(phase=phase)
+        if self.lighting_recommender.recommendation:
+            property_recommendations.append(self.lighting_recommender.recommendation)
+            phase += 1
 
         # Solar recommendations
-        self.solar_recommender.recommend()
+        self.solar_recommender.recommend(phase=phase)
         if self.solar_recommender.recommendation:
             property_recommendations.append(self.solar_recommender.recommendation)
+            phase += 1
 
         # We insert temporary ids into the recommendations which is important for the optimiser later
         property_recommendations = self.insert_temp_recommendation_id(property_recommendations)
diff --git a/recommendations/RoofRecommendations.py b/recommendations/RoofRecommendations.py
index 0bbfd69d..eb1c6c4f 100644
--- a/recommendations/RoofRecommendations.py
+++ b/recommendations/RoofRecommendations.py
@@ -53,7 +53,7 @@ class RoofRecommendations:
             ]
         ]
 
-    def recommend(self):
+    def recommend(self, phase):
 
         if self.property.roof["has_dwelling_above"]:
             return
@@ -98,11 +98,11 @@ class RoofRecommendations:
             return
 
         if self.property.roof["is_pitched"] or self.property.roof["is_flat"]:
-            self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof)
+            self.recommend_roof_insulation(u_value, insulation_thickness, self.property.roof, phase)
             return
 
         if self.property.roof["is_roof_room"]:
-            self.recommend_room_roof_insulation(u_value)
+            self.recommend_room_roof_insulation(u_value, phase)
             return
 
         raise NotImplementedError("Implement me")
@@ -124,7 +124,7 @@ class RoofRecommendations:
         raise ValueError("Invalid material type")
 
     def recommend_roof_insulation(
-        self, u_value, insulation_thickness, roof
+        self, u_value, insulation_thickness, roof, phase
     ):
 
         """
@@ -217,6 +217,7 @@ class RoofRecommendations:
 
                     recommendations.append(
                         {
+                            "phase": phase,
                             "parts": [
                                 get_recommended_part(
                                     part=material.to_dict(),
@@ -236,7 +237,7 @@ class RoofRecommendations:
 
         self.recommendations = recommendations
 
-    def recommend_room_roof_insulation(self, u_value):
+    def recommend_room_roof_insulation(self, u_value, phase):
         """
         This method recommends room in roof insulation for properties that have been identified
         to possess a room in roof.
@@ -314,6 +315,7 @@ class RoofRecommendations:
 
                     recommendations.append(
                         {
+                            "phase": phase,
                             "parts": [
                                 get_recommended_part(
                                     part=material,
diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py
index 9995bbe1..5fae09b3 100644
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@@ -18,7 +18,7 @@ class SolarPvRecommendations:
 
         self.recommendation = []
 
-    def recommend(self):
+    def recommend(self, phase):
         """
         We check if a property is potentially suitable for solar PV based on the following criteria:
         - The property is a house or bungalow
@@ -77,6 +77,7 @@ class SolarPvRecommendations:
 
             self.recommendation.append(
                 {
+                    "phase": phase,
                     "parts": [],
                     "type": "solar_pv",
                     "description": description,
@@ -86,22 +87,6 @@ class SolarPvRecommendations:
                     **cost_result,
                     # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale
                     # back up here
-                    "photo_supply": 100 * scenario
+                    "photo_supply": 100 * roof_coverage
                 }
             )
-
-        self.recommendation = [
-            {
-                "parts": [],
-                "type": "solar_pv",
-                "description": f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) panel system on "
-                               f"{roof_coverage_percent}% the roof",
-                "starting_u_value": None,
-                "new_u_value": None,
-                "sap_points": None,
-                **cost_result,
-                # This is required for simulating the SAP impact. solar_pv_percentage is between 0 & 1 so we scale
-                # back up here
-                "photo_supply": 100 * self.property.solar_pv_percentage
-            }
-        ]
diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py
index 6c61f27c..7241cdec 100644
--- a/recommendations/VentilationRecommendations.py
+++ b/recommendations/VentilationRecommendations.py
@@ -62,6 +62,7 @@ class VentilationRecommendations(Definitions):
         # We recommend installing two mechanical ventilation systems
         self.recommendation = [
             {
+                "phase": None,
                 "parts": part,
                 "type": part[0]["type"],
                 "description": f"Install {n_units} {part[0]['description']} units",
diff --git a/recommendations/WallRecommendations.py b/recommendations/WallRecommendations.py
index 6e2d64ec..467c6ad3 100644
--- a/recommendations/WallRecommendations.py
+++ b/recommendations/WallRecommendations.py
@@ -97,7 +97,7 @@ class WallRecommendations(Definitions):
 
         return True
 
-    def recommend(self):
+    def recommend(self, phase=0):
         # if building built after 1990 + we're able to identify U-value +
         # U-value less than 0.18 and if in or close to a conversation area,
         # recommend internal wall insulation as a possible measure
@@ -146,19 +146,19 @@ class WallRecommendations(Definitions):
         if is_cavity_wall:
             if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
                 # Test filling cavity
-                self.find_cavity_insulation(u_value, insulation_thickness)
+                self.find_cavity_insulation(u_value, insulation_thickness, phase)
 
             return
 
         # Remaining wall types are treated with IWI or EWI
         if u_value >= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
-            self.find_insulation(u_value)
+            self.find_insulation(u_value, phase)
             return
 
         # If the u-value is within regulations, we don't do anything
         return
 
-    def find_cavity_insulation(self, u_value, insulation_thickness):
+    def find_cavity_insulation(self, u_value, insulation_thickness, phase):
         """
         This method tests different materials to fill the cavity wall, determining which
         material will give us the best U-value.
@@ -210,6 +210,7 @@ class WallRecommendations(Definitions):
 
                 recommendations.append(
                     {
+                        "phase": phase,
                         "parts": [
                             get_recommended_part(
                                 part=material.to_dict(),
@@ -229,7 +230,7 @@ class WallRecommendations(Definitions):
 
         self.recommendations = recommendations
 
-    def _find_insulation(self, u_value, insulation_materials, non_insulation_materials):
+    def _find_insulation(self, u_value, insulation_materials, non_insulation_materials, phase):
 
         lowest_selected_u_value = None
         recommendations = []
@@ -274,6 +275,7 @@ class WallRecommendations(Definitions):
 
                     recommendations.append(
                         {
+                            "phase": phase,
                             "parts": [
                                 get_recommended_part(
                                     part=material.to_dict(),
@@ -293,7 +295,7 @@ class WallRecommendations(Definitions):
 
         return recommendations
 
-    def find_insulation(self, u_value):
+    def find_insulation(self, u_value, phase):
         """
         This function contains the logic for finding potential insulation measures for a property, depending
         on the parts available and whether the property can have external wall insulation installed
@@ -310,13 +312,15 @@ class WallRecommendations(Definitions):
             ewi_recommendations = self._find_insulation(
                 u_value=u_value,
                 insulation_materials=pd.DataFrame(self.external_wall_insulation_materials),
-                non_insulation_materials=self.external_wall_non_insulation_materials
+                non_insulation_materials=self.external_wall_non_insulation_materials,
+                phase=phase
             )
 
         iwi_recommendations = self._find_insulation(
             u_value=u_value,
             insulation_materials=pd.DataFrame(self.internal_wall_insulation_materials),
-            non_insulation_materials=self.internal_wall_non_insulation_materials
+            non_insulation_materials=self.internal_wall_non_insulation_materials,
+            phase=phase
         )
 
         self.recommendations += ewi_recommendations + iwi_recommendations
diff --git a/recommendations/WindowsRecommendations.py b/recommendations/WindowsRecommendations.py
index b6ecd099..d7404e3b 100644
--- a/recommendations/WindowsRecommendations.py
+++ b/recommendations/WindowsRecommendations.py
@@ -30,7 +30,7 @@ class WindowsRecommendations:
             raise ValueError("There should only be one window glazing material")
         self.glazing_material = self.glazing_material[0]
 
-    def recommend(self):
+    def recommend(self, phase=0):
         """
         This method will recommend the best possible glazing options for a property.
 
@@ -85,6 +85,7 @@ class WindowsRecommendations:
 
         self.recommendation = [
             {
+                "phase": phase,
                 "parts": [],
                 "type": "windows_glazing",
                 "description": description,

From 0f60082ba1bd4346f07b8ba5f9e417600a2737cc Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Wed, 14 Feb 2024 19:15:41 +0000
Subject: [PATCH 09/16] Added in the sequential scoring code

---
 backend/Property.py                     | 257 ++++++++++++++----------
 backend/app/plan/router.py              |  16 +-
 recommendations/FloorRecommendations.py |   1 +
 recommendations/Recommendations.py      |  47 ++++-
 4 files changed, 207 insertions(+), 114 deletions(-)

diff --git a/backend/Property.py b/backend/Property.py
index e57d8326..109f70a9 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -150,158 +150,195 @@ class Property:
 
         # self.base_difference_record.df
 
-    def adjust_difference_record_with_recommendations(self, property_recommendations):
+    def adjust_difference_record_with_recommendations(
+        self, property_recommendations,
+        property_representative_recommendations
+    ):
         """
         This method will adjust the difference record, based on the recommendations made for the property
+
+        In order to score the measures, we need to consider the phase of the retrofit.
+
         :param property_recommendations: dictionary of recommendations for the property
+        :param property_representative_recommendations: dictionary of representative recommendations for the property
         """
 
         self.recommendations_scoring_data = []
+        phases = sorted([r[0]["phase"] for r in property_recommendations if r[0]["phase"] is not None])
+
+        for phase in phases:
+            property_recommendations_by_phase = [r for r in property_recommendations if r[0]["phase"] == phase][0]
+            previous_phases = [p for p in phases if p < phase]
+            previous_phase_representatives = [
+                r for r in property_representative_recommendations if r["phase"] in previous_phases
+            ]
+            recommendation_record = self.base_difference_record.df.to_dict("records")[0].copy()
+
+            for rec in property_recommendations_by_phase:
+                # We simulate the impact of the recommendation at this current phase, and all of the prior phases
 
-        for recommendations_by_type in property_recommendations:
-            for i, rec in enumerate(recommendations_by_type):
-                recommendation_record = self.base_difference_record.df.to_dict("records")[0].copy()
                 scoring_dict = self.create_recommendation_scoring_data(
-                    property_id=self.id, recommendation_record=recommendation_record, recommendation=rec,
+                    property_id=self.id,
+                    recommendation_record=recommendation_record,
+                    recommendations=previous_phase_representatives + [rec],
+                    primary_recommendation_id=rec["recommendation_id"]
                 )
-
                 self.recommendations_scoring_data.append(scoring_dict)
 
     @staticmethod
-    def create_recommendation_scoring_data(property_id, recommendation_record, recommendation: dict):
+    def create_recommendation_scoring_data(
+        property_id, recommendation_record, recommendations: list, primary_recommendation_id: int
+    ):
+        """
+        This function will iterate through a list of recommendations and apply a simulation for each recommendation
+        This allows us to later multiple measures and see the impact of the measures on the property
+        :param property_id: The id of the property
+        :param recommendation_record: The record of the property, which will be updated
+        :param recommendations: The list of recommendations to apply
+        :param primary_recommendation_id: The id of the primary recommendation, which is used to identify the record
+        :return: The updated recommendation record
+        """
+
+        output = recommendation_record.copy()
 
         for col in [
             "walls_insulation_thickness", "floor_insulation_thickness", "roof_insulation_thickness"
         ]:
-            if recommendation_record[col] is None:
-                recommendation_record[col] = "none"
+            if output[col] is None:
+                output[col] = "none"
 
-        # We update the description to indicate it's insulated
-        if recommendation["type"] in ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"]:
-            # The upgrade made here is to the u-value of the walls and the description of the
-            # insulation thickness
-            recommendation_record["walls_thermal_transmittance_ending"] = recommendation["new_u_value"]
-            recommendation_record["walls_insulation_thickness_ending"] = "above average"
-            recommendation_record["walls_energy_eff_ending"] = "Good"
+        for recommendation in recommendations:
+            # For the list of recommendations we have, we iteratively update the output
 
-            # Note: often when the wall is insulatied, the internal/external insulation is not noted so we should
-            #       test the impact of using these booleans
-            if recommendation["type"] == "external_wall_insulation":
-                recommendation_record["external_insulation"] = True
-                recommendation_record["internal_insulation"] = False
+            # We update the description to indicate it's insulated
+            if recommendation["type"] in [
+                "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"
+            ]:
+                # The upgrade made here is to the u-value of the walls and the description of the
+                # insulation thickness
+                output["walls_thermal_transmittance_ending"] = recommendation["new_u_value"]
+                output["walls_insulation_thickness_ending"] = "above average"
+                output["walls_energy_eff_ending"] = "Good"
 
-            if recommendation["type"] == "internal_wall_insulation":
-                recommendation_record["external_insulation"] = False
-                recommendation_record["internal_insulation"] = True
+                # Note: often when the wall is insulatied, the internal/external insulation is not noted so we should
+                #       test the impact of using these booleans
+                if recommendation["type"] == "external_wall_insulation":
+                    output["external_insulation"] = True
+                    output["internal_insulation"] = False
 
-        else:
-            if recommendation_record["walls_thermal_transmittance_ending"] is None:
-                raise ValueError("We should not have a None value for the u value")
+                if recommendation["type"] == "internal_wall_insulation":
+                    output["external_insulation"] = False
+                    output["internal_insulation"] = True
 
-            if recommendation_record["walls_insulation_thickness_ending"] is None:
-                recommendation_record["walls_insulation_thickness_ending"] = "none"
+                # When making a recommendation for the wall, we will also update the ventilation
+                if output["mechanical_ventilation_ending"] == 'natural':
+                    output["mechanical_ventilation_ending"] = 'mechanical, extract only'
 
-        # Update description to indicate it's insulate
-        if recommendation["type"] in [
-            "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation"
-        ]:
-            if len(recommendation["parts"]) > 1:
-                raise NotImplementedError("Have more than 1 floor insulation part - handle this case")
+            else:
+                if output["walls_thermal_transmittance_ending"] is None:
+                    raise ValueError("We should not have a None value for the u value")
 
-            # recommendation_record["floor_thermal_transmittance_ending"] = recommendation["new_u_value"]
-            # We don't really see above average for this in the training data
-            recommendation_record["floor_insulation_thickness_ending"] = "average"
-            # This is rarely ever populated in the training data
-            # recommendation_record["floor_energy_eff_ending"] = "Good"
-        else:
-            if recommendation_record["floor_thermal_transmittance_ending"] is None:
-                raise ValueError("We should not have a None value for the u value")
+                if output["walls_insulation_thickness_ending"] is None:
+                    output["walls_insulation_thickness_ending"] = "none"
 
-            if recommendation_record["floor_insulation_thickness_ending"] is None:
-                recommendation_record["floor_insulation_thickness_ending"] = "none"
+            # Update description to indicate it's insulate
+            if recommendation["type"] in [
+                "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation"
+            ]:
+                if len(recommendation["parts"]) > 1:
+                    raise NotImplementedError("Have more than 1 floor insulation part - handle this case")
 
-        if recommendation["type"] in ["loft_insulation", "room_roof_insulation", "flat_roof_insulation"]:
-            recommendation_record["roof_thermal_transmittance_ending"] = recommendation["new_u_value"]
+                # output["floor_thermal_transmittance_ending"] = recommendation["new_u_value"]
+                # We don't really see above average for this in the training data
+                output["floor_insulation_thickness_ending"] = "average"
+                # This is rarely ever populated in the training data
+                # output["floor_energy_eff_ending"] = "Good"
+            else:
+                if output["floor_thermal_transmittance_ending"] is None:
+                    raise ValueError("We should not have a None value for the u value")
 
-            parts = recommendation["parts"]
-            if len(parts) != 1:
-                raise ValueError("More than one part for roof insulation - investiage me")
+                if output["floor_insulation_thickness_ending"] is None:
+                    output["floor_insulation_thickness_ending"] = "none"
 
-            # This is based on the values we have in the training data
-            valid_numeric_values = [
-                12, 25, 50, 75, 100, 150, 200, 250, 270, 300, 350, 400
-            ]
+            if recommendation["type"] in ["loft_insulation", "room_roof_insulation", "flat_roof_insulation"]:
+                output["roof_thermal_transmittance_ending"] = recommendation["new_u_value"]
 
-            proposed_depth = int(parts[0]["depth"])
-            if proposed_depth not in valid_numeric_values:
-                # Take the nearest value for scoring
-                proposed_depth = min(valid_numeric_values, key=lambda x: abs(x - proposed_depth))
+                parts = recommendation["parts"]
+                if len(parts) != 1:
+                    raise ValueError("More than one part for roof insulation - investiage me")
 
-            recommendation_record["roof_insulation_thickness_ending"] = str(proposed_depth)
-            if recommendation["type"] == "loft_insulation":
-                if proposed_depth >= 270:
-                    recommendation_record["roof_energy_eff_ending"] = "Very Good"
+                # This is based on the values we have in the training data
+                valid_numeric_values = [
+                    12, 25, 50, 75, 100, 150, 200, 250, 270, 300, 350, 400
+                ]
+
+                proposed_depth = int(parts[0]["depth"])
+                if proposed_depth not in valid_numeric_values:
+                    # Take the nearest value for scoring
+                    proposed_depth = min(valid_numeric_values, key=lambda x: abs(x - proposed_depth))
+
+                output["roof_insulation_thickness_ending"] = str(proposed_depth)
+                if recommendation["type"] == "loft_insulation":
+                    if proposed_depth >= 270:
+                        output["roof_energy_eff_ending"] = "Very Good"
+                    else:
+                        output["roof_energy_eff_ending"] = "Good"
                 else:
-                    recommendation_record["roof_energy_eff_ending"] = "Good"
+                    output["roof_energy_eff_ending"] = "Very Good"
             else:
-                recommendation_record["roof_energy_eff_ending"] = "Very Good"
-        else:
-            # Fill missing roof u-values - this fill is not based on recommended upgrades
-            if recommendation_record["roof_thermal_transmittance_ending"] is None:
-                raise ValueError("We should not have a None value for the u value")
+                # Fill missing roof u-values - this fill is not based on recommended upgrades
+                if output["roof_thermal_transmittance_ending"] is None:
+                    raise ValueError("We should not have a None value for the u value")
 
-            if recommendation_record["roof_insulation_thickness_ending"] is None:
-                recommendation_record["roof_insulation_thickness_ending"] = "none"
+                if output["roof_insulation_thickness_ending"] is None:
+                    output["roof_insulation_thickness_ending"] = "none"
 
-        if recommendation["type"] == "mechanical_ventilation":
-            recommendation_record["mechanical_ventilation_ending"] = 'mechanical, extract only'
+            if recommendation["type"] == "sealing_open_fireplace":
+                output["number_open_fireplaces_ending"] = 0
 
-        if recommendation["type"] == "sealing_open_fireplace":
-            recommendation_record["number_open_fireplaces_ending"] = 0
+            if recommendation["type"] == "low_energy_lighting":
+                output["low_energy_lighting_ending"] = 100
+                output["lighting_energy_eff_starting"] = "Very Good"
 
-        if recommendation["type"] == "low_energy_lighting":
-            recommendation_record["low_energy_lighting_ending"] = 100
-            recommendation_record["lighting_energy_eff_starting"] = "Very Good"
+            if recommendation["type"] == "windows_glazing":
+                output["multi_glaze_proportion_ending"] = 100
+                output["windows_energy_eff_ending"] = "Average"
 
-        if recommendation["type"] == "windows_glazing":
-            recommendation_record["multi_glaze_proportion_ending"] = 100
-            recommendation_record["windows_energy_eff_ending"] = "Average"
+                is_secondary_glazing = recommendation["is_secondary_glazing"]
 
-            is_secondary_glazing = recommendation["is_secondary_glazing"]
+                if output["glazing_type_ending"] == "multiple":
+                    pass
+                elif output["glazing_type_ending"] == "single":
+                    output["glazing_type_ending"] = "secondary" if is_secondary_glazing else "double"
+                elif output["glazing_type_ending"] == "double":
+                    output["glazing_type_ending"] = "multiple" if is_secondary_glazing else "double"
+                elif output["glazing_type_ending"] == "secondary":
+                    output["glazing_type_ending"] = "secondary" if is_secondary_glazing else "multiple"
+                elif output["glazing_type_ending"] in ["triple", "high performance"]:
+                    output["glazing_type_ending"] = "multiple"
+                else:
+                    raise ValueError("Invalid glazing type - implement me")
 
-            if recommendation_record["glazing_type_ending"] == "multiple":
-                pass
-            elif recommendation_record["glazing_type_ending"] == "single":
-                recommendation_record["glazing_type_ending"] = "secondary" if is_secondary_glazing else "double"
-            elif recommendation_record["glazing_type_ending"] == "double":
-                recommendation_record["glazing_type_ending"] = "multiple" if is_secondary_glazing else "double"
-            elif recommendation_record["glazing_type_ending"] == "secondary":
-                recommendation_record["glazing_type_ending"] = "secondary" if is_secondary_glazing else "multiple"
-            elif recommendation_record["glazing_type_ending"] in ["triple", "high performance"]:
-                recommendation_record["glazing_type_ending"] = "multiple"
-            else:
-                raise ValueError("Invalid glazing type - implement me")
+                if is_secondary_glazing:
+                    output["glazed_type_ending"] = "secondary glazing"
+                else:
+                    output["glazed_type_ending"] = "double glazing installed during or after 2002 "
 
-            if is_secondary_glazing:
-                recommendation_record["glazed_type_ending"] = "secondary glazing"
-            else:
-                recommendation_record["glazed_type_ending"] = "double glazing installed during or after 2002 "
+            if recommendation["type"] == "solar_pv":
+                output["photo_supply_ending"] = recommendation["photo_supply"]
 
-        if recommendation["type"] == "solar_pv":
-            recommendation_record["photo_supply_ending"] = recommendation["photo_supply"]
+            if recommendation["type"] not in [
+                "mechanical_ventilation", "sealing_open_fireplace", "low_energy_lighting",
+                "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
+                "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
+                "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation",
+                "windows_glazing", "solar_pv"
+            ]:
+                raise NotImplementedError("Implement me")
 
-        if recommendation["type"] not in [
-            "mechanical_ventilation", "sealing_open_fireplace", "low_energy_lighting",
-            "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
-            "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
-            "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation",
-            "windows_glazing", "solar_pv"
-        ]:
-            raise NotImplementedError("Implement me")
+        output['id'] = "+".join([str(property_id), str(primary_recommendation_id)])
 
-        recommendation_record['id'] = "+".join([str(property_id), str(recommendation["recommendation_id"])])
-
-        return recommendation_record
+        return output
 
     def get_components(self, cleaned, photo_supply_lookup, floor_area_decile_thresholds):
         """
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 811c3c09..08d3f048 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -136,22 +136,32 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         recommendations = {}
         recommendations_scoring_data = []
-
+        representive_recommendations = {}
         for p in input_properties:
 
             # Property recommendations
             p.get_components(cleaned, photo_supply_lookup, floor_area_decile_thresholds)
 
+            # TODO: For the private customer, we should probably NOT allow floor insulation, because it often requires
+            #       decanting the tenant
             recommender = Recommendations(property_instance=p, materials=materials)
-            property_recommendations = recommender.recommend()
+            property_recommendations, property_representative_recommendations = recommender.recommend()
 
             if not property_recommendations:
                 continue
 
             recommendations[p.id] = property_recommendations
+            representive_recommendations[p.id] = property_representative_recommendations
 
             p.create_base_difference_epc_record(cleaned_lookup=cleaned)
-            p.adjust_difference_record_with_recommendations(property_recommendations)
+            p.adjust_difference_record_with_recommendations(
+                property_recommendations, property_representative_recommendations
+            )
+
+            p.recommendations_scoring_data[0]["id"]
+            p.recommendations_scoring_data[0]["walls_thermal_transmittance"]
+            p.recommendations_scoring_data[0]["walls_thermal_transmittance_ending"]
+            p.recommendations_scoring_data[0]["walls_thermal_transmittance_ending"]
 
             recommendations_scoring_data.extend(p.recommendations_scoring_data)
 
diff --git a/recommendations/FloorRecommendations.py b/recommendations/FloorRecommendations.py
index a19ee025..40d9fb10 100644
--- a/recommendations/FloorRecommendations.py
+++ b/recommendations/FloorRecommendations.py
@@ -118,6 +118,7 @@ class FloorRecommendations(Definitions):
         if self.property.floor["is_suspended"]:
             # Given the U-value, we recommend underfloor insulation
             self.recommend_floor_insulation(
+                phase=phase,
                 u_value=u_value,
                 insulation_materials=self.suspended_floor_insulation_materials,
                 non_insulation_materials=self.suspended_floor_non_insulation_materials
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index b4b764c2..f0cf7806 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -107,7 +107,52 @@ class Recommendations:
         # We insert temporary ids into the recommendations which is important for the optimiser later
         property_recommendations = self.insert_temp_recommendation_id(property_recommendations)
 
-        return property_recommendations
+        # We also need to create the representative recommendations for each recommendation type
+        property_representative_recommendations = self.create_representative_recommendations(property_recommendations)
+
+        return property_recommendations, property_representative_recommendations
+
+    @staticmethod
+    def create_representative_recommendations(property_recommendations):
+        """
+        This method will create a representative recommendation for each recommendation type
+        In order to create a representative recommendation, we choose the recommendation that has:
+        1) Where a U-value is available, has the best U-value to cost ratio
+        2) Where SAP points are available, has the best SAP points to cost ratio
+
+        We don't include mechanical ventilation in the representative recommendations, since we don't attribute a
+        SAP impact to this recommendation
+        :return:
+        """
+        property_representative_recommendations = []
+
+        for recommendations_by_type in property_recommendations:
+
+            if recommendations_by_type[0].get("type") == "mechanical_ventilation":
+                continue
+
+            has_u_value = recommendations_by_type[0].get("new_u_value") is not None
+            has_sap_points = recommendations_by_type[0].get("sap_points") is not None
+
+            if has_u_value:
+                # We sort by the cost per U-value improvement - the lower the better
+                recommendations_by_type.sort(
+                    key=lambda x: x["total"] / x["starting_u_value"] - x["new_u_value"]
+                )
+            elif not has_u_value and has_sap_points:
+                # Sort the options by the cost per SAP point improvement - the lower the better
+                recommendations_by_type.sort(
+                    key=lambda x: x["total"] / x["sap_points"]
+                )
+            else:
+                # Sort the options by cost - the lower the better
+                recommendations_by_type.sort(
+                    key=lambda x: x["total"]
+                )
+
+            property_representative_recommendations.append(recommendations_by_type[0])
+
+        return property_representative_recommendations
 
     @staticmethod
     def insert_temp_recommendation_id(property_recommendations):

From b39e9c989943f2a6752935d72b21e5a7dea8b361 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Thu, 15 Feb 2024 10:57:57 +0000
Subject: [PATCH 10/16] working on updating the HA code for new file formats

---
 .idea/Model.iml                               |   2 +-
 .idea/misc.xml                                |   2 +-
 backend/app/plan/router.py                    |   5 -
 .../ha_15_32/ha_analysis_batch_3.py           | 171 +++++++++---------
 4 files changed, 84 insertions(+), 96 deletions(-)

diff --git a/.idea/Model.iml b/.idea/Model.iml
index 4413bb06..b0f9c00d 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 6f308057..1122b380 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 08d3f048..3799d43f 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -158,11 +158,6 @@ async def trigger_plan(body: PlanTriggerRequest):
                 property_recommendations, property_representative_recommendations
             )
 
-            p.recommendations_scoring_data[0]["id"]
-            p.recommendations_scoring_data[0]["walls_thermal_transmittance"]
-            p.recommendations_scoring_data[0]["walls_thermal_transmittance_ending"]
-            p.recommendations_scoring_data[0]["walls_thermal_transmittance_ending"]
-
             recommendations_scoring_data.extend(p.recommendations_scoring_data)
 
         logger.info("Preparing data for scoring in sap change api")
diff --git a/etl/eligibility/ha_15_32/ha_analysis_batch_3.py b/etl/eligibility/ha_15_32/ha_analysis_batch_3.py
index e94babcd..5ed7d6f2 100644
--- a/etl/eligibility/ha_15_32/ha_analysis_batch_3.py
+++ b/etl/eligibility/ha_15_32/ha_analysis_batch_3.py
@@ -24,27 +24,6 @@ load_dotenv(ENV_FILE)
 
 
 class DataLoader:
-    COLOUR_CONFIG = {
-        "ha_1": {
-            "asset_list": {"red": "FFFF0000", "green": "FF00B050"},
-        },
-        "ha_6": {
-            "asset_list": {"red": "FFFF0000", "green": "FF00B050"},
-            "survey_list": {
-                "green": "FF92D050", "purple": "FF7030A0", "red": "FFFF0000", "blue": "FF00B0F0"
-            }
-        },
-        "ha_14": {
-            "asset_list": {"red": "FFFF0000", "green": "FF00B050"},
-        },
-        "ha_39": {
-            "asset_list": {"red": "FFFF0000", "green": "FF00B050"},
-        },
-        "ha_107": {
-            "asset_list": {"red": "FFFF0000", "green": "FF00B050"},
-        }
-    }
-
     MIN_ROWS = {
         "ha_1": 2,
         "ha_6": 2,
@@ -53,12 +32,87 @@ class DataLoader:
         "ha_107": 2,
     }
 
+    COLUMN_CONFIG = {
+        "ha_1": {
+            "address": "Address",
+            "postcode": "Address - Postcode"
+        }
+    }
+
     def __init__(self, files, use_cache):
         self.files = files
         self.use_cache = use_cache
 
         self.data = {}
 
+    def create_asset_list_matching_address(self, ha_name, asset_list):
+
+        if ha_name in ["ha_1", "ha_6"]:
+            asset_list["matching_address"] = asset_list[
+                self.COLUMN_CONFIG[ha_name]["address"]
+            ].str.lower().str.strip()
+            asset_list["matching_postcode"] = asset_list[
+                self.COLUMN_CONFIG[ha_name]["postcode"]
+            ].str.lower().str.strip()
+        elif ha_name == "ha_14":
+            # Create matching_address by concatenating Address 1, Address 2, Address 3, Address 4, Postcode
+            asset_list["matching_address"] = asset_list["Address 1"].str.lower().str.strip() + ", " + \
+                                             asset_list["Address 2"].str.lower().str.strip() + ", " + \
+                                             asset_list["Address 3"].str.lower().str.strip() + ", " + \
+                                             asset_list["Address 4"].str.lower().str.strip() + ", " + \
+                                             asset_list["Postcode"].str.lower().str.strip()
+            asset_list["matching_postcode"] = asset_list["Postcode"].str.lower().str.strip()
+        elif ha_name == "ha_39":
+            # Create matching_address by concatenating add_1, add_2, add_3, add_4, add_5, post_code
+            asset_list["matching_address"] = asset_list["add_1"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["add_2"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["add_3"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["add_4"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["add_5"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["post_code"].astype(str).str.lower().str.strip()
+            asset_list["matching_postcode"] = asset_list["post_code"].str.lower().str.strip()
+        elif ha_name == "ha_107":
+            # Create matching_address by concatenating House No, Street, Town, District, Postcode
+            asset_list["matching_address"] = asset_list["House No"].astype(str).str.lower().str.strip() + ", " + \
+                                             asset_list["Street"].str.lower().str.strip() + ", " + \
+                                             asset_list["Town"].str.lower().str.strip() + ", " + \
+                                             asset_list["District"].str.lower().str.strip() + ", " + \
+                                             asset_list["Postcode"].str.lower().str.strip()
+            asset_list["matching_postcode"] = asset_list["Postcode"].str.lower().str.strip()
+        else:
+            raise NotImplementedError("implement me")
+
+        return asset_list
+
+    def append_asset_list_built_form(self, ha_name, asset_list):
+
+        # Finally, we process property_type or built form, where needed
+        if ha_name == "ha_6":
+            asset_list["built_form"] = asset_list["Property Type"].apply(self.identify_built_form_ha6)
+
+        return asset_list
+
+    @staticmethod
+    def create_asset_list_house_no(ha_name, asset_list):
+        """
+        This function will append the House number onto the asset list
+        :return:
+        """
+
+        if ha_name in ["ha_107"]:
+            asset_list["HouseNo"] = asset_list["House No"].copy()
+        else:
+            split_addresses = asset_list['matching_address'].str.split(',', expand=True)
+            house_numbers = split_addresses[0].str.split(' ', expand=True)
+            # THe first column should be HouseNo - we aren't interested in the other columns, but we don't know how
+            # many columns there might be
+            house_numbers = house_numbers.iloc[:, 0:1]
+            house_numbers.columns = ['HouseNo']
+
+            asset_list = pd.concat([asset_list, house_numbers[["HouseNo"]]], axis=1)
+
+        return asset_list
+
     def load_asset_list(self, file_path, ha_name, sheet_name=None):
         workbook = openpyxl.load_workbook(file_path)
         if sheet_name is not None:
@@ -87,74 +141,15 @@ class DataLoader:
         # Remove entirely empty roww - consider all rows apart from row_color
         asset_list = asset_list.loc[asset_list.loc[:, asset_list.columns != 'row_color'].notnull().any(axis=1)]
 
-        asset_list_colours = self.COLOUR_CONFIG[ha_name]["asset_list"]
-
-        asset_list["row_colour_name"] = np.where(
-            asset_list["row_color"] == asset_list_colours["red"], "red",
-            np.where(asset_list["row_color"] == asset_list_colours["green"], "green", "yellow")
-        )
-
-        asset_list["row_meaning"] = np.where(
-            asset_list["row_colour_name"] == "red", "does not meet criteria",
-            np.where(
-                asset_list["row_colour_name"] == "green", "identified potential eco works (CWI)", "maybe in the future"
-            )
-        )
-
         # Add in asset_list_row_id
         asset_list["asset_list_row_id"] = [ha_name + str(i) for i in range(0, len(asset_list))]
 
-        # Prepare the asset list
-        # Depending on the HA, we need to rename some columns
-        if ha_name == "ha_1":
-            asset_list["matching_address"] = asset_list["Address"].str.lower().str.strip()
-            asset_list["matching_postcode"] = asset_list["Address - Postcode"].str.lower().str.strip()
-        elif ha_name == "ha_6":
-            asset_list["matching_address"] = asset_list["propertyaddress"].str.lower().str.strip()
-            asset_list["matching_postcode"] = asset_list["Post Code"].str.lower().str.strip()
-        elif ha_name == "ha_14":
-            # Create matching_address by concatenating Address 1, Address 2, Address 3, Address 4, Postcode
-            asset_list["matching_address"] = asset_list["Address 1"].str.lower().str.strip() + ", " + \
-                                             asset_list["Address 2"].str.lower().str.strip() + ", " + \
-                                             asset_list["Address 3"].str.lower().str.strip() + ", " + \
-                                             asset_list["Address 4"].str.lower().str.strip() + ", " + \
-                                             asset_list["Postcode"].str.lower().str.strip()
-            asset_list["matching_postcode"] = asset_list["Postcode"].str.lower().str.strip()
-        elif ha_name == "ha_39":
-            # Create matching_address by concatenating add_1, add_2, add_3, add_4, add_5, post_code
-            asset_list["matching_address"] = asset_list["add_1"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["add_2"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["add_3"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["add_4"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["add_5"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["post_code"].astype(str).str.lower().str.strip()
-            asset_list["matching_postcode"] = asset_list["post_code"].str.lower().str.strip()
-        elif ha_name == "ha_107":
-            # Create matching_address by concatenating House No, Street, Town, District, Postcode
-            asset_list["matching_address"] = asset_list["House No"].astype(str).str.lower().str.strip() + ", " + \
-                                             asset_list["Street"].str.lower().str.strip() + ", " + \
-                                             asset_list["Town"].str.lower().str.strip() + ", " + \
-                                             asset_list["District"].str.lower().str.strip() + ", " + \
-                                             asset_list["Postcode"].str.lower().str.strip()
-            asset_list["matching_postcode"] = asset_list["Postcode"].str.lower().str.strip()
-        else:
-            raise NotImplementedError("implement me")
+        # Create matching address and matching postcode
+        asset_list = self.create_asset_list_matching_address(ha_name=ha_name, asset_list=asset_list)
 
-        if ha_name in ["ha_107"]:
-            asset_list["HouseNo"] = asset_list["House No"].copy()
-        else:
-            split_addresses = asset_list['matching_address'].str.split(',', expand=True)
-            house_numbers = split_addresses[0].str.split(' ', expand=True)
-            # THe first column should be HouseNo - we aren't interested in the other columns, but we don't know how
-            # many columns there might be
-            house_numbers = house_numbers.iloc[:, 0:1]
-            house_numbers.columns = ['HouseNo']
+        asset_list = self.create_asset_list_house_no(ha_name=ha_name, asset_list=asset_list)
 
-            asset_list = pd.concat([asset_list, house_numbers[["HouseNo"]]], axis=1)
-
-        # Finally, we process property_type or built form, where needed
-        if ha_name == "ha_6":
-            asset_list["built_form"] = asset_list["Property Type"].apply(self.identify_built_form_ha6)
+        asset_list = self.append_asset_list_built_form(ha_name=ha_name, asset_list=asset_list)
 
         return asset_list
 
@@ -177,9 +172,7 @@ class DataLoader:
         survey_list = pd.DataFrame(survey_rows, columns=[cell.value for cell in survey_sheet[1]])
         # Remove columns that are None
         survey_list = survey_list.loc[:, survey_list.columns.notnull()]
-
         survey_list["row_colour"] = survey_colors
-        survey_list_colours = self.COLOUR_CONFIG[ha_name]["survey_list"]
 
         # The survey list has 4 possible colours:
         # PURPLE - Installer advised install complete and a complimentary post works EPC has been completed.
@@ -1252,13 +1245,13 @@ def app():
     :return:
     """
 
-    use_cache = True
+    use_cache = False
 
     files = {
         "ha_1": {
             "asset_list": {
-                "filepath": "etl/eligibility/ha_15_32/HA 1 - ASSET LIST.xlsx",
-                "sheetname": "HA 1"
+                "filepath": "local_data/ha_data/HA1/ACCENT GROUP.xlsx",
+                "sheetname": "Energy data"
             }
         },
         "ha_6": {

From 82d19fc6fc1b3429d2f2331dcbc84e3f9d91234f Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Thu, 15 Feb 2024 19:32:33 +0000
Subject: [PATCH 11/16] implementing new sequenced scoring approach for
 recommendations

---
 .idea/Model.iml                               |  2 +-
 .idea/misc.xml                                |  2 +-
 backend/Property.py                           | 35 +++++---
 backend/ml_models/api.py                      |  8 +-
 recommendations/LightingRecommendations.py    |  3 +
 recommendations/Recommendations.py            | 84 +++++++++++++------
 recommendations/VentilationRecommendations.py |  5 +-
 7 files changed, 94 insertions(+), 45 deletions(-)

diff --git a/.idea/Model.iml b/.idea/Model.iml
index b0f9c00d..4413bb06 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 1122b380..6f308057 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>
diff --git a/backend/Property.py b/backend/Property.py
index 109f70a9..fddea1b1 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -1,20 +1,15 @@
-from datetime import datetime
-import re
 import os
-
-import numpy as np
+from itertools import groupby
 import pandas as pd
 
-from etl.epc.DataProcessor import EPCDataProcessor
 from etl.epc.Dataset import TrainingDataset
-from etl.epc.settings import LATEST_FIELD, MANDATORY_FIXED_FEATURES, POTENTIAL_COLUMNS, EFFICIENCY_FEATURES, \
-    BUILT_FORM_REMAP
+from etl.epc.settings import LATEST_FIELD, MANDATORY_FIXED_FEATURES
 from etl.epc_clean.epc_attributes.all_cleaners import all_cleaner_map
 from etl.solar.SolarPhotoSupply import SolarPhotoSupply
 from utils.logger import setup_logger
 from utils.s3 import read_dataframe_from_s3_parquet
 from etl.epc.settings import DATA_ANOMALY_MATCHES
-from recommendations.rdsap_tables import england_wales_age_band_lookup, FLOOR_LEVEL_MAP
+from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
 from recommendations.recommendation_utils import (
     estimate_perimeter, get_wall_type, estimate_external_wall_area, esimtate_pitched_roof_area, estimate_windows
 )
@@ -172,11 +167,28 @@ class Property:
             previous_phase_representatives = [
                 r for r in property_representative_recommendations if r["phase"] in previous_phases
             ]
+            # For solid wall insulation, we will actually have 2 representative recommendations, since we consider
+            # both internal and external wall insulation as possible measures. We will use the representative that
+            # has the lowest efficiency.
+            # Take the representative with the lowest efficiency, by phase
+
+            # To be safe, we sort by phase
+            previous_phase_representatives = sorted(previous_phase_representatives, key=lambda x: x['phase'])
+
+            previous_phase_representatives = [
+                min(group, key=lambda x: x['efficiency']) for _, group in groupby(
+                    previous_phase_representatives, key=lambda x: x['phase']
+                )
+            ]
+
             recommendation_record = self.base_difference_record.df.to_dict("records")[0].copy()
 
             for rec in property_recommendations_by_phase:
                 # We simulate the impact of the recommendation at this current phase, and all of the prior phases
 
+                if rec["type"] == "mechanical_ventilation":
+                    continue
+
                 scoring_dict = self.create_recommendation_scoring_data(
                     property_id=self.id,
                     recommendation_record=recommendation_record,
@@ -230,9 +242,10 @@ class Property:
                     output["external_insulation"] = False
                     output["internal_insulation"] = True
 
+                # TODO: perhaps detrimental
                 # When making a recommendation for the wall, we will also update the ventilation
-                if output["mechanical_ventilation_ending"] == 'natural':
-                    output["mechanical_ventilation_ending"] = 'mechanical, extract only'
+                # if output["mechanical_ventilation_ending"] == 'natural':
+                #     output["mechanical_ventilation_ending"] = 'mechanical, extract only'
 
             else:
                 if output["walls_thermal_transmittance_ending"] is None:
@@ -328,7 +341,7 @@ class Property:
                 output["photo_supply_ending"] = recommendation["photo_supply"]
 
             if recommendation["type"] not in [
-                "mechanical_ventilation", "sealing_open_fireplace", "low_energy_lighting",
+                "sealing_open_fireplace", "low_energy_lighting",
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation",
                 "loft_insulation", "room_roof_insulation", "flat_roof_insulation",
                 "solid_floor_insulation", "suspended_floor_insulation", "exposed_floor_insulation",
diff --git a/backend/ml_models/api.py b/backend/ml_models/api.py
index bc09f26c..bdc7c178 100644
--- a/backend/ml_models/api.py
+++ b/backend/ml_models/api.py
@@ -130,8 +130,14 @@ class ModelApi:
                 )
             )
 
-            predictions_df["predictions"] = predictions_df["predictions"].astype(float).round(1)
+            predictions_df['predictions'] = predictions_df["predictions"].astype(float).round(1)
             predictions_df[['property_id', 'recommendation_id']] = predictions_df['id'].str.split('+', expand=True)
+            # To grab the phase, we pull the integer after "phase=" in the recommendation_id. We can do this with a
+            # string split on phase= and then grab the second element of the resulting list. We could also use a
+            # regular expression to do this but we use the string split method here, for safety.
+            predictions_df['phase'] = predictions_df['recommendation_id'].str.split('phase=').str[1].str[0]
+            # Convert back to int
+            predictions_df['phase'] = predictions_df['phase'].astype(int)
 
             predictions[model_prefix] = predictions_df
 
diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py
index 6d50f0a2..352c4d8a 100644
--- a/recommendations/LightingRecommendations.py
+++ b/recommendations/LightingRecommendations.py
@@ -4,6 +4,9 @@ from recommendations.Costs import Costs
 
 
 class LightingRecommendations:
+    # We introduce a SAP limit to lighting, which is based on empirical findings. We do see cases where lighting is
+    # worth more than 2 points, but this is unlikely in the context of other upgrades that can be made to the property
+    SAP_LIMIT = 2
 
     def __init__(self, property_instance: Property, materials: List):
         """
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index f0cf7806..93472068 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -1,5 +1,6 @@
 from backend.Property import Property
 from typing import List
+from itertools import groupby
 from recommendations.FloorRecommendations import FloorRecommendations
 from recommendations.WallRecommendations import WallRecommendations
 from recommendations.RoofRecommendations import RoofRecommendations
@@ -134,23 +135,32 @@ class Recommendations:
             has_u_value = recommendations_by_type[0].get("new_u_value") is not None
             has_sap_points = recommendations_by_type[0].get("sap_points") is not None
 
-            if has_u_value:
-                # We sort by the cost per U-value improvement - the lower the better
-                recommendations_by_type.sort(
-                    key=lambda x: x["total"] / x["starting_u_value"] - x["new_u_value"]
-                )
-            elif not has_u_value and has_sap_points:
-                # Sort the options by the cost per SAP point improvement - the lower the better
-                recommendations_by_type.sort(
-                    key=lambda x: x["total"] / x["sap_points"]
-                )
-            else:
-                # Sort the options by cost - the lower the better
-                recommendations_by_type.sort(
-                    key=lambda x: x["total"]
-                )
+            # When check if these recommendations have two different types, such as solid wall insulation
+            # If we have multiple types, we group by type and then select the best recommendation for each type
 
-            property_representative_recommendations.append(recommendations_by_type[0])
+            recommendations_by_type = sorted(recommendations_by_type, key=lambda x: x["type"])
+            representative_recommendations = []
+            for type, recommendations in groupby(recommendations_by_type, key=lambda x: x["type"]):
+                recommendations = list(recommendations)
+                # We also create an efficiency key, which is used to sort the recommendations
+                if has_u_value:
+                    # We sort by the cost per U-value improvement - the lower the better
+                    for rec in recommendations:
+                        rec["efficiency"] = rec["total"] / rec["starting_u_value"] - rec["new_u_value"]
+                elif not has_u_value and has_sap_points:
+                    # Sort the options by the cost per SAP point improvement - the lower the better
+                    for rec in recommendations:
+                        rec["efficiency"] = rec["total"] / rec["sap_points"]
+                else:
+                    # Sort the options by cost - the lower the better
+                    for rec in recommendations:
+                        rec["efficiency"] = rec["total"]
+
+                recommendations.sort(
+                    key=lambda x: x["efficiency"]
+                )
+                representative_recommendations.append(recommendations[0])
+            property_representative_recommendations.extend(representative_recommendations)
 
         return property_representative_recommendations
 
@@ -168,7 +178,7 @@ class Recommendations:
 
         for recs in property_recommendations:
             for rec in recs:
-                rec["recommendation_id"] = idx
+                rec["recommendation_id"] = f"{str(idx)}_phase={str(rec['phase'])}"
                 idx += 1
 
         return property_recommendations
@@ -198,11 +208,16 @@ class Recommendations:
 
         property_recommendations = recommendations[property_instance.id].copy()
 
+        # We calculate the impact by phase
+        sap_phase_impact = property_sap_predictions.groupby("phase")["predictions"].median().reset_index()
+        heat_phase_impact = property_heat_predictions.groupby("phase")["predictions"].median().reset_index()
+        carbon_phase_impact = property_carbon_predictions.groupby("phase")["predictions"].median().reset_index()
+
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
 
-                # We don't use the model for low energy lighting at the moment
-                if rec["type"] == "low_energy_lighting":
+                if rec["type"] == "mechanical_ventilation":
+                    # We don't have a percieved sap impact of mechanical ventilation
                     continue
 
                 new_heat_demand = property_heat_predictions[property_heat_predictions["recommendation_id"] == str(
@@ -216,22 +231,37 @@ class Recommendations:
                 new_sap = property_sap_predictions[property_sap_predictions["recommendation_id"] == str(
                     rec["recommendation_id"]
                 )]["predictions"].values[0]
-                rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"])
 
-                if rec["type"] == "mechanical_ventilation":
+                if rec["phase"] == 0:
+                    rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"])
+                    rec["co2_equivalent_savings"] = float(property_instance.data["co2-emissions-current"]) - new_carbon
+                    rec["heat_demand"] = property_instance.floor_area * (
+                        float(property_instance.data["energy-consumption-current"]) - new_heat_demand
+                    )
+                else:
+
+                    previous_phase = rec["phase"] - 1
+                    rec["sap_points"] = (
+                        new_sap - sap_phase_impact[sap_phase_impact["phase"] == previous_phase]["predictions"].values[0]
+                    )
+                    rec["co2_equivalent_savings"] = (
+                        carbon_phase_impact[carbon_phase_impact["phase"] == previous_phase]["predictions"].values[0] -
+                        new_carbon
+                    )
+                    rec["heat_demand"] = property_instance.floor_area * (
+                        heat_phase_impact[heat_phase_impact["phase"] == previous_phase]["predictions"].values[0] -
+                        new_heat_demand
+                    )
+
+                if rec["type"] == "low_energy_lighting":
                     # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2
-                    rec["sap_points"] = min(rec["sap_points"], VentilationRecommendations.SAP_LIMIT)
+                    rec["sap_points"] = min(rec["sap_points"], LightingRecommendations.SAP_LIMIT)
 
                 # Round to 2 decimal places
                 rec["sap_points"] = round(rec["sap_points"], 2)
-                rec["co2_equivalent_savings"] = float(property_instance.data["co2-emissions-current"]) - new_carbon
 
                 # Energy consumption current is per meter squared, so we need to multiply by the floor area to get
                 # an absolute figure for the home
-                rec["heat_demand"] = (
-                    (float(property_instance.data["energy-consumption-current"]) - new_heat_demand
-                     ) * property_instance.floor_area)
-
                 rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
 
                 if (rec["sap_points"] is None) and (rec["co2_equivalent_savings"] is None) or (
diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py
index 7241cdec..19aad0e1 100644
--- a/recommendations/VentilationRecommendations.py
+++ b/recommendations/VentilationRecommendations.py
@@ -15,9 +15,6 @@ class VentilationRecommendations(Definitions):
         'mechanical, supply and extract'
     ]
 
-    # We introduce a SAP limit, to prevent over-predicting the SAP impact of mechanical ventilation
-    SAP_LIMIT = 2
-
     def __init__(
         self,
         property_instance: Property,
@@ -68,7 +65,7 @@ class VentilationRecommendations(Definitions):
                 "description": f"Install {n_units} {part[0]['description']} units",
                 "starting_u_value": None,
                 "new_u_value": None,
-                "sap_points": None,
+                "sap_points": 0,
                 "total": estimated_cost,
                 # We use a very simple and rough estimate of 4 hours per unit
                 "labour_hours": 4 * n_units,

From 5d45243e990243178297da9280eeca9773f97253 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Fri, 16 Feb 2024 13:01:19 +0000
Subject: [PATCH 12/16] implementing new process to adjust energy savings and
 reduce complexity in router

---
 backend/Property.py                           |  19 ++
 .../db/functions/recommendations_functions.py |   1 +
 backend/app/plan/router.py                    | 188 ++----------------
 recommendations/Recommendations.py            |  83 ++++++--
 4 files changed, 105 insertions(+), 186 deletions(-)

diff --git a/backend/Property.py b/backend/Property.py
index fddea1b1..b20d409a 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -42,6 +42,7 @@ class Property:
     walls = None
     windows = None
     lighting = None
+    energy_source = None
 
     spatial = None
     base_difference_record = None
@@ -417,6 +418,7 @@ class Property:
         self.set_solar_panel_area(
             photo_supply_lookup=photo_supply_lookup, floor_area_decile_thresholds=floor_area_decile_thresholds
         )
+        self.set_energy_source()
 
     def set_spatial(self, spatial: pd.DataFrame):
         """
@@ -749,3 +751,20 @@ class Property:
             self.insulation_floor_area * percentage_of_roof if self.roof["is_flat"] else
             self.pitched_roof_area * percentage_of_roof
         )
+
+    def set_energy_source(self):
+        """
+        This method sets the energy source of the property, based on the mains gas flag and energy tariff.
+        """
+        # Default to "electricity_and_gas" to cover most scenarios including when mains_gas_flag is True
+        energy_source = "electricity_and_gas"
+
+        # If the tariff explicitly indicates electricity use without a dual indication and mains_gas_flag is not True
+        # We check for the common electricity tariffs
+        if not self.data["mains_gas_flag"] and self.data["energy_tariff"] in [
+            "Single", "off-peak 7 hour", "off-peak 10 hour", "off-peak 18 hour", "standard tariff", "24 hour"
+        ]:
+            energy_source = "electricity"
+
+        # Set the energy source based on the conditions above
+        self.energy_source = energy_source
diff --git a/backend/app/db/functions/recommendations_functions.py b/backend/app/db/functions/recommendations_functions.py
index f7fcb7a3..1426e339 100644
--- a/backend/app/db/functions/recommendations_functions.py
+++ b/backend/app/db/functions/recommendations_functions.py
@@ -81,6 +81,7 @@ def upload_recommendations(session: Session, recommendations_to_upload, property
             "new_u_value": rec.get("new_u_value"),
             "sap_points": rec["sap_points"],
             "heat_demand": rec["heat_demand"],
+            "adjusted_heat_demand": rec["adjusted_heat_demand"],
             "co2_equivalent_savings": rec["co2_equivalent_savings"],
             "total_work_hours": rec["labour_hours"],
             "energy_cost_savings": rec["energy_cost_savings"],
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 3799d43f..3b010d12 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -136,7 +136,7 @@ async def trigger_plan(body: PlanTriggerRequest):
 
         recommendations = {}
         recommendations_scoring_data = []
-        representive_recommendations = {}
+        representative_recommendations = {}
         for p in input_properties:
 
             # Property recommendations
@@ -151,7 +151,7 @@ async def trigger_plan(body: PlanTriggerRequest):
                 continue
 
             recommendations[p.id] = property_recommendations
-            representive_recommendations[p.id] = property_representative_recommendations
+            representative_recommendations[p.id] = property_representative_recommendations
 
             p.create_base_difference_epc_record(cleaned_lookup=cleaned)
             p.adjust_difference_record_with_recommendations(
@@ -185,10 +185,18 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             property_instance = [p for p in input_properties if p.id == property_id][0]
 
-            recommendations_with_impact = Recommendations.calculate_recommendation_impact(
-                property_instance=property_instance,
-                all_predictions=all_predictions,
-                recommendations=recommendations
+            recommendations_with_impact, current_adjusted_energy, expected_adjusted_energy = (
+                Recommendations.calculate_recommendation_impact(
+                    property_instance=property_instance,
+                    all_predictions=all_predictions,
+                    recommendations=recommendations
+                )
+            )
+
+            # Store the resulting adjusted energy in the property instance
+            property_instance.set_adjusted_energy(
+                current_adjusted_energy=current_adjusted_energy,
+                expected_adjusted_energy=expected_adjusted_energy
             )
 
             input_measures = prepare_input_measures(recommendations_with_impact, body.goal)
@@ -242,174 +250,6 @@ async def trigger_plan(body: PlanTriggerRequest):
             ]
             recommendations[property_id] = final_recommendations
 
-        # This is a temporary step, to estimate the impact of the measured on heat demand and carbon
-        # TODO: This needs to be cleaned up, if it happens to be kept
-        representative_recs = {}
-        for property_id, property_recommendations in recommendations.items():
-            default_recommendations = [r for r in property_recommendations if r["default"]]
-            default_types = {x["type"] for x in default_recommendations}
-
-            # Missing types
-            missing_types = list(set([r["type"] for r in property_recommendations if r["type"] not in default_types]))
-            # We might have a missing type as one of the solid wall options because for a solid wall, you might
-            # have ewi or iwi but only one of them will be a default
-            if ("internal_wall_insulation" in default_types) or ("external_wall_insaultion" in default_types):
-                missing_types = [
-                    t for t in missing_types if t not in ["internal_wall_insulation", "external_wall_insulation"]
-                ]
-
-            # We check if NO wall insulation was selected but iwi and ewi are available
-            # This condition will check
-            # 1) iwi and ewi are both in missing_types
-            # 2) iwi and ewi are not in default_types
-            # If both of these are true, it means that no wall insulation was selected via the optimisation routine
-            # but both are possible, so we need to select a default. We default to iwi because it's usually cheaper
-            if (("internal_wall_insulation" in missing_types) and ("external_wall_insulation" in missing_types)) and (
-                ("internal_wall_insulation" not in default_types) and ("external_wall_insulation" not in default_types)
-            ):
-                missing_types = [t for t in missing_types if t != "external_wall_insulation"]
-
-            if missing_types:
-                for missed_type in missing_types:
-                    missed = [r for r in property_recommendations if r["type"] == missed_type]
-                    min_cost = min([r["total"] for r in missed])
-                    # Grab a representative, based on cheapest cost
-
-                    representative_rec = [r for r in property_recommendations if np.isclose(r["total"], min_cost)]
-                    default_recommendations.append(representative_rec[0])
-
-            representative_recs[property_id] = default_recommendations
-
-        # We update the carbon and heat demand predictions
-        # TODO: The api call producing all_combined_predictions has been removed so we can potentially completely
-        #       refactor this block to just perform the energy adjustments
-        for property_id, property_recommendations in recommendations.items():
-
-            property_instance = [p for p in input_properties if p.id == property_id][0]
-
-            heat_demand_change = sum(
-                x.get("heat_demand", 0) for x in representative_recs[property_id] if
-                x["type"] not in ["mechanical_ventilation", "low_energy_lighting"]
-            )
-            carbon_change = sum(
-                x.get("co2_equivalent_savings", 0) for x in representative_recs[property_id] if
-                x["type"] not in ["mechanical_ventilation", "low_energy_lighting"]
-            )
-
-            starting_heat_demand = (
-                float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area
-            )
-            expected_heat_demand = starting_heat_demand - heat_demand_change
-
-            # We don't want to adjust the heat demand for mechanical ventilation so we add it back on
-
-            # We adjust the heat demand figures to align to the UCL paper
-            current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
-                epc_energy_consumption=starting_heat_demand,
-                current_epc_rating=property_instance.data["current-energy-rating"],
-            )
-
-            # We sum up the SAP points of the default recommendations and calculate a new EPC category. This
-            # category is then used to produce adjusted energy figures
-
-            expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
-                epc_energy_consumption=expected_heat_demand,
-                current_epc_rating=property_instance.data["current-energy-rating"],
-            )
-
-            heat_demand_change = (
-                current_adjusted_energy - expected_adjusted_energy
-            )
-
-            # update the recommendations
-            # We need to totals for the representative recommendations
-            representative_rec_data = [
-                {
-                    "recommendation_id": r["recommendation_id"],
-                    "co2_equivalent_savings": r.get("co2_equivalent_savings"),
-                    "heat_demand": r.get("heat_demand"),
-                    "type": r["type"]
-                } for r
-                in representative_recs[property_id]
-            ]
-            representative_rec_data = pd.DataFrame(representative_rec_data)
-            # Suppress mechanical ventilation to have zero heat demand and co2
-            representative_rec_data.loc[
-                representative_rec_data["type"] == "mechanical_ventilation", "co2_equivalent_savings"
-            ] = 0
-            representative_rec_data.loc[
-                representative_rec_data["type"] == "mechanical_ventilation", "heat_demand"
-            ] = 0
-            # Supress low energy lighting to have zero heat demand and co2 - this does not get affected by this process
-            representative_rec_data.loc[
-                representative_rec_data["type"] == "low_energy_lighting", "co2_equivalent_savings"
-            ] = 0
-            representative_rec_data.loc[
-                representative_rec_data["type"] == "low_energy_lighting", "heat_demand"
-            ] = 0
-
-            # Convert co2 and heat demand to proportions of their column sums
-            representative_rec_data["co2_equivalent_savings_percent"] = (
-                representative_rec_data["co2_equivalent_savings"] /
-                representative_rec_data["co2_equivalent_savings"].sum()
-            )
-
-            representative_rec_data["heat_demand_percent"] = (
-                representative_rec_data["heat_demand"] / representative_rec_data["heat_demand"].sum()
-            )
-
-            # We'll use the proportions to update the carbon and heat demand
-            representative_rec_data["co2_equivalent_savings"] = (
-                carbon_change * representative_rec_data["co2_equivalent_savings_percent"]
-            )
-
-            representative_rec_data["heat_demand"] = (
-                heat_demand_change * representative_rec_data["heat_demand_percent"]
-            )
-
-            # Finally, insert these values into the final recommendations
-            for rec in property_recommendations:
-                if rec["type"] in ["external_wall_insulation", "internal_wall_insulation"]:
-                    change_data = representative_rec_data[
-                        representative_rec_data["type"].isin(["external_wall_insulation", "internal_wall_insulation"])
-                    ]
-                else:
-                    change_data = representative_rec_data[representative_rec_data["type"] == rec["type"]]
-
-                if rec["type"] == "mechanical_ventilation":
-                    rec["co2_equivalent_savings"] = 0
-                    rec["heat_demand"] = 0
-                    rec["energy_cost_savings"] = 0
-                elif rec["type"] == "low_energy_lighting":
-                    # We do not convert, we just calculate energy cost savings
-                    rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"])
-                    continue
-                else:
-                    rec["co2_equivalent_savings"] = change_data["co2_equivalent_savings"].values[0]
-                    rec["heat_demand"] = change_data["heat_demand"].values[0]
-                    # If the recommendation is solar, the savings are entirely in electricity
-                    if rec["type"] == "solar_pv":
-                        rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"])
-                    else:
-                        rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
-
-            # Update recommendations
-            recommendations[property_id] = property_recommendations
-
-            # For expected adjust energy, we don't include mechanical ventilation so we'll add it back on
-            mechanical_ventilation_rec = representative_rec_data[
-                representative_rec_data["type"] == "mechanical_ventilation"
-                ]
-            if not mechanical_ventilation_rec.empty:
-                expected_adjusted_energy = (
-                    expected_adjusted_energy + mechanical_ventilation_rec["heat_demand"].values[0]
-                )
-
-            property_instance.set_adjusted_energy(
-                current_adjusted_energy=current_adjusted_energy,
-                expected_adjusted_energy=expected_adjusted_energy
-            )
-
         # 1) the property data
         # 2) the property details (epc)
         # 3) the recommendations
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index 93472068..06d98a69 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -1,3 +1,5 @@
+import numpy as np
+
 from backend.Property import Property
 from typing import List
 from itertools import groupby
@@ -213,6 +215,44 @@ class Recommendations:
         heat_phase_impact = property_heat_predictions.groupby("phase")["predictions"].median().reset_index()
         carbon_phase_impact = property_carbon_predictions.groupby("phase")["predictions"].median().reset_index()
 
+        ## TODO: NEW
+
+        # The heat demand change is the difference between the starting heat demand and the value at the final phase
+        expected_heat_demand = property_instance.floor_area * (
+            heat_phase_impact[heat_phase_impact["phase"] == max(heat_phase_impact["phase"])]["predictions"].values[0]
+        )
+
+        expected_carbon = (
+            carbon_phase_impact[carbon_phase_impact["phase"] == max(carbon_phase_impact["phase"])][
+                "predictions"].values[0]
+        )
+
+        starting_heat_demand = (
+            float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area
+        )
+
+        # This is the unadjusted resulting heat demand
+        predicted_heat_demand_change = starting_heat_demand - expected_heat_demand
+
+        starting_carbon = float(property_instance.data["co2-emissions-current"])
+
+        # We don't want to adjust the heat demand for mechanical ventilation so we add it back on
+
+        # We adjust the heat demand figures to align to the UCL paper
+        current_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+            epc_energy_consumption=starting_heat_demand,
+            current_epc_rating=property_instance.data["current-energy-rating"],
+        )
+
+        expected_adjusted_energy = AnnualBillSavings.adjust_energy_to_metered(
+            epc_energy_consumption=expected_heat_demand,
+            current_epc_rating=property_instance.data["current-energy-rating"],
+        )
+
+        adjusted_heat_demand_change = (
+            current_adjusted_energy - expected_adjusted_energy
+        )
+
         for recommendations_by_type in property_recommendations:
             for rec in recommendations_by_type:
 
@@ -233,39 +273,58 @@ class Recommendations:
                 )]["predictions"].values[0]
 
                 if rec["phase"] == 0:
-                    rec["sap_points"] = new_sap - float(property_instance.data["current-energy-efficiency"])
-                    rec["co2_equivalent_savings"] = float(property_instance.data["co2-emissions-current"]) - new_carbon
-                    rec["heat_demand"] = property_instance.floor_area * (
+                    predicted_sap_points = new_sap - float(property_instance.data["current-energy-efficiency"])
+                    predicted_co2_savings = float(property_instance.data["co2-emissions-current"]) - new_carbon
+                    predicted_heat_demand = property_instance.floor_area * (
                         float(property_instance.data["energy-consumption-current"]) - new_heat_demand
                     )
                 else:
-
                     previous_phase = rec["phase"] - 1
-                    rec["sap_points"] = (
+                    predicted_sap_points = (
                         new_sap - sap_phase_impact[sap_phase_impact["phase"] == previous_phase]["predictions"].values[0]
                     )
-                    rec["co2_equivalent_savings"] = (
+                    predicted_co2_savings = (
                         carbon_phase_impact[carbon_phase_impact["phase"] == previous_phase]["predictions"].values[0] -
                         new_carbon
                     )
-                    rec["heat_demand"] = property_instance.floor_area * (
+                    predicted_heat_demand = property_instance.floor_area * (
                         heat_phase_impact[heat_phase_impact["phase"] == previous_phase]["predictions"].values[0] -
                         new_heat_demand
                     )
 
                 if rec["type"] == "low_energy_lighting":
                     # For the moment, we cap the number of SAP points that can be achieved by ventilation at 2
-                    rec["sap_points"] = min(rec["sap_points"], LightingRecommendations.SAP_LIMIT)
+                    rec["sap_points"] = min(predicted_sap_points, LightingRecommendations.SAP_LIMIT)
+                    rec["co2_equivalent_savings"] = min(predicted_co2_savings, rec["co2_equivalent_savings"])
+                    rec["heat_demand"] = min(predicted_heat_demand, rec["heat_demand"])
+                else:
+                    rec["sap_points"] = predicted_sap_points
+                    rec["co2_equivalent_savings"] = predicted_co2_savings
+                    rec["heat_demand"] = predicted_heat_demand
 
                 # Round to 2 decimal places
                 rec["sap_points"] = round(rec["sap_points"], 2)
 
-                # Energy consumption current is per meter squared, so we need to multiply by the floor area to get
-                # an absolute figure for the home
-                rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
+                # We now calculate the adjusted heat demand for this recommendation, which is simply the percentage
+                # of the total adjusted heat demand change. The percentage we use is this recommendation's percentage
+                # of the total heat demand per square meter change
+
+                rec["adjusted_heat_demand"] = adjusted_heat_demand_change * (
+                    rec["heat_demand"] / predicted_heat_demand_change
+                )
+                # We make sure this is NOT below 0
+                rec["adjusted_heat_demand"] = max(0, rec["heat_demand"])
+
+                # Depending on the property's tarriff, we calculate the amount of energy savings this measure will bring
+                if property_instance.energy_source == "electricity":
+                    rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"])
+                elif property_instance.energy_source == "electricity_and_gas":
+                    rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
+                else:
+                    raise ValueError("Invalid value for energy source")
 
                 if (rec["sap_points"] is None) and (rec["co2_equivalent_savings"] is None) or (
                     rec["heat_demand"] is None) or (rec["energy_cost_savings"] is None):
                     raise ValueError("sap points, co2 or heat demand is missing")
 
-        return property_recommendations
+        return property_recommendations, current_adjusted_energy, expected_adjusted_energy

From 4fb479d3c1936d5fbb108d9f0676392f6c2d3397 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Fri, 16 Feb 2024 13:02:15 +0000
Subject: [PATCH 13/16] Added adjusted_heat_demand to Recommnendations model

---
 backend/app/db/models/recommendations.py | 1 +
 recommendations/Recommendations.py       | 8 --------
 2 files changed, 1 insertion(+), 8 deletions(-)

diff --git a/backend/app/db/models/recommendations.py b/backend/app/db/models/recommendations.py
index ff7aa642..42ecbddf 100644
--- a/backend/app/db/models/recommendations.py
+++ b/backend/app/db/models/recommendations.py
@@ -22,6 +22,7 @@ class Recommendation(Base):
     new_u_value = Column(Float)
     sap_points = Column(Float)
     heat_demand = Column(Float)
+    adjusted_heat_demand = Column(Float)
     co2_equivalent_savings = Column(Float)
     energy_savings = Column(Float)
     energy_cost_savings = Column(Float)
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index 06d98a69..7bdfbdea 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -221,12 +221,6 @@ class Recommendations:
         expected_heat_demand = property_instance.floor_area * (
             heat_phase_impact[heat_phase_impact["phase"] == max(heat_phase_impact["phase"])]["predictions"].values[0]
         )
-
-        expected_carbon = (
-            carbon_phase_impact[carbon_phase_impact["phase"] == max(carbon_phase_impact["phase"])][
-                "predictions"].values[0]
-        )
-
         starting_heat_demand = (
             float(property_instance.data["energy-consumption-current"]) * property_instance.floor_area
         )
@@ -234,8 +228,6 @@ class Recommendations:
         # This is the unadjusted resulting heat demand
         predicted_heat_demand_change = starting_heat_demand - expected_heat_demand
 
-        starting_carbon = float(property_instance.data["co2-emissions-current"])
-
         # We don't want to adjust the heat demand for mechanical ventilation so we add it back on
 
         # We adjust the heat demand figures to align to the UCL paper

From 1f0dea980ff65a88a68483532f50779acec3a07f Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Fri, 16 Feb 2024 14:28:13 +0000
Subject: [PATCH 14/16] Debugging set_energy_source

---
 backend/Property.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/Property.py b/backend/Property.py
index b20d409a..75cfaef1 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -761,7 +761,7 @@ class Property:
 
         # If the tariff explicitly indicates electricity use without a dual indication and mains_gas_flag is not True
         # We check for the common electricity tariffs
-        if not self.data["mains_gas_flag"] and self.data["energy_tariff"] in [
+        if not self.data["mains-gas-flag"] and self.data["energy-tariff"] in [
             "Single", "off-peak 7 hour", "off-peak 10 hour", "off-peak 18 hour", "standard tariff", "24 hour"
         ]:
             energy_source = "electricity"

From 132bf98eeb8791aa015313078a179f32f560c478 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Fri, 16 Feb 2024 15:18:09 +0000
Subject: [PATCH 15/16] small tweaks to fill missing data in ventilation recs
 and percentage in solar description

---
 backend/app/plan/router.py                    | 2 +-
 recommendations/Costs.py                      | 8 ++++----
 recommendations/Recommendations.py            | 8 +++-----
 recommendations/SolarPvRecommendations.py     | 4 ++--
 recommendations/VentilationRecommendations.py | 4 ++++
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 3b010d12..28dcaa68 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -220,7 +220,7 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             selected_recommendations = {r["id"] for r in solution}
 
-            # If wall ventilation is selected, we also include mechanical ventilation as a best practice measure
+            # If wall insulation is selected, we also include mechanical ventilation as a best practice measure
             if any(x in [r["type"] for r in solution] for x in [
                 "internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"
             ]):
diff --git a/recommendations/Costs.py b/recommendations/Costs.py
index dd1f1a21..6ea17dce 100644
--- a/recommendations/Costs.py
+++ b/recommendations/Costs.py
@@ -860,12 +860,12 @@ class Costs:
         kw = wattage / 1000
         total_cost = kw * regional_cost
 
-        subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
-
         if has_battery:
             # The battery cost is based on the £3500 quote, recieved from installers
-            subtotal_before_vat += BATTERY_COST
-            
+            total_cost += BATTERY_COST
+
+        subtotal_before_vat = total_cost / (1 + self.VAT_RATE)
+
         vat = total_cost - subtotal_before_vat
 
         # Labour hours are based on estimates from online research but an average team seems to consist of 3 people
diff --git a/recommendations/Recommendations.py b/recommendations/Recommendations.py
index 7bdfbdea..d1eec41b 100644
--- a/recommendations/Recommendations.py
+++ b/recommendations/Recommendations.py
@@ -215,8 +215,6 @@ class Recommendations:
         heat_phase_impact = property_heat_predictions.groupby("phase")["predictions"].median().reset_index()
         carbon_phase_impact = property_carbon_predictions.groupby("phase")["predictions"].median().reset_index()
 
-        ## TODO: NEW
-
         # The heat demand change is the difference between the starting heat demand and the value at the final phase
         expected_heat_demand = property_instance.floor_area * (
             heat_phase_impact[heat_phase_impact["phase"] == max(heat_phase_impact["phase"])]["predictions"].values[0]
@@ -305,13 +303,13 @@ class Recommendations:
                     rec["heat_demand"] / predicted_heat_demand_change
                 )
                 # We make sure this is NOT below 0
-                rec["adjusted_heat_demand"] = max(0, rec["heat_demand"])
+                rec["adjusted_heat_demand"] = max(0, rec["adjusted_heat_demand"])
 
                 # Depending on the property's tarriff, we calculate the amount of energy savings this measure will bring
                 if property_instance.energy_source == "electricity":
-                    rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["heat_demand"])
+                    rec["energy_cost_savings"] = AnnualBillSavings.estimate_electric(rec["adjusted_heat_demand"])
                 elif property_instance.energy_source == "electricity_and_gas":
-                    rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["heat_demand"])
+                    rec["energy_cost_savings"] = AnnualBillSavings.estimate(rec["adjusted_heat_demand"])
                 else:
                     raise ValueError("Invalid value for energy source")
 
diff --git a/recommendations/SolarPvRecommendations.py b/recommendations/SolarPvRecommendations.py
index 5fae09b3..3a89b213 100644
--- a/recommendations/SolarPvRecommendations.py
+++ b/recommendations/SolarPvRecommendations.py
@@ -70,10 +70,10 @@ class SolarPvRecommendations:
 
             if has_battery:
                 description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) panel system on "
-                               f"{round(roof_coverage_percent * 100)}% the roof, with a battery storage system.")
+                               f"{round(roof_coverage_percent)}% the roof, with a battery storage system.")
             else:
                 description = (f"Install a {kw} kilowatt-peak (kWp) solar photovoltaic (PV) p"
-                               f"anel system on {round(roof_coverage_percent * 100)}% the roof.")
+                               f"anel system on {round(roof_coverage_percent)}% the roof.")
 
             self.recommendation.append(
                 {
diff --git a/recommendations/VentilationRecommendations.py b/recommendations/VentilationRecommendations.py
index 19aad0e1..1657b759 100644
--- a/recommendations/VentilationRecommendations.py
+++ b/recommendations/VentilationRecommendations.py
@@ -66,6 +66,10 @@ class VentilationRecommendations(Definitions):
                 "starting_u_value": None,
                 "new_u_value": None,
                 "sap_points": 0,
+                "heat_demand": 0,
+                "adjusted_heat_demand": 0,
+                "co2_equivalent_savings": 0,
+                "energy_cost_savings": 0,
                 "total": estimated_cost,
                 # We use a very simple and rough estimate of 4 hours per unit
                 "labour_hours": 4 * n_units,

From 7ff3d8799991f1f1a46344c4d4b5c3dfba820e77 Mon Sep 17 00:00:00 2001
From: Khalim Conn-Kowlessar <kconnkowlessar@gmail.com>
Date: Fri, 16 Feb 2024 17:04:43 +0000
Subject: [PATCH 16/16] Trying and failing to implement the newest model

---
 backend/Property.py                      |  2 ++
 backend/app/plan/router.py               |  2 ++
 etl/epc/generate_scenarios_data.py       | 42 ++++++++++++++--------
 etl/testing_data/sap_model_simulation.py | 46 +++++++++++++++++-------
 4 files changed, 66 insertions(+), 26 deletions(-)

diff --git a/backend/Property.py b/backend/Property.py
index 75cfaef1..418a35a1 100644
--- a/backend/Property.py
+++ b/backend/Property.py
@@ -230,6 +230,8 @@ class Property:
                 # The upgrade made here is to the u-value of the walls and the description of the
                 # insulation thickness
                 output["walls_thermal_transmittance_ending"] = recommendation["new_u_value"]
+                # Setting the insulation thickness here to above average should be tested further because we
+                # don't see a high volume of instances for this
                 output["walls_insulation_thickness_ending"] = "above average"
                 output["walls_energy_eff_ending"] = "Good"
 
diff --git a/backend/app/plan/router.py b/backend/app/plan/router.py
index 28dcaa68..07fdbe94 100644
--- a/backend/app/plan/router.py
+++ b/backend/app/plan/router.py
@@ -160,6 +160,8 @@ async def trigger_plan(body: PlanTriggerRequest):
 
             recommendations_scoring_data.extend(p.recommendations_scoring_data)
 
+        # TODO: Make sure that number_habitable_rooms has been dropped
+
         logger.info("Preparing data for scoring in sap change api")
         recommendations_scoring_data = pd.DataFrame(recommendations_scoring_data)
         recommendations_scoring_data = recommendations_scoring_data.drop(
diff --git a/etl/epc/generate_scenarios_data.py b/etl/epc/generate_scenarios_data.py
index af3ed4a4..afe9ab98 100644
--- a/etl/epc/generate_scenarios_data.py
+++ b/etl/epc/generate_scenarios_data.py
@@ -50,9 +50,9 @@ scenario_properties = [
         "postcode": "NN1 5JY",
         "lmk-key": "1459796789102016070507274146560098",
         "measures": [
-            [["internal_wall_insulation"], "11", None],
-            [["external_wall_insulation"], "10", None],
-            [["solar", "windows"], "12-15", {"photo_supply_ending": 50}],
+            [["internal_wall_insulation"], "11", None, [0]],
+            [["external_wall_insulation"], "10", None, [0]],
+            [["solar", "windows"], "12-15", {"photo_supply_ending": 50}, [0, 1]],
         ],
     },
     {
@@ -60,7 +60,7 @@ scenario_properties = [
         "postcode": "HP1 2HA",
         "lmk-key": "c14029235739827d5f627dc8aa9bb567d026b267e851e0db0001db24638667b1",
         "measures": [
-            [["cavity_wall_insulation", "loft_insulation"], "15", None],
+            [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]],
         ],
     },
     {
@@ -68,7 +68,7 @@ scenario_properties = [
         "postcode": "HP1 2HE",
         "lmk-key": "99296a6dda21314fef3a61cda59e441e9a2aacf115eb96f4a0fa85696bf7b117",
         "measures": [
-            [["cavity_wall_insulation", "loft_insulation"], "15", None],
+            [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]],
         ],
     },
     {
@@ -76,7 +76,7 @@ scenario_properties = [
         "postcode": "HP1 2AN",
         "lmk-key": "d1e0534be3a44c33003323b21d0e322e3daddc65b5ee71936f89c59ddab96b50",
         "measures": [
-            [["cavity_wall_insulation", "loft_insulation"], "15", None],
+            [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]],
         ],
     },
     {
@@ -84,7 +84,7 @@ scenario_properties = [
         "postcode": "HP1 2HX",
         "lmk-key": "1eae354db522a95188018d9cd0502ed8c609910b6c88f8797d3a25f59b11770a",
         "measures": [
-            [["cavity_wall_insulation", "loft_insulation"], "15", None],
+            [["cavity_wall_insulation", "loft_insulation"], "15", None, [0, 1]],
         ],
     },
 ]
@@ -195,18 +195,18 @@ for scenario_property in scenario_properties:
             recommendation_record = p.base_difference_record.df.to_dict("records")[
                 0
             ].copy()
-            for rec in combi:
-                recommendation_record = p.create_recommendation_scoring_data(
-                    property_id=rec["type"],
-                    recommendation_record=recommendation_record,
-                    recommendation=rec,
-                )
+            recommendation_record = p.create_recommendation_scoring_data(
+                property_id=i,
+                primary_recommendation_id=i,
+                recommendation_record=recommendation_record,
+                recommendations=combi,
+            )
 
             if override is not None:
                 for key, value in override.items():
                     recommendation_record[key] = value
 
-            recommendation_record["id"] = "+".join(measure) + "+" + str(i)
+            recommendation_record["id"] = "&".join(measure) + "+" + str(i)
             recommendation_record["impact"] = impact
             scoring_list.append(recommendation_record)
 
@@ -230,6 +230,20 @@ recommendations_scoring_data.insert(0, "impact", impact_col)
 id_col = recommendations_scoring_data.pop("id")
 recommendations_scoring_data.insert(0, "id", id_col)
 
+from backend.ml_models.api import ModelApi
+
+model_api = ModelApi(portfolio_id="generate-scenarios-data", timestamp=created_at)
+
+all_predictions = model_api.predict_all(
+    df=recommendations_scoring_data,
+    bucket=get_settings().DATA_BUCKET,
+    prediction_buckets={
+        "sap_change_predictions": get_settings().SAP_PREDICTIONS_BUCKET,
+        "heat_demand_predictions": get_settings().HEAT_PREDICTIONS_BUCKET,
+        "carbon_change_predictions": get_settings().CARBON_PREDICTIONS_BUCKET
+    }
+)
+
 save_dataframe_to_s3_parquet(
     recommendations_scoring_data,
     "retrofit-data-dev",
diff --git a/etl/testing_data/sap_model_simulation.py b/etl/testing_data/sap_model_simulation.py
index 726c2428..12b26249 100644
--- a/etl/testing_data/sap_model_simulation.py
+++ b/etl/testing_data/sap_model_simulation.py
@@ -1935,10 +1935,10 @@ def app():
             "address": "73 Long Chaulden",
             "postcode": "HP1 2HX",
         },
-        {
-            "address": "77 Simmons Drive",
-            "postcode": "B32 1SL",
-        },
+        # {
+        #     "address": "77 Simmons Drive",
+        #     "postcode": "B32 1SL",
+        # },
         {
             "address": "139 School Road",
             "postcode": "B28 8JF",
@@ -1984,6 +1984,19 @@ def app():
         floor_recommendations = recommender.floor_recommender.recommendations
         solar_recommendations = recommender.solar_recommender.recommendation
 
+        # We set wall recommendations to phase 0, loft to phase 1, floor to phase 2 and solar to phase 3
+        for rec in wall_recommendations:
+            rec["phase"] = 0
+
+        for rec in loft_recommendations:
+            rec["phase"] = 1
+
+        for rec in floor_recommendations:
+            rec["phase"] = 2
+
+        for rec in solar_recommendations:
+            rec["phase"] = 3
+
         # TODO: TEMP!
         solar_recommendations[0]["photo_supply"] = 50
 
@@ -1994,7 +2007,8 @@ def app():
         for wall_rec in wall_recommendations:
             recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
             scoring_dict = p.create_recommendation_scoring_data(
-                property_id=p.id, recommendation_record=recommendation_record, recommendation=wall_rec,
+                property_id=p.id, recommendation_record=recommendation_record, recommendations=[wall_rec],
+                primary_recommendation_id=wall_rec["recommendation_id"]
             )
             wall_scoring_data.append(scoring_dict)
 
@@ -2002,7 +2016,8 @@ def app():
         for roof_rec in loft_recommendations:
             recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
             scoring_dict = p.create_recommendation_scoring_data(
-                property_id=p.id, recommendation_record=recommendation_record, recommendation=roof_rec,
+                property_id=p.id, recommendation_record=recommendation_record, recommendations=[roof_rec],
+                primary_recommendation_id=roof_rec["recommendation_id"]
             )
             roof_scoring_data.append(scoring_dict)
 
@@ -2010,7 +2025,8 @@ def app():
         for floor_rec in floor_recommendations:
             recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
             scoring_dict = p.create_recommendation_scoring_data(
-                property_id=p.id, recommendation_record=recommendation_record, recommendation=floor_rec,
+                property_id=p.id, recommendation_record=recommendation_record, recommendations=[floor_rec],
+                primary_recommendation_id=floor_rec["recommendation_id"]
             )
             floor_scoring_data.append(scoring_dict)
 
@@ -2018,7 +2034,8 @@ def app():
         for solar_rec in solar_recommendations:
             recommendation_record = p.base_difference_record.df.to_dict("records")[0].copy()
             scoring_dict = p.create_recommendation_scoring_data(
-                property_id=p.id, recommendation_record=recommendation_record, recommendation=solar_rec,
+                property_id=p.id, recommendation_record=recommendation_record, recommendations=[solar_rec],
+                primary_recommendation_id=solar_rec["recommendation_id"]
             )
             solar_scoring_data.append(scoring_dict)
 
@@ -2029,25 +2046,30 @@ def app():
         scoring_dict_with_wall = p.create_recommendation_scoring_data(
             property_id=p.id,
             recommendation_record=starting_record.copy(),
-            recommendation=wall_recommendations[0],
+            recommendations=[wall_recommendations[0]],
+            primary_recommendation_id=wall_recommendations[0]["recommendation_id"]
         ) if wall_recommendations else starting_record.copy()
 
         scoring_dict_with_wall_and_roof = p.create_recommendation_scoring_data(
             property_id=p.id,
             recommendation_record=scoring_dict_with_wall.copy(),
-            recommendation=loft_recommendations[0],
+            recommendations=[wall_recommendations[0], loft_recommendations[0], ],
+            primary_recommendation_id=loft_recommendations[0]["recommendation_id"]
         ) if loft_recommendations else scoring_dict_with_wall.copy()
 
         scoring_dict_with_wall_roof_floor = p.create_recommendation_scoring_data(
             property_id=p.id,
             recommendation_record=scoring_dict_with_wall_and_roof.copy(),
-            recommendation=floor_recommendations[0],
+            recommendations=[wall_recommendations[0], loft_recommendations[0], floor_recommendations[0]],
+            primary_recommendation_id=floor_recommendations[0]["recommendation_id"]
         ) if floor_recommendations else scoring_dict_with_wall_and_roof.copy()
 
         scoring_dict_with_wall_roof_floor_solar = p.create_recommendation_scoring_data(
             property_id=p.id,
             recommendation_record=scoring_dict_with_wall_roof_floor.copy(),
-            recommendation=solar_recommendations[0],
+            recommendations=[wall_recommendations[0], loft_recommendations[0], floor_recommendations[0],
+                             solar_recommendations[0]],
+            primary_recommendation_id=solar_recommendations[0]["recommendation_id"]
         ) if solar_recommendations else scoring_dict_with_wall_roof_floor.copy()
 
         # We score each dataset with the model