Updating the floor recommendations class for new cost data

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Costs" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyNamespacePackagesService">

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Costs" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/plan/router.py

@@ -21,7 +21,7 @@ from backend.app.db.models.portfolio import rating_lookup
 from backend.app.dependencies import validate_token
 from backend.app.plan.schemas import PlanTriggerRequest
 from backend.app.plan.utils import (
-    create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id
+    create_recommendation_scoring_data, prepare_materials, get_cleaned, insert_temp_recommendation_id
 )
 from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3
 
@@ -114,7 +114,7 @@ async def trigger_plan(body: PlanTriggerRequest):
         # the same data
         logger.info("Reading in materials and cleaned datasets")
         materials = get_materials(session)
-        materials_by_type = filter_materials(materials)
+        materials = prepare_materials(materials)
         cleaned = get_cleaned()
 
         logger.info("Getting components and epc recommendations")
@@ -126,13 +126,14 @@ async def trigger_plan(body: PlanTriggerRequest):
         recommendations_scoring_data = []
 
         for p in input_properties:
+
             # Property recommendations
             p.get_components(cleaned)
 
             property_recommendations = []
 
             # Floor recommendations
-            floor_recommender = FloorRecommendations(property_instance=p, materials=materials_by_type["floor"])
+            floor_recommender = FloorRecommendations(property_instance=p, materials=materials)
             floor_recommender.recommend()
 
             if floor_recommender.recommendations:

backend/app/plan/utils.py

@@ -1,6 +1,5 @@
 import pandas as pd
 from backend.Property import Property
-from collections import defaultdict
 from utils.s3 import read_from_s3
 
 from recommendations.recommendation_utils import get_wall_u_value, get_floor_u_value, get_roof_u_value
@@ -10,22 +9,13 @@ from backend.app.config import get_settings
 import msgpack
 
 
-def filter_materials(materials):
+def prepare_materials(materials):
-    materials_by_type = defaultdict(list)
+    """
-
+    This function will prepare the materials for recommendations
-    mapping = {
+    :param materials: list of materials, as retrieved from the database
-        "walls": ["internal_wall_insulation", "external_wall_insulation", "cavity_wall_insulation"],
+    :return:
-        "floor": ["suspended_floor_insulation", "solid_floor_insulation", "exposed_floor_insulation"],
+    """
-        "ventilation": ["mechanical_ventilation"],
+    return [row2dict(material) for material in materials]
-        "roof": ["loft_insulation", "flat_roof_insulation", "room_roof_insulation"]
-    }
-
-    materials = [row2dict(material) for material in materials]
-
-    for component, types in mapping.items():
-        materials_by_type[component] = [part for part in materials if part["type"] in types]
-
-    return dict(materials_by_type)
 
 
 def insert_temp_recommendation_id(property_recommendations):
@@ -173,7 +163,7 @@ def create_recommendation_scoring_data(
         parts = recommendation["parts"]
         if len(parts) != 1:
             raise ValueError("More than one part for roof insulation - investiage me")
-        
+
         scoring_dict["roof_insulation_thickness_ENDING"] = str(parts[0]["depths"][0])
         scoring_dict["ROOF_ENERGY_EFF_ENDING"] = "Very Good"
     else:

etl/costs/app.py

@@ -1,12 +1,12 @@
 import os
 import dotenv
-import json
 import pandas as pd
 import numpy as np
 from pathlib import Path
 from sqlalchemy.orm import Session
 from sqlalchemy import create_engine
 from backend.app.db.models.materials import Material
+from recommendations.recommendation_utils import calculate_r_value_per_mm
 
 DATA_DIRECTORY = Path(__file__).parent / "local_data" / "Hestia Materials.xlsx"
 # Environment file is at the same level as this file
@@ -90,6 +90,14 @@ def app():
     costs["depth"] = costs["depth"].fillna(0)
     costs["depth"] = costs["depth"].astype(str)
 
+    costs["r_value_per_mm"] = costs.apply(
+        lambda row: calculate_r_value_per_mm(float(row["depth"]), row["thermal_conductivity"]), axis=1
+    )
+    costs["r_value_unit"] = "square_meter_kelvin_per_watt"
+
+    for col in ["material_cost", "labour_cost", "labour_hours_per_unit", "plant_cost"]:
+        costs[col] = costs[col].fillna(0)
+
     # Push the costs to the database
     push_costs_to_db(db_engine, costs)
 

recommendations/FloorRecommendations.py

@@ -1,5 +1,8 @@
 import math
 from typing import List
+
+import pandas as pd
+
 from BaseUtility import Definitions
 from datatypes.enums import QuantityUnits
 from backend.Property import Property
@@ -8,6 +11,7 @@ from recommendations.recommendation_utils import (
     get_recommended_part, get_floor_u_value
 )
 from recommendations.rdsap_tables import FLOOR_LEVEL_MAP
+from recommendations.Costs import Costs
 
 
 class FloorRecommendations(Definitions):
@@ -30,25 +34,41 @@ class FloorRecommendations(Definitions):
         materials: List,
     ):
         self.property = property_instance
+        self.costs = Costs(self.property)
         # For audit purposes, when estimating u values we'll store it
         self.estimated_u_value = None
 
         # Will contains a list of recommended measures
         self.recommendations = []
 
-        self.materials = materials
+        self.suspended_floor_insulation_materials = [
-
+            part for part in materials if part["type"] == "suspended_floor_insulation"
-        self.suspended_floor_insulation_parts = [
-            part for part in self.materials if part["type"] == "suspended_floor_insulation"
-        ]
-        self.solid_floor_insulation_parts = [
-            part for part in self.materials if part["type"] == "solid_floor_insulation"
         ]
 
-        self.exposed_floor_insulation_parts = [
+        self.suspended_floor_non_insulation_materials = [
-            part for part in self.materials if part["type"] == "exposed_floor_insulation"
+            part for part in materials if part["type"] in [
+                "suspended_floor_demolition", "suspended_floor_redecoration", "suspended_floor_vapour_barrier"
+            ]
         ]
 
+        self.solid_floor_insulation_materials = [
+            part for part in materials if part["type"] == "solid_floor_insulation"
+        ]
+
+        self.solid_floor_non_insulation_materials = [
+            part for part in materials if part["type"] in [
+                "solid_floor_demolition", "solid_floor_preparation", "solid_floor_vapour_barrier",
+                "solid_floor_redecoration"
+            ]
+        ]
+
+        self.exposed_floor_insulation_materials = [
+            part for part in materials if part["type"] == "exposed_floor_insulation"
+        ]
+
+        # TODO: To be completed
+        self.exposed_floor_non_insulation_materials = []
+
     def recommend(self):
         u_value = self.property.floor["thermal_transmittance"]
 
@@ -98,7 +118,11 @@ class FloorRecommendations(Definitions):
 
         if self.property.floor["is_suspended"]:
             # Given the U-value, we recommend underfloor insulation
-            self.recommend_floor_insulation(u_value=u_value, parts=self.suspended_floor_insulation_parts)
+            self.recommend_floor_insulation(
+                u_value=u_value,
+                insulation_materials=self.suspended_floor_insulation_materials,
+                non_insulation_materials=self.suspended_floor_non_insulation_materials
+            )
             return
 
         if self.property.floor["is_solid"]:
@@ -113,20 +137,23 @@ class FloorRecommendations(Definitions):
         raise NotImplementedError("Implement me!")
 
     @staticmethod
-    def _make_floor_description(part, depth):
+    def _make_floor_description(material):
-        return f"Install {depth}{part['depth_unit']} {part['description']} insulation"
+        return f"Install {int(material['depth'])}{material['depth_unit']} {material['description']} insulation"
 
-    def recommend_floor_insulation(self, u_value, parts):
+    def recommend_floor_insulation(self, u_value, insulation_materials, non_insulation_materials):
         """
         This method is tasked with estimating the impact of performing suspended floor insulation
         :return:
         """
 
-        lowest_selected_u_value = None
+        insulation_materials = pd.DataFrame(insulation_materials)
-        for part in parts:
-            for depth, cost_per_unit in zip(part["depths"], part["cost"]):
 
-                part_u_value = r_value_per_mm_to_u_value(depth, part["r_value_per_mm"])
+        lowest_selected_u_value = None
+        for _, insulation_material_group in insulation_materials.groupby("description"):
+
+            for _, material in insulation_material_group.iterrows():
+
+                part_u_value = r_value_per_mm_to_u_value(material["depth"], material["r_value_per_mm"])
                 _, new_u_value = calculate_u_value_uplift(u_value, part_u_value)
                 new_u_value = math.ceil(new_u_value * 100.0) / 100.0
 
@@ -137,26 +164,37 @@ class FloorRecommendations(Definitions):
 
                 if new_u_value <= self.BUILDING_REGULATIONS_PART_L_MAX_U_VALUE:
                     lowest_selected_u_value = update_lowest_selected_u_value(lowest_selected_u_value, new_u_value)
-                    quantity = self.property.insulation_floor_area
 
-                    estimated_cost = cost_per_unit * quantity
+                    if material["type"] == "suspended_floor_insulation":
+                        cost_result = self.costs.suspended_floor_insulation(
+                            insulation_floor_area=self.property.insulation_floor_area,
+                            material=material.to_dict(),
+                            non_insulation_materials=non_insulation_materials
+                        )
+                    elif material["type"] == "solid_floor_insulation":
+                        cost_result = self.costs.solid_floor_insulation(
+                            insulation_floor_area=self.property.insulation_floor_area,
+                            material=material.to_dict(),
+                            non_insulation_materials=non_insulation_materials
+                        )
+                    else:
+                        raise NotImplementedError("Implement me!")
 
                     self.recommendations.append(
                         {
                             "parts": [
                                 get_recommended_part(
-                                    part=part,
+                                    part=material.to_dict(),
-                                    selected_depth=depth,
+                                    quantity=self.property.insulation_floor_area,
-                                    quantity=quantity,
                                     quantity_unit=QuantityUnits.m2.value,
-                                    selected_total_cost=estimated_cost
+                                    cost_result=cost_result
                                 ),
                             ],
                             "type": "floor_insulation",
-                            "description": self._make_floor_description(part, depth),
+                            "description": self._make_floor_description(material),
                             "starting_u_value": u_value,
                             "new_u_value": new_u_value,
                             "sap_points": None,
-                            "cost": estimated_cost,
+                            **cost_result
                         }
                     )

recommendations/recommendation_utils.py

@@ -109,22 +109,21 @@ def update_lowest_selected_u_value(lowest_selected_u_value, new_u_value):
     return lowest_selected_u_value
 
 
-def get_recommended_part(part, selected_depth, selected_total_cost, quantity, quantity_unit):
+def get_recommended_part(part, cost_result, quantity, quantity_unit):
     """
     Utility function to return a recommended part with the selected depth.
     :param part: part to be recommended
-    :param selected_depth: depth of the selected part
+    :param cost_result: Total cost of the selected part, as returned by the Cost class
-    :param selected_total_cost: Total cost of the selected part
     :param quantity: Quantity of the selected part
     :param quantity_unit: Unit of the quantity
     :return:
     """
     recommended_part = deepcopy(part)
-    recommended_part["depths"] = [selected_depth]
-    recommended_part["estimated_cost"] = selected_total_cost
     recommended_part["quantity"] = quantity
     recommended_part["quantity_unit"] = quantity_unit
 
+    recommended_part.update(cost_result)
+
     return recommended_part
 
 
@@ -563,6 +562,9 @@ def calculate_r_value_per_mm(thickness_mm, thermal_conductivity_w_mK):
     :return:
     """
 
+    if thermal_conductivity_w_mK is None:
+        return None
+
     r_value_m2k_w = (thickness_mm / 1000) / thermal_conductivity_w_mK
 
     # Calculate R-value per mm