fixed merge conflicts

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Python 3.10 (model_data)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (backend)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (model_data)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (backend)" project-jdk-type="Python SDK" />
   <component name="PythonCompatibilityInspectionAdvertiser">
     <option name="version" value="3" />
   </component>

backend/app/db/functions/materials_functions.py

@@ -0,0 +1,15 @@
+from backend.app.db.connection import db_engine
+from backend.app.db.models.materials import Material
+from sqlalchemy.orm import sessionmaker
+
+
+def get_materials():
+    """
+    This function will retrieve all materials from the database.
+    :return: A list of Material objects if successful, an empty list otherwise.
+    """
+    Session = sessionmaker(bind=db_engine)
+    with Session() as session:
+        materials = session.query(Material).all()
+
+    return materials if materials else []

backend/app/db/models/materials.py

@@ -0,0 +1,51 @@
+import enum
+
+from sqlalchemy import Column, Integer, String, Float, Enum, TIMESTAMP
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.sql import func
+
+Base = declarative_base()
+
+
+class MaterialType(enum.Enum):
+    suspended_floor_insulation = "suspended_floor_insulation"
+    solid_floor_insulation = "solid_floor_insulation"
+    external_wall_insulation = "external_wall_insulation"
+    internal_wall_insulation = "internal_wall_insulation"
+
+
+class DepthUnit(enum.Enum):
+    mm = "mm"
+
+
+class CostUnit(enum.Enum):
+    gbp_sq_meter = "gbp_sq_meter"
+
+
+class RValueUnit(enum.Enum):
+    square_meter_kelvin_per_watt = "square_meter_kelvin_per_watt"
+
+
+class ThermalConductivityUnit(enum.Enum):
+    watt_per_meter_kelvin = "watt_per_meter_kelvin"
+
+
+class Material(Base):
+    __tablename__ = 'material'
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    type = Column(Enum(MaterialType, values_callable=lambda x: [e.value for e in x]), nullable=False)
+    description = Column(String, nullable=False)
+    depths = Column(String)  # You may want to use a specific JSON type depending on the database
+    depth_unit = Column(Enum(DepthUnit, values_callable=lambda x: [e.value for e in x]), nullable=False)
+    cost = Column(Float)
+    cost_unit = Column(Enum(CostUnit, values_callable=lambda x: [e.value for e in x]), nullable=False)
+    r_value_per_mm = Column(Float)
+    r_value_unit = Column(Enum(RValueUnit, values_callable=lambda x: [e.value for e in x]), nullable=False)
+    thermal_conductivity = Column(Float)
+    thermal_conductivity_unit = Column(
+        Enum(ThermalConductivityUnit, values_callable=lambda x: [e.value for e in x]),
+        nullable=False
+    )
+    link = Column(String)
+    created_at = Column(TIMESTAMP, nullable=False, server_default=func.now())

backend/app/db/utils.py

@@ -0,0 +1,18 @@
+import enum
+
+
+def row2dict(row):
+    """
+    Generic function to convert a SQLAlchemy row to a dictionary.
+    May not be the best practice implementing like this but works for the moment
+    """
+
+    d = {}
+    for column in row.__table__.columns:
+        val = getattr(row, column.name)
+        if isinstance(val, enum.Enum):
+            val = val.value
+
+        d[column.name] = val
+
+    return d

backend/app/plan/router.py

@@ -11,17 +11,19 @@ from utils.logger import setup_logger
 from recommendations.FloorRecommendations import FloorRecommendations
 from recommendations.WallRecommendations import WallRecommendations
 from utils.uvalue_estimates import classify_decile_newvalues
+from backend.app.db.utils import row2dict
+from starlette.responses import Response
 
 # database interaction functions
 from backend.app.db.functions.property_functions import (
     create_property, create_property_targets, update_property_data, create_property_details_epc
 )
+from backend.app.db.functions.materials_functions import get_materials
 
 # TODO: This is placeholder until data is stored in DB
 from backend.app.plan.uvalue_estimates_walls import uvalue_estimates_walls
 from backend.app.plan.uvalue_estimates_floors import uvalue_estimates_floors
 from backend.app.plan.temp_cleaned_data import cleaned
-from backend.app.plan.temp_materials_db import materials
 
 logger = setup_logger()
 
@@ -81,10 +83,11 @@ lighting_averages = [
 ]
 
 
-def get_materials(materials):
+def filter_materials(materials):
     materials_by_type = defaultdict(list)
 
     for material in materials:
+        material = row2dict(material)
         material_type = material["type"]
         materials_by_type[material_type].append(material)
 
@@ -134,6 +137,9 @@ async def trigger_plan(body: PlanTriggerRequest):
             )
         )
 
+    if not input_properties:
+        return Response(status_code=204)
+
     logger.info("Getting EPC data")
     for p in input_properties:
         p.search_address_epc()
@@ -155,11 +161,19 @@ async def trigger_plan(body: PlanTriggerRequest):
     # The materials data could be cached or local so we don't need to make
     # consistent requrests to the backend for
     # the same data
-    materials_by_type = get_materials(materials)
+    # TODO: It might not be the best choice to store the materials data in a database table since thi
+    #       table probably won't be very large and won't be updated that often. It might be better to
+    #       store this data in s3 load it into memory when the app starts up. We will test this
+
+    materials = get_materials()
+    materials_by_type = filter_materials(materials)
 
     logger.info("Getting components and properties recommendations")
-    recommendations = []
+
-    for property_id, p in enumerate(input_properties):
+    recommendations = {}
+    for p in input_properties:
+        property_recommendations = []
+
         # For each property, classiy floor area decide
         total_floor_area_group_decile = classify_decile_newvalues(
             decile_boundaries=floors_decile_data["decile_boundaries"],
@@ -187,11 +201,8 @@ async def trigger_plan(body: PlanTriggerRequest):
             total_floor_area_group_decile=total_floor_area_group_decile
         )
         floor_recommender.recommend()
-        # insert property id
-        for rec in floor_recommender.recommendations:
-            rec["property_id"] = property_id
 
-        recommendations.extend(floor_recommender.recommendations)
+        property_recommendations.extend(floor_recommender.recommendations)
 
         # Wall recommendations
         # We would make this u-value query directly to the database
@@ -219,13 +230,12 @@ async def trigger_plan(body: PlanTriggerRequest):
             materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"]
         )
         wall_recomendations.recommend()
-        # insert property id
-        for rec in wall_recomendations.recommendations:
-            rec["property_id"] = property_id
 
-        recommendations.extend(wall_recomendations.recommendations)
+        property_recommendations.extend(wall_recomendations.recommendations)
 
-    # Once we're done, we'll store:
+        recommendations[p.id] = property_recommendations
+
+        # Once we're done, we'll store:
     # 1) the property data
     # 2) the property details (epc)
     # 3) the recommendations
@@ -238,4 +248,10 @@ async def trigger_plan(body: PlanTriggerRequest):
         property_data = p.get_full_property_data()
         update_property_data(property_id=p.id, portfolio_id=body.portfolio_id, property_data=property_data)
 
-    return {"recommendations": recommendations}
+        # Upload recommendations
+        recommendations_to_upload = recommendations[p.id]
+        if not recommendations:
+            continue
+        # Create a plan
+
+    return Response(status_code=200)

backend/app/plan/temp_materials_db.py

@@ -1,242 +0,0 @@
-suspended_floor_insulation_parts = [
-    {
-        # Example product
-        # All product types here:
-        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/recticel/filterby/application/floors.html
-        "id": 1,
-        "type": "suspended_floor_insulation",
-        "description": "Rigid Insulation Foam Boards",
-        "depths": [25, 30, 40, 50, 60, 70, 75, 80, 90, 100, 110, 120, 130, 140, 150],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.04545454545454546,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.022,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationsuperstore.co.uk/product/recticel-eurothane-general-purpose-pir-insulation"
-                "-board-2400-x-1200-x-100mm.html"
-    },
-    {
-        # All product types here:
-        # https://www.insulationsuperstore.co.uk/browse/insulation/brand/rockwool/filterby/application/floors
-        # /material/mineral-wool.html
-        "id": 2,
-        "type": "suspended_floor_insulation",
-        "description": "Mineral Wool Floor Insulation",
-        "depths": [25, 40, 50, 60, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.035,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationsuperstore.co.uk/product/rockwool-rwa45-acoustic-insulation-slab-100mm-2-88m2"
-                "-pack.html"
-    },
-]
-
-solid_floor_insulation_parts = [
-    {
-        # All product types here:
-        # https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation?brand=7015&p=1
-        # Example screed https://www.screwfix.com/p/mapei-ultraplan-3240-self-levelling-compound-25kg/4959f
-        "id": 3,
-        "type": "solid_floor_insulation",
-        "description": "Rigid Insulation Foam Boards with floor screed",
-        "depths": [25, 50, 70, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.04545454545454546,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.052631578947368425,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationexpress.co.uk/floor-insulation/solid-floor-insulation/k103-100mm"
-    },
-
-]
-
-external_wall_insulation_parts = [
-    {
-        "id": 4,
-        "type": "external_wall_insulation",
-        "description": "Mineral Wool External Wall Insulation",
-        "depths": [30, 50, 70, 80, 90, 100, 150, 200],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.0278,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.036,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://insulationgo.co.uk/100mm-rockwool-external-wall-insulation-dual-density-slabs-a1-non"
-                "-combustible-slab-ewi-render-fire/"
-    },
-    {
-        "id": 5,
-        "type": "external_wall_insulation",
-        "description": "Expanded Polystyrene External Wall Insulation",
-        "depths": [25, 50, 100, 125],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02703,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.037,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationking.co.uk/products/polystyrene-eps70?variant=44156186558759"
-    },
-    {
-        "id": 6,
-        "type": "external_wall_insulation",
-        "description": "Phenolic Foam External Wall Insulation",
-        "depths": [20, 50, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.043478260869565216,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.023,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationshop.co/20mm_kooltherm_k5_external_wall_kingspan.html"
-    },
-    {
-        "id": 7,
-        "type": "external_wall_insulation",
-        "description": "Polyisocyanurate/Polyurethane Foam External Wall Insulation",
-        "depths": [],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": None,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": None,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": None
-    },
-    {
-        "id": 8,
-        "type": "external_wall_insulation",
-        "description": "Wood Fiber External Wall Insulation",
-        "depths": [40, 60],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.023255813953488375,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.043,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.mikewye.co.uk/product/steico-duo-dry/"
-    },
-    {
-        "id": 9,
-        "type": "external_wall_insulation",
-        "description": "Aerogel External Wall Insulation",
-        "depths": [10, 20, 30, 40, 50, 60, 70],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.06666666666666667,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.015,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.thermablok.co.uk/site/wp-content/uploads/2022/09/Thermablok-Aerogel-Insulation-Blanket"
-                "-TDS-AIS-and-Steel-Related-Details.pdf"
-    },
-    {
-        "id": 10,
-        "type": "external_wall_insulation",
-        "description": "Vacuum Insulation Panels External Wall Insulation",
-        "depths": [45, 60],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.16666666666666666,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.006,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": None
-    }
-]
-
-internal_wall_insulation_parts = [
-    {
-        "id": 11,
-        "type": "internal_wall_insulation",
-        "description": "Rigid Insulation Boards Internal Wall Insulation",
-        "depths": [25, 40, 50, 75, 100],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.026315789473684213,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.038,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationshop.co/25mm_polystyrene_insulation_eps_70jablite.html"
-    },
-    {
-        "id": 12,
-        "type": "internal_wall_insulation",
-        "description": "Mineral Wool Internal Wall Insulation",
-        "depths": [140],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.035,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.rockwool.com/siteassets/rw-uk/downloads/datasheets/flexi.pdf"
-    },
-    {
-        "id": 13,
-        "type": "internal_wall_insulation",
-        "description": "Insulated Plasterboard Internal Wall Insulation",
-        "depths": [25, 80],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.02857142857142857,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.019,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.kingspan.com/gb/en/products/insulation-boards/wall-insulation-boards/kooltherm-k118"
-                "-insulated-plasterboard/"
-    },
-    {
-        "id": 14,
-        "type": "internal_wall_insulation",
-        "description": "Reflective Internal Wall Insulation",
-        "depths": [],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": None,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": None,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": None
-    },
-    {
-        "id": 15,
-        "type": "internal_wall_insulation",
-        "description": "Vacuum Insulation Panels Wall Insulation",
-        "depths": [20, 30],
-        "depth_unit": "mm",
-        "cost": None,
-        "cost_unit": None,
-        "r_value_per_mm": 0.125,
-        "r_value_unit": "square_meter_kelvin_per_watt",
-        "thermal_conductivity": 0.008,
-        "thermal_conductivity_unit": "watt_per_meter_kelvin",
-        "link": "https://www.insulationsuperstore.co.uk/product/vacutherm-vacupor-nt-b2-vacuum-insulated-panel-1m-x"
-                "-600mm-x-30mm.html"
-    },
-]
-
-materials = (
-    suspended_floor_insulation_parts + solid_floor_insulation_parts + external_wall_insulation_parts + \
-    internal_wall_insulation_parts
-)

model_data/simulation_system/app.py

@@ -299,6 +299,7 @@ def app():
                         # Clean using averages
 
                         avgs = iterative_filtering(cleaning_averages, property_data)
+                        # TODO: Should probably do a mean/median?
                         field_value = avgs[field].iloc[0]
 
                         if pd.isnull(field_value):
@@ -343,6 +344,7 @@ def app():
                 rdsap_change = ending_record[RDSAP_RESPONSE] - starting_record[RDSAP_RESPONSE]
                 heat_demand_change = ending_record[HEAT_DEMAND_RESPONSE] - starting_record[HEAT_DEMAND_RESPONSE]
 
+                # TODO: Should this be <= 0?
                 if rdsap_change == 0:
                     # Assumption: We aren't interested in records that exhibit no change
                     continue