set up pre-flight

backend/Property.py

@@ -9,7 +9,7 @@ from utils.s3 import read_dataframe_from_s3_parquet
 from epc_api.client import EpcClient
 from BaseUtility import Definitions
 from recommendations.rdsap_tables import england_wales_age_band_lookup
-from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type
+from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area
 
 ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev')
 EPC_AUTH_TOKEN = os.environ.get('EPC_AUTH_TOKEN')
@@ -268,12 +268,9 @@ class Property(Definitions):
         self.set_count_variables()
         self.set_heat_loss_corridor()
         self.set_mains_gas()
-        self.set_floor_height()
-        self.set_wall_area()
         self.set_age_band()
 
-        self.set_basic_property_attributes()
+        self.set_basic_property_dimensions()
-        self.set_wall_type()
 
         for description, attribute in cleaned.items():
 
@@ -292,6 +289,8 @@ class Property(Definitions):
                 raise ValueError("Either No attributes or multiple found for %s" % description)
             setattr(self, self.ATTRIBUTE_MAP[description], attributes[0])
 
+        self.set_wall_type()
+
     def set_age_band(self):
         """
         Sets a cleaned version of the age band of the property given the EPC data
@@ -381,17 +380,6 @@ class Property(Definitions):
         else:
             self.mains_gas = map[self.data["mains-gas-flag"]]
 
-    def set_floor_height(self):
-        """
-        Sets the floor height of the property
-        :return:
-        """
-
-        if self.data["floor-height"] == "" or self.data["floor-height"] in self.DATA_ANOMALY_MATCHES:
-            self.floor_height = None
-        else:
-            self.floor_height = float(self.data["floor-height"])
-
     def _clean_upload_data(self, to_update):
         for k, v in to_update.items():
             if v in self.DATA_ANOMALY_MATCHES:
@@ -475,13 +463,6 @@ class Property(Definitions):
 
         return property_details_epc
 
-    def set_wall_area(self):
-        """
-        This method is placeholder
-        It implements our floor area model to produce an estimate of the property's insulatable wall area
-        While we do not have the
-        """
-
     def get_spatial_data(self, uprn_filenames):
 
         """
@@ -509,7 +490,7 @@ class Property(Definitions):
         # Pull out spatial features
         self.set_spatial(spatial)
 
-    def set_basic_property_attributes(self):
+    def set_basic_property_dimensions(self):
         """
         This method sets the number of floors of the property, using a simple approach based on an estimate for
         average room size, number of rooms and total floor area
@@ -526,10 +507,6 @@ class Property(Definitions):
 
         number_of_rooms = float(self.data["number-habitable-rooms"])
 
-        self.perimeter = estimate_perimeter(
-            self.floor_area / self.number_of_floors, number_of_rooms / self.number_of_floors
-        )
-
         if self.data["property-type"] == "House":
             self.number_of_floors = estimate_floors(self.floor_area, number_of_rooms)
         elif self.data["property-type"] == "Flat":
@@ -537,6 +514,20 @@ class Property(Definitions):
         else:
             raise NotImplementedError("Implement me")
 
+        if self.data["floor-height"] == "" or self.data["floor-height"] in self.DATA_ANOMALY_MATCHES:
+            self.floor_height = 2.3
+            print("This is where we should fill with cleaned data")
+        else:
+            self.floor_height = float(self.data["floor-height"])
+
+        self.perimeter = estimate_perimeter(
+            self.floor_area / self.number_of_floors, number_of_rooms / self.number_of_floors
+        )
+
+        self.insulation_wall_area = estimate_wall_area(
+            num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter
+        )
+
     def set_wall_type(self):
         """
         This method sets the wall type of the property, using a simple approach based on the wall description

backend/app/plan/temp_script_for_flight.py

@@ -1,14 +1,160 @@
-local_data = {
+from datetime import datetime
-    "plan_input": plan_input,
+
-    "uprn_filenames": uprn_filenames,
+import pandas as pd
-    "local_property_data": local_property_data,
+from epc_api.client import EpcClient
-    "materials": materials,
+from fastapi import APIRouter, Depends
-    "materials_by_type": materials_by_type,
+from sqlalchemy.exc import IntegrityError, OperationalError
-    "cleaned": cleaned,
+from sqlalchemy.orm import sessionmaker
-    "cleaning_data": cleaning_data
+from starlette.responses import Response
-}
+
+from backend.app.config import get_settings
+from backend.app.db.connection import db_engine
+from backend.app.db.functions.materials_functions import get_materials
+from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations
+from backend.app.db.functions.property_functions import (
+    create_property, create_property_details_epc, create_property_targets, update_property_data
+)
+from backend.app.db.functions.recommendations_functions import (
+    create_plan, create_plan_recommendations, upload_recommendations
+)
+from backend.app.db.models.portfolio import rating_lookup
+from backend.app.dependencies import validate_token
+from backend.app.plan.schemas import PlanTriggerRequest
+from backend.app.plan.utils import (
+    create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id
+)
+from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3
+
+from backend.ml_models.sap_change_model.api import SAPChangeModelAPI
+from backend.Property import Property
+from etl.epc.DataProcessor import DataProcessor
+from etl.epc.settings import COLUMNS_TO_MERGE_ON
+from recommendations.FloorRecommendations import FloorRecommendations
+from recommendations.optimiser.CostOptimiser import CostOptimiser
+from recommendations.optimiser.GainOptimiser import GainOptimiser
+from recommendations.optimiser.optimiser_functions import prepare_input_measures
+from recommendations.WallRecommendations import WallRecommendations
+from utils.logger import setup_logger
+from utils.s3 import read_dataframe_from_s3_parquet
+
+logger = setup_logger()
 
 import pickle
 
-with open('local_data.pickle', 'wb') as f:
+with open('local_data.pickle', 'rb') as f:
-    pickle.dump(local_data, f)
+    local_data = pickle.load(f)
+
+with open("property_dimensions.pickle", "rb") as f:
+    property_dimensions = pickle.load(f)
+
+with open("sap_change_dataset.pickle", "rb") as f:
+    sap_change_dataset = pickle.load(f)
+
+created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+
+plan_input = local_data["plan_input"]
+uprn_filenames = local_data["uprn_filenames"]
+local_property_data = local_data["local_property_data"]
+materials = local_data["materials"]
+materials_by_type = local_data["materials_by_type"]
+cleaned = local_data["cleaned"]
+cleaning_data = local_data["cleaning_data"]
+
+epc_client = EpcClient(auth_token="NO-TOKEN")
+
+input_properties = []
+for i, config in enumerate(plan_input):
+    property_id = local_property_data[i]["id"]
+    input_properties.append(
+        Property(
+            postcode=config['postcode'],
+            address1=config['address'],
+            epc_client=epc_client,
+            id=property_id
+        )
+    )
+
+logger.info("Getting EPC, and spatial data")
+for i, p in enumerate(input_properties):
+    p.data = local_property_data[i]["data"]
+    p.uprn = local_property_data[i]["uprn"]
+    p.id = local_property_data[i]["id"]
+    p.full_sap_epc = local_property_data[i]["full_sap_epc"]
+    p.old_data = local_property_data[i]["old_data"]
+    p.is_listed = False
+    p.in_conservation_area = False
+    p.is_heritage = False
+
+    p.set_year_built()
+
+    # TODO: TESTING
+    p.data['number-habitable-rooms'] = 3
+
+recommendations = {}
+recommendations_scoring_data = []
+
+for p in input_properties:
+    property_recommendations = []
+
+    # Property recommendations
+    p.get_components(cleaned)
+
+    # Floor recommendations
+    floor_recommender = FloorRecommendations(
+        property_instance=p,
+        materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"],
+    )
+    floor_recommender.recommend()
+
+    if floor_recommender.recommendations:
+        property_recommendations.append(floor_recommender.recommendations)
+
+    # Wall recommendations
+
+    wall_recomender = WallRecommendations(
+        property_instance=p,
+        materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"]
+    )
+    wall_recomender.recommend()
+
+    if wall_recomender.recommendations:
+        property_recommendations.append(wall_recomender.recommendations)
+
+    # We insert temporary ids into the recommendations which is important for the optimiser later
+    property_recommendations = insert_temp_recommendation_id(property_recommendations)
+
+    if not property_recommendations:
+        continue
+
+    recommendations[p.id] = property_recommendations
+
+    # Finally, we'll prepare data for predicting the impact on SAP
+    # TODO: We should use the cleaned data from get_components in the data rather than the raw
+    #       values. We should create a method in Property which takes the EPC data and inserts the cleaned
+    #       data
+
+    data_processor = DataProcessor(None, newdata=True)
+    data_processor.insert_data(pd.DataFrame([p.data.copy()]))
+    data_processor.pre_process()
+
+    starting_epc_data = data_processor.get_component_features(suffix="_STARTING")
+    ending_epc_data = data_processor.get_component_features(suffix="_ENDING")
+    fixed_data = data_processor.get_fixed_features()
+
+    # We update the ending record with the recommended updates and we set lodgement date to today
+    ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at
+
+    for recommendations_by_type in property_recommendations:
+        for rec in recommendations_by_type:
+            scoring_dict = create_recommendation_scoring_data(
+                property=p,
+                recommendation=rec,
+                starting_epc_data=starting_epc_data,
+                ending_epc_data=ending_epc_data,
+                fixed_data=fixed_data,
+            )
+
+            recommendations_scoring_data.append(scoring_dict)
+
+# cleanup
+del data_processor

recommendations/rdsap_tables.py

@@ -482,6 +482,7 @@ FLOOR_LEVEL_MAP = {
     "Basement": -1,
     "Ground": 0,
     "ground floor": 0,
+    "mid floor": 1,
     "20+": 20,
     "21st or above": 21,
     **{str(i).zfill(2): i for i in range(0, 21)},

recommendations/recommendation_utils.py

@@ -493,3 +493,11 @@ def estimate_floors(floor_area, num_rooms):
     floors = round(floors)
 
     return floors
+
+
+def estimate_wall_area(num_floors, floor_height, perimeter):
+    wall_area_one_floor = perimeter * floor_height
+
+    total_wall_area = wall_area_one_floor * num_floors
+
+    return total_wall_area