From cbef2b5e45573291f236abdf683c3b81896b58bd Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Fri, 6 Oct 2023 15:14:14 +0100 Subject: [PATCH] set up pre-flight --- backend/Property.py | 47 +++--- backend/app/plan/temp_script_for_flight.py | 168 +++++++++++++++++++-- recommendations/rdsap_tables.py | 1 + recommendations/recommendation_utils.py | 8 + 4 files changed, 185 insertions(+), 39 deletions(-) diff --git a/backend/Property.py b/backend/Property.py index 92fc41e9..79c79f8a 100644 --- a/backend/Property.py +++ b/backend/Property.py @@ -9,7 +9,7 @@ from utils.s3 import read_dataframe_from_s3_parquet from epc_api.client import EpcClient from BaseUtility import Definitions from recommendations.rdsap_tables import england_wales_age_band_lookup -from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type +from recommendations.recommendation_utils import estimate_floors, estimate_perimeter, get_wall_type, estimate_wall_area ENVIRONMENT = os.environ.get('ENVIRONMENT', 'dev') EPC_AUTH_TOKEN = os.environ.get('EPC_AUTH_TOKEN') @@ -268,12 +268,9 @@ class Property(Definitions): self.set_count_variables() self.set_heat_loss_corridor() self.set_mains_gas() - self.set_floor_height() - self.set_wall_area() self.set_age_band() - self.set_basic_property_attributes() - self.set_wall_type() + self.set_basic_property_dimensions() for description, attribute in cleaned.items(): @@ -292,6 +289,8 @@ class Property(Definitions): raise ValueError("Either No attributes or multiple found for %s" % description) setattr(self, self.ATTRIBUTE_MAP[description], attributes[0]) + self.set_wall_type() + def set_age_band(self): """ Sets a cleaned version of the age band of the property given the EPC data @@ -381,17 +380,6 @@ class Property(Definitions): else: self.mains_gas = map[self.data["mains-gas-flag"]] - def set_floor_height(self): - """ - Sets the floor height of the property - :return: - """ - - if self.data["floor-height"] == "" or self.data["floor-height"] in self.DATA_ANOMALY_MATCHES: - self.floor_height = None - else: - self.floor_height = float(self.data["floor-height"]) - def _clean_upload_data(self, to_update): for k, v in to_update.items(): if v in self.DATA_ANOMALY_MATCHES: @@ -475,13 +463,6 @@ class Property(Definitions): return property_details_epc - def set_wall_area(self): - """ - This method is placeholder - It implements our floor area model to produce an estimate of the property's insulatable wall area - While we do not have the - """ - def get_spatial_data(self, uprn_filenames): """ @@ -509,7 +490,7 @@ class Property(Definitions): # Pull out spatial features self.set_spatial(spatial) - def set_basic_property_attributes(self): + def set_basic_property_dimensions(self): """ This method sets the number of floors of the property, using a simple approach based on an estimate for average room size, number of rooms and total floor area @@ -526,10 +507,6 @@ class Property(Definitions): number_of_rooms = float(self.data["number-habitable-rooms"]) - self.perimeter = estimate_perimeter( - self.floor_area / self.number_of_floors, number_of_rooms / self.number_of_floors - ) - if self.data["property-type"] == "House": self.number_of_floors = estimate_floors(self.floor_area, number_of_rooms) elif self.data["property-type"] == "Flat": @@ -537,6 +514,20 @@ class Property(Definitions): else: raise NotImplementedError("Implement me") + if self.data["floor-height"] == "" or self.data["floor-height"] in self.DATA_ANOMALY_MATCHES: + self.floor_height = 2.3 + print("This is where we should fill with cleaned data") + else: + self.floor_height = float(self.data["floor-height"]) + + self.perimeter = estimate_perimeter( + self.floor_area / self.number_of_floors, number_of_rooms / self.number_of_floors + ) + + self.insulation_wall_area = estimate_wall_area( + num_floors=self.number_of_floors, floor_height=self.floor_height, perimeter=self.perimeter + ) + def set_wall_type(self): """ This method sets the wall type of the property, using a simple approach based on the wall description diff --git a/backend/app/plan/temp_script_for_flight.py b/backend/app/plan/temp_script_for_flight.py index 1a251f26..8de3ad3c 100644 --- a/backend/app/plan/temp_script_for_flight.py +++ b/backend/app/plan/temp_script_for_flight.py @@ -1,14 +1,160 @@ -local_data = { - "plan_input": plan_input, - "uprn_filenames": uprn_filenames, - "local_property_data": local_property_data, - "materials": materials, - "materials_by_type": materials_by_type, - "cleaned": cleaned, - "cleaning_data": cleaning_data -} +from datetime import datetime + +import pandas as pd +from epc_api.client import EpcClient +from fastapi import APIRouter, Depends +from sqlalchemy.exc import IntegrityError, OperationalError +from sqlalchemy.orm import sessionmaker +from starlette.responses import Response + +from backend.app.config import get_settings +from backend.app.db.connection import db_engine +from backend.app.db.functions.materials_functions import get_materials +from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations +from backend.app.db.functions.property_functions import ( + create_property, create_property_details_epc, create_property_targets, update_property_data +) +from backend.app.db.functions.recommendations_functions import ( + create_plan, create_plan_recommendations, upload_recommendations +) +from backend.app.db.models.portfolio import rating_lookup +from backend.app.dependencies import validate_token +from backend.app.plan.schemas import PlanTriggerRequest +from backend.app.plan.utils import ( + create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id +) +from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 + +from backend.ml_models.sap_change_model.api import SAPChangeModelAPI +from backend.Property import Property +from etl.epc.DataProcessor import DataProcessor +from etl.epc.settings import COLUMNS_TO_MERGE_ON +from recommendations.FloorRecommendations import FloorRecommendations +from recommendations.optimiser.CostOptimiser import CostOptimiser +from recommendations.optimiser.GainOptimiser import GainOptimiser +from recommendations.optimiser.optimiser_functions import prepare_input_measures +from recommendations.WallRecommendations import WallRecommendations +from utils.logger import setup_logger +from utils.s3 import read_dataframe_from_s3_parquet + +logger = setup_logger() import pickle -with open('local_data.pickle', 'wb') as f: - pickle.dump(local_data, f) +with open('local_data.pickle', 'rb') as f: + local_data = pickle.load(f) + +with open("property_dimensions.pickle", "rb") as f: + property_dimensions = pickle.load(f) + +with open("sap_change_dataset.pickle", "rb") as f: + sap_change_dataset = pickle.load(f) + +created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + +plan_input = local_data["plan_input"] +uprn_filenames = local_data["uprn_filenames"] +local_property_data = local_data["local_property_data"] +materials = local_data["materials"] +materials_by_type = local_data["materials_by_type"] +cleaned = local_data["cleaned"] +cleaning_data = local_data["cleaning_data"] + +epc_client = EpcClient(auth_token="NO-TOKEN") + +input_properties = [] +for i, config in enumerate(plan_input): + property_id = local_property_data[i]["id"] + input_properties.append( + Property( + postcode=config['postcode'], + address1=config['address'], + epc_client=epc_client, + id=property_id + ) + ) + +logger.info("Getting EPC, and spatial data") +for i, p in enumerate(input_properties): + p.data = local_property_data[i]["data"] + p.uprn = local_property_data[i]["uprn"] + p.id = local_property_data[i]["id"] + p.full_sap_epc = local_property_data[i]["full_sap_epc"] + p.old_data = local_property_data[i]["old_data"] + p.is_listed = False + p.in_conservation_area = False + p.is_heritage = False + + p.set_year_built() + + # TODO: TESTING + p.data['number-habitable-rooms'] = 3 + +recommendations = {} +recommendations_scoring_data = [] + +for p in input_properties: + property_recommendations = [] + + # Property recommendations + p.get_components(cleaned) + + # Floor recommendations + floor_recommender = FloorRecommendations( + property_instance=p, + materials=materials_by_type["suspended_floor_insulation"] + materials_by_type["solid_floor_insulation"], + ) + floor_recommender.recommend() + + if floor_recommender.recommendations: + property_recommendations.append(floor_recommender.recommendations) + + # Wall recommendations + + wall_recomender = WallRecommendations( + property_instance=p, + materials=materials_by_type["external_wall_insulation"] + materials_by_type["internal_wall_insulation"] + ) + wall_recomender.recommend() + + if wall_recomender.recommendations: + property_recommendations.append(wall_recomender.recommendations) + + # We insert temporary ids into the recommendations which is important for the optimiser later + property_recommendations = insert_temp_recommendation_id(property_recommendations) + + if not property_recommendations: + continue + + recommendations[p.id] = property_recommendations + + # Finally, we'll prepare data for predicting the impact on SAP + # TODO: We should use the cleaned data from get_components in the data rather than the raw + # values. We should create a method in Property which takes the EPC data and inserts the cleaned + # data + + data_processor = DataProcessor(None, newdata=True) + data_processor.insert_data(pd.DataFrame([p.data.copy()])) + data_processor.pre_process() + + starting_epc_data = data_processor.get_component_features(suffix="_STARTING") + ending_epc_data = data_processor.get_component_features(suffix="_ENDING") + fixed_data = data_processor.get_fixed_features() + + # We update the ending record with the recommended updates and we set lodgement date to today + ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at + + for recommendations_by_type in property_recommendations: + for rec in recommendations_by_type: + scoring_dict = create_recommendation_scoring_data( + property=p, + recommendation=rec, + starting_epc_data=starting_epc_data, + ending_epc_data=ending_epc_data, + fixed_data=fixed_data, + ) + + recommendations_scoring_data.append(scoring_dict) + +# cleanup +del data_processor diff --git a/recommendations/rdsap_tables.py b/recommendations/rdsap_tables.py index 589d3bb4..a622a4cc 100644 --- a/recommendations/rdsap_tables.py +++ b/recommendations/rdsap_tables.py @@ -482,6 +482,7 @@ FLOOR_LEVEL_MAP = { "Basement": -1, "Ground": 0, "ground floor": 0, + "mid floor": 1, "20+": 20, "21st or above": 21, **{str(i).zfill(2): i for i in range(0, 21)}, diff --git a/recommendations/recommendation_utils.py b/recommendations/recommendation_utils.py index 71fa9e53..8289624d 100644 --- a/recommendations/recommendation_utils.py +++ b/recommendations/recommendation_utils.py @@ -493,3 +493,11 @@ def estimate_floors(floor_area, num_rooms): floors = round(floors) return floors + + +def estimate_wall_area(num_floors, floor_height, perimeter): + wall_area_one_floor = perimeter * floor_height + + total_wall_area = wall_area_one_floor * num_floors + + return total_wall_area