from datetime import datetime import pandas as pd from epc_api.client import EpcClient from fastapi import APIRouter, Depends from sqlalchemy.exc import IntegrityError, OperationalError from sqlalchemy.orm import sessionmaker from starlette.responses import Response from backend.app.config import get_settings from backend.app.db.connection import db_engine from backend.app.db.functions.materials_functions import get_materials from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations from backend.app.db.functions.property_functions import ( create_property, create_property_details_epc, create_property_targets, update_property_data ) from backend.app.db.functions.recommendations_functions import ( create_plan, create_plan_recommendations, upload_recommendations ) from backend.app.db.models.portfolio import rating_lookup from backend.app.dependencies import validate_token from backend.app.plan.schemas import PlanTriggerRequest from backend.app.plan.utils import ( create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id ) from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3 from backend.ml_models.sap_change_model.api import SAPChangeModelAPI from backend.Property import Property from etl.epc.DataProcessor import DataProcessor from etl.epc.settings import COLUMNS_TO_MERGE_ON from recommendations.FloorRecommendations import FloorRecommendations from recommendations.optimiser.CostOptimiser import CostOptimiser from recommendations.optimiser.GainOptimiser import GainOptimiser from recommendations.optimiser.optimiser_functions import prepare_input_measures from recommendations.WallRecommendations import WallRecommendations from utils.logger import setup_logger from utils.s3 import read_dataframe_from_s3_parquet logger = setup_logger() import pickle with open('local_data.pickle', 'rb') as f: local_data = pickle.load(f) with open("property_dimensions.pickle", "rb") as f: property_dimensions = pickle.load(f) with open("sap_change_dataset.pickle", "rb") as f: sap_change_dataset = pickle.load(f) created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") plan_input = local_data["plan_input"] uprn_filenames = local_data["uprn_filenames"] local_property_data = local_data["local_property_data"] materials = local_data["materials"] materials_by_type = filter_materials(materials) cleaned = local_data["cleaned"] cleaning_data = local_data["cleaning_data"] # Need to find some proper materials materials_by_type["walls"] += [ {'id': 4, 'type': 'cavity_wall_insulation', 'description': 'Example Material 1', 'depths': None, 'depth_unit': None, 'cost': 20, 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.0278, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.036, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': None, 'created_at': None, 'is_active': True}, {'id': 10, 'type': "cavity_wall_insulation", 'description': 'Example Material 2', 'depths': None, 'depth_unit': None, 'cost': 25, 'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.02631579, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038, 'thermal_conductivity_unit': 'watt_per_meter_kelvin', 'link': None, 'created_at': None, 'is_active': True} ] epc_client = EpcClient(auth_token="NO-TOKEN") input_properties = [] for i, config in enumerate(plan_input): property_id = local_property_data[i]["id"] input_properties.append( Property( postcode=config['postcode'], address1=config['address'], epc_client=epc_client, id=property_id ) ) logger.info("Getting EPC, and spatial data") for i, p in enumerate(input_properties): p.data = local_property_data[i]["data"] p.uprn = local_property_data[i]["uprn"] p.id = local_property_data[i]["id"] p.full_sap_epc = local_property_data[i]["full_sap_epc"] p.old_data = local_property_data[i]["old_data"] p.is_listed = False p.in_conservation_area = False p.is_heritage = False p.set_year_built() # TODO: TESTING p.data['number-habitable-rooms'] = 3 recommendations = {} recommendations_scoring_data = [] for p in input_properties: property_recommendations = [] # Property recommendations p.get_components(cleaned) # Floor recommendations floor_recommender = FloorRecommendations( property_instance=p, materials=materials_by_type["floor"], ) floor_recommender.recommend() if floor_recommender.recommendations: property_recommendations.append(floor_recommender.recommendations) # Wall recommendations wall_recomender = WallRecommendations( property_instance=p, materials=materials_by_type["walls"] ) wall_recomender.recommend() if wall_recomender.recommendations: property_recommendations.append(wall_recomender.recommendations) # We insert temporary ids into the recommendations which is important for the optimiser later property_recommendations = insert_temp_recommendation_id(property_recommendations) if not property_recommendations: continue recommendations[p.id] = property_recommendations # Finally, we'll prepare data for predicting the impact on SAP # TODO: We should use the cleaned data from get_components in the data rather than the raw # values. We should create a method in Property which takes the EPC data and inserts the cleaned # data data_processor = DataProcessor(None, newdata=True) data_processor.insert_data(pd.DataFrame([p.data.copy()])) data_processor.pre_process() starting_epc_data = data_processor.get_component_features(suffix="_STARTING") ending_epc_data = data_processor.get_component_features(suffix="_ENDING") fixed_data = data_processor.get_fixed_features() # We update the ending record with the recommended updates and we set lodgement date to today ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at for recommendations_by_type in property_recommendations: for rec in recommendations_by_type: scoring_dict = create_recommendation_scoring_data( property=p, recommendation=rec, starting_epc_data=starting_epc_data, ending_epc_data=ending_epc_data, fixed_data=fixed_data, ) recommendations_scoring_data.append(scoring_dict) # cleanup del data_processor