Model/backend/app/plan/temp_script_for_flight.py

from datetime import datetime

import pandas as pd
from epc_api.client import EpcClient
from fastapi import APIRouter, Depends
from sqlalchemy.exc import IntegrityError, OperationalError
from sqlalchemy.orm import sessionmaker
from starlette.responses import Response

from backend.app.config import get_settings
from backend.app.db.connection import db_engine
from backend.app.db.functions.materials_functions import get_materials
from backend.app.db.functions.portfolio_functions import aggregate_portfolio_recommendations
from backend.app.db.functions.property_functions import (
    create_property, create_property_details_epc, create_property_targets, update_property_data
)
from backend.app.db.functions.recommendations_functions import (
    create_plan, create_plan_recommendations, upload_recommendations
)
from backend.app.db.models.portfolio import rating_lookup
from backend.app.dependencies import validate_token
from backend.app.plan.schemas import PlanTriggerRequest
from backend.app.plan.utils import (
    create_recommendation_scoring_data, filter_materials, get_cleaned, insert_temp_recommendation_id
)
from backend.app.utils import epc_to_sap_lower_bound, read_csv_from_s3, read_parquet_from_s3

from backend.ml_models.sap_change_model.api import SAPChangeModelAPI
from backend.Property import Property
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from recommendations.FloorRecommendations import FloorRecommendations
from recommendations.optimiser.CostOptimiser import CostOptimiser
from recommendations.optimiser.GainOptimiser import GainOptimiser
from recommendations.optimiser.optimiser_functions import prepare_input_measures
from recommendations.WallRecommendations import WallRecommendations
from utils.logger import setup_logger
from utils.s3 import read_dataframe_from_s3_parquet

logger = setup_logger()

import pickle

with open('local_data.pickle', 'rb') as f:
    local_data = pickle.load(f)

with open("property_dimensions.pickle", "rb") as f:
    property_dimensions = pickle.load(f)

with open("sap_change_dataset.pickle", "rb") as f:
    sap_change_dataset = pickle.load(f)

created_at = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")

plan_input = local_data["plan_input"]
uprn_filenames = local_data["uprn_filenames"]
local_property_data = local_data["local_property_data"]
materials = local_data["materials"]
materials_by_type = filter_materials(materials)
cleaned = local_data["cleaned"]
cleaning_data = local_data["cleaning_data"]

# Need to find some proper materials
materials_by_type["walls"] += [
    {'id': 4, 'type': 'cavity_wall_insulation', 'description': 'Example Material 1',
     'depths': None,
     'depth_unit': None, 'cost': 20,
     'cost_unit': 'gbp_sq_meter', 'r_value_per_mm': 0.0278, 'r_value_unit': 'square_meter_kelvin_per_watt',
     'thermal_conductivity': 0.036, 'thermal_conductivity_unit': 'watt_per_meter_kelvin',
     'link': None, 'created_at': None, 'is_active': True},
    {'id': 10, 'type': "cavity_wall_insulation", 'description': 'Example Material 2',
     'depths': None, 'depth_unit': None, 'cost': 25, 'cost_unit': 'gbp_sq_meter',
     'r_value_per_mm': 0.02631579, 'r_value_unit': 'square_meter_kelvin_per_watt', 'thermal_conductivity': 0.038,
     'thermal_conductivity_unit': 'watt_per_meter_kelvin',
     'link': None,
     'created_at': None, 'is_active': True}
]

epc_client = EpcClient(auth_token="NO-TOKEN")

input_properties = []
for i, config in enumerate(plan_input):
    property_id = local_property_data[i]["id"]
    input_properties.append(
        Property(
            postcode=config['postcode'],
            address1=config['address'],
            epc_client=epc_client,
            id=property_id
        )
    )

logger.info("Getting EPC, and spatial data")
for i, p in enumerate(input_properties):
    p.data = local_property_data[i]["data"]
    p.uprn = local_property_data[i]["uprn"]
    p.id = local_property_data[i]["id"]
    p.full_sap_epc = local_property_data[i]["full_sap_epc"]
    p.old_data = local_property_data[i]["old_data"]
    p.is_listed = False
    p.in_conservation_area = False
    p.is_heritage = False

    p.set_year_built()

    # TODO: TESTING
    p.data['number-habitable-rooms'] = 3

recommendations = {}
recommendations_scoring_data = []

for p in input_properties:
    property_recommendations = []

    # Property recommendations
    p.get_components(cleaned)

    # Floor recommendations
    floor_recommender = FloorRecommendations(
        property_instance=p,
        materials=materials_by_type["floor"],
    )
    floor_recommender.recommend()

    if floor_recommender.recommendations:
        property_recommendations.append(floor_recommender.recommendations)

    # Wall recommendations

    wall_recomender = WallRecommendations(
        property_instance=p,
        materials=materials_by_type["walls"]
    )
    wall_recomender.recommend()

    if wall_recomender.recommendations:
        property_recommendations.append(wall_recomender.recommendations)

    # We insert temporary ids into the recommendations which is important for the optimiser later
    property_recommendations = insert_temp_recommendation_id(property_recommendations)

    if not property_recommendations:
        continue

    recommendations[p.id] = property_recommendations

    # Finally, we'll prepare data for predicting the impact on SAP
    # TODO: We should use the cleaned data from get_components in the data rather than the raw
    #       values. We should create a method in Property which takes the EPC data and inserts the cleaned
    #       data

    data_processor = DataProcessor(None, newdata=True)
    data_processor.insert_data(pd.DataFrame([p.data.copy()]))
    data_processor.pre_process()

    starting_epc_data = data_processor.get_component_features(suffix="_STARTING")
    ending_epc_data = data_processor.get_component_features(suffix="_ENDING")
    fixed_data = data_processor.get_fixed_features()

    # We update the ending record with the recommended updates and we set lodgement date to today
    ending_epc_data["LODGEMENT_DATE_ENDING"] = created_at

    for recommendations_by_type in property_recommendations:
        for rec in recommendations_by_type:
            scoring_dict = create_recommendation_scoring_data(
                property=p,
                recommendation=rec,
                starting_epc_data=starting_epc_data,
                ending_epc_data=ending_epc_data,
                fixed_data=fixed_data,
            )

            recommendations_scoring_data.append(scoring_dict)

# cleanup
del data_processor