Model/etl/customers/ksquared/Wave3 Modelling.py

import os
import time

from dotenv import load_dotenv
from tqdm import tqdm
import pandas as pd
from etl.find_my_epc.RetrieveFindMyEpc import RetrieveFindMyEpc
from backend.SearchEpc import SearchEpc
from utils.s3 import save_csv_to_s3

load_dotenv(dotenv_path="backend/.env")
EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
USER_ID = 8
PORTFOLIO_ID = 117


def app():
    """
    This script prepares the asset lists for the additional housing associations, CAHA and Hornsey Housing Trust,
    that are forming a consortium led by AIHA
    :return:
    """

    hornsey_asset_list = pd.read_excel(
        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/AIHA/SHDF - Template - EOI - Hornsey Housing "
        "Trust.xlsx",
        sheet_name="Ksquared-All units information",
        header=3
    )

    # We don't need the first row
    hornsey_asset_list = hornsey_asset_list.iloc[1:]
    # Fill NA values with empty strings
    hornsey_asset_list = hornsey_asset_list.fillna("")
    hornsey_asset_list["Address letter or number"] = hornsey_asset_list["Address letter or number"].astype(
        str
    ).str.strip()
    hornsey_asset_list["Postcode"] = hornsey_asset_list["Postcode"].astype(str).str.strip()
    hornsey_asset_list["Street address"] = hornsey_asset_list["Street address"].astype(str).str.strip()
    # Replace double spaces
    for col in ["Address letter or number", "Street address", "Postcode"]:
        hornsey_asset_list[col] = hornsey_asset_list[col].str.replace("  ", " ")

    hornsey_asset_list = hornsey_asset_list[hornsey_asset_list["Address letter or number"] != ""]

    missed_uprns = {
        "Flat 13A Stowell House": 100021213098,
        "Flat 24 Stowell House": 100021213110,
        "Flat 1 36 Haringey Park": None
    }
    extracted_data = []
    asset_list = []
    for _, home in tqdm(hornsey_asset_list.iterrows(), total=len(hornsey_asset_list)):

        if home["Address letter or number"] == "Flat 1 36 Haringey Park":
            continue

        # Some properties do not have an epc
        if not home["Energy starting band (EPC)"]:
            asset_list.append(
                {
                    "uprn": missed_uprns[home["Address letter or number"]],
                    "address": home["Address letter or number"],
                    "postcode": home["Postcode"],
                    "property_type": "Flat",  # They're all flats
                }
            )
            continue

        unit_number = home["Address letter or number"]
        street = home["Street address"]
        postcode = home["Postcode"]
        address = ", ".join([x for x in [unit_number, street] if x])
        find_epc_searcher = RetrieveFindMyEpc(address=address, postcode=postcode)
        find_epc_data = find_epc_searcher.retrieve_newest_find_my_epc_data()
        time.sleep(0.5)
        # We need uprn
        searcher = SearchEpc(
            address1=address,
            postcode=postcode,
            auth_token=EPC_AUTH_TOKEN,
            os_api_key="",
            full_address=address,
        )
        searcher.find_property(skip_os=True)
        newest_epc = searcher.newest_epc
        if newest_epc["current-energy-efficiency"] != home["Energy starting band (EPC)"].split("-")[1]:
            raise Exception("Something went wrong with the EPC data")

        extracted_data.append(
            {
                "uprn": newest_epc["uprn"],
                **find_epc_data,
                "hotwater-description": newest_epc["hotwater-description"],
            }
        )

        asset_list.append(
            {
                "uprn": newest_epc["uprn"],
                "address": home["Address letter or number"],
                "postcode": home["Postcode"],
                "property_type": "Flat",  # They're all flats
            }
        )

    # We format the extracted data so that is has the same structure as non-intrusive recommendations
    # We then get the UPRNs and create the asset list

    non_invasive_recommendations = [
        {
            "uprn": r["uprn"],
            "recommendations": r["recommendations"]
        } for r in extracted_data
    ]
    for r in non_invasive_recommendations:
        new_recommendations = []
        extracted = [r for r in extracted_data if r["uprn"] == r["uprn"]][0]
        for rec in r["recommendations"]:
            if extracted["hotwater-description"] == "Gas boiler/circulator, no cylinder thermostat":
                if rec["type"] in ["hot_water_tank_insulation", "cylinder_thermostat"]:
                    continue
            rec["survey"] = False
            new_recommendations.append(rec)
        r["recommendations"] = new_recommendations

    # Store the asset list in s3
    filename = f"{USER_ID}/{PORTFOLIO_ID}/asset_list.csv"
    save_csv_to_s3(
        dataframe=pd.DataFrame(asset_list),
        bucket_name="retrofit-plan-inputs-dev",
        file_name=filename
    )

    # Store the non-invasive recommendations in s3
    non_invasive_recommendations_filename = f"{USER_ID}/{PORTFOLIO_ID}/non_invasive_recommendations.csv"
    save_csv_to_s3(
        dataframe=pd.DataFrame(non_invasive_recommendations),
        bucket_name="retrofit-plan-inputs-dev",
        file_name=non_invasive_recommendations_filename
    )

    body = {
        "portfolio_id": str(PORTFOLIO_ID),
        "housing_type": "Social",
        "goal": "Increasing EPC",
        "goal_value": "C",
        "trigger_file_path": filename,
        "already_installed_file_path": "",
        "patches_file_path": "",
        "non_invasive_recommendations_file_path": non_invasive_recommendations_filename,
        "valuation_file_path": "",
        "scenario_name": "Wave 3 Packages",
        "multi_plan": True,
        "budget": None,
        "exclusions": ["boiler_upgrade"]
    }
    print(body)