Model/etl/eligibility/ha_15_32/ha4_app.py

import os
import msgpack
from pathlib import Path
from datetime import datetime
import numpy as np
import pandas as pd
from utils.s3 import read_from_s3
from utils.logger import setup_logger
from dotenv import load_dotenv
from utils.s3 import read_dataframe_from_s3_parquet
from tqdm import tqdm
from backend.SearchEpc import SearchEpc
from etl.eligibility.Eligibility import Eligibility
from etl.eligibility.ha_15_32.app import prepare_model_data_row
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from backend.ml_models.api import ModelApi
from etl.solar.SolarPhotoSupply import SolarPhotoSupply
from recommendations.recommendation_utils import calculate_cavity_age
from recommendation_utils import convert_thickness_to_numeric

import re

EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"

logger = setup_logger()
load_dotenv(ENV_FILE)


def load_ha_4():
    pd.set_option('display.max_rows', 500)
    pd.set_option('display.max_columns', 500)
    pd.set_option('display.width', 1000)

    data = pd.read_csv(f"etl/eligibility/ha_15_32/HA 4 Asset List.csv", low_memory=False)
    return data


def standardise_ha_4(data):
    # Location name contains some strings like {0664} which we remove
    data['Location Name'] = data['Location Name'].str.replace('\{.*?\}', '', regex=True)

    # Trim whitespace from either end of location name
    data["Location Name"] = data["Location Name"].str.strip()

    # Remove any unusable postcodes
    data = data[data["Post Code"] != '\\\\'].copy()

    # Some specific replacements
    data["Location Name"] = np.where(
        data["Location Name"] == "Calderbrook Pl & Cog La",
        "Calderbrook Place",
        data["Location Name"]
    )

    return data


def get_ha_4_data(data, cleaned, cleaning_data, created_at, photo_supply_lookup, floor_area_decile_thresholds):
    scoring_data = []
    results = []
    nodata = []
    for _, property_meta in tqdm(data.iterrows(), total=len(data)):
        # For many of the entries in this dataset, we're actually given an entire building, so we EPCs for every
        # building
        searcher = SearchEpc(
            address1=property_meta["Address Line 1"],
            postcode=property_meta["Post Code"],
            auth_token=EPC_AUTH_TOKEN,
            os_api_key=None,
            property_type=property_type_lookup.get(house["Archetype"]),
        )

        searcher.find_property(skip_os=True)

        if searcher.newest_epc is None:
            searcher = SearchEpc(
                address1=property_meta["Location Name"],
                postcode=property_meta["Post Code"],
                auth_token=EPC_AUTH_TOKEN,
                os_api_key=None,
                property_type=property_type_lookup.get(house["Archetype"]),
            )
            searcher.search()

        if searcher.newest_epc is None:
            nodata.append(house["row_id"])
            continue

        newest_epc = searcher.newest_epc
        older_epcs = searcher.older_epcs
        full_sap_epc = searcher.full_sap_epc

        searcher.search()

        if searcher.data is None:
            nodata.append(property_meta.to_dict())
            continue

        epcs = searcher.data["rows"]
        epcs = pd.DataFrame(epcs)

        # Take the newest EPC by UPRN
        epcs = epcs.sort_values(by=["lodgement-date"], ascending=False)
        newest_epcs = epcs.drop_duplicates(subset=["uprn"], keep="first")

        # For each EPC, we now check eligibility
        for _, epc in newest_epcs.iterrows():
            eligibility = Eligibility(epc=epc.to_dict(), cleaned=cleaned)
            eligibility.check_gbis_warmfront()
            eligibility.check_eco4_warmfront()

            # If the house is not identified, we do a full gbis and eco4 check
            eligibility.check_gbis()
            eligibility.check_eco4()

            if eligibility.eco4_warmfront["eligible"]:
                # We get old_eps
                old_data = epcs[
                    (epcs["uprn"] == epc["uprn"]) &
                    (epcs["lmk-key"] != epc["lmk-key"])
                    ].to_dict("records")

                full_sap_epc = epcs[
                    (epcs["uprn"] == epc["uprn"]) &
                    (epcs["transaction-type"] == "new dwelling")
                    ].to_dict("records")

                scoring_dictionary = prepare_model_data_row(
                    property_id=eligibility.epc["uprn"],
                    modelling_epc=eligibility.epc,
                    cleaned=cleaned,
                    cleaning_data=cleaning_data,
                    created_at=created_at,
                    old_data=old_data,
                    full_sap_epc=full_sap_epc
                )
                scoring_data.extend(scoring_dictionary)

            results.append(
                {
                    "uprn": epc["uprn"],
                    "Location Name": property_meta["Location Name"],
                    "Post Code": property_meta["Post Code"],
                    "property_type": eligibility.epc["property-type"],
                    "gbis_eligible": eligibility.gbis_warmfront,
                    "eco4_eligible": eligibility.eco4_warmfront["eligible"],
                    "eco4_message": eligibility.eco4_warmfront["message"],
                    "sap": float(eligibility.epc["current-energy-efficiency"]),
                    "gbis_eligible_future": eligibility.gbis["eligible"],
                    "gbis_eligible_future_message": eligibility.gbis["message"],
                    "eco4_eligible_future": eligibility.eco4["eligible"],
                    "eco4_eligible_future_message": eligibility.eco4["message"],
                    # Property components
                    "roof": eligibility.roof["clean_description"],
                    "walls": eligibility.walls["clean_description"],
                    "cavity_type": eligibility.cavity["type"],
                    "heating": eligibility.epc["mainheat-description"],
                    "tenure": eligibility.tenure,
                    "date_epc": eligibility.epc["lodgement-date"],
                }
            )

    scoring_df = pd.DataFrame(scoring_data)

    # Perform the same cleaning as in the model - first clean number of room variables though
    scoring_df = DataProcessor.apply_averages_cleaning(
        data_to_clean=scoring_df,
        cleaning_data=cleaning_data,
        cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'],
        colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"],
    )

    scoring_df = DataProcessor.apply_averages_cleaning(
        data_to_clean=scoring_df,
        cleaning_data=cleaning_data,
        cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"],
    ).drop(columns=["LOCAL_AUTHORITY"])

    scoring_df = DataProcessor.clean_missings_after_description_process(
        scoring_df,
        ignore_cols=[c for c in scoring_df.columns if ("thermal_transmittance" in c) or (
            "insulation_thickness" in c) or ("ENERGY_EFF" in c)]
    )

    scoring_df = DataProcessor.clean_efficiency_variables(scoring_df)

    model_api = ModelApi(portfolio_id="ha33-eligibility", timestamp=created_at)
    all_predictions = model_api.predict_all(
        df=scoring_df,
        bucket="retrofit-data-dev",
        prediction_buckets={
            "sap_change_predictions": "retrofit-sap-predictions-dev",
            "heat_demand_predictions": "retrofit-heat-predictions-dev",
            "carbon_change_predictions": "retrofit-carbon-predictions-dev"
        }
    )

    predictions = all_predictions["sap_change_predictions"].copy()

    results_df = pd.DataFrame(results)

    predictions = predictions.rename(columns={"property_id": "uprn"}).merge(
        results_df[["uprn", "sap"]], how="left", on="uprn"
    )
    predictions["sap_uplift"] = predictions["predictions"] - predictions["sap"]
    predictions = predictions.groupby("uprn")["sap_uplift"].sum().reset_index()

    results_df = results_df.merge(
        predictions[["sap_uplift", "uprn"]],
        how="left",
        on="uprn"
    )
    results_df["post_install_sap"] = results_df["sap"] + results_df["sap_uplift"]

    results_df = results_df[~pd.isnull(results_df["uprn"])]

    eligibility_assessment = []
    for _, row in results_df[results_df["eco4_eligible"] == True].iterrows():
        # The upgrade requirements are dependent on the current SAP

        # If the property is an F or G, it only needs to upgrade to an %
        if row["sap"] <= 38:
            if row["post_install_sap"] >= 57:
                eligibility_classification = "highest confidence"
            elif row["post_install_sap"] >= 55:
                eligibility_classification = "high confidence"
            elif row["post_install_sap"] >= 53:
                eligibility_classification = "medium confidence"
            else:
                eligibility_classification = "unlikely"
        else:

            if row["post_install_sap"] >= 71:
                eligibility_classification = "highest confidence"
            elif row["post_install_sap"] >= 69:
                eligibility_classification = "high confidence"
            elif row["post_install_sap"] >= 67:
                eligibility_classification = "medium confidence"
            else:
                eligibility_classification = "unlikely"

        eligibility_assessment.append(
            {
                "uprn": row["uprn"],
                "eligibility_classification": eligibility_classification
            }
        )

    eligibility_assessment = pd.DataFrame(eligibility_assessment)

    results_df = results_df.merge(
        eligibility_assessment, how="left", on="uprn"
    )
    # We have some properties that are duplicated so we take just one instance
    results_df = results_df.drop_duplicates(subset=["uprn"])

    return results_df, scoring_data, nodata


def analyse_ha_4(results_df, data):
    n_identified = (results_df["gbis_eligible"] | results_df["eco4_eligible"]).sum()
    n_eco4 = results_df["eco4_eligible"].sum()
    n_gbis = results_df[~results_df["eco4_eligible"]]["gbis_eligible"].sum()

    eco_eligibile = results_df[results_df["eco4_eligible"]]
    eco_eligibile["eligibility_classification"].value_counts()

    future_possibilities_eco = results_df[
        (results_df["eco4_eligible_future"] == True) & (~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
        ].copy()

    future_possibilities_gbis = results_df[
        (results_df["gbis_eligible_future"] == True) & (results_df["eco4_eligible_future"] == False) & (
            ~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
        ].copy()

    total_future_possibilities = future_possibilities_eco.shape[0] + future_possibilities_gbis.shape[0]


def app():
    data = load_ha_4()

    data = standardise_ha_4(data)

    data["row_id"] = ["h4" + str(i) for i in range(0, len(data))]

    cleaned = read_from_s3(
        s3_file_name="cleaned_epc_data/cleaned.bson",
        bucket_name="retrofit-data-dev"
    )
    cleaned = msgpack.unpackb(cleaned, raw=False)

    cleaning_data = read_dataframe_from_s3_parquet(
        bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
    )

    created_at = datetime.now().isoformat()

    photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")

    results_df, scoring_data, nodata = get_ha_4_data(
        data=data,
        cleaned=cleaned,
        cleaning_data=cleaning_data,
        created_at=created_at,
        photo_supply_lookup=photo_supply_lookup,
        floor_area_decile_thresholds=floor_area_decile_thresholds
    )

    # Store the data locally as a pickle
    # import pickle
    # with open("ha_4.pickle", "wb") as f:
    #     pickle.dump(
    #         {
    #             "results_df": results_df,
    #             "scoring_data": scoring_data,
    #             "nodata": nodata
    #         }, f)

    # Read in
    # import pickle
    # with open("ha_4.pickle", "rb") as f:
    #     data = pickle.load(f)
    # results_df = data["results_df"]
    # scoring_data = data["scoring_data"]
    # nodata = data["nodata"]