Model/etl/eligibility/ha_15_32/ha7_app.py

import os
import msgpack
import openpyxl
from openpyxl.styles.colors import COLOR_INDEX
from pathlib import Path
from datetime import datetime
import pandas as pd
import numpy as np
from utils.s3 import read_from_s3, read_dataframe_from_s3_parquet
from utils.logger import setup_logger
from dotenv import load_dotenv
from tqdm import tqdm
from backend.SearchEpc import SearchEpc
from etl.eligibility.Eligibility import Eligibility
from etl.eligibility.ha_15_32.app import prepare_model_data_row
from etl.epc.DataProcessor import DataProcessor
from etl.epc.settings import COLUMNS_TO_MERGE_ON
from backend.ml_models.api import ModelApi
from etl.solar.SolarPhotoSupply import SolarPhotoSupply
from recommendations.recommendation_utils import calculate_cavity_age
from recommendation_utils import convert_thickness_to_numeric

ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"

logger = setup_logger()
load_dotenv(ENV_FILE)

EPC_AUTH_TOKEN = os.getenv("EPC_AUTH_TOKEN")
OS_API_KEY = os.getenv("ORDNANCE_SURVEY_API_KEY")


def load_data():
    """
    Load the data from the excel
    """

    workbook = openpyxl.load_workbook('etl/eligibility/ha_15_32/HESTIA - HA 7 ASSET LIST.xlsx')
    sheet = workbook.active

    # Prepare lists to collect rows data and their colors
    rows_data = []
    rows_colors = []
    for row in sheet.iter_rows(min_row=2, values_only=False):  # Assuming the first row is headers
        row_data = [cell.value for cell in row]  # This will get you the cell values

        row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None
        row_color = COLOR_INDEX[row_color]
        rows_data.append(row_data)
        rows_colors.append(row_color)

    df = pd.DataFrame(rows_data, columns=[cell.value for cell in sheet[1]])

    # Add the row colors as a new column
    df['row_color'] = rows_colors
    df.columns.values[8] = "is_active"

    # Remove None columns
    df = df.dropna(axis=1, how='all')
    # We now parse the colours
    df["row_color"].unique()
    df["row_colour_name"] = np.where(
        df["row_color"] == "0000FFFF", "red",
        np.where(df["row_color"] == "00FF00FF", "green", "yellow")
    )
    df["row_code"] = np.where(
        df["row_colour_name"] == "red", "invalid",
        np.where(df["row_colour_name"] == "green", "potential ECO4", "needs criteria change")
    )

    return df


def get_ha7_data(data, cleaned, cleaning_data, created_at, photo_supply_lookup, floor_area_decile_thresholds):
    property_type_lookup = {
        # "Mid Terrace": "Mid-Terrace",
        # "End Terrace": "End-Terrace",
        # "Semi Detached": "Semi-Detached",
        # "Detached": "Detached",
        "House": "House",
        "Flat": "Flat",
        "Bungalow": "Bungalow",
        "Maisonette": "Maisonette",
    }

    scoring_data = []
    results = []
    nodata = []
    for _, house in tqdm(data.iterrows(), total=len(data)):

        if house["Address"]:
            address = house["Address"]
        else:
            address = house["Address2"]

        searcher = SearchEpc(
            address1=address,
            postcode=house["Postcode"],
            auth_token=EPC_AUTH_TOKEN,
            os_api_key=None,
            property_type=property_type_lookup.get(house["Archetype"]),
        )

        searcher.find_property(skip_os=True)

        if searcher.newest_epc is None:
            nodata.append(house["row_id"])
            continue

        newest_epc = searcher.newest_epc
        older_epcs = searcher.older_epcs
        full_sap_epc = searcher.full_sap_epc

        eligibility = Eligibility(epc=newest_epc, cleaned=cleaned)
        eligibility.check_gbis_warmfront()
        eligibility.check_eco4_warmfront()

        # If the property is a cavity wall and it's filled, we produce an estimate for the age of the cavity

        # Loft MUST be suitable
        cavity_age = None
        if (
            eligibility.walls["is_cavity_wall"] and
            eligibility.walls["is_filled_cavity"] and
            eligibility.loft["suitability"] and
            eligibility.eco4_warmfront["message"] == "Failed due to full cavity - check cavity age"
        ):
            # We check the age of the cavity and if it's particularly old, we flag it
            cavity_age = calculate_cavity_age(newest_epc, older_epcs, cleaned)

        # If the house is not identified, we do a full gbis and eco4 check
        eligibility.check_gbis()
        eligibility.check_eco4()

        if eligibility.eco4_warmfront["eligible"]:
            scoring_dictionary = prepare_model_data_row(
                property_id=house["row_id"],
                modelling_epc=eligibility.epc,
                cleaned=cleaned,
                cleaning_data=cleaning_data,
                created_at=created_at,
                old_data=older_epcs,
                full_sap_epc=full_sap_epc,
                photo_supply_lookup=photo_supply_lookup,
                floor_area_decile_thresholds=floor_area_decile_thresholds
            )
            scoring_data.extend(scoring_dictionary)

        # If nothing is eligible or gbis is eligible, then we make a record this
        results.append(
            {
                "row_id": house["row_id"],
                "address": house["Address"],
                "postcode": house["Postcode"],
                "gbis_eligible": eligibility.gbis_warmfront,
                "eco4_eligible": eligibility.eco4_warmfront["eligible"],
                "eco4_message": eligibility.eco4_warmfront["message"],
                "sap": float(eligibility.epc["current-energy-efficiency"]),
                "gbis_eligible_future": eligibility.gbis["eligible"],
                "gbis_eligible_future_message": eligibility.gbis["message"],
                "eco4_eligible_future": eligibility.eco4["eligible"],
                "eco4_eligible_future_message": eligibility.eco4["message"],
                # Property components
                "roof": eligibility.roof["clean_description"],
                "walls": eligibility.walls["clean_description"],
                "heating": eligibility.epc["mainheat-description"],
                "tenure": eligibility.tenure,
                "date_epc": eligibility.epc["lodgement-date"],
                **newest_epc,
                "cavity_age": cavity_age,
                **eligibility.walls,
                **eligibility.roof,
            }
        )

    scoring_df = pd.DataFrame(scoring_data)
    # Implement the same process that is being used in the recommendation engine to cleaning scoring_df

    # Perform the same cleaning as in the model - first clean number of room variables though
    scoring_df = DataProcessor.apply_averages_cleaning(
        data_to_clean=scoring_df,
        cleaning_data=cleaning_data,
        cols_to_merge_on=['PROPERTY_TYPE', 'BUILT_FORM', 'CONSTRUCTION_AGE_BAND', 'LOCAL_AUTHORITY'],
        colnames=["NUMBER_HABITABLE_ROOMS", "NUMBER_HEATED_ROOMS"],
    )

    scoring_df = DataProcessor.apply_averages_cleaning(
        data_to_clean=scoring_df,
        cleaning_data=cleaning_data,
        cols_to_merge_on=COLUMNS_TO_MERGE_ON + ["LOCAL_AUTHORITY"],
    ).drop(columns=["LOCAL_AUTHORITY"])

    scoring_df = DataProcessor.clean_missings_after_description_process(
        scoring_df,
        ignore_cols=[c for c in scoring_df.columns if ("thermal_transmittance" in c) or (
            "insulation_thickness" in c) or ("ENERGY_EFF" in c)]
    )

    scoring_df = DataProcessor.clean_efficiency_variables(scoring_df)

    model_api = ModelApi(portfolio_id="ha33-eligibility", timestamp=created_at)
    all_predictions = model_api.predict_all(
        df=scoring_df,
        bucket="retrofit-data-dev",
        prediction_buckets={
            "sap_change_predictions": "retrofit-sap-predictions-dev",
            "heat_demand_predictions": "retrofit-heat-predictions-dev",
            "carbon_change_predictions": "retrofit-carbon-predictions-dev"
        }
    )

    predictions = all_predictions["sap_change_predictions"].copy()

    results_df = pd.DataFrame(results)

    predictions = predictions.rename(columns={"property_id": "row_id"}).merge(
        results_df[["row_id", "sap"]], how="left", on="row_id"
    )
    predictions["sap_uplift"] = predictions["predictions"] - predictions["sap"]
    predictions = predictions.groupby("row_id")["sap_uplift"].sum().reset_index()

    results_df = results_df.merge(
        predictions[["sap_uplift", "row_id"]],
        how="left",
        on="row_id"
    )

    results_df["post_install_sap"] = results_df["sap"] + results_df["sap_uplift"]

    eligibility_assessment = []
    for _, row in results_df[results_df["eco4_eligible"] == True].iterrows():
        # The upgrade requirements are dependent on the current SAP

        # If the property is an F or G, it only needs to upgrade to an %
        if row["sap"] <= 38:
            if row["post_install_sap"] >= 57:
                eligibility_classification = "highest confidence"
            elif row["post_install_sap"] >= 55:
                eligibility_classification = "high confidence"
            elif row["post_install_sap"] >= 53:
                eligibility_classification = "medium confidence"
            else:
                eligibility_classification = "unlikely"
        else:

            if row["post_install_sap"] >= 71:
                eligibility_classification = "highest confidence"
            elif row["post_install_sap"] >= 69:
                eligibility_classification = "high confidence"
            elif row["post_install_sap"] >= 67:
                eligibility_classification = "medium confidence"
            else:
                eligibility_classification = "unlikely"

        eligibility_assessment.append(
            {
                "row_id": row["row_id"],
                "eligibility_classification": eligibility_classification
            }
        )

    eligibility_assessment = pd.DataFrame(eligibility_assessment)

    results_df = results_df.merge(
        eligibility_assessment, how="left", on="row_id"
    )

    return results_df, scoring_data, nodata


def analyse_ha_7(results_df, data):
    analysis_data = results_df.merge(
        data[["row_id", "row_code", "Property Type", "Construction Year Band"]], how="left", on="row_id"
    )

    analysis_data["row_code"].value_counts()

    # NEW

    analysis_data["roof_insulation_thickness"] = np.where(
        pd.isnull(analysis_data["roof_insulation_thickness"]), None, analysis_data["roof_insulation_thickness"]
    )
    analysis_data["roof_insulation_thickness_numeric"] = analysis_data["roof_insulation_thickness"].apply(
        lambda x: convert_thickness_to_numeric(x, is_flat=False, is_pitched=True)
    )

    ideal_eco4 = analysis_data[
        (analysis_data["eco4_eligible"] == True) & (
            analysis_data["roof_insulation_thickness_numeric"] <= 100)
        ]

    secondary_eco4_warmfront_not_sold = analysis_data[
        (analysis_data["eco4_eligible"] == True) & (
            analysis_data["roof_insulation_thickness_numeric"] > 100)
        ]

    # underperforming cavities
    underperforming_cavities = analysis_data[
        (analysis_data["eco4_message"] == "Failed due to full cavity - check cavity age") & (
            analysis_data["cavity_age"] > 9 * 365
        ) & (analysis_data["roof_insulation_thickness_numeric"] <= 100)
        ]

    identified_gbis_not_sold = analysis_data[
        (analysis_data["gbis_eligible"] == True) & (
            analysis_data["eco4_eligible"] == False
        )
        ]

    wf_identified = analysis_data[
        (analysis_data["row_code"] == "potential ECO4")
    ]

    # END NEW

    warmfront_identification = analysis_data["row_code"].value_counts()
    warmfront_identified = analysis_data[analysis_data["row_code"] == "potential ECO4"]
    warmfront_identified["walls"].value_counts(normalize=True)

    analysis_data["Construction Year Band"].value_counts(normalize=True)

    # Number of days from today

    days_to_today = (datetime.now() - pd.to_datetime(warmfront_identified["date_epc"])).dt.days
    days_to_today.mean()

    property_types = analysis_data["Property Type"].value_counts()

    n_identified = (results_df["gbis_eligible"] | results_df["eco4_eligible"]).sum()

    eco_identified = results_df[results_df["eco4_eligible"]]
    n_eco4 = eco_identified["eco4_eligible"].sum()
    gbis_identified = results_df[~results_df["eco4_eligible"] & results_df["gbis_eligible"]]
    n_gbis = results_df[~results_df["eco4_eligible"]]["gbis_eligible"].sum()

    eco_eligibile = results_df[results_df["eco4_eligible"]]
    eco_eligibile["eligibility_classification"].value_counts()

    future_possibilities_eco = results_df[
        (results_df["eco4_eligible_future"] == True) & (~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
        ].copy()

    future_possibilities_gbis = results_df[
        (results_df["gbis_eligible_future"] == True) & (results_df["eco4_eligible_future"] == False) & (
            ~(results_df["gbis_eligible"] | results_df["eco4_eligible"]))
        ].copy()

    total_future_possibilities = future_possibilities_eco.shape[0] + future_possibilities_gbis.shape[0]


def app():
    data = load_data()
    data["row_id"] = ["ha7" + str(i) for i in range(0, len(data))]

    cleaned = read_from_s3(
        s3_file_name="cleaned_epc_data/cleaned.bson",
        bucket_name="retrofit-data-dev"
    )
    cleaned = msgpack.unpackb(cleaned, raw=False)

    cleaning_data = read_dataframe_from_s3_parquet(
        bucket_name="retrofit-data-dev", file_key="sap_change_model/cleaning_dataset.parquet",
    )

    photo_supply_lookup, floor_area_decile_thresholds = SolarPhotoSupply.load(bucket="retrofit-data-dev")

    created_at = datetime.now().isoformat()

    results_df, scoring_data, nodata = get_ha7_data(
        data, cleaned, cleaning_data, created_at, photo_supply_lookup, floor_area_decile_thresholds
    )

    # Pickle results
    # import pickle
    # with open("ha7_results_jan_10.pkl", "wb") as f:
    #     pickle.dump({"results_df": results_df, "scoring_data": scoring_data, "nodata": nodata}, f)

    # Read in the old data
    # import pickle
    # with open("ha7_results_jan_10.pkl", "rb") as f:
    #     old_data = pickle.load(f)
    # results_df = old_data["results_df"]
    # scoring_data = old_data["scoring_data"]
    # nodata = old_data["nodata"]