Completed HA24

etl/eligibility/ha_15_32/ha24_app.py

@@ -345,6 +345,53 @@ def get_epc_data(data, cleaned, cleaning_data, created_at):
     return results_df, scoring_data, nodata
 
 
+def analyse_results(results_df, data, survey_list):
+    analysis_data = data[["row_id", "survey_key", "warmfront_identified"]].merge(
+        results_df, how="left", on="row_id"
+    ).merge(
+        survey_list[["survey_key", survey_list.columns[0]]].rename(columns={survey_list.columns[0]: "funding_scheme"}),
+        how="left", on="survey_key"
+    )
+
+    warmfront_identified = analysis_data[analysis_data["warmfront_identified"]]
+
+    # Of the ECO jobs, what proportion to we get right
+    warmfront_identified_eco = warmfront_identified[
+        warmfront_identified["funding_scheme"].isin(["ECO4 A/W", "AFFORDABLE WARMTH"])
+    ]
+
+    eco_success_rate = warmfront_identified_eco["eco4_eligible"].sum() / warmfront_identified_eco.shape[0]
+
+    warmfront_identified_gbis = warmfront_identified[
+        warmfront_identified["funding_scheme"].isin(["ECO4 GBIS (ECO+)"])
+    ]
+
+    # No gbis for this
+    # gbis_success_rate = warmfront_identified_gbis["gbis_eligible"].sum() / warmfront_identified_gbis.shape[0]
+
+    # Additional identified
+    additional_identified_eco = analysis_data[
+        (analysis_data["eco4_eligible"] == True) & (analysis_data["warmfront_identified"] == False)
+        ]
+
+    additional_identified_eco["eligibility_classification"].value_counts()
+
+    additional_identified_gbis = analysis_data[
+        (analysis_data["gbis_eligible"] == True) & (analysis_data["eco4_eligible"] == False) & (
+            analysis_data["warmfront_identified"] == False
+        )
+        ].shape[0]
+    # Future
+    additional_identified_eco_future = analysis_data[
+        (analysis_data["eco4_eligible_future"] == True) & (analysis_data["warmfront_identified"] == False)
+        ].shape[0]
+    additional_identified_gbis_future = analysis_data[
+        (analysis_data["gbis_eligible_future"] == True) & (analysis_data["eco4_eligible_future"] == False) & (
+            analysis_data["warmfront_identified"] == False
+        )
+        ].shape[0]
+
+
 def app():
     data, survey_list = load_data()
 
@@ -363,3 +410,14 @@ def app():
     created_at = datetime.now().isoformat()
 
     results_df, scoring_data, nodata = get_epc_data(data, cleaned, cleaning_data, created_at)
+
+    # Pickle results just in case
+    # import pickle
+    # with open("ha24.pickle", "wb") as f:
+    #     pickle.dump(
+    #         {
+    #             "scoring_data": scoring_data,
+    #             "results": results_df,
+    #             "nodata": nodata
+    #         }, f
+    #     )

etl/eligibility/ha_15_32/ha25_app.py

@@ -0,0 +1,76 @@
+import msgpack
+import openpyxl
+from openpyxl.styles.colors import COLOR_INDEX
+from pathlib import Path
+from datetime import datetime
+import pandas as pd
+import numpy as np
+from utils.s3 import read_from_s3
+from utils.logger import setup_logger
+from dotenv import load_dotenv
+from backend.app.utils import read_parquet_from_s3
+from tqdm import tqdm
+from backend.SearchEpc import SearchEpc
+from etl.eligibility.Eligibility import Eligibility
+from etl.eligibility.ha_15_32.app import prepare_model_data_row
+from etl.epc.DataProcessor import DataProcessor
+from etl.epc.settings import COLUMNS_TO_MERGE_ON
+from backend.ml_models.api import ModelApi
+
+import re
+
+ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"
+
+logger = setup_logger()
+load_dotenv(ENV_FILE)
+
+
+def load_data():
+    workbook = openpyxl.load_workbook('etl/eligibility/ha_15_32/HESTIA - HA 25 ASSET LIST.xlsx')
+    sheet = workbook.active
+    sheet_colnames = [cell.value for cell in sheet[1]]
+
+    rows_data = []
+    rows_colors = []
+    for row in sheet.iter_rows(min_row=2, values_only=False):  # Assuming the first row is headers
+        row_data = [cell.value for cell in row]  # This will get you the cell values
+        row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None
+        # row_color = COLOR_INDEX[row_color]
+        rows_data.append(row_data)
+        rows_colors.append(row_color)
+
+    asset_list = pd.DataFrame(rows_data, columns=sheet_colnames)
+    asset_list['row_color'] = rows_colors
+
+    asset_list["row_colour_name"] = np.where(
+        asset_list["row_color"] == "FFFF0000", "red",
+        np.where(asset_list["row_color"] == "FF00B050", "green", "yellow")
+    )
+
+    asset_list["row_colour_code"] = np.where(
+        asset_list["row_colour_name"] == "red", "does not meet criteria",
+        np.where(asset_list["row_colour_name"] == "green", "identified potential eco", "maybe in the future")
+    )
+
+    # We analysis historical ECO3 survey list
+    eco3_survey_workbook = openpyxl.load_workbook(f'etl/eligibility/ha_15_32/HESTIA - HA 25 ECO3 SURVEY LIST.xlsx')
+    dir(eco3_survey_workbook)
+    eco3_survey_sheet = eco3_survey_workbook.active
+
+    eco3_survey_rows = []
+    eco3_survey_colors = []
+
+    for row in eco3_survey_sheet.iter_rows(min_row=2, values_only=False):  # Assuming the first row is headers
+        row_data = [cell.value for cell in row]  # This will get you the cell values
+        row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None
+        # row_color = COLOR_INDEX[row_color]
+        eco3_survey_rows.append(row_data)
+        eco3_survey_colors.append(row_color)
+
+    # Some adhoc analysis on the eco3 survey list, just to get completion and cancellation rates historically
+    eco3_survey_list = pd.DataFrame(eco3_survey_rows, columns=[cell.value for cell in eco3_survey_sheet[1]])
+    eco3_survey_list["row_colour"] = eco3_survey_colors
+    # Remove rows where street name is missing
+    eco3_survey_list = eco3_survey_list[~pd.isnull(eco3_survey_list["Street / Block Name"])]
+
+    eco3_survey_list["INSTALLED OR CANCELLED"]