working on merge between asset list and survey list

etl/eligibility/ha_15_32/ha16_app.py

@@ -0,0 +1,115 @@
+import msgpack
+import openpyxl
+from openpyxl.styles.colors import COLOR_INDEX
+from pathlib import Path
+from datetime import datetime
+import pandas as pd
+import numpy as np
+from utils.s3 import read_from_s3
+from utils.logger import setup_logger
+from dotenv import load_dotenv
+from backend.app.utils import read_parquet_from_s3
+from tqdm import tqdm
+from backend.SearchEpc import SearchEpc
+from etl.eligibility.Eligibility import Eligibility
+from etl.eligibility.ha_15_32.app import prepare_model_data_row
+from etl.epc.DataProcessor import DataProcessor
+from etl.epc.settings import COLUMNS_TO_MERGE_ON
+from backend.ml_models.api import ModelApi
+
+import re
+
+ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env"
+
+logger = setup_logger()
+load_dotenv(ENV_FILE)
+
+
+def load_data():
+    # This asset list is spread across two sheets, which we need to combine
+
+    asset_list_filenames = [
+        "HESTIA - HA 16 ASSET LIST PART 1 OF 2.xlsx",
+        "HESTIA - HA 16 ASSET LIST PART 2 OF 2.xlsx",
+    ]
+
+    # Prepare lists to collect rows data and their colors
+    rows_data = []
+    rows_colors = []
+    colnames = []
+    for asset_list_filename in asset_list_filenames:
+        workbook = openpyxl.load_workbook(f'etl/eligibility/ha_15_32/{asset_list_filename}')
+        sheet = workbook.active
+        sheet_colnames = [cell.value for cell in sheet[1]]
+        colnames.append(sheet_colnames)
+
+        for row in sheet.iter_rows(min_row=2, values_only=False):  # Assuming the first row is headers
+            row_data = [cell.value for cell in row]  # This will get you the cell values
+            row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None
+            # row_color = COLOR_INDEX[row_color]
+            rows_data.append(row_data)
+            rows_colors.append(row_color)
+
+    asset_list = pd.DataFrame(rows_data, columns=colnames[0])
+    # Remove None columns
+    asset_list = asset_list.iloc[:, 0:12]
+    asset_list['row_color'] = rows_colors
+
+    asset_list["row_colour_name"] = np.where(
+        asset_list["row_color"] == "FFFF0000", "red",
+        np.where(asset_list["row_color"] == "FF92D050", "green", "yellow")
+    )
+
+    # Split up the address on commas, which is useful for matching later
+    split_addresses = asset_list['Address'].str.split(',', expand=True)
+    split_addresses.columns = ['temp', 'address2', 'address3', 'address4', 'address5']
+
+    asset_list = pd.concat([asset_list, split_addresses], axis=1)
+    # There is no commas separating house number and address 1
+    split_addresses2 = asset_list['temp'].str.split(' ', expand=True)
+    split_addresses2.columns = ['HouseNo', 'part1', 'part2', "part3", "part4"]
+    # We could re-concatenate but we only care about HouseNo for the moment
+    asset_list = pd.concat([asset_list, split_addresses2[["HouseNo"]]], axis=1)
+
+    # We now read in the survey list
+    survey_workbook = openpyxl.load_workbook(f'etl/eligibility/ha_15_32/HESTIA- HA 16 ECO4 SURVEY LIST.xlsx')
+    survey_sheet = survey_workbook.active
+
+    survey_rows = []
+    survey_colors = []
+
+    for row in survey_sheet.iter_rows(min_row=2, values_only=False):  # Assuming the first row is headers
+        row_data = [cell.value for cell in row]  # This will get you the cell values
+        row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None
+        # row_color = COLOR_INDEX[row_color]
+        survey_rows.append(row_data)
+        survey_colors.append(row_color)
+
+    survey_list = pd.DataFrame(survey_rows, columns=[cell.value for cell in survey_sheet[1]])
+
+    # For the survey list, we don't need the colours, since there is a column called "INSTALLED OR CANCELLED"
+    # which describes the status of the property
+    survey_list["row_colour"] = survey_colors
+    survey_list["survey_key"] = ["survey_" + str(i) for i in range(0, len(survey_list))]
+    # Tidy up the street/block name a bit
+    survey_list["Street / Block Name"] = survey_list["Street / Block Name"].str.replace("/", ", ")
+
+    # We now need to merge the survey list onto the asset list
+    # Could be easier just to do a search on each row, even though it's much slower
+    matched = []
+    for _, row in tqdm(survey_list.iterrows(), total=len(survey_list)):
+        # Filter on the first line of the address
+        df = asset_list[asset_list["Address"].str.lower().str.contains(row["Street / Block Name"].lower())].copy()
+        df = df[df["Postcode"].str.contains(row["Post Code"])]
+        df = df[df["Address"].str.contains(str(row["NO."]))]
+        if df.shape[0] != 1:
+            df = df[df["HouseNo"] == str(row["NO."])]
+            if df.shape[0] != 1:
+                raise ValueError("Investigate")
+
+        matched.append(
+            {
+                "survey_key": row["survey_key"],
+                "matched_address": df["Address"].values[0]
+            }
+        )