From 43004a5d8beac085d6eca1c6a588941abea97173 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Sun, 24 Dec 2023 15:48:36 +0000 Subject: [PATCH] working on merge between asset list and survey list --- etl/eligibility/ha_15_32/ha16_app.py | 115 +++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 etl/eligibility/ha_15_32/ha16_app.py diff --git a/etl/eligibility/ha_15_32/ha16_app.py b/etl/eligibility/ha_15_32/ha16_app.py new file mode 100644 index 00000000..e347d47c --- /dev/null +++ b/etl/eligibility/ha_15_32/ha16_app.py @@ -0,0 +1,115 @@ +import msgpack +import openpyxl +from openpyxl.styles.colors import COLOR_INDEX +from pathlib import Path +from datetime import datetime +import pandas as pd +import numpy as np +from utils.s3 import read_from_s3 +from utils.logger import setup_logger +from dotenv import load_dotenv +from backend.app.utils import read_parquet_from_s3 +from tqdm import tqdm +from backend.SearchEpc import SearchEpc +from etl.eligibility.Eligibility import Eligibility +from etl.eligibility.ha_15_32.app import prepare_model_data_row +from etl.epc.DataProcessor import DataProcessor +from etl.epc.settings import COLUMNS_TO_MERGE_ON +from backend.ml_models.api import ModelApi + +import re + +ENV_FILE = Path(__file__).parent / "etl" / "eligibility" / "ha_15_32" / ".env" + +logger = setup_logger() +load_dotenv(ENV_FILE) + + +def load_data(): + # This asset list is spread across two sheets, which we need to combine + + asset_list_filenames = [ + "HESTIA - HA 16 ASSET LIST PART 1 OF 2.xlsx", + "HESTIA - HA 16 ASSET LIST PART 2 OF 2.xlsx", + ] + + # Prepare lists to collect rows data and their colors + rows_data = [] + rows_colors = [] + colnames = [] + for asset_list_filename in asset_list_filenames: + workbook = openpyxl.load_workbook(f'etl/eligibility/ha_15_32/{asset_list_filename}') + sheet = workbook.active + sheet_colnames = [cell.value for cell in sheet[1]] + colnames.append(sheet_colnames) + + for row in sheet.iter_rows(min_row=2, values_only=False): # Assuming the first row is headers + row_data = [cell.value for cell in row] # This will get you the cell values + row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None + # row_color = COLOR_INDEX[row_color] + rows_data.append(row_data) + rows_colors.append(row_color) + + asset_list = pd.DataFrame(rows_data, columns=colnames[0]) + # Remove None columns + asset_list = asset_list.iloc[:, 0:12] + asset_list['row_color'] = rows_colors + + asset_list["row_colour_name"] = np.where( + asset_list["row_color"] == "FFFF0000", "red", + np.where(asset_list["row_color"] == "FF92D050", "green", "yellow") + ) + + # Split up the address on commas, which is useful for matching later + split_addresses = asset_list['Address'].str.split(',', expand=True) + split_addresses.columns = ['temp', 'address2', 'address3', 'address4', 'address5'] + + asset_list = pd.concat([asset_list, split_addresses], axis=1) + # There is no commas separating house number and address 1 + split_addresses2 = asset_list['temp'].str.split(' ', expand=True) + split_addresses2.columns = ['HouseNo', 'part1', 'part2', "part3", "part4"] + # We could re-concatenate but we only care about HouseNo for the moment + asset_list = pd.concat([asset_list, split_addresses2[["HouseNo"]]], axis=1) + + # We now read in the survey list + survey_workbook = openpyxl.load_workbook(f'etl/eligibility/ha_15_32/HESTIA- HA 16 ECO4 SURVEY LIST.xlsx') + survey_sheet = survey_workbook.active + + survey_rows = [] + survey_colors = [] + + for row in survey_sheet.iter_rows(min_row=2, values_only=False): # Assuming the first row is headers + row_data = [cell.value for cell in row] # This will get you the cell values + row_color = row[0].fill.start_color.index if row[0].fill.start_color.index != '00000000' else None + # row_color = COLOR_INDEX[row_color] + survey_rows.append(row_data) + survey_colors.append(row_color) + + survey_list = pd.DataFrame(survey_rows, columns=[cell.value for cell in survey_sheet[1]]) + + # For the survey list, we don't need the colours, since there is a column called "INSTALLED OR CANCELLED" + # which describes the status of the property + survey_list["row_colour"] = survey_colors + survey_list["survey_key"] = ["survey_" + str(i) for i in range(0, len(survey_list))] + # Tidy up the street/block name a bit + survey_list["Street / Block Name"] = survey_list["Street / Block Name"].str.replace("/", ", ") + + # We now need to merge the survey list onto the asset list + # Could be easier just to do a search on each row, even though it's much slower + matched = [] + for _, row in tqdm(survey_list.iterrows(), total=len(survey_list)): + # Filter on the first line of the address + df = asset_list[asset_list["Address"].str.lower().str.contains(row["Street / Block Name"].lower())].copy() + df = df[df["Postcode"].str.contains(row["Post Code"])] + df = df[df["Address"].str.contains(str(row["NO."]))] + if df.shape[0] != 1: + df = df[df["HouseNo"] == str(row["NO."])] + if df.shape[0] != 1: + raise ValueError("Investigate") + + matched.append( + { + "survey_key": row["survey_key"], + "matched_address": df["Address"].values[0] + } + )