diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index b7dd8d70..529aef3d 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -1235,11 +1235,11 @@ class AssetList: elif self.old_format_non_intrusives_present: non_intrusives_wall_filter = ( self.standardised_asset_list['non-intrusives: WFT Findings'].str.lower().str.strip().isin( - ["empty cavity", "partial fill", "empty", "EMPTY CAVITY 70MM", "partial"] + ["empty cavity", "partial fill", "empty", "EMPTY CAVITY 70MM", "partial", "empty cav"] ) | ( ( self.standardised_asset_list['non-intrusives: WFT Findings'] - .str.lower().str.strip().str.contains("empty cavity|partial fill|empty|partial") & + .str.lower().str.strip().str.contains("empty cavity|partial fill") & ~self.standardised_asset_list['non-intrusives: WFT Findings'] .astype(str).str.lower().str.strip().str.contains("major access issues") ) @@ -1368,8 +1368,16 @@ class AssetList: print("Review these categories!!!!") extraction_wall_filter = ( self.standardised_asset_list['non-intrusives: WFT Findings'].str.lower().str.strip().isin( - ["retro drilled", "retro filled", "fibre from build", "polybead", "retro drilled and filled", - "retro drilled & filled", "blown in white wool", "blown in yellow wool"] + [ + 'blown in yellow wool', 'retro drilled & filled', 'white fibre from build', + 'foam filled from build', 'retro drilled gas in block', 'block in rock wool', 'rdf / tilehung', + 'fibre from build', 'blown in rock wool', 'rdf / tile hung', 'retro drilled', + 'rock wool from build', 'part rendered retro drilled', 'white fibtr from build.', + 'retro drilled and filled', 'blown in white wool', 'blown in yellow fibre from build', 'rdf', + 'polybead', 'foam filled', 'blown in white bead from build', 'blown in yellow fibre', + 'retro drilled det', 'blown in rockwool', 'retro drilled det empty cav', 'retro drilled end', + 'retro filled extension', 'retro filled', 'foam' + ] ) ) @@ -1433,7 +1441,18 @@ class AssetList: self.standardised_asset_list["solar_epc_data_indicates_correct_heating_system"] & self.standardised_asset_list["solar_epc_data_indicates_requires_heating_upgrade"] ).sum(): - raise ValueError("Both heating system checks are true - this should not be possible") + logger.info("We have an example of both heating system checks being true - checking known cases") + known_edge_cases = ['Ground source heat pump, radiators, electric, Electric storage heaters'] + error_cases = self.standardised_asset_list[ + ( + self.standardised_asset_list["solar_epc_data_indicates_correct_heating_system"] & + self.standardised_asset_list["solar_epc_data_indicates_requires_heating_upgrade"] + ) + ] + if all(error_cases[self.EPC_API_DATA_NAMES["mainheat-description"]].isin(known_edge_cases)): + logger.info("Within known edge cases") + else: + raise ValueError("Both heating system checks are true - this should not be possible") # Check 3: Does the property meet the fabric condition # Solar PV installs are subject to the minimum insulation requirements which means: diff --git a/asset_list/app.py b/asset_list/app.py index d5ce7226..e3c612a7 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -64,7 +64,7 @@ def app(): # Thurrock data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thurrock" - data_filename = "THURROCK COUNCIL.xlsx" + data_filename = "THURROCK COUNCIL - For analysis.xlsx" sheet_name = "Assets" postcode_column = 'Postcode' fulladdress_column = "Full Address" diff --git a/etl/customers/medway/flag_reviewed.py b/etl/customers/medway/flag_reviewed.py new file mode 100644 index 00000000..e2b27670 --- /dev/null +++ b/etl/customers/medway/flag_reviewed.py @@ -0,0 +1,104 @@ +""" +This script marks which properties have been reviewed by the Medway. +""" +import pandas as pd +import numpy as np +from tqdm import tqdm +import os + +asset_list = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/MEDWAY Asset List - Standardised.xlsx", + sheet_name="Standardised Asset List", +) +flat_data = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/MEDWAY Asset List - Standardised.xlsx", + sheet_name="Flat Data", +) + +reviewed_assets = pd.read_excel( + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/Programme Final Check.xlsx", +) +exclude_from_programme = reviewed_assets.copy() # [reviewed_assets["Khalim - include in programme"] == "No"].copy() +exclude_from_programme["exclusion_reason"] = None +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme["UPRN"] == "SOLD"), + "Sold", + exclude_from_programme["exclusion_reason"], +) +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme["Include in SHDF Bid?"] == "Definite"), + "Included in SHDF Bid", + exclude_from_programme["exclusion_reason"], +) + +exclude_from_programme["exclusion_reason"] = np.where( + (exclude_from_programme['Move Forward'] == "EPC C"), + "Excluded from Programme", + exclude_from_programme["exclusion_reason"], +) + +# exclude_from_programme = exclude_from_programme[~pd.isnull(exclude_from_programme["exclusion_reason"])] +exclude_from_programme = exclude_from_programme.reset_index(drop=True) +exclude_from_programme["row_id"] = exclude_from_programme.index + +# Match to asset list +matched = [] +for _, x in tqdm(exclude_from_programme.iterrows(), total=len(exclude_from_programme)): + + if x["No."] == 218 and x["Postcode"] == "ME8 6QB": + pc = "ME8 6QP" + elif x["No."] == 198 and x["Postcode"] == "ME8 6HL": + pc = "ME8 6LU" + elif x["No."] == "39a" and x["Postcode"] == "ME7 2BU": + pc = "ME7 2BU" + else: + pc = x["Postcode"] + + hn = x["No."] + + m = asset_list[ + (asset_list["domna_address_1"] == str(hn)) & + (asset_list["domna_postcode"] == str(pc)) + ] + + if m.empty: + m = asset_list[ + (asset_list["domna_full_address"].str.replace(",", "").str.lower().str.contains( + x["Address"].lower().strip())) + ] + + if m.shape[0] == 1: + matched.append( + { + "full_address": m["domna_full_address"].values[0], + "postcode": m["domna_postcode"].values[0], + "review_no": x["No."], + "review_address": x["Address"], + "review_postcode": x["Postcode"], + "exclusion_reason": x["exclusion_reason"], + "landlord_property_id": m["landlord_property_id"].values[0], + "ciga_guarantee": x["Unnamed: 21"] + } + ) + continue + + raise NotImplementedError("FIX ME") + +matched = pd.DataFrame(matched) + +matched = matched.rename( + columns={"review_address": "ciga_check_address"} +) + +asset_list = asset_list.merge( + matched[["landlord_property_id", "exclusion_reason", "ciga_guarantee", "ciga_check_address"]], + how="left", on="landlord_property_id" +) + +# Store as an excel +filename = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Medway/Reviewed Standardised Programme.xlsx" +# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data + +with pd.ExcelWriter(filename) as writer: + asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False) + flat_data.to_excel(writer, sheet_name="Flat Data", index=False)