diff --git a/.idea/Model.iml b/.idea/Model.iml index df6c4faa..c6561970 100644 --- a/.idea/Model.iml +++ b/.idea/Model.iml @@ -10,11 +10,4 @@ - - - \ No newline at end of file diff --git a/asset_list/app.py b/asset_list/app.py index e3c612a7..bb898c09 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -182,6 +182,34 @@ def app(): # master_filepaths = [] # master_to_asset_list_filepath = None + data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West" + data_filename = "Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE.xlsx" + sheet_name = "CHECKED" + postcode_column = 'Postcode' + fulladdress_column = None + address1_column = "AddressLine1" + address1_method = None + address_cols_to_concat = ["AddressLine1", "AddressLine2", "AddressLine3"] + missing_postcodes_method = None + landlord_year_built = None + landlord_os_uprn = None + landlord_property_type = "Archetype (PFP)" + landlord_built_form = "Archetype (PFP)" + landlord_wall_construction = None + landlord_roof_construction = None + landlord_heating_system = None + landlord_existing_pv = None + landlord_property_id = "Uprn" + outcomes_filename = None + outcomes_sheetname = None + outcomes_postcode = None + outcomes_houseno = None + outcomes_id = None + master_filepaths = [] + master_to_asset_list_filepath = None + landlord_sap = None + phase = None + # Maps addresses to uprn in problematic cases manual_uprn_map = {} diff --git a/etl/customers/places_for_people/finalise_programme.py b/etl/customers/places_for_people/finalise_programme.py new file mode 100644 index 00000000..bb612ebf --- /dev/null +++ b/etl/customers/places_for_people/finalise_programme.py @@ -0,0 +1,143 @@ +""" +Having produced the 4 standardsied asset lists for PFP, this script performs a final review +on those assets, reconciling against a list of properties that they sent us that indicates the +properties that they have retained, acquired and then the list will also include some properties that we +have never seen before and so might require additional inspections +""" + +import pandas as pd +import numpy as np +import os +from tqdm import tqdm + + +def match_to_list(pfp_reconciliation_list, asset_list): + lookup = [] + for _, asset in tqdm(pfp_reconciliation_list.iterrows(), total=pfp_reconciliation_list.shape[0]): + _id = str(asset['PRO PROPREF']) + # Match to the asset list - we check the bas ID and then we test removing leading zeros + matched = asset_list[asset_list["landlord_property_id"] == _id] + if matched.empty: + _id_stripped = _id.lstrip("0") + matched = asset_list[asset_list["landlord_property_id"] == _id_stripped] + + if not matched.empty: + lookup.append( + { + "reconciliation_id": _id, + "landlord_property_id": matched["landlord_property_id"].values[0], + } + ) + + lookup = pd.DataFrame(lookup) + asset_list["reconciliation"] = np.where( + asset_list["landlord_property_id"].isin( + lookup["landlord_property_id"].values + ), + "Property still owned by PFP", + "Property not owned by PFP" + ) + + return asset_list, lookup + + +data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/Finalise Programme" + +pfp_reconciliation_list = pd.read_excel( + os.path.join(data_folder, "PFP properties w repair responsibility.xlsx"), +) +# London +pfp_london = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - areas surrounding London - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_london["landlord_property_id"] = pfp_london["landlord_property_id"].astype(str) + +# North-East +pfp_ne = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - North East - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_ne["landlord_property_id"] = pfp_ne["landlord_property_id"].astype(str) + +# North-West +pfp_nw = pd.read_excel( + os.path.join( + data_folder, + "Standardised Asset Lists/Places for People NORTH WEST - INSPECTIONS MASTER - UPDATE - " + "Standardised.xlsx" + ), + sheet_name="Standardised Asset List" +) +pfp_nw["landlord_property_id"] = pfp_nw["landlord_property_id"].astype(str) + +# East +pfp_east = pd.read_excel( + os.path.join(data_folder, "Standardised Asset Lists/PFP - East - Standardised.xlsx"), + sheet_name="Standardised Asset List" +) +pfp_east["landlord_property_id"] = pfp_east["landlord_property_id"].astype(str) + +pfp_london, lookup_london = match_to_list(pfp_reconciliation_list, pfp_london) +pfp_ne, lookup_ne = match_to_list(pfp_reconciliation_list, pfp_ne) +pfp_nw, lookup_nw = match_to_list(pfp_reconciliation_list, pfp_nw) +pfp_east, lookup_east = match_to_list(pfp_reconciliation_list, pfp_east) + +pfp_london["reconciliation"].value_counts() +pfp_ne["reconciliation"].value_counts() +pfp_nw["reconciliation"].value_counts() +pfp_east["reconciliation"].value_counts() + +# We store the reconciled datasets +pfp_london.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - areas surrounding London - reconciled.csv"), + index=False +) +pfp_ne.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - North East - reconciled.csv"), + index=False +) +pfp_nw.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - North West - reconciled.csv"), + index=False +) +pfp_east.to_csv( + os.path.join(data_folder, "Reconciled Programme/PFP - East - reconciled.csv"), + index=False +) + +pd.set_option('display.max_columns', None) +pd.set_option('display.width', 1000) + +# We look at what was on the reconciled list, that was NOT on the original list +all_ids = lookup_london["reconciliation_id"].tolist() + \ + lookup_ne["reconciliation_id"].tolist() + \ + lookup_nw["reconciliation_id"].tolist() + \ + lookup_east["reconciliation_id"].tolist() +missed_inspections = pd.read_excel( + os.path.join( + data_folder, + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/North-West/Places for People NORTH " + "WEST - INSPECTIONS MASTER - UPDATE.xlsx" + ), + sheet_name="MISSING STILL" +) +missed_inspections.columns = ["landlord_id", "address"] + +not_seen = pfp_reconciliation_list[ + ~pfp_reconciliation_list["PRO PROPREF"].astype(str).isin(all_ids) +].copy() + +not_seen["Note"] = None +not_seen["Note"] = np.where( + not_seen["PRO PROPREF"].astype(str).isin(missed_inspections["landlord_id"].astype(str).values) | + not_seen["PRO PROPREF"].astype(str).str.lstrip("0").isin(missed_inspections["landlord_id"].astype(str).values), + "Property not inspected", + not_seen["Note"] +) +not_seen["Note"] = not_seen["Note"].fillna("Property not in original lists") + +# Store +not_seen = os.path.join( + data_folder, "Reconciled Programme/Properties not inspected by Domna.xlsx" +) diff --git a/recommendations/tests/test_costs.py b/recommendations/tests/test_costs.py index 74a210c1..4b8d74db 100644 --- a/recommendations/tests/test_costs.py +++ b/recommendations/tests/test_costs.py @@ -1,6 +1,5 @@ from recommendations.Costs import Costs from unittest.mock import Mock -import datetime import pytest @@ -298,10 +297,10 @@ class TestCosts: # Test for different wattages @pytest.mark.parametrize("n_panels, expected_cost", [ - (7, 4055.0), - (10, 4540.0), - (12, 4863.0), - (15, 5707.0), + (7, 5458.727999999999), + (10, 6013.139999999999), + (12, 6386.447999999999), + (15, 7594.451999999999), ]) def test_solar_pv_different_wattages(self, n_panels, expected_cost): mock_property = Mock() diff --git a/recommendations/tests/test_data/heating_recommendations_data.py b/recommendations/tests/test_data/heating_recommendations_data.py index 53f8bd25..a7f4121e 100644 --- a/recommendations/tests/test_data/heating_recommendations_data.py +++ b/recommendations/tests/test_data/heating_recommendations_data.py @@ -179,13 +179,10 @@ testing_examples = [ 'uprn': 100021560521.0, 'uprn-source': 'Address Matched', }, "heating_measure_types": [ - 'boiler_upgrade', 'roomstat_programmer_trvs', 'time_temperature_zone_control', ], - "notes": "Because of this property is a maisonette, which already has a boiler (but an inefficient one due to " - "the current water heating efficiency) the only recommendation we expect is for " - "a boiler upgrade. The heating controls are programmer and thermostat, so we can also recommend" + "notes": "The heating controls are programmer and thermostat, so we can also recommend" "better heating controls" }, { diff --git a/recommendations/tests/test_floor_recommendations.py b/recommendations/tests/test_floor_recommendations.py index 17f1f82e..eb4f30d2 100644 --- a/recommendations/tests/test_floor_recommendations.py +++ b/recommendations/tests/test_floor_recommendations.py @@ -65,6 +65,7 @@ class TestFloorRecommendations: input_properties[2].number_of_floors = 1 input_properties[2].floor_level = 0 input_properties[2].already_installed = [] + input_properties[2].non_invasive_recommendations = {} recommender = FloorRecommendations(property_instance=input_properties[2], materials=materials) assert recommender.estimated_u_value is None @@ -115,6 +116,7 @@ class TestFloorRecommendations: input_properties[4].number_of_floors = 1 input_properties[4].floor_level = 0 input_properties[4].already_installed = [] + input_properties[4].non_invasive_recommendations = {} # In this case, we have no county, so in this case, it should yse the local-authority-label if possible input_properties[4].data["county"] = ""