diff --git a/etl/customers/stonewater/Wave 3 Preparation.py b/etl/customers/stonewater/Wave 3 Preparation.py index 5ebb06e26..974cd9084 100644 --- a/etl/customers/stonewater/Wave 3 Preparation.py +++ b/etl/customers/stonewater/Wave 3 Preparation.py @@ -1719,6 +1719,72 @@ def propsed_wave_3_sample(): # Tier 2: We have a property in the same archetype that was surveyed and is below EPC D # + def match_property_to_surveyed(property, survey_results_with_original_features): + surveyed = survey_results_with_original_features[ + ( + survey_results_with_original_features["Property Type"] == + property["Property Type"] + ) & + ( + survey_results_with_original_features["Wall Type"] == + property["Wall Type"] + ) & + ( + survey_results_with_original_features["Roof Type"] == + property["Roof Type"] + ) & + ( + survey_results_with_original_features["Heating"] == + property["Heating"] + ) + ].copy() + + if not surveyed.empty: + return surveyed + + surveyed = survey_results_with_original_features[ + ( + survey_results_with_original_features["Property Type"] == + property["Property Type"] + ) & + ( + survey_results_with_original_features["Wall Type"] == + property["Wall Type"] + ) & + ( + survey_results_with_original_features["Roof Type"].str.split(":").str[0] == + property["Roof Type"].split(":")[0] + ) & + ( + survey_results_with_original_features["Heating"] == + property["Heating"] + ) + ].copy() + + if not surveyed.empty: + return surveyed + + surveyed = survey_results_with_original_features[ + ( + survey_results_with_original_features["Property Type"] == + property["Property Type"] + ) & + ( + survey_results_with_original_features["Wall Type"] == + property["Wall Type"] + ) & + ( + survey_results_with_original_features["Roof Type"].str.split(":").str[0] == + property["Roof Type"].split(":")[0] + ) & + ( + survey_results_with_original_features["Heating"].str.split(":").str[0] == + property["Heating"].split(":")[0] + ) + ].copy() + + return surveyed + results = [] for region in tqdm(unique_postal_regions): # Take all of the properties in that region @@ -1757,6 +1823,7 @@ def propsed_wave_3_sample(): ][["Archetype ID", "Current EPC Band"]].drop_duplicates() if region_surveyed["Archetype ID"].duplicated().sum(): + region_surveyed = [] for arch_id in archetypes: for _, property in region_assets[region_assets["Archetype ID"] == arch_id].iterrows(): @@ -1765,6 +1832,12 @@ def propsed_wave_3_sample(): ].copy() if archetype_data.empty: continue + if archetype_data.shape[0] > 1: + # Look for an exact match, or as close as possible + archetype_data_filtered = match_property_to_surveyed(property, archetype_data) + if not archetype_data_filtered.empty: + archetype_data = archetype_data_filtered + archetype_data["distance_meters"] = haversine( lat1=property.latitude, lon1=property.longitude, lat2=archetype_data["latitude"].values, lon2=archetype_data["longitude"].values @@ -1899,28 +1972,15 @@ def propsed_wave_3_sample(): # This means that this archetype was never surveyed and so we need to find a sufficiently similar property final_missed_matches = [] for a_id in missed_addressids: + + match_type = "3 - compared to similar properties" + property = asset_list[asset_list["Address ID"] == a_id].squeeze() - surveyed = survey_results_with_original_features[ - ( - survey_results_with_original_features["Property Type"] == - property["Property Type"] - ) & - ( - survey_results_with_original_features["Wall Type"] == - property["Wall Type"] - ) & - ( - survey_results_with_original_features["Roof Type"] == - property["Roof Type"] - ) & - ( - survey_results_with_original_features["Heating"] == - property["Heating"] - ) - ].copy() + surveyed = match_property_to_surveyed(property, survey_results_with_original_features) if surveyed.empty: + match_type = "3 - compared to similar properties, relaxed" # In this case, we do one additional check where we filter on everything the same apart from heating, # where we do a slightly more rough match surveyed = survey_results_with_original_features[ @@ -2026,14 +2086,12 @@ def propsed_wave_3_sample(): expected_epc = sap_to_epc(expected_sap) if expected_epc in ["C", "B", "A"]: - tier = "5 - EPC C or above" - else: - tier = "3 - similar property, weighted on distance" + match_type = "5 - EPC C or above" final_missed_matches.append( { "Address ID": a_id, - "Confidence Tier": tier, + "Confidence Tier": match_type, "Current EPC Band": expected_epc } ) @@ -2197,22 +2255,9 @@ def propsed_wave_3_sample(): # '2 - same archetype', # '3 - similar property, weighted on distance' - gain_columns = [ - '1 - Archetype surveyed', - '1 - property was surveyed', - '2 - same archetype', - '3 - similar property, weighted on distance' - ] - # - # Loss is the sum of these columns: - # '4 - no similar property, needs survey to confirm', - # '5 - EPC C or above', '5 - property was surveyed' + gain_columns = sorted([x for x in results["Confidence Tier"].unique() if "1 - " in x or "2 - " in x or "3 - " in x]) + loss_columns = sorted([x for x in results["Confidence Tier"].unique() if "4 - " in x or "5 - " in x]) - loss_columns = [ - '4 - no similar property, needs survey to confirm', - '5 - EPC C or above', - '5 - property was surveyed' - ] geographic_summary["Gain"] = geographic_summary[gain_columns].sum(axis=1) geographic_summary["Loss"] = geographic_summary[loss_columns].sum(axis=1) @@ -2283,7 +2328,7 @@ def propsed_wave_3_sample(): # Remaining loss allowed # remaining_loss_constraint = 230 - region_totals["Loss"] - remaining_loss_constraint = 250 + remaining_loss_constraint = 220 postcode_selected_rows, _ = optimise( gain=postcode_summary_unselected_regions["Gain"].values, loss=postcode_summary_unselected_regions["Loss"].values,