done with ha25 matching for now

etl/eligibility/ha_15_32/ha_analysis_batch_3.py

@@ -172,7 +172,7 @@ class DataLoader:
     }
 
     UNMATCHED_ECO3 = {
-        "HA25": 94
+        "HA25": 119
     }
 
     def __init__(self, directories, december_figures_filepath, use_cache, rebuild):
@@ -478,7 +478,7 @@ class DataLoader:
         # For HA1 and HA25, there is an exception in the structure of the data. We don't have any survey or ciga
         # lists, and so
         # we can return the asset list now
-        if ha_name in ["HA1", "HA25"]:
+        if ha_name in ["HA1"]:
             return asset_list, pd.DataFrame(), pd.DataFrame()
 
         # If we have ECO3 surveys, we need to match them, because any properties treated under ECO3 won't be
@@ -1460,10 +1460,8 @@ class DataLoader:
 
         return survey_list
 
-    def merge_eco3_to_assets(self, asset_list, eco3_list, ha_name):
+    @staticmethod
-
+    def correct_ha25_eco3_list(eco3_list):
-        # May need an eco3 list correction function
-
         # NEADS DRIVE, postcode with bs305dt, is not found in the asset list
         eco3_list = eco3_list[
             ~(eco3_list["Post Code"] == "BS305DT")
@@ -1483,6 +1481,29 @@ class DataLoader:
             "Hall Road", "Hall Rd"
         )
 
+        eco3_list["Street / Block Name"] = eco3_list["Street / Block Name"].str.replace(
+            "SPRINGFIELD WAY SAINT DAY", "SPRINGFIELD WAY ST DAY"
+        )
+        eco3_list["Street / Block Name"] = eco3_list["Street / Block Name"].str.replace(
+            "BOND SPEAR COURT", "BOND-SPEAR COURT"
+        )
+        eco3_list["Street / Block Name"] = eco3_list["Street / Block Name"].str.replace(
+            "ST.MARYS HILL", "ST MARYS HILL"
+        )
+        # Correct the postcode for edmund road
+        eco3_list["Post Code"] = np.where(
+            (eco3_list["Street / Block Name"] == "EDMUND ROAD") &
+            (eco3_list["Post Code"] == "TR14 8QJ"),
+            "TR15 1BY",
+            eco3_list["Post Code"]
+        )
+        return eco3_list
+
+    def merge_eco3_to_assets(self, asset_list, eco3_list, ha_name):
+
+        eco3_list_correction_function = getattr(self, f"correct_{ha_name.lower()}_eco3_list")
+        eco3_list = eco3_list_correction_function(eco3_list)
+
         if ha_name == "HA25":
             missed_postcodes = {
                 postcode.lower() for postcode in eco3_list["Post Code"] if
@@ -1492,8 +1513,9 @@ class DataLoader:
 
         # For the asset list, we create a matching address without any punctuation
         # TODO: We should generally just remove puncutation from addresses when matching
-        asset_list['matching_address_no_punctuation'] = asset_list['matching_address'].str.replace(r'[^\w\s]', '',
+        asset_list['matching_address_no_punctuation'] = asset_list['matching_address'].str.replace(
-                                                                                                   regex=True)
+            r'[^\w\s]', '', regex=True
+        )
         # Remove double spaces
         asset_list["matching_address_no_punctuation"] = asset_list["matching_address_no_punctuation"].str.replace(
             "  ", " "
@@ -1502,6 +1524,8 @@ class DataLoader:
         matching_lookup = []
         missed = []
         for _, row in tqdm(eco3_list.iterrows(), total=len(eco3_list)):
+            # if row["eco3_list_row_id"] == "HA25_Eco3_5422":
+            #     raise Exception()
             postcode = row["Post Code"].lower().strip()
 
             # df will never be empty, since we've already done a check for common postcodes
@@ -1553,38 +1577,24 @@ class DataLoader:
             )
 
         # We verify the missed
-        # -HA25 contains 88 missed entries. These are actually 8 unique postcodes, where surveys were conducted
+        # HA25 contains 119 missed entries. These are actually 24 unique postcodes, and the majority belong to 2
-        # on properties that had house numbers outside of the asset list
+        # where many surveys were conducted on house numbers, not in the asset list
         if len(missed) != self.UNMATCHED_ECO3[ha_name]:
             raise ValueError(
                 f"Unmatched addresses for {ha_name} is not as expected, got {len(missed)} unmatched"
             )
 
-        # TODO: 194 missed
-
         matching_lookup = pd.DataFrame(matching_lookup)
         # Check dupes as this will cause problems later on
         if matching_lookup["asset_list_row_id"].duplicated().any():
             raise ValueError("Duplicated asset list row ids")
 
-        missed_df = eco3_list[eco3_list["eco3_list_row_id"].isin(missed)]
+        # Merge onto eco3 list
-        missed_df.head(3).tail(1)["eco3_list_row_id"]
+        eco3_list = eco3_list.merge(matching_lookup, how="left", on="eco3_list_row_id")
 
-        duped_ids = matching_lookup[matching_lookup["asset_list_row_id"].duplicated()]["asset_list_row_id"].tolist()
+        asset_list = asset_list.drop(columns=["matching_address_no_punctuation"])
-        duped_df = matching_lookup[
-            matching_lookup["asset_list_row_id"].isin(duped_ids)
-        ]
-        duped_surveys = eco3_list[
-            eco3_list["eco3_list_row_id"].isin(duped_df["eco3_list_row_id"].values)
-        ].copy()
 
-        duped_surveys = duped_surveys.merge(matching_lookup, how="left", on="eco3_list_row_id")
+        return eco3_list
-
-        duped_surveys[
-            ["NO ", "Street / Block Name", "Post Code", "eco3_list_row_id", "asset_list_row_id"]
-        ].sort_values("asset_list_row_id").head()
-
-        asset_list[asset_list["asset_list_row_id"] == "HA2515145"]["matching_address"].values
 
     @staticmethod
     def extract_streetname(address, house_number=None, postcode=None):