refining detection for solar and breakdown counts

asset_list/AssetList.py

@@ -1012,7 +1012,8 @@ class AssetList:
                 )
             )
         else:
-            raise NotImplementedError("need to implement the case for non-intrusives")
+            # We set the filter to False, as we have no non-intrusives
+            non_intrusives_wall_filter = False
 
         self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] = (
             (~self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE].isin(["bedsit"])) &
@@ -1110,7 +1111,8 @@ class AssetList:
             )
 
         else:
-            raise NotImplementedError("need to implement the case for non-intrusives")
+            self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] = False
+            self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"] = False
 
         # Adjust
         self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"] = np.where(
@@ -1131,7 +1133,7 @@ class AssetList:
         )
         self.standardised_asset_list["solar_landlord_data_indicates_needs_heating_upgrade"] = (
             self.standardised_asset_list[self.STANDARD_HEATING_SYSTEM].isin(
-                ["electric storage heaters", "room heaters"]
+                ["electric storage heaters", "room heaters", "electric radiators"]
             )
         )
 
@@ -1179,7 +1181,8 @@ class AssetList:
                 )
             )
         else:
-            raise NotImplementedError("need to implement the case for non-intrusives")
+            # We don't have an indication
+            existing_solar_non_intrusives_check = False
 
         self.standardised_asset_list["property_has_solar"] = (
             (self.standardised_asset_list[self.STANDARD_EXISTING_PV] == "already has PV") |
@@ -1208,7 +1211,7 @@ class AssetList:
         print("Should we include cavity properties where they might be uninsulated?")
         self.standardised_asset_list["solar_landlord_walls_insulated"] = (
             self.standardised_asset_list[self.STANDARD_WALL_CONSTRUCTION].isin(
-                ["filled cavity", "insulated solid brick"]
+                ["filled cavity", "insulated solid brick", "insulated timber frame"]
             )
         )
 
@@ -1225,7 +1228,7 @@ class AssetList:
                 )
             )
         else:
-            raise NotImplementedError("need to implement the case for non-intrusives")
+            self.standardised_asset_list["solar_non_intrusives_walls_insulated"] = False
 
         # TODO: We don't have information about the roof from this landlord
 
@@ -1294,7 +1297,6 @@ class AssetList:
             lambda x: int(x) < 200 if str(x).isdigit() else False
         )
 
-        # TODO: Fill with False - should be temp!
         self.standardised_asset_list["epc_has_floor_recommendation"] = (
             self.standardised_asset_list["epc_has_floor_recommendation"].fillna(False)
         )
@@ -1339,36 +1341,6 @@ class AssetList:
             )
         )
 
-        # Check for other floor types, insulated
-        self.standardised_asset_list["solar_epc_floor_is_other_insulated"] = (
-            # The floor is suspended and insulated
-            (
-                (
-                    self.standardised_asset_list[self.EPC_API_DATA_NAMES["floor-description"]].str
-                    .lower().str.contains("suspended")
-                ) & (
-                    ~self.standardised_asset_list["epc_has_floor_recommendation"]
-                ) & (
-                    # We do not utilise estimated EPCs for this method because we will always find that
-                    # "epc_has_floor_recommendation" is False
-                    self.standardised_asset_list["estimated"] == False
-                )
-            ) | (
-                (
-                    self.standardised_asset_list[
-                        self.EPC_API_DATA_NAMES["floor-description"]
-                    ].str.lower().str.contains("suspended")
-                ) & (
-                    self.standardised_asset_list[
-                        self.EPC_API_DATA_NAMES["floor-description"]
-                    ].str.lower().str.contains(", insulated")
-                )
-            ) | (
-                self.standardised_asset_list["floor_u_value"].apply(
-                    lambda x: x <= 0.5 if not pd.isnull(x) else False
-                )
-            )
-        )
         ####################################
         # Check solar eligibility
         ####################################
@@ -1390,7 +1362,13 @@ class AssetList:
             self.standardised_asset_list["solar_non_intrusives_walls_insulated"]
         )
 
+        not_a_flat = (
+            self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "flat"
+        )
+
         self.standardised_asset_list["solar_eligible_solid_floor"] = (
+            # Property isn't a flag
+            not_a_flat &
             # Landlord data or EPC data indicates the heating system is appropriate
             correct_heating_system &
             # The property doesn't currently have solar
@@ -1399,11 +1377,32 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_roof_insulated"] &
-            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"]
+            # Floor type check
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+
+        self.standardised_asset_list["solar_eligible_solid_floor_sap_above_threshold"] = (
+            # Property isn't a flag
+            not_a_flat &
+            # Landlord data or EPC data indicates the heating system is appropriate
+            correct_heating_system &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_roof_insulated"] &
+            # Floor type check
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
         # With heating upgrade
         self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"] = (
+            not_a_flat &
             # Needs heating upgrade
             needs_heating_upgrade &
             # The property doesn't currently have solar
@@ -1412,14 +1411,43 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_roof_insulated"] &
-            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"]
+            # Floor type check
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP Below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] &
+            # SAP above threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
+        # With heating upgrade, above threshold
+        self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"] = (
+            not_a_flat &
+            # Needs heating upgrade
+            needs_heating_upgrade &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_roof_insulated"] &
+            # Floor type check
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP Below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+
         # Because the EPC data can be contradictrory, we remove any overlap
         self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"] = np.where(
             self.standardised_asset_list["solar_eligible_solid_floor"],
             False,
             self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"]
         )
+        self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"] = np.where(
+            self.standardised_asset_list["solar_eligible_solid_floor_sap_above_threshold"],
+            False,
+            self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold"]
+        )
 
         # We shouldn't have an overlap
         if (
@@ -1430,6 +1458,7 @@ class AssetList:
 
         # Solid floor but needs a loft top-up
         self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"] = (
+            not_a_flat &
             # Landlord data or EPC data indicates the heating system is appropriate
             correct_heating_system &
             # The property doesn't currently have solar
@@ -1438,10 +1467,31 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_loft_needs_topup"] &
-            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"]
+            # Check floor
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
+        # Solid floor, needs loft, above SAP thresold
+        self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_sap_above_threshold"] = (
+            not_a_flat &
+            # Landlord data or EPC data indicates the heating system is appropriate
+            correct_heating_system &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_loft_needs_topup"] &
+            # Check floor
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+        # Needs loft & heating
         self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_needs_heating_upgrade"] = (
+            not_a_flat &
             # Needs heating upgrade
             needs_heating_upgrade &
             # The property doesn't currently have solar
@@ -1450,11 +1500,33 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_loft_needs_topup"] &
-            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"]
+            # Floor type
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+
+        self.standardised_asset_list[
+            "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade_sap_above_threshold"
+        ] = (
+            not_a_flat &
+            # Needs heating upgrade
+            needs_heating_upgrade &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_loft_needs_topup"] &
+            # Floor type
+            self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
         # Other floor type, fully insulated
         self.standardised_asset_list["solar_eligible_other_floor"] = (
+            not_a_flat &
             # Landlord data or EPC data indicates the heating system is appropriate
             correct_heating_system &
             # The property doesn't currently have solar
@@ -1463,11 +1535,30 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_roof_insulated"] &
-            self.standardised_asset_list["solar_epc_floor_is_other_insulated"]
+            # Floor type
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+        self.standardised_asset_list["solar_eligible_other_floor_sap_above_threshold"] = (
+            not_a_flat &
+            # Landlord data or EPC data indicates the heating system is appropriate
+            correct_heating_system &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_roof_insulated"] &
+            # Floor type - other types
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
         # With heating upgrade
         self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"] = (
+            not_a_flat &
             # Needs heating upgrade
             needs_heating_upgrade &
             # The property doesn't currently have solar
@@ -1476,11 +1567,37 @@ class AssetList:
             walls_are_insulated &
             # Roof is insulated
             self.standardised_asset_list["solar_epc_roof_insulated"] &
-            self.standardised_asset_list["solar_epc_floor_is_other_insulated"]
+            # Other floor types
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
+        # With heating upgrade, SAP above threshold
+        self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold"] = (
+            not_a_flat &
+            # Needs heating upgrade
+            needs_heating_upgrade &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof is insulated
+            self.standardised_asset_list["solar_epc_roof_insulated"] &
+            # Other floor types
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+        # Check for overlap
+        if (
+            self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"] &
+            self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold"]
+        ).sum():
+            raise ValueError("Both heating upgrade and no heating upgrade are true - this should not be possible")
 
         # Other floor type, needs loft top-up
         self.standardised_asset_list["solar_eligible_other_floor_needs_loft"] = (
+            not_a_flat &
             # Landlord data or EPC data indicates the heating system is appropriate
             correct_heating_system &
             # The property doesn't currently have solar
@@ -1489,12 +1606,31 @@ class AssetList:
             walls_are_insulated &
             # Roof need loft top-up
             self.standardised_asset_list["solar_epc_loft_needs_topup"] &
-            # Floor is not solid, but is insulated
+            # Other floor types
-            self.standardised_asset_list["solar_epc_floor_is_other_insulated"]
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+        # Other floor type, needs loft top-up, SAP above threshold
+        self.standardised_asset_list["solar_eligible_other_floor_needs_loft_sap_above_threshold"] = (
+            not_a_flat &
+            # Landlord data or EPC data indicates the heating system is appropriate
+            correct_heating_system &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof need loft top-up
+            self.standardised_asset_list["solar_epc_loft_needs_topup"] &
+            # Other floor types
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
         # With heating upgrade
         self.standardised_asset_list["solar_eligible_other_floor_needs_loft_needs_heating_upgrade"] = (
+            not_a_flat &
             # Landlord data or EPC data indicates the heating system is appropriate
             needs_heating_upgrade &
             # The property doesn't currently have solar
@@ -1503,8 +1639,28 @@ class AssetList:
             walls_are_insulated &
             # Roof need loft top-up
             self.standardised_asset_list["solar_epc_loft_needs_topup"] &
-            # Floor is not solid, but is insulated
+            # Other floor types
-            self.standardised_asset_list["solar_epc_floor_is_other_insulated"]
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP below threshold
+            self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
+        )
+
+        self.standardised_asset_list[
+            "solar_eligible_other_floor_needs_loft_needs_heating_upgrade_sap_above_threshold"
+        ] = (
+            not_a_flat &
+            # Landlord data or EPC data indicates the heating system is appropriate
+            needs_heating_upgrade &
+            # The property doesn't currently have solar
+            ~self.standardised_asset_list["property_has_solar"] &
+            # The walls are insulated
+            walls_are_insulated &
+            # Roof need loft top-up
+            self.standardised_asset_list["solar_epc_loft_needs_topup"] &
+            # Other floor types
+            ~self.standardised_asset_list["solar_epc_floor_is_solid_no_recommendation"] &
+            # SAP above threshold
+            ~self.standardised_asset_list[self.ATTRIBUTE_SAP_THRESHOLD_AND_BELOW]
         )
 
         # Drop anything we don't need
@@ -1529,66 +1685,6 @@ class AssetList:
             self.standardised_asset_list[self.STANDARD_PROPERTY_TYPE] != "block of flats"
             ]
 
-        # Produce some aggregate figures
-        self.work_type_figures = {
-            # Empty cavity from non-intrusives
-            "Empty Cavity (non-intrusives)": non_blocks_of_flats["non_intrusive_indicates_empty_cavity"].sum(),
-            "Empty Cavity (non-intrusives, blocks of flats)": (
-                blocks_of_flats["non_intrusive_indicates_empty_cavity"].sum()
-            ),
-            "Empty Cavity (non-intrusives, no SAP filter)": (
-                non_blocks_of_flats["non_intrusive_indicates_empty_cavity_no_sap_filter"].sum()
-            ),
-            "Empty Cavity (non-intrusives, no SAP filter, blocks of flats)": (
-                blocks_of_flats["non_intrusive_indicates_empty_cavity_no_sap_filter"].sum()
-            ),
-            "Empty Cavity (EPC)": (
-                (
-                    non_blocks_of_flats["epc_indicates_empty_cavity"] &
-                    ~non_blocks_of_flats["non_intrusive_indicates_empty_cavity"]
-                ).sum()
-            ),
-            "Empty Cavity (EPC, blocks of flat)": (
-                (
-                    blocks_of_flats["epc_indicates_empty_cavity"] &
-                    ~blocks_of_flats["non_intrusive_indicates_empty_cavity"]
-                ).sum()
-            ),
-            "Cavity Extraction": (
-                (
-                    ~non_blocks_of_flats["non_intrusive_indicates_empty_cavity"] &
-                    ~non_blocks_of_flats["epc_indicates_empty_cavity"] &
-                    non_blocks_of_flats["non_intrusive_indicates_cavity_extraction"]
-                ).sum()
-            ),
-            "Cavity Extraction (blocks of flats)": (
-                (
-                    ~blocks_of_flats["non_intrusive_indicates_empty_cavity"] &
-                    ~blocks_of_flats["epc_indicates_empty_cavity"] &
-                    blocks_of_flats["non_intrusive_indicates_cavity_extraction"]
-                ).sum()
-            ),
-            "Cavity Extraction (no SAP filter)": (
-                (
-                    ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
-                    ~self.standardised_asset_list["epc_indicates_empty_cavity"] &
-                    self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_sap_filter"]
-                ).sum()
-            ),
-            "Solar PV (Solid Floor)": (
-                self.standardised_asset_list["solar_eligible_solid_floor"].sum()
-            ),
-            "Solar PV (Solid Floor, Needs Loft Top-up)": (
-                self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"].sum()
-            ),
-            "Solar PV (Other Floor)": (
-                self.standardised_asset_list["solar_eligible_other_floor"].sum()
-            ),
-            "Solar PV (Other Floor, Needs Loft Top-up)": (
-                self.standardised_asset_list["solar_eligible_other_floor_needs_loft"].sum()
-            )
-        }
-
         # Finally, we note why each property has been flagged
         self.standardised_asset_list["cavity_reason"] = None
         self.standardised_asset_list["cavity_reason"] = np.where(
@@ -1628,51 +1724,55 @@ class AssetList:
             self.standardised_asset_list["cavity_reason"]
         )
 
+        ######################################################
         # Flag solar
+        ######################################################
         self.standardised_asset_list["solar_reason"] = None
-        self.standardised_asset_list["solar_reason"] = np.where(
-            self.standardised_asset_list["solar_eligible_solid_floor"],
-            "Solid Floor, Insulated, No Existing Solar",
-            self.standardised_asset_list["solar_reason"]
-        )
-        self.standardised_asset_list["solar_reason"] = np.where(
-            self.standardised_asset_list["solar_eligible_solid_floor_needs_heating_upgrade"],
-            "Solid Floor, Insulated, No Existing Solar, Needs Heating Upgrade",
-            self.standardised_asset_list["solar_reason"]
-        )
 
-        self.standardised_asset_list["solar_reason"] = np.where(
+        # Map of variables and fill values for the solar_reason variable
-            self.standardised_asset_list["solar_eligible_solid_floor_needs_loft"],
+        solar_reason_map = {
-            "Solid Floor, Insulated, Needs Loft, No Existing Solar",
+            "solar_eligible_solid_floor": "Solar Eligible, Solid Floor",
-            self.standardised_asset_list["solar_reason"]
+            "solar_eligible_solid_floor_sap_above_threshold": "Solar Eligible, Solid Floor, SAP Above Threshold",
-        )
+            "solar_eligible_solid_floor_needs_heating_upgrade": (
-        self.standardised_asset_list["solar_reason"] = np.where(
+                "Solar Eligible, Solid Floor, Needs Heating Upgrade"
-            self.standardised_asset_list["solar_eligible_solid_floor_needs_loft_needs_heating_upgrade"],
+            ),
-            "Solid Floor, Insulated, Needs Loft, No Existing Solar, Needs Heating Upgrade",
+            "solar_eligible_solid_floor_needs_heating_upgrade_sap_above_threshold": (
-            self.standardised_asset_list["solar_reason"]
+                "Solar Eligible, Solid Floor, Needs Heating Upgrade, SAP Above Threshold"
-        )
+            ),
+            "solar_eligible_solid_floor_needs_loft": "Solar Eligible, Solid Floor, Needs Loft",
+            "solar_eligible_solid_floor_needs_loft_sap_above_threshold": (
+                "Solar Eligible, Solid Floor, Needs Loft, SAP Above Threshold"
+            ),
+            "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade": (
+                "Solar Eligible, Solid Floor, Needs Loft, Needs Heating Upgrade"
+            ),
+            "solar_eligible_solid_floor_needs_loft_needs_heating_upgrade_sap_above_threshold": (
+                "Solar Eligible, Solid Floor, Needs Loft, Needs Heating Upgrade, SAP Above Threshold"
+            ),
+            "solar_eligible_other_floor": "Solar Eligible, Other Floor",
+            "solar_eligible_other_floor_sap_above_threshold": "Solar Eligible, Other Floor, SAP Above Threshold",
+            "solar_eligible_other_floor_needs_heating_upgrade": "Solar Eligible, Other Floor, Needs Heating Upgrade",
+            "solar_eligible_other_floor_needs_heating_upgrade_sap_above_threshold": (
+                "Solar Eligible, Other Floor, Needs Heating Upgrade, SAP Above Threshold"
+            ),
+            "solar_eligible_other_floor_needs_loft": "Solar Eligible, Other Floor, Needs Loft",
+            "solar_eligible_other_floor_needs_loft_sap_above_threshold": (
+                "Solar Eligible, Other Floor, Needs Loft, SAP Above Threshold"
+            ),
+            "solar_eligible_other_floor_needs_loft_needs_heating_upgrade": (
+                "Solar Eligible, Other Floor, Needs Loft, Needs Heating Upgrade"
+            ),
+            "solar_eligible_other_floor_needs_loft_needs_heating_upgrade_sap_above_threshold": (
+                "Solar Eligible, Other Floor, Needs Loft, Needs Heating Upgrade, SAP Above Threshold"
+            )
+        }
 
-        self.standardised_asset_list["solar_reason"] = np.where(
+        for variable, reason in solar_reason_map.items():
-            self.standardised_asset_list["solar_eligible_other_floor"],
+            self.standardised_asset_list["solar_reason"] = np.where(
-            "Other Floor, Insulated, No Existing Solar",
+                self.standardised_asset_list[variable],
-            self.standardised_asset_list["solar_reason"]
+                reason,
-        )
+                self.standardised_asset_list["solar_reason"]
-        self.standardised_asset_list["solar_reason"] = np.where(
+            )
-            self.standardised_asset_list["solar_eligible_other_floor_needs_heating_upgrade"],
-            "Other Floor, Insulated, No Existing Solar, Needs Heating Upgrade",
-            self.standardised_asset_list["solar_reason"]
-        )
-
-        self.standardised_asset_list["solar_reason"] = np.where(
-            self.standardised_asset_list["solar_eligible_other_floor_needs_loft"],
-            "Other Floor, Insulated, Needs Loft, No Existing Solar",
-            self.standardised_asset_list["solar_reason"]
-        )
-        self.standardised_asset_list["solar_reason"] = np.where(
-            self.standardised_asset_list["solar_eligible_other_floor_needs_loft_needs_heating_upgrade"],
-            "Other Floor, Insulated, Needs Loft, No Existing Solar, Needs Heating Upgrade",
-            self.standardised_asset_list["solar_reason"]
-        )
 
         # Flag anything that has existing outcomes
         if self.outcomes is not None:
@@ -1694,6 +1794,12 @@ class AssetList:
                 self.standardised_asset_list["cavity_reason"]
             )
 
+        # Produce some aggregate figures
+        self.work_type_figures = {
+            **self.standardised_asset_list["cavity_reason"].value_counts().to_dict(),
+            **self.standardised_asset_list["solar_reason"].value_counts().to_dict()
+        }
+
     def flat_analysis(self):
 
         # We need to deduce the building name - we strip out the house number
@@ -2028,7 +2134,7 @@ class AssetList:
         outcomes_houseno
     ):
         if outcomes_filepath is None:
-            pass
+            return
 
         # ToDO: Parameterise for future use?
         self.outcomes = pd.read_excel(outcomes_filepath, sheet_name=outcomes_sheetname)
@@ -2119,6 +2225,8 @@ class AssetList:
             self.outcomes[["row_id", "Outcome", "Notes", date_col]], how="left", on="row_id"
         )
 
+        df = lookup[lookup["domna_property_id"] == "44beckettavenuegainsboroughdn211en-1d4811cbb046"]
+
         visit_counts = (
             lookup.groupby(self.DOMNA_PROPERTY_ID)["row_id"]
             .count()
@@ -2153,6 +2261,9 @@ class AssetList:
     ):
         # TODO: This probably needs further expansion
 
+        if not master_filepaths:
+            return
+
         if master_to_asset_list_filepath is not None:
             id_map = pd.read_csv(master_to_asset_list_filepath)
         else:

asset_list/app.py

@@ -247,6 +247,30 @@ def app():
     # - We want: fully insulated property (all wall types), EPC D or below (floors should be solid)
     # - Or the insulation required is loft/cavity (floors should be solid)
 
+    # Wates
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Wates - "
+    data_filename = "ECO 4 Wates.xlsx"
+    sheet_name = "Roadmap Homes"
+    postcode_column = 'Postcode'
+    fulladdress_column = None
+    address1_column = "Address Line 1"
+    address1_method = None
+    address_cols_to_concat = ["Address Line 1", "Address Line 2", "Address Line 3"]
+    missing_postcodes_method = None
+    landlord_year_built = "Build Year"
+    landlord_os_uprn = None
+    landlord_property_type = "Archetype"
+    landlord_wall_construction = "Wall"
+    landlord_heating_system = "Heating Type"
+    landlord_existing_pv = None
+    landlord_property_id = "UPRN"
+    outcomes_filename = None
+    outcomes_sheetname = None
+    outcomes_postcode = None
+    outcomes_houseno = None
+    master_filepaths = []
+    master_to_asset_list_filepath = None
+
     # Ealing
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Ealing/Programme data - 04032025"
     # data_filename = "Ealing BC - Property Plus Tenure 25.02.2025.xlsx"
@@ -265,6 +289,29 @@ def app():
     # landlord_existing_pv = None
     # landlord_property_id = "Property ref"
 
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Colchester"
+    # data_filename = "Warmfront data- Colchester Borough Homes (Complete).xlsx"
+    # sheet_name = "Sheet1"
+    # postcode_column = 'Full Address.1'
+    # fulladdress_column = "Full Address"
+    # address1_column = None
+    # address1_method = "first_word"
+    # address_cols_to_concat = []
+    # missing_postcodes_method = None
+    # landlord_year_built = "Build Date"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Property Type"
+    # landlord_wall_construction = "Wallinsul"
+    # landlord_heating_system = "HeatSorc"
+    # landlord_existing_pv = None
+    # landlord_property_id = "Property Reference"
+    # outcomes_filename = None
+    # outcomes_sheetname = None
+    # outcomes_postcode = None
+    # outcomes_houseno = None
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
+
     # For Westward
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Westward"
     # data_filename = "WESTWARD - completed list..xlsx"
@@ -282,6 +329,12 @@ def app():
     # landlord_heating_system = "Heat Source"
     # landlord_existing_pv = "PV (Y/N)"
     # landlord_property_id = "Place ref"
+    # outcomes_filename = None
+    # outcomes_sheetname = None
+    # outcomes_postcode = None
+    # outcomes_houseno = None
+    # master_filepaths = []
+    # master_to_asset_list_filepath = None
 
     # For ACIS - programme re-build
     # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/ACIS/ACIS Full Programme Review March 2025"
@@ -393,7 +446,7 @@ def app():
 
     # We now flag properties that have been treated under existing programmes
     asset_list.flag_outcomes(
-        outcomes_filepath=os.path.join(data_folder, outcomes_filename),
+        outcomes_filepath=os.path.join(data_folder, outcomes_filename) if outcomes_filename else None,
         outcomes_sheetname=outcomes_sheetname,
         outcomes_postcode=outcomes_postcode,
         outcomes_houseno=outcomes_houseno
@@ -566,6 +619,22 @@ def app():
 
     pprint(asset_list.work_type_figures)
 
+    # TODO: Characterise the properties that didn't qualify
+    eg = asset_list.standardised_asset_list[
+        pd.isnull(asset_list.standardised_asset_list["solar_reason"])
+    ]
+    eg[asset_list.EPC_API_DATA_NAMES["floor-description"]].value_counts()
+
+    # TODO: Look into the estimated ones
+    eg["estimated"].value_counts()
+
+    eg = eg[eg[asset_list.STANDARD_HEATING_SYSTEM] == "high heat retention storage heaters"]
+    eg[asset_list.STANDARD_WALL_CONSTRUCTION].value_counts()
+    eg = eg[eg[asset_list.STANDARD_WALL_CONSTRUCTION] == "filled cavity"]
+    eg[asset_list.EPC_API_DATA_NAMES["roof-description"]].value_counts()
+    eg[asset_list.EPC_API_DATA_NAMES["floor-description"]].value_counts()
+    eg["epc_has_floor_recommendation"].value_counts()
+
     asset_list.flat_analysis()
 
     asset_list.load_contact_details(
@@ -614,6 +683,7 @@ def app():
     with pd.ExcelWriter(filename) as writer:
         asset_list.standardised_asset_list.to_excel(writer, sheet_name="Standardised Asset List", index=False)
         asset_list.flat_data.to_excel(writer, sheet_name="Flat Data", index=False)
+        # If we have outcomes, we add a tab with the outcomes
 
     # Store the Hubspot export as a csv
     hubspot_data.to_csv(os.path.join(data_folder, "Hubspot Export.csv"), index=False)

asset_list/mappings/heating_systems.py

@@ -95,5 +95,16 @@ HEATING_MAPPINGS = {
     'Boiler Solid fuel': 'boiler - other fuel',
     'Community heating Community (mains gas)': 'communal gas boiler',
     'Boiler Biomass': 'boiler - other fuel',
-    'No heating system Mains gas': 'unknown'
+    'No heating system Mains gas': 'unknown',
+
+    'Storage heaters': 'electric storage heaters',
+    'Air Source': 'air source heat pump',
+    'Ground source': 'ground source heat pump',
+    'OIl': 'boiler - other fuel',
+    'Quantum storage heaters (old sh on EPC)': 'high heat retention storage heaters',
+    'Quanum Storage heaters': 'high heat retention storage heaters',
+    'Quantum storage heaters (Old SH on EPC)': 'high heat retention storage heaters',
+    'Quantum storage heaters': 'high heat retention storage heaters',
+    'Air Source (EPC says SH)': 'air source heat pump',
+    'ASHP - Was logged as oil': 'air source heat pump'
 }

asset_list/mappings/property_type.py

@@ -63,5 +63,13 @@ PROPERTY_MAPPING = {
     '2 Bed 1st Floor Sheltered Flat': 'flat',
     '1 Bed First Floor Flat': 'flat',
     '3 Bed First Floor Flat': 'flat',
-    'ND': 'unknown'
+    'ND': 'unknown',
+    'House (Mid Terrace)': 'house',
+    'First Floor Flat General': 'flat',
+    'House (End Terrace)': 'house',
+    'House (Mid terrace)': 'house',
+    'Bungalow (Semi)': 'bungalow',
+    'Ground Floor Flat General': 'flat',
+    'House (Semi)': 'house'
+
 }

asset_list/mappings/walls.py

@@ -1,10 +1,14 @@
 import numpy as np
 
 STANDARD_WALL_CONSTRUCTIONS = {
+    # Cavity
     "uninsulated cavity", "filled cavity", "partial insulated cavity", "cavity unknown insulation",
+    # Solic Brick
     "uninsulated solid brick", "insulated solid brick", "solid brick unknown insulation",
-    "timber frame",
+    # Timber Frame
-    "system built", "granite or whinstone", "other", "unknown", "sandstone or limestone",
+    "timber frame unknown insulation", "insulated timber frame", "uninsulated timber frame",
+    "system built", "granite or whinstone", "other",
+    "unknown", "sandstone or limestone",
     "cob",
     "new build - average thermal transmittance",
 }
@@ -117,5 +121,18 @@ WALL_CONSTRUCTION_MAPPINGS = {
     'Solid brick Internal': 'insulated solid brick',
     'Cavity Internal': 'filled cavity',
     'System build Internal': 'system built',
-    'Solid brick As-built': 'solid brick unknown insulation'
+    'Solid brick As-built': 'solid brick unknown insulation',
+
+    'Cavity ': 'cavity unknown insulation',
+    'Solid brick ': 'solid brick unknown insulation',
+    'Timber frame Timber frame (good insulation)': 'insulated timber frame',
+    ' ': 'unknown',
+    'Cavity No data': 'cavity unknown insulation',
+    'Non trad ': 'other',
+    'Solid brick / Multiple Attributes ': 'solid brick unknown insulation',
+    'Cavity Believe CWI done by Dyson': 'filled cavity',
+    'Cavity CWI required': 'uninsulated cavity',
+    'Solid brick EWI installed': 'insulated solid brick',
+    'Cavity Cavity batts': 'filled cavity',
+    'Cavity CWI Completed by Dyson': 'filled cavity'
 }

etl/customers/mod/pilot/1. Create Sample.py

@@ -0,0 +1,34 @@
+import pandas as pd
+
+
+def app():
+    """
+    Given the sample data and additonal properties, this function prepares the data
+    :return:
+    """
+    folder_path = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/MOD/Pilot Programme"
+    sample_list = pd.read_excel(f"{folder_path}/20250227_DIO_Accommodation_Sample_Properties.xlsx")
+    asset_data = pd.read_excel(f"{folder_path}/20250303_DIO_Accommodation_Property_Attribution.xlsx")
+    asset_data["BLNDG_GOVERMENT_UPRN"] = asset_data["BLNDG_GOVERMENT_UPRN"].astype("Int64")
+
+    asset_data["BLNDG_GOVERMENT_UPRN"].nunique()
+    for _id in asset_data["ESTB_ID"].unique():
+        data = asset_data[asset_data["ESTB_ID"] == _id]
+        z = data["BLNDG_GOVERMENT_UPRN"]
+
+        data["BLNDG_GOVERMENT_UPRN"].unique()
+
+    asset_data["BLNDG_GOVERMENT_UPRN"].unique()
+
+    df = asset_data.groupby("BLNDG_GOVERMENT_UPRN")["ESTB_ID"].nunique().sort_values(ascending=False).reset_index()
+
+    example = asset_data[asset_data["BLNDG_GOVERMENT_UPRN"] == df.head(1)["BLNDG_GOVERMENT_UPRN"].values[0]]
+
+    asset_data[asset_data["BLNDG_GOVERMENT_UPRN"]]
+
+    asset_data = asset_data[asset_data["ESTB_ID"].isin(sample_list["ESTB_ID"].values)]
+    asset_data.drop_duplicates("ESTB_ID", inplace=True)
+
+    [x for x in asset_data.columns if "uprn" in x.lower()]
+
+    example = asset_data[asset_data["ESTB_ID"] == 1547072]