cleaning up hubspot prepare code

asset_list/AssetList.py

@@ -2442,7 +2442,9 @@ class AssetList:
 
         # We check if all products are covered in the lookup table
         cavity_products = self.standardised_asset_list["cavity_reason"].unique().tolist()
+        cavity_products = [x for x in cavity_products if not pd.isnull(x)]
         solar_products = self.standardised_asset_list["solar_reason"].unique().tolist()
+        solar_products = [x for x in solar_products if not pd.isnull(x)]
 
         product_map = {}
         for identified_product in cavity_products + solar_products:

asset_list/app.py

@@ -60,42 +60,76 @@ def app():
     Property UPRN
     """
 
-    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Calico"
-    data_filename = "07.04 CALICO - Final List.xlsx"
-    asset_list_header = 2
-    sheet_name = "Final List"
+    # NCHA
+    data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/NCHA"
+    data_filename = "Energy Information MASTER June 2025.xlsx"
+    sheet_name = "Data"
     postcode_column = 'Postcode'
-    fulladdress_column = None
-    address1_column = "Property Number / Name"
-    address1_method = None
-    address_cols_to_concat = [
-        "Property Number / Name",
-        "Street",
-        "Town"
-    ]
+    fulladdress_column = "Address"
+    address1_column = None
+    address1_method = "house_number_extraction"
+    address_cols_to_concat = []
     missing_postcodes_method = None
-    landlord_year_built = "NROSH Estimated Build Date"
+    landlord_year_built = "Build Date (HAR10)"
     landlord_os_uprn = None
-    landlord_property_type = "Asset Type"
-    landlord_built_form = None
-    landlord_wall_construction = "Wall Type"
-    landlord_heating_system = "Boiler Type"
-    landlord_existing_pv = None
-    landlord_property_id = "Asset Reference"
-    outcomes_filename = []
-    outcomes_sheetname = []
-    outcomes_postcode = []
-    outcomes_houseno = []
-    outcomes_id = []
-    outcomes_address = []
-    master_filepaths = []
-    master_id_colnames = []
-    master_to_asset_list_filepath = None
+    landlord_property_type = "Property Type (HAR10)"
+    landlord_built_form = "Build Form (EPC)"
+    landlord_wall_construction = "Wall Description"
     landlord_roof_construction = None
-    landlord_block_reference = None
-    landlord_sap = "Current Efficiency Rating - Score"
-    phase = None
+    landlord_heating_system = "HEAT Code"
+    landlord_existing_pv = None
+    landlord_property_id = "Place ref"
+    landlord_sap = "EPC SAP"
+    outcomes_filename = None
+    outcomes_sheetname = None
+    outcomes_postcode = None
+    outcomes_houseno = None
+    outcomes_id = None
+    outcomes_address = None
+    master_filepaths = []
+    master_to_asset_list_filepath = None
+    phase = False
     ecosurv_landlords = None
+    asset_list_header = 0
+    landlord_block_reference = None
+    master_id_colnames = []
+
+    # data_folder = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Calico"
+    # data_filename = "07.04 CALICO - Final List.xlsx"
+    # asset_list_header = 2
+    # sheet_name = "Final List"
+    # postcode_column = 'Postcode'
+    # fulladdress_column = None
+    # address1_column = "Property Number / Name"
+    # address1_method = None
+    # address_cols_to_concat = [
+    #     "Property Number / Name",
+    #     "Street",
+    #     "Town"
+    # ]
+    # missing_postcodes_method = None
+    # landlord_year_built = "NROSH Estimated Build Date"
+    # landlord_os_uprn = None
+    # landlord_property_type = "Asset Type"
+    # landlord_built_form = None
+    # landlord_wall_construction = "Wall Type"
+    # landlord_heating_system = "Boiler Type"
+    # landlord_existing_pv = None
+    # landlord_property_id = "Asset Reference"
+    # outcomes_filename = []
+    # outcomes_sheetname = []
+    # outcomes_postcode = []
+    # outcomes_houseno = []
+    # outcomes_id = []
+    # outcomes_address = []
+    # master_filepaths = []
+    # master_id_colnames = []
+    # master_to_asset_list_filepath = None
+    # landlord_roof_construction = None
+    # landlord_block_reference = None
+    # landlord_sap = "Current Efficiency Rating - Score"
+    # phase = None
+    # ecosurv_landlords = None
 
     # data_folder = (
     #     "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Programme Update - March 2025/2018 Asset

asset_list/hubspot/prepare_for_hubspot.py

@@ -19,19 +19,19 @@ def app():
 
     # inputs:
     reconcile_programme = False  # If True, the hubspot upload will include all properties with a project code
-    customer_domain = "https://livewest.co.uk"
+    customer_domain = "https://sandwell.gov.uk"
     installer_name = "J & J CRUMP"
     asset_list_filepath = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Hubspot/Livewest South-West - Standardised "
-        "V2.xlsx"
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Sandwell/Hubspot/Sandwell BC - Full Asset List MAIN - "
+        "Standardised.xlsx"
     )
-    asset_list_sheet_name = "Standardised Asset List"
-    asset_list_header = 0
+    asset_list_sheet_name = "Proposed Program"
+    asset_list_header = 1
 
     contact_details_filepath = (
-        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Livewest/Hubspot/23.06 Livewest Contacts.xlsx"
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Sandwell/Hubspot/Sandwell Contact Details.xlsx"
     )
-    contacts_sheet_name = "Contact Information"
+    contacts_sheet_name = "Sheet1"
     contacts_landlord_property_id = "landlord_property_id"
     contacts_phone_number_column = "phone_number"
     contacts_secondary_phone_number_column = "secondary_phone_number"
@@ -41,6 +41,10 @@ def app():
     contacts_firstname_column = "firstname"
     contacts_lastname_column = "lastname"
 
+    existing_programme_filepath = (
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Sandwell/Hubspot/property-status.csv"
+    )
+
     asset_list = AssetList.load_standardised_asset_list(
         asset_list_filepath, asset_list_sheet_name, asset_list_header
     )
@@ -63,6 +67,14 @@ def app():
         reconcile_programme=reconcile_programme
     )
 
+    # Remove the existing programme
+    existing_programme = pd.read_csv(existing_programme_filepath, encoding="utf-8-sig")
+    asset_list.hubspot_data = asset_list.hubspot_data[
+        ~asset_list.hubspot_data["Domna Property ID <LISTING domna_property_id>"].isin(
+            existing_programme['Domna Property ID'].values
+        )
+    ]
+
     # Get the filepath and the filename. Append hubspot upload to the filename. We also change the file type to csv
     directory, filename = os.path.split(asset_list_filepath)
     name, ext = os.path.splitext(filename)

asset_list/mappings/walls.py

@@ -248,5 +248,7 @@ WALL_CONSTRUCTION_MAPPINGS = {
     'No Fines': 'system built',
     'Granite/Whinstone': 'granite or whinstone',
     'Not applicable to this asset type': 'unknown',
-    'Steel Frame': 'system built'
+    'Steel Frame': 'system built',
+    'Solid Wall As Built': 'uninsulated solid brick',
+    'Solid As Built': 'uninsulated solid brick'
 }

asset_list/utils.py

@@ -80,7 +80,7 @@ def get_data(
             )
             # Force the skipping of estimating the EPC
             # We check if the property was split
-            if home["is_expended_block"]:
+            if home.get("is_expended_block"):
                 searcher.ordnance_survey_client.property_type = "Flat"
                 searcher.property_type = "Flat"
                 searcher.set_strict_property_type_search()