working on hubspot upload

.idea/Model.iml

@@ -7,7 +7,7 @@
       <sourceFolder url="file://$MODULE_DIR$/open_uprn" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/recommendations" isTestSource="false" />
     </content>
-    <orderEntry type="jdk" jdkName="Fastapi-backend" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="AssetList" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/misc.xml

@@ -3,7 +3,7 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (backend)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Fastapi-backend" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="AssetList" project-jdk-type="Python SDK" />
   <component name="PyCharmProfessionalAdvertiser">
     <option name="shown" value="true" />
   </component>

asset_list/AssetList.py

@@ -350,6 +350,34 @@ class AssetList:
         "cavity wall, as built, partial insulation",
     ]
 
+    # Work type prefixes:
+    # Empties
+    EMPTY_CAVITY_NON_INTRUSIVE = "Non-Intrusive Data Shows Empty Cavity"
+    EPC_EMPTY_INSPECTIONS_RETRO_DRILLED = "EPC Shows Empty Cavity, inspections show retro drilled"
+    EPC_EMPTY_INSPECTIONS_FILLED = "EPC Shows Empty Cavity, inspections show filled or other"
+    EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD = "EPC Shows Empty Cavity, inspections show filled at build"
+    EPC_EMPTY_INSPECTIONS_NON_CAVITY = "EPC Shows Empty Cavity, inspections show non-cavity build"
+    EPC_EMPTY = "EPC Shows Empty Cavity"
+    LANDLORD_EMPTY_INSPECTIONS_OTHER = ("Landlord Data Shows Empty Cavity, EPC & Inspections Shows Filled or "
+                                        "Non-cavity")
+    # Extraction
+    EXTRACTION_NON_INTRUSIVE = "Non-Intrusive Data Shows Cavity Extraction"
+
+    # Solar
+    SOLAR_ELIGIBLE = "Solar Eligible"
+    SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED = "Solar Eligible, Solid Wall Uninsulated, EPC E or Below"
+    SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE = "Solar Eligible, Needs Heating Upgrade"
+
+    CRM_PRODUCTS = {
+        "Empty Cavity - ECO4": {"id": 82733738177, "unit_price": 1000, "name": "Empty Cavity & Loft - ECO4"},
+        "Extract & Fill - ECO4": {"id": 100307905778, "unit_price": 500, "name": "Extract & Fill - ECO4"},
+        "Solar PV - ECO4": {"id": 82623589564, "unit_price": 1608, "name": "Solar PV - ECO4"},
+        "Solar PV + HHRSH - ECO4": {"id": 155529972924, "unit_price": 1608, "name": "Solar PV + HHRSH - ECO4"},
+        "Solar PV + Heating Upgrade - ECO4": {
+            "id": 109265426665, "unit_price": 1608, "name": "Solar PV + Heating Upgrade - ECO4"
+        },
+    }
+
     def __init__(
         self,
         local_filepath,
@@ -1719,10 +1747,10 @@ class AssetList:
         self.standardised_asset_list["cavity_reason"] = None
 
         empty_cavity_map = {
-            "non_intrusive_indicates_empty_cavity": "Non-Intrusive Data Shows Empty Cavity: ",
+            "non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE_PREFIX + ": ",
-            "non_intrusive_indicates_empty_cavity_has_solar": "Non-Intrusive Data Shows Empty Cavity - property "
+            "non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property "
                                                               "already has solar: ",
-            "non_intrusive_indicates_empty_cavity_no_year_filter": f"Non-Intrusive Data Shows Empty Cavity, "
+            "non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, "
                                                                    f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
 
         }
@@ -1747,7 +1775,7 @@ class AssetList:
                     )) &
                     pd.isnull(self.standardised_asset_list["cavity_reason"])
                 ),
-                "EPC Shows Empty Cavity, inspections show retro drilled: " + self.standardised_asset_list[
+                f"{EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
                     "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
@@ -1759,7 +1787,7 @@ class AssetList:
                     self.standardised_asset_list['non_intrusive_indicates_cavity_extraction'] &
                     pd.isnull(self.standardised_asset_list["cavity_reason"])
                 ),
-                "EPC Shows Empty Cavity, inspections show filled or other: " + self.standardised_asset_list[
+                f"{self.EPC_EMPTY_INSPECTIONS_FILLED}: " + self.standardised_asset_list[
                     "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
@@ -1771,7 +1799,7 @@ class AssetList:
                     (self.standardised_asset_list['non-intrusives: Insulated'] == "RETRO DRILLED") &
                     pd.isnull(self.standardised_asset_list["cavity_reason"])
                 ),
-                "EPC Shows Empty Cavity, inspections show retro drilled: " + self.standardised_asset_list[
+                f"{self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
                     "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
@@ -1783,8 +1811,7 @@ class AssetList:
                     (self.standardised_asset_list['non-intrusives: Insulated'] == "FILLED AT BUILD") &
                     pd.isnull(self.standardised_asset_list["cavity_reason"])
                 ),
-                "EPC Shows Empty Cavity, inspections show filled at build: " + self.standardised_asset_list[
+                f"{self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD}: " + self.standardised_asset_list["SAP Category"],
-                    "SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
         else:
@@ -1794,7 +1821,7 @@ class AssetList:
                     ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
                     pd.isnull(self.standardised_asset_list["cavity_reason"])
                 ),
-                "EPC Shows Empty Cavity: " + self.standardised_asset_list["SAP Category"],
+                f"{self.EPC_EMPTY}: " + self.standardised_asset_list["SAP Category"],
                 self.standardised_asset_list["cavity_reason"]
             )
 
@@ -1804,10 +1831,12 @@ class AssetList:
                 ~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
                 pd.isnull(self.standardised_asset_list["cavity_reason"])
             ),
-            "EPC Shows Empty Cavity, inspections show non-cavity build: " + self.standardised_asset_list[
+            f"{self.EPC_EMPTY_INSPECTIONS_NON_CAVITY}: " + self.standardised_asset_list["SAP Category"],
-                "SAP Category"],
             self.standardised_asset_list["cavity_reason"]
         )
+
+        # Work type prefixes
+
         # Landlord data: The landlord's data indicates that the wall is an uninsulated cavity wall, but EPC and
         # inspections show filled
         self.standardised_asset_list["cavity_reason"] = np.where(
@@ -1817,7 +1846,7 @@ class AssetList:
                 ~self.standardised_asset_list["epc_indicates_empty_cavity"] &
                 pd.isnull(self.standardised_asset_list["cavity_reason"])
             ),
-            "Landlord Data Shows Empty Cavity, EPC & Inspections Shows Filled or Non-cavity: " +
+            f"{self.LANDLORD_EMPTY_INSPECTIONS_OTHER}: " +
             self.standardised_asset_list["SAP Category"],
             self.standardised_asset_list["cavity_reason"]
         )
@@ -1828,7 +1857,7 @@ class AssetList:
                 self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] &
                 pd.isnull(self.standardised_asset_list["cavity_reason"])
             ),
-            "Non-Intrusive Data Shows Cavity Extraction: " + self.standardised_asset_list["SAP Category"],
+            f"{self.EXTRACTION_NON_INTRUSIVE}: " + self.standardised_asset_list["SAP Category"],
             self.standardised_asset_list["cavity_reason"]
         )
 
@@ -1837,7 +1866,7 @@ class AssetList:
                 self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_year_filter"] &
                 pd.isnull(self.standardised_asset_list["cavity_reason"])
             ),
-            f"Non-Intrusive Data Shows Cavity Extraction, built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: " +
+            f"{self.EXTRACTION_NON_INTRUSIVE}, built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: " +
             self.standardised_asset_list["SAP Category"],
             self.standardised_asset_list["cavity_reason"]
         )
@@ -1850,11 +1879,9 @@ class AssetList:
         # Map of variables and fill values for the solar_reason variable
         # ordering of this map is important, where we flag our prioritised work types first
         solar_reason_map = {
-            "solar_eligible": "Solar Eligible: ",
+            "solar_eligible": f"{self.SOLAR_ELIGIBLE}: ",
-            "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
+            "solar_eligible_solid_wall_uninsulated": f"{self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED}: ",
-            "solar_eligible_needs_heating_upgrade": (
+            "solar_eligible_needs_heating_upgrade": f"{self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE}: "
-                "Solar Eligible, Needs Heating Upgrade: "
-            )
         }
 
         for variable, reason in solar_reason_map.items():
@@ -2079,68 +2106,97 @@ class AssetList:
                 *contact_details[fullname_column].apply(self.split_full_name)
             )
         else:
-            raise NotImplementedError("Implement me")
+            contact_details["title"] = None
 
         self.contact_details = contact_details
 
-    def prepare_for_crm(self, company_domain, crm_pipeline_name, first_dealstage, assigned_surveyors):
+    @classmethod
+    def load_standardised_asset_list(cls, filepath):
+        """
+        This function is designed to load the standardised asset list from a file
+        :return:
+        """
+        # This is a placeholder for now
+        # instantiate the class
+        instance = cls(
+            local_filepath=filepath,
+            sheet_name="Standardised Asset List",
+            address1_colname=cls.STANDARD_ADDRESS_1,
+            postcode_colname=cls.STANDARD_POSTCODE,
+            full_address_colname=cls.STANDARD_FULL_ADDRESS,
+            landlord_property_id=cls.STANDARD_LANDLORD_PROPERTY_ID,
+            full_address_cols_to_concat=[],
+            missing_postcodes_method=None,
+            address1_extraction_method=None,
+            landlord_year_built=None,
+            landlord_uprn=None,
+            landlord_property_type=None,
+            landlord_built_form=None,
+            landlord_wall_construction=None,
+            landlord_roof_construction=None,
+            landlord_heating_system=None,
+            landlord_existing_pv=None,
+            landlord_sap=None,
+            landlord_block_reference=None,
+            phase=False,
+            header=0
+        )
+        return instance
+
+    def prepare_for_crm(self, company_domain, installer_name):
         """
         This function prepares the data for upload into Hubspot
         :return:
         """
-        # This is a placeholder for now
-
         # This maps the opportunities as we reference them, to the product data as stored in Hubspot
-        product_lookup_table = {
+
-            "Non-Intrusive Data Showed Cavity Extraction": {
+        prefixes_to_products = {
-                "name": "Extract & Fill - ECO4", "id": 100307905778, "unit_price": 500
+            # Empty
-            },
+            self.EMPTY_CAVITY_NON_INTRUSIVE: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-            "Non-Intrusive Data Showed Empty Cavity": {
+            self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-                "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
+            self.EPC_EMPTY_INSPECTIONS_FILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-            },
+            self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-            "Non-Intrusive Data Showed Empty Cavity but all SAP scores allowed": {
+            self.EPC_EMPTY_INSPECTIONS_NON_CAVITY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-                "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
+            self.EPC_EMPTY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-            },
+            self.LANDLORD_EMPTY_INSPECTIONS_OTHER: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
-            "Non-Intrusive Data Showed Cavity Extraction but all SAP scores allowed": {
+            # Extraction
-                "name": "Extract & Fill - ECO4", "id": 100307905778, "unit_price": 500
+            self.EXTRACTION_NON_INTRUSIVE: self.CRM_PRODUCTS["Extract & Fill - ECO4"],
-            },
+            # Solar
-            "EPC Data Showed Empty Cavity": {
+            self.SOLAR_ELIGIBLE: self.CRM_PRODUCTS["Solar PV - ECO4"],
-                "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
+            self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED: self.CRM_PRODUCTS["Solar PV - ECO4"],
-            },
+            self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE: self.CRM_PRODUCTS["Solar PV + Heating Upgrade - ECO4"],
-            "Solid Floor, Insulated, No Solar": {
-                "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
-            },
-            "Solid Floor, Insulated, Needs Loft": {
-                "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
-            },
-            "Other Floor, Insulated, No Solar": {
-                "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
-            },
-            "Other Floor, Insulated, Needs Loft": {
-                "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
-            }
         }
+
         # We check if all products are covered in the lookup table
-        cavity_products = self.standardised_asset_list["cavity_reason"].unique()
+        cavity_products = self.standardised_asset_list["cavity_reason"].unique().tolist()
-        solar_products = self.standardised_asset_list["solar_reason"].unique()
+        solar_products = self.standardised_asset_list["solar_reason"].unique().tolist()
-        # Check if there any options not in out lookup table
+
-        if (
+        product_map = {}
-            any(x for x in cavity_products if x not in product_lookup_table) or
+        for identified_product in cavity_products + solar_products:
-            any(x for x in solar_products if x not in product_lookup_table)
+            if pd.isnull(identified_product):
-        ):
+                continue
-            raise ValueError("We have products not referenced in the lookup table - check this")
+
+            matched_product = None
+            for product_prefix, crm_product in prefixes_to_products.items():
+                if identified_product.startswith(product_prefix):
+                    matched_product = crm_product
+
+            product_map[identified_product] = matched_product
+
+        # For each cavity and solar product, we iterate through the prexies and map to the products
+
+        # # Check if there any options not in out lookup table
+        # if (
+        #     any(x for x in cavity_products if x not in product_lookup_table) or
+        #     any(x for x in solar_products if x not in product_lookup_table)
+        # ):
+        #     raise ValueError("We have products not referenced in the lookup table - check this")
 
         programme_data = self.standardised_asset_list.copy()
-
+        # We take rows that have a survyor and a date for the survey
-        # Exclusions - these are properties we won't treat for the moment
+        programme_data = programme_data[
-        product_exclusions = [
+            ~pd.isnull(programme_data["survey_week"]) & ~pd.isnull(programme_data["surveyor"])
-            "Other Floor, Insulated, No Solar",
+            ]
-            "Other Floor, Insulated, Needs Loft"
-        ]
-        if product_exclusions:
-            logger.warning("Excluding products: %s", product_exclusions)
-
-        programme_data = programme_data[programme_data["solar_reason"].isin(product_exclusions) == False]
 
         # Merge on the contact details
         programme_data = programme_data.merge(
@@ -2153,26 +2209,34 @@ class AssetList:
         programme_data["Company Domain Name <COMPANY domain>"] = company_domain
         # Append the product data onto the programme data
         programme_data["cavity_product"] = programme_data["cavity_reason"].map(
-            lambda x: product_lookup_table.get(x, {"name": None})["name"]
+            lambda x: product_map.get(x, {"name": None})["name"]
         )
         programme_data["solar_product"] = programme_data["solar_reason"].map(
-            lambda x: product_lookup_table.get(x, {"name": None})["name"]
+            lambda x: product_map.get(x, {"name": None})["name"]
         )
 
-        programme_data["domna_product"] = programme_data["solar_reason"].copy()
+        # We check if we have any missings
+        cavity_missing = pd.isnull(programme_data[~pd.isnull(programme_data["cavity_reason"])]["cavity_product"]).sum()
+        solar_missing = pd.isnull(programme_data[~pd.isnull(programme_data["solar_reason"])]["solar_product"]).sum()
+
+        if cavity_missing > 0 or solar_missing > 0:
+            raise ValueError(
+                f"We have {cavity_missing} cavity products and {solar_missing} solar products that are not "
+                "mapped to a product in the lookup table. Please check the mapping."
+            )
+
+        programme_data["domna_product"] = programme_data["solar_product"].copy()
         programme_data["domna_product"] = np.where(
             pd.isnull(programme_data["domna_product"]),
-            programme_data["solar_product"],
+            programme_data["cavity_product"],
             programme_data["domna_product"]
         )
         # We filter just on rows where we have a product
-        programme_data = programme_data[
+        programme_data = programme_data[~pd.isnull(programme_data["domna_product"])]
-            ~pd.isnull(programme_data["domna_product"])
-        ]
         programme_data = programme_data.drop(columns=["solar_product", "cavity_product"])
 
         product_df = (
-            pd.DataFrame(product_lookup_table).T[["name", "id", "unit_price"]]
+            pd.DataFrame(self.CRM_PRODUCTS).T[["name", "id", "unit_price"]]
             .reset_index()
             .rename(
                 columns={
@@ -2194,21 +2258,27 @@ class AssetList:
         )
 
         # Add in deal and pipeline information
-        programme_data["dealname"] = programme_data[self.STANDARD_FULL_ADDRESS] + " : " + programme_data[
+        programme_data["dealname"] = (
-            "domna_product"]
+            programme_data[self.STANDARD_FULL_ADDRESS] + " : " + programme_data["domna_product"]
-        programme_data['Pipeline <DEAL pipeline>'] = crm_pipeline_name
+        )
-        programme_data['Deal Stage <DEAL dealstage>'] = first_dealstage
+        programme_data['Pipeline <DEAL pipeline>'] = hubspot_config.CRM_PIPELINE_NAME
+        programme_data['Deal Stage <DEAL dealstage>'] = hubspot_config.CRM_PIPELINE_FIRST_STAGE_NAME
         programme_data['Associations: Listing'] = "Property Owner"
 
-        programme_data = programme_data.merge(
+        # programme_data = programme_data.merge(
-            assigned_surveyors.rename(
+        #     assigned_surveyors.rename(
-                columns={self.landlord_property_id: self.STANDARD_LANDLORD_PROPERTY_ID}
+        #         columns={self.landlord_property_id: self.STANDARD_LANDLORD_PROPERTY_ID}
-            ), how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID
+        #     ), how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID
+        # )
+
+        # Add in some columns if we have them
+        date_of_inspections = (
+            "Non-Intrusives: Date of Inspection" if
+            "Non-Intrusives: Date of Inspection" in programme_data.columns else None
         )
 
         # This maps the hubspot schema to the template. Anything that is not covered in this will be flagged
         schema_mappings = {
-            'Name <LISTING hs_name>': self.DOMNA_PROPERTY_ID,  # TODO: Maybe change this?
             'Company Domain Name <COMPANY domain>': 'Company Domain Name <COMPANY domain>',
             'Email <CONTACT email>': (
                 self.contact_detail_fields["email"] if self.contact_detail_fields["email"] else None
@@ -2227,9 +2297,10 @@ class AssetList:
             'Address 2 <LISTING hs_address_2>': None,  # TODO: Don't have this for the moment
             'Postcode <LISTING hs_zip>': self.STANDARD_POSTCODE,
             'Property Type <LISTING property_type>': self.STANDARD_PROPERTY_TYPE,
-            'Property Sub Type <LISTING property_sub_type>': None,  # TODO: Don't have this for the moment
+            'Property Sub Type <LISTING property_sub_type>': self.STANDARD_BUILT_FORM,
             'Bedroom(s) <LISTING hs_bedrooms>': None,  # TODO: Don't have this for the moment
             'Domna Property ID <LISTING domna_property_id>': self.DOMNA_PROPERTY_ID,
+            # We populate this with the column that we have
             'National UPRN <LISTING national_uprn>': (
                 self.STANDARD_UPRN if self.STANDARD_UPRN is not None else self.EPC_API_DATA_NAMES["uprn"]
             ),
@@ -2239,8 +2310,7 @@ class AssetList:
             'Year Built <LISTING hs_year_built>': self.STANDARD_YEAR_BUILT,
             'Boiler Make <LISTING boiler_make>': None,  # TODO: Don't have this for the moment
             'Boiler Model <LISTING boiler_model>': None,  # TODO: Don't have this for the moment
-            'Non-Intrusives: Date Checked <LISTING non_intrusives__date_checked>': None,
+            'Non-Intrusives: Date Checked <LISTING non_intrusives__date_checked>': date_of_inspections,
-            # TODO: Don't have this for the moment
             'Non-Intrusives: Wall Type <LISTING non_intrusives__wall_type>': (
                 "non-intrusives: Construction" if self.non_intrusives_present else None
             ),
@@ -2283,16 +2353,22 @@ class AssetList:
             'Last EPC: Age Band <LISTING last_epc__age_band>': self.EPC_API_DATA_NAMES["construction-age-band"],
             'Deal Stage <DEAL dealstage>': 'Deal Stage <DEAL dealstage>',
             'Pipeline <DEAL pipeline>': 'Pipeline <DEAL pipeline>',
-            'Expected Commencement Date <DEAL expected_commencement_date>': None,  # TODO: Need to set this,
+            'Expected Commencement Date <DEAL expected_commencement_date>': "survey_week",
             'Deal Name <DEAL dealname>': "dealname",  # Need to create this,
             'Product ID <LINE_ITEM hs_product_id>': 'Product ID <LINE_ITEM hs_product_id>',
             'Name <LINE_ITEM name>': 'Name <LINE_ITEM name>',
             'Unit price <LINE_ITEM price>': 'Unit price <LINE_ITEM price>',
             'Quantity <LINE_ITEM quantity>': 'Quantity <LINE_ITEM quantity>',
             'Deal Owner': 'surveyor_email',
-            'Amount <DEAL amount>': 'Unit price <LINE_ITEM price>',
+            'Project Code <DEAL project_code>': 'project_code',
+            'Associations: Listing': 'Associations: Listing',
         }
 
+        # We sometimes columns if the landlord never provided them
+        missed_mapping_cols = [c for c in schema_mappings.values() if c not in programme_data.columns if c is not None]
+        for c in missed_mapping_cols:
+            programme_data[c] = None
+
         # We now create the finalised dataset to be uploaded into Hubspot
         variables_required = list(schema_mappings.values())
         variables_required = [v for v in variables_required if v is not None]
@@ -2307,6 +2383,22 @@ class AssetList:
             columns={v: k for k, v in schema_mappings.items() if v is not None}
         )
 
+        programme_data['Installer <DEAL installer>'] = installer_name
+        programme_data['Name <LISTING hs_name>'] = (
+            programme_data['Address 1 <LISTING hs_address_1>'] + " ," + programme_data['Postcode <LISTING hs_zip>']
+        )
+        # The listing owner email is the same as the surveyor email (deal owner), so they can see the listing
+        programme_data['Listing Owner Email <LISTING hubspot_owner_id>'] = programme_data['Deal Owner']
+        programme_data['Amount <DEAL amount>'] = 0
+
+        # We make sure we have all of the columns that we need
+        missed_columns = [c for c in hubspot_config.CRM_UPLOAD_COLUMNS if c not in programme_data.columns]
+        if missed_columns:
+            raise ValueError(
+                f"We have the following columns that are not in the programme data: {missed_columns}. "
+                "Please check the mapping and ensure all required columns are present."
+            )
+
         self.hubspot_data = programme_data
 
     def flag_ecosurv(self, ecosurv_landlords=None, landlords_to_ignore=None):

asset_list/hubspot/config.py

@@ -1,5 +1,8 @@
 from enum import IntEnum
 
+CRM_PIPELINE_NAME = 'Operations - Housing Associations'
+CRM_PIPELINE_FIRST_STAGE_NAME = 'READY TO BE SCHEDULED'
+
 
 class HubspotProcessStatus(IntEnum):
     def __new__(cls, value, label):
@@ -26,3 +29,43 @@ class HubspotProcessStatus(IntEnum):
     LODGEMENT_COMPLETE = 7, "LODGEMENT COMPLETE"
     # The property has been cancelled
     INSTALLER_CANCELLED_FINALIZED = 8, "INSTALLER CANCELLED - FINALIZED"
+
+
+CRM_UPLOAD_COLUMNS = [
+    'Name <LISTING hs_name>', 'Associations: Listing', 'Company Domain Name <COMPANY domain>',
+    'Email <CONTACT email>', 'First Name <CONTACT firstname>', 'Last Name <CONTACT lastname>',
+    'Phone <CONTACT phone>', 'Listing Owner Email <LISTING hubspot_owner_id>',
+    'Full Address <LISTING full_address>', 'Address 1 <LISTING hs_address_1>',
+    'Address 2 <LISTING hs_address_2>', 'Postcode <LISTING hs_zip>',
+    'Property Type <LISTING property_type>', 'Property Sub Type <LISTING property_sub_type>',
+    'Bedroom(s) <LISTING hs_bedrooms>', 'Domna Property ID <LISTING domna_property_id>',
+    'National UPRN <LISTING national_uprn>', 'Owner Property ID <LISTING owner_property_id>',
+    'Wall Construction <LISTING wall_construction>', 'Heating System <LISTING heating_system>',
+    'Year Built <LISTING hs_year_built>', 'Boiler Make <LISTING boiler_make>',
+    'Boiler Model <LISTING boiler_model>',
+    'Non-Intrusives: Date Checked <LISTING non_intrusives__date_checked>',
+    'Non-Intrusives: Wall Type <LISTING non_intrusives__wall_type>',
+    'Non-intrusives: Insulation <LISTING non_intrusives__insulation>',
+    'Non-intrusives: Insulation Material <LISTING non_intrusives__insulation_material>',
+    'Non-Intrusives: CIGA Check Required <LISTING non_intrusives__ciga_check_required>',
+    'Non-Intrusives: PV Access Issues <LISTING non_intrusives__access_issues>',
+    'Non-Intrusives: Roof Orientation <LISTING non_intrusives__roof_orientation>',
+    'Non-Intrusives: Surveyor Notes <LISTING non_intrusives__surveyor_notes>',
+    'Non-Intrusives: Surveyor Name <LISTING non_intrusives__surveyor_name>',
+    'CIGA: Date Requested <LISTING ciga__date_requested>',
+    'CIGA: Cavity Guarantee Found <LISTING ciga__cavity_guarantee_found>',
+    'Last EPC: Is Estimated <LISTING last_epc__is_estimated>',
+    'Last EPC: EPC Rating <LISTING last_epc__epc_rating>',
+    'Last EPC: SAP Rating <LISTING last_epc__sap_rating>',
+    'Last EPC: Main Heating Description <LISTING last_epc__main_heating_description>',
+    'Last EPC: Heating Controls <LISTING last_epc__heating_controls>',
+    'Last EPC: Lodgement Date <LISTING last_epc__lodgement_date>',
+    'Last EPC: Floor Area <LISTING last_epc__floor_area>', 'Last EPC: Wall <LISTING last_epc__wall>',
+    'Last EPC: Roof <LISTING last_epc__roof>', 'Last EPC: Floor <LISTING last_epc__floor>',
+    'Last EPC: Room Height <LISTING last_epc__room_height>',
+    'Last EPC: Age Band <LISTING last_epc__age_band>', 'Deal Stage <DEAL dealstage>',
+    'Pipeline <DEAL pipeline>', 'Expected Commencement Date <DEAL expected_commencement_date>',
+    'Deal Name <DEAL dealname>', 'Project Code <DEAL project_code>',
+    'Product ID <LINE_ITEM hs_product_id>', 'Name <LINE_ITEM name>', 'Unit price <LINE_ITEM price>',
+    'Quantity <LINE_ITEM quantity>', 'Deal Owner', 'Amount <DEAL amount>', 'Installer <DEAL installer>'
+]

asset_list/hubspot/prepare_for_hubspot.py

@@ -1,4 +1,5 @@
 import pandas as pd
+from asset_list.AssetList import AssetList
 
 
 def app():
@@ -9,10 +10,42 @@ def app():
           cavity_reason and solar_reason are populated, as if we want to include historical surveys, this will remove
           them
 
+
+    TODO: If we wish to upload deals in batches
+
     :return:
     """
 
-    filepath = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Thrive "
+    # inputs:
-                "Programme - reconciled.xlsx")
+    customer_domain = "https://thrivehomes.org.uk"
+    asset_list_filepath = (
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Hubspot Upload - "
+        "Sample.xlsx"
+    )
+    contact_details_filepath = (
+        "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Sample contact "
+        "details.xlsx"
+    )
+    contacts_sheet_name = "Sheet1"
+    contacts_landlord_property_id = "landlord_property_id"
+    contacts_phone_number_column = "phone_number"
+    contacts_email_column = "email"
+    contacts_fullname_column = "fullname"
+    contacts_firstname_column = "firstname"
+    contacts_lastname_column = "lastname"
 
-    standardised_asset_list = pd.read_excel(filepath, sheet_name="Standardised Asset List")
+    asset_list = AssetList.load_standardised_asset_list(asset_list_filepath)
+    asset_list.load_contact_details(
+        local_filepath=contact_details_filepath,
+        sheet_name=contacts_sheet_name,
+        landlord_property_id=contacts_landlord_property_id,
+        phone_number_column=contacts_phone_number_column,
+        email_column=contacts_email_column,
+        fullname_column=contacts_fullname_column,
+        firstname_column=contacts_firstname_column,
+        lastname_column=contacts_lastname_column
+    )
+
+    asset_list.prepare_for_crm(
+        company_domain=customer_domain
+    )

backend/Funding.py

@@ -411,3 +411,123 @@ class Funding:
         self.gbis()
         # self.eco4()
         self.whlg()
+
+
+class Funding2:
+    """
+    New class to handle funding calculation
+    """
+
+    def __init__(self, tenure: HousingType):
+        self.tenure = tenure
+
+    @staticmethod
+    def get_sap_band(sap_score_number):
+        bands = [
+            ("High_A", 96, float("inf")),
+            ("Low_A", 92, 96),
+            ("High_B", 86, 92),
+            ("Low_B", 81, 86),
+            ("High_C", 74.5, 81),
+            ("Low_C", 69, 74.5),
+            ("High_D", 61.5, 69),
+            ("Low_D", 55, 61.5),
+            ("High_E", 46.5, 55),
+            ("Low_E", 39, 46.5),
+            ("High_F", 29.5, 39),
+            ("Low_F", 21, 29.5),
+            ("High_G", 10.5, 21),
+            ("Low_G", 1, 10.5),
+        ]
+
+        for band, lower, upper in bands:
+            if lower <= sap_score_number < upper:
+                return band
+
+        return None
+
+    def eco4_prs_eligibility(
+        self, starting_sap: int, measures: List, mainheat_description: str, heating_control_description: str
+    ):
+        """
+        Handles the eligibility criteria for private rental properties under eco
+        :return: 
+        """
+
+        # Help to heat group
+        # 1) EPC E - G
+        # 2) Must receive one of SWI, FTCH, renewable heating or DHC
+        # 3) Tenant must be on benefits
+
+        # We don't consider the tenant being on benefits - we just notify the end user that this is a requirement
+
+        meets_epc = starting_sap <= 54
+        has_solid_wall = "internal_wall_insulation" in measures or "external_wall_insulation" in measures
+        # We check if the property has a heating system that means solar pv counts as a renewable heating system
+
+        has_eligible_electric_heating = any(x in mainheat_description for x in [
+            "air source heat pump", "ground source heat pump", "boiler and radiators, electric"
+        ]) | (("electric storage heaters" in mainheat_description) and
+              (heating_control_description.lower() == "controls for high heat retention storage heaters")
+              )
+
+        # Counts as renewable heating
+        solar_renweable_heating = has_eligible_electric_heating & ("solar_pv" in measures)
+        # Is a renewable heating
+        ashp = "air_source_heat_pump" in measures
+
+        if meets_epc & (solar_renweable_heating or ashp or has_solid_wall):
+            return True
+
+        return False
+
+    def check_funding(
+        self, measures: List,
+        starting_sap: int,
+        ending_sap: int,
+        mainheat_description: str,
+        heating_control_description: str
+    ):
+        """
+        Given a list of measures, this function will check if the package of measures is fundable
+        :param measures:
+        :param starting_sap:
+        :param ending_sap:
+        :return:
+        """
+
+        starting_band = self.get_sap_band(starting_sap)
+        ending_band = self.get_sap_band(ending_sap)
+
+        # For ECO4 eligibility, the property needs to end at a C if it starts at a D or E, otherwise should end at a
+        # D
+
+        if starting_band <= 38 & ending_band >= 55:
+            # F or G should get to D
+            raise NotImplementedError("Implement F or G to D eligibility")
+
+        ########################
+        # Private
+        ########################
+        # 1) ECO4
+        # 2) GBIS
+
+        if self.tenure == "Private":
+            is_eligible = self.eco4_prs_eligibility(
+                starting_sap=starting_sap,
+                measures=measures,
+                mainheat_description=mainheat_description,
+                heating_control_description=heating_control_description
+            )
+            pass
+
+        ########################
+        # Social
+        ########################
+        # 1) ECO4
+        # 2) GBIS
+
+        if self.tenure == "Social":
+            pass
+
+        raise NotImplementedError("Only implemented for Private or Social housing")

etl/customers/cambridge/surveys.py

@@ -0,0 +1,24 @@
+import pandas as pd
+from backend.ml_models.Valuation import PropertyValuation
+from backend.app.utils import sap_to_epc
+
+# Read in the survey data
+surveys = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Cambridge/Survey Data.xlsx",
+    sheet_name="Survey data",
+)
+
+increases = []
+for _, x in surveys.iterrows():
+    current_epc = sap_to_epc(x["Pre SAP"])
+    target_epc = sap_to_epc(x["Scenario 1 Post SAP"])
+    current_value = x["Valuation"]
+
+    val = PropertyValuation.estimate_valuation_improvement(
+        current_value,
+        current_epc,
+        target_epc,
+        total_cost=None
+    )
+    avg_increase = val["average_increase"]
+    increases.append(round(avg_increase))

etl/customers/places_for_people/abs.py

@@ -0,0 +1,199 @@
+"""
+This script is to calculate the ABS for the Places for People London project
+"""
+
+import os
+import pandas as pd
+
+# London
+pfp_london_cav = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_areas_surrounding_london_reviewed_standardised_15052025.xlsx",
+    sheet_name="Cav Route",
+    header=1
+)
+pfp_london_cav = pfp_london_cav.rename(columns={"Route": "Route March"})
+pfp_london_pv = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_areas_surrounding_london_reviewed_standardised_15052025.xlsx",
+    sheet_name="PV Route",
+    header=1
+)
+pfp_london_pv = pfp_london_pv.rename(columns={"Route": "Route March"})
+pfp_london_cav["location"] = "London"
+pfp_london_pv["location"] = "London"
+# East
+pfp_east_cav = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_east_reviewed_standarised_15052025.xlsx",
+    sheet_name="Cav Route",
+    header=1
+)
+pfp_east_cav = pfp_east_cav.rename(columns={"Route": "Route March"})
+pfp_east_pv = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_east_reviewed_standarised_15052025.xlsx",
+    sheet_name="PV Route",
+    header=1
+)
+pfp_east_pv = pfp_east_pv.rename(columns={"Route": "Route March"})
+pfp_east_cav["location"] = "East"
+pfp_east_pv["location"] = "East"
+# North east
+pfp_north_east_cav = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_north_east_reviewed_standardised_15052025.xlsx",
+    sheet_name="Cav Route",
+    header=1
+)
+pfp_north_east_cav = pfp_north_east_cav.rename(columns={"Route": "Route March"})
+pfp_north_east_pv = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_north_east_reviewed_standardised_15052025.xlsx",
+    sheet_name="PV Route",
+    header=1
+)
+pfp_north_east_pv = pfp_north_east_pv.rename(columns={"Route": "Route March"})
+pfp_north_east_cav["location"] = "North East"
+pfp_north_east_pv["location"] = "North East"
+# North West
+pfp_north_west_cav = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_north_west_reviewed_standardised_15052025.xlsx",
+    sheet_name="Cav Route",
+    header=1
+)
+pfp_north_west_cav = pfp_north_west_cav.rename(columns={"Route": "Route March"})
+pfp_north_west_pv = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+    "rates/PFP_north_west_reviewed_standardised_15052025.xlsx",
+    sheet_name="PV Route",
+    header=1
+)
+pfp_north_west_pv = pfp_north_west_pv.rename(columns={"Route": "Route March"})
+pfp_north_west_cav["location"] = "North West"
+pfp_north_west_pv["location"] = "North West"
+
+cav_route = pd.concat(
+    [
+        pfp_london_cav,
+        pfp_east_cav,
+        pfp_north_east_cav,
+        pfp_north_west_cav
+    ]
+)
+solar_route = pd.concat(
+    [
+        pfp_london_pv,
+        pfp_east_pv,
+        pfp_north_east_pv,
+        pfp_north_west_pv
+    ]
+)
+
+
+def get_band(sap_score_number):
+    bands = [
+        ("High_A", 96, float("inf")),
+        ("Low_A", 92, 96),
+        ("High_B", 86, 92),
+        ("Low_B", 81, 86),
+        ("High_C", 74.5, 81),
+        ("Low_C", 69, 74.5),
+        ("High_D", 61.5, 69),
+        ("Low_D", 55, 61.5),
+        ("High_E", 46.5, 55),
+        ("Low_E", 39, 46.5),
+        ("High_F", 29.5, 39),
+        ("Low_F", 21, 29.5),
+        ("High_G", 10.5, 21),
+        ("Low_G", 1, 10.5),
+    ]
+
+    for band, lower, upper in bands:
+        if lower <= sap_score_number < upper:
+            return band
+
+    return None
+
+
+def classify_floor_area(floor_area):
+    if floor_area <= 72:
+        return "0-72"
+
+    if floor_area <= 97:
+        return "73-97"
+
+    if floor_area <= 199:
+        return "98-199"
+
+    return "200+"
+
+
+# We classify the abs bounds
+solar_route["starting_abs_band"] = solar_route["epc_sap_score_on_register"].apply(get_band)
+solar_route["ending_abs_band_scenario1"] = "High_C"
+solar_route["ending_abs_band_scenario2"] = "Low_B"
+solar_route["epc_total_floor_area"] = solar_route["epc_total_floor_area"].fillna(90)
+solar_route["floor_area_band"] = solar_route["epc_total_floor_area"].apply(classify_floor_area)
+
+# We classify the abs bounds
+cav_route["epc_sap_score_on_register"] = cav_route["epc_sap_score_on_register"].fillna(68)
+cav_route["starting_abs_band"] = cav_route["epc_sap_score_on_register"].apply(get_band)
+cav_route["floor_area_band"] = cav_route["epc_total_floor_area"].apply(classify_floor_area)
+cav_route["ending_abs_band"] = "Low_C"
+
+abs_matrix = pd.read_csv(
+    "/Users/khalimconn-kowlessar/Downloads/ECO4 Full Project Scores Matrix.csv"
+)
+
+cav_route = cav_route.merge(
+    abs_matrix.rename(columns={"Cost Savings": "ABS Rate"}),
+    how="left",
+    left_on=["starting_abs_band", "ending_abs_band", "floor_area_band"],
+    right_on=["Starting Band", "Finishing Band", "Floor Area Segment"],
+)
+solar_route = solar_route.merge(
+    abs_matrix.rename(columns={"Cost Savings": "ABS Rate"}),
+    how="left",
+    left_on=["starting_abs_band", "ending_abs_band_scenario1", "floor_area_band"],
+    right_on=["Starting Band", "Finishing Band", "Floor Area Segment"],
+)
+cav_route["ABS Rate"] = cav_route["ABS Rate"].fillna(0)
+solar_route["ABS Rate"] = solar_route["ABS Rate"].fillna(0)
+
+cav_abs_agg = (
+    cav_route.groupby("Route March").agg(
+        {
+            "ABS Rate": "sum",
+            "landlord_property_id": "count",
+        }
+    ).reset_index()
+)
+cav_abs_agg["Week Number"] = cav_abs_agg["Route March"].str.extract(r"(\d+)").astype(int)
+cav_abs_agg = cav_abs_agg.sort_values("Week Number", ascending=True)
+cav_abs_agg = cav_abs_agg.rename(columns={"landlord_property_id": "Number of Properties"})
+
+solar_abs_agg = (
+    solar_route.groupby("Route March").agg(
+        {
+            "ABS Rate": "sum",
+            "landlord_property_id": "count",
+        }
+    ).reset_index()
+)
+solar_abs_agg["Week Number"] = solar_abs_agg["Route March"].str.extract(r"(\d+)").astype(int)
+solar_abs_agg = solar_abs_agg.rename(columns={"landlord_property_id": "Number of Properties"})
+solar_abs_agg = solar_abs_agg.sort_values("Week Number", ascending=True)
+
+# We store the data
+# Store as an excel
+filename = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs rates/pfp programme rates.xlsx"
+# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data
+
+with pd.ExcelWriter(filename) as writer:
+    solar_abs_agg.to_excel(writer, sheet_name="Solar ABS", index=False)
+    cav_abs_agg.to_excel(writer, sheet_name="Cav ABS", index=False)
+
+    cav_route.to_excel(writer, sheet_name="Cavity data", index=False)
+    solar_route.to_excel(writer, sheet_name="Solar data", index=False)

etl/customers/thrive/Project codes.py

@@ -38,10 +38,10 @@ PROJECT_CODE_MAP = {
     'Phase 8': "THRIVE-008",
     'Phase 9': "THRIVE-009",
     'Phase 10': "THRIVE-010",
-    "Week1": "THRIVE-WEEK-001",
+    "Week 1": "THRIVE-WEEK-001",
-    "Week2": "THRIVE-WEEK-002",
+    "Week 2": "THRIVE-WEEK-002",
-    "Week4": "THRIVE-WEEK-004",
+    "Week 4": "THRIVE-WEEK-004",
-    "Week7": "THRIVE-WEEK-007",
+    "Week 7": "THRIVE-WEEK-007",
 }
 programme_codes["project_code"] = programme_codes["programme_reference"].map(PROJECT_CODE_MAP)
 
@@ -102,7 +102,29 @@ with pd.ExcelWriter(filename) as writer:
     block_analysis.to_excel(writer, sheet_name="Block Analysis", index=False)
     # If we have outcomes, we add a tab with the outcomes
     outcomes.to_excel(writer, sheet_name="Outcomes", index=False)
-
     unmatched_submissions.to_excel(writer, sheet_name="Unmatched Submissions", index=False)
-
     unmatched_ecosurv.to_excel(writer, sheet_name="Unmatched Ecosurv", index=False)
+
+# A check, just comparing against the master tracker to make sure I have all of the installs
+asset_list = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Thrive Asset List - "
+    "Complete - Updated May 2025 - Standardised.xlsx",
+    sheet_name="Standardised Asset List",
+)
+
+master_tracker = pd.read_excel(
+    "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Master Tracker (Thrive - "
+    "Warmfront).xlsx",
+    sheet_name="Master Tracker",
+    header=1
+)
+
+df = asset_list[["landlord_property_id", "hubspot_status"]].merge(
+    master_tracker[~pd.isnull(master_tracker['Date Completed'])][["UPRN", "Date Completed"]],
+    how="inner",
+    left_on="landlord_property_id",
+    right_on="UPRN"
+)
+
+df["hubspot_status"].value_counts()
+df[df["hubspot_status"] == "SUBMITTED TO INSTALLER"]