diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py index 3f5ef7ff..ef125110 100644 --- a/asset_list/AssetList.py +++ b/asset_list/AssetList.py @@ -368,14 +368,19 @@ class AssetList: SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED = "Solar Eligible, Solid Wall Uninsulated, EPC E or Below" SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE = "Solar Eligible, Needs Heating Upgrade" + CRM_HISTORICAL_CAVITY_PRODUCT = { + "id": 156989182176, "unit_price": 0, "name": "Historical ECO Cavity" + } + CRM_PRODUCTS = { - "Empty Cavity - ECO4": {"id": 82733738177, "unit_price": 1000, "name": "Empty Cavity & Loft - ECO4"}, + "Empty Cavity - ECO4": {"id": 82733738177, "unit_price": 1000, "name": "Empty Cavity - ECO4"}, "Extract & Fill - ECO4": {"id": 100307905778, "unit_price": 500, "name": "Extract & Fill - ECO4"}, "Solar PV - ECO4": {"id": 82623589564, "unit_price": 1608, "name": "Solar PV - ECO4"}, "Solar PV + HHRSH - ECO4": {"id": 155529972924, "unit_price": 1608, "name": "Solar PV + HHRSH - ECO4"}, "Solar PV + Heating Upgrade - ECO4": { "id": 109265426665, "unit_price": 1608, "name": "Solar PV + Heating Upgrade - ECO4" }, + "Historical ECO Cavity": CRM_HISTORICAL_CAVITY_PRODUCT } def __init__( @@ -2128,27 +2133,33 @@ class AssetList: full_address_cols_to_concat=[], missing_postcodes_method=None, address1_extraction_method=None, - landlord_year_built=None, - landlord_uprn=None, - landlord_property_type=None, - landlord_built_form=None, - landlord_wall_construction=None, - landlord_roof_construction=None, - landlord_heating_system=None, - landlord_existing_pv=None, - landlord_sap=None, - landlord_block_reference=None, + landlord_year_built=cls.STANDARD_YEAR_BUILT, + landlord_uprn=cls.STANDARD_UPRN, + landlord_property_type=cls.STANDARD_PROPERTY_TYPE, + landlord_built_form=cls.STANDARD_BUILT_FORM, + landlord_wall_construction=cls.STANDARD_WALL_CONSTRUCTION, + landlord_roof_construction=cls.STANDARD_ROOF_CONSTRUCTION, + landlord_heating_system=cls.STANDARD_HEATING_SYSTEM, + landlord_existing_pv=cls.STANDARD_EXISTING_PV, + landlord_sap=cls.STANDARD_SAP, + landlord_block_reference=cls.STANDARD_BLOCK_REFERENCE, phase=False, header=0 ) return instance - def prepare_for_crm(self, company_domain, installer_name): + def prepare_for_crm(self, company_domain, installer_name, reconcile_programme=False): """ This function prepares the data for upload into Hubspot + :param company_domain: The company domain name to be used in the CRM + :param installer_name: The name of the installer to be used in the CRM + :param reconcile_programme: If True, will include all properties with a project code, regardless of status + :raises ValueError: If the installer name is not valid or if there are missing products :return: """ # This maps the opportunities as we reference them, to the product data as stored in Hubspot + if not hubspot_config.Installer.is_valid_value(installer_name): + raise ValueError(f"Installer name {installer_name} is not valid. Please check the installer name.") prefixes_to_products = { # Empty @@ -2185,18 +2196,37 @@ class AssetList: # For each cavity and solar product, we iterate through the prexies and map to the products - # # Check if there any options not in out lookup table - # if ( - # any(x for x in cavity_products if x not in product_lookup_table) or - # any(x for x in solar_products if x not in product_lookup_table) - # ): - # raise ValueError("We have products not referenced in the lookup table - check this") - programme_data = self.standardised_asset_list.copy() + # Format the two date columns + programme_data["survey_week"] = pd.to_datetime(programme_data["survey_week"], errors="coerce") + programme_data[self.EPC_API_DATA_NAMES["inspection-date"]] = pd.to_datetime( + programme_data[self.EPC_API_DATA_NAMES["inspection-date"]], + errors="coerce" + ) + # Convert to dd/mm/yyyy format + programme_data["survey_week"] = programme_data["survey_week"].dt.strftime("%d/%m/%Y") + programme_data[self.EPC_API_DATA_NAMES["inspection-date"]] = ( + programme_data[self.EPC_API_DATA_NAMES["inspection-date"]].dt.strftime("%d/%m/%Y") + ) + # We take rows that have a survyor and a date for the survey - programme_data = programme_data[ - ~pd.isnull(programme_data["survey_week"]) & ~pd.isnull(programme_data["surveyor"]) - ] + # We include properties under 2 circumstances: + # 1) The hubspot status is ready to be scheduled and there is an assigned surveyor and week for survey + # 2) The hubspot status is something else, meaning this has been included in an existing programme + # 3) reconcile programme is true, and therefore all proeprties with a project code will be included + + if reconcile_programme: + programme_data = programme_data[~pd.isnull(programme_data["project_code"])] + else: + ready_to_be_scheduled = ( + ( + programme_data["hubspot_status"] == hubspot_config.HubspotProcessStatus.READY_TO_BE_SCHEDULED.label + ) & (~pd.isnull(programme_data["survey_week"]) & ~pd.isnull(programme_data["surveyor"])) + ) + completed_works = ( + programme_data["hubspot_status"] != hubspot_config.HubspotProcessStatus.READY_TO_BE_SCHEDULED.label + ) + programme_data = programme_data[ready_to_be_scheduled | completed_works] # Merge on the contact details programme_data = programme_data.merge( @@ -2232,8 +2262,16 @@ class AssetList: programme_data["domna_product"] ) # We filter just on rows where we have a product - programme_data = programme_data[~pd.isnull(programme_data["domna_product"])] - programme_data = programme_data.drop(columns=["solar_product", "cavity_product"]) + if reconcile_programme: + # We include historical works, which will include hisorical cavity so we set these as extraction (as + # this is the main work mix) + programme_data["domna_product"] = programme_data["domna_product"].fillna( + self.CRM_HISTORICAL_CAVITY_PRODUCT["name"] + ) + else: + + programme_data = programme_data[~pd.isnull(programme_data["domna_product"])] + programme_data = programme_data.drop(columns=["solar_product", "cavity_product"]) product_df = ( pd.DataFrame(self.CRM_PRODUCTS).T[["name", "id", "unit_price"]] @@ -2251,25 +2289,24 @@ class AssetList: product_df['Quantity '] = 1 # Append on the product data - programme_data = programme_data.merge( - product_df, - how="left", - on="domna_product", - ) + programme_data = programme_data.merge(product_df, how="left", on="domna_product") # Add in deal and pipeline information programme_data["dealname"] = ( programme_data[self.STANDARD_FULL_ADDRESS] + " : " + programme_data["domna_product"] ) programme_data['Pipeline '] = hubspot_config.CRM_PIPELINE_NAME - programme_data['Deal Stage '] = hubspot_config.CRM_PIPELINE_FIRST_STAGE_NAME programme_data['Associations: Listing'] = "Property Owner" - # programme_data = programme_data.merge( - # assigned_surveyors.rename( - # columns={self.landlord_property_id: self.STANDARD_LANDLORD_PROPERTY_ID} - # ), how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID - # ) + # We determine which column we should use for the UPRN + if self.STANDARD_UPRN not in programme_data.columns: + uprn_column = self.EPC_API_DATA_NAMES["uprn"] + else: + # Use the value that has the most coverage + uprn_column = "hubspot_uprn" + programme_data[uprn_column] = programme_data[self.STANDARD_UPRN].fillna( + programme_data[self.EPC_API_DATA_NAMES["uprn"]] + ) # Add in some columns if we have them date_of_inspections = ( @@ -2277,6 +2314,67 @@ class AssetList: "Non-Intrusives: Date of Inspection" in programme_data.columns else None ) + # Ammend the property type and built form columns + programme_data["hubspot_property_type"] = programme_data[self.STANDARD_PROPERTY_TYPE].copy() + programme_data["hubspot_built_form"] = programme_data[self.STANDARD_BUILT_FORM].copy() + + def _replace_property_description_data(programme_data, column_name): + """ + Helper function to replace property type or built form data with a specified value. + """ + + if column_name == "hubspot_property_type": + valid_values = ["house", "bungalow", "flat", "maisonette"] + epc_fill_col = "property-type" + elif column_name == "hubspot_built_form": + valid_values = ["detached", "semi-detached", "mid-terrace", "end-terrace"] + epc_fill_col = "built-form" + else: + raise ValueError(f"Invalid column name: {column_name}. Must be 'hubspot_property_type' or " + f"'hubspot_built_form'.") + + # Any vakue that is not house, bungalow, flat or maisonette is set to None + programme_data[column_name] = np.where( + ~programme_data[column_name].isin(valid_values), + None, + programme_data[column_name] + ) + # We fill with the EPC property type + programme_data[column_name] = np.where( + pd.isnull(programme_data[column_name]), + programme_data[self.EPC_API_DATA_NAMES[epc_fill_col]], + programme_data[column_name] + ) + + programme_data[column_name] = programme_data[column_name].fillna("unknown") + + return programme_data + + # Clean up the property type and built form columns + programme_data = _replace_property_description_data(programme_data, "hubspot_property_type") + programme_data = _replace_property_description_data(programme_data, "hubspot_built_form") + + # We accomodate the old vs new inspections format + if "non-intrusives: WFT Findings" in programme_data.columns: + # We have the old format - we only have notes + non_intrusives_surveyor_notes = "non-intrusives: WFT Findings" + non_intrusives_construction = None + non_intrusives_insulated = None + non_intrusives_insulation_material = None + non_intrusives_ciga_check_required = None + non_intrusives_pv_access = None + non_intrusives_roof_orientation = None + non_intrusives_surveyor_name = None + else: + non_intrusives_surveyor_notes = 'non-intrusives: Any further surveyor notes' + non_intrusives_construction = "non-intrusives: Construction" + non_intrusives_insulated = "non-intrusives: Insulated" + non_intrusives_insulation_material = "non-intrusives: Material" + non_intrusives_ciga_check_required = 'non-intrusives: CIGA Check Required' + non_intrusives_pv_access = 'non-intrusives: PV, ACCESS ISSUE, SEE NOTES' + non_intrusives_roof_orientation = 'non-intrusives: OFF GAS - ROOF ORIENTATION' + non_intrusives_surveyor_name = 'non-intrusives: Surveyors Name' + # This maps the hubspot schema to the template. Anything that is not covered in this will be flagged schema_mappings = { 'Company Domain Name ': 'Company Domain Name ', @@ -2296,14 +2394,12 @@ class AssetList: 'Address 1 ': self.STANDARD_ADDRESS_1, 'Address 2 ': None, # TODO: Don't have this for the moment 'Postcode ': self.STANDARD_POSTCODE, - 'Property Type ': self.STANDARD_PROPERTY_TYPE, - 'Property Sub Type ': self.STANDARD_BUILT_FORM, + 'Property Type ': "hubspot_property_type", + 'Property Sub Type ': "hubspot_built_form", 'Bedroom(s) ': None, # TODO: Don't have this for the moment 'Domna Property ID ': self.DOMNA_PROPERTY_ID, # We populate this with the column that we have - 'National UPRN ': ( - self.STANDARD_UPRN if self.STANDARD_UPRN is not None else self.EPC_API_DATA_NAMES["uprn"] - ), + 'National UPRN ': uprn_column, 'Owner Property ID ': self.STANDARD_LANDLORD_PROPERTY_ID, 'Wall Construction ': self.STANDARD_WALL_CONSTRUCTION, 'Heating System ': self.STANDARD_HEATING_SYSTEM, @@ -2311,30 +2407,17 @@ class AssetList: 'Boiler Make ': None, # TODO: Don't have this for the moment 'Boiler Model ': None, # TODO: Don't have this for the moment 'Non-Intrusives: Date Checked ': date_of_inspections, - 'Non-Intrusives: Wall Type ': ( - "non-intrusives: Construction" if self.non_intrusives_present else None - ), - 'Non-intrusives: Insulation ': ( - "non-intrusives: Insulated" if self.non_intrusives_present else None - ), - 'Non-intrusives: Insulation Material ': ( - "non-intrusives: Material" if self.non_intrusives_present else None - ), - 'Non-Intrusives: CIGA Check Required ': ( - 'non-intrusives: CIGA Check Required' if self.non_intrusives_present else None - ), - 'Non-Intrusives: PV Access Issues ': ( - 'non-intrusives: PV, ACCESS ISSUE, SEE NOTES' if self.non_intrusives_present else None - ), - 'Non-Intrusives: Roof Orientation ': ( - 'non-intrusives: OFF GAS - ROOF ORIENTATION' if self.non_intrusives_present else None - ), - 'Non-Intrusives: Surveyor Notes ': ( - 'non-intrusives: Any further surveyor notes' if self.non_intrusives_present else None - ), - 'Non-Intrusives: Surveyor Name ': ( - 'non-intrusives: Surveyors Name' if self.non_intrusives_present else None - ), + 'Non-Intrusives: Wall Type ': non_intrusives_construction, + 'Non-intrusives: Insulation ': non_intrusives_insulated, + 'Non-intrusives: Insulation Material ': + non_intrusives_insulation_material, + 'Non-Intrusives: CIGA Check Required ': + non_intrusives_ciga_check_required, + 'Non-Intrusives: PV Access Issues ': non_intrusives_pv_access, + 'Non-Intrusives: Roof Orientation ': + non_intrusives_roof_orientation, + 'Non-Intrusives: Surveyor Notes ': non_intrusives_surveyor_notes, + 'Non-Intrusives: Surveyor Name ': non_intrusives_surveyor_name, 'CIGA: Date Requested ': None, # TODO: Don't have this for the moment 'CIGA: Cavity Guarantee Found ': None, 'Last EPC: Is Estimated ': self.EPC_API_DATA_NAMES["estimated"], @@ -2351,7 +2434,6 @@ class AssetList: 'Last EPC: Floor ': self.EPC_API_DATA_NAMES["floor-description"], 'Last EPC: Room Height ': self.EPC_API_DATA_NAMES["floor-height"], 'Last EPC: Age Band ': self.EPC_API_DATA_NAMES["construction-age-band"], - 'Deal Stage ': 'Deal Stage ', 'Pipeline ': 'Pipeline ', 'Expected Commencement Date ': "survey_week", 'Deal Name ': "dealname", # Need to create this, @@ -2362,6 +2444,7 @@ class AssetList: 'Deal Owner': 'surveyor_email', 'Project Code ': 'project_code', 'Associations: Listing': 'Associations: Listing', + 'Deal Stage ': "hubspot_status", } # We sometimes columns if the landlord never provided them diff --git a/asset_list/hubspot/config.py b/asset_list/hubspot/config.py index 6e16279a..01540b7b 100644 --- a/asset_list/hubspot/config.py +++ b/asset_list/hubspot/config.py @@ -1,7 +1,6 @@ -from enum import IntEnum +from enum import IntEnum, Enum CRM_PIPELINE_NAME = 'Operations - Housing Associations' -CRM_PIPELINE_FIRST_STAGE_NAME = 'READY TO BE SCHEDULED' class HubspotProcessStatus(IntEnum): @@ -31,6 +30,19 @@ class HubspotProcessStatus(IntEnum): INSTALLER_CANCELLED_FINALIZED = 8, "INSTALLER CANCELLED - FINALIZED" +class Installer(Enum): + SCIS = "SCIS" + JJ_CRUMP = "J & J CRUMP" + SGEC = "SGEC" + + @classmethod + def is_valid_value(cls, value): + """ + Check if the value is a valid installer. + """ + return value in cls._value2member_map_ + + CRM_UPLOAD_COLUMNS = [ 'Name ', 'Associations: Listing', 'Company Domain Name ', 'Email ', 'First Name ', 'Last Name ', diff --git a/asset_list/hubspot/prepare_for_hubspot.py b/asset_list/hubspot/prepare_for_hubspot.py index 8ed654f3..ee3bc65d 100644 --- a/asset_list/hubspot/prepare_for_hubspot.py +++ b/asset_list/hubspot/prepare_for_hubspot.py @@ -1,3 +1,4 @@ +import os import pandas as pd from asset_list.AssetList import AssetList @@ -17,10 +18,12 @@ def app(): """ # inputs: + reconcile_programme = True # If True, the hubspot upload will include all properties with a project code customer_domain = "https://thrivehomes.org.uk" + installer_name = "J & J CRUMP" asset_list_filepath = ( - "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Hubspot Upload - " - "Sample.xlsx" + "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Thrive Programme - " + "Hubspot Upload 3.xlsx" ) contact_details_filepath = ( "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Sample contact " @@ -47,5 +50,22 @@ def app(): ) asset_list.prepare_for_crm( - company_domain=customer_domain + company_domain=customer_domain, + installer_name=installer_name, + reconcile_programme=reconcile_programme ) + + # Get the filepath and the filename. Append hubspot upload to the filename. We also change the file type to csv + directory, filename = os.path.split(asset_list_filepath) + name, ext = os.path.splitext(filename) + output_filename = f"{name} - Hubspot Upload.csv" + output_filepath = os.path.join(directory, output_filename) + + if pd.isnull(asset_list.hubspot_data['Project Code ']).sum(): + raise ValueError("FIX MEEE") + + if pd.isnull(asset_list.hubspot_data['Deal Stage ']).any(): + raise ValueError("Warning: Some rows have missing project codes. These will not be uploaded to HubSpot.") + + # Just store locally + asset_list.hubspot_data.to_csv(output_filepath, index=False, encoding="utf-8-sig") diff --git a/etl/customers/l_and_g/risk_matrix.py b/etl/customers/l_and_g/risk_matrix.py index c800117e..8f5451fc 100644 --- a/etl/customers/l_and_g/risk_matrix.py +++ b/etl/customers/l_and_g/risk_matrix.py @@ -81,6 +81,7 @@ def app(): # We need to calculate the costs cost_data = [] for _, row in epr_data.iterrows(): + epc = row["EPC"][0] sap = int(row["EPC"][1:])