diff --git a/.idea/Model.iml b/.idea/Model.iml
index c6561970..09f2e496 100644
--- a/.idea/Model.iml
+++ b/.idea/Model.iml
@@ -7,7 +7,7 @@
-
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
index 50cad4ca..fb10c6b0 100644
--- a/.idea/misc.xml
+++ b/.idea/misc.xml
@@ -3,7 +3,7 @@
-
+
diff --git a/asset_list/AssetList.py b/asset_list/AssetList.py
index fea0f59e..3f5ef7ff 100644
--- a/asset_list/AssetList.py
+++ b/asset_list/AssetList.py
@@ -350,6 +350,34 @@ class AssetList:
"cavity wall, as built, partial insulation",
]
+ # Work type prefixes:
+ # Empties
+ EMPTY_CAVITY_NON_INTRUSIVE = "Non-Intrusive Data Shows Empty Cavity"
+ EPC_EMPTY_INSPECTIONS_RETRO_DRILLED = "EPC Shows Empty Cavity, inspections show retro drilled"
+ EPC_EMPTY_INSPECTIONS_FILLED = "EPC Shows Empty Cavity, inspections show filled or other"
+ EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD = "EPC Shows Empty Cavity, inspections show filled at build"
+ EPC_EMPTY_INSPECTIONS_NON_CAVITY = "EPC Shows Empty Cavity, inspections show non-cavity build"
+ EPC_EMPTY = "EPC Shows Empty Cavity"
+ LANDLORD_EMPTY_INSPECTIONS_OTHER = ("Landlord Data Shows Empty Cavity, EPC & Inspections Shows Filled or "
+ "Non-cavity")
+ # Extraction
+ EXTRACTION_NON_INTRUSIVE = "Non-Intrusive Data Shows Cavity Extraction"
+
+ # Solar
+ SOLAR_ELIGIBLE = "Solar Eligible"
+ SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED = "Solar Eligible, Solid Wall Uninsulated, EPC E or Below"
+ SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE = "Solar Eligible, Needs Heating Upgrade"
+
+ CRM_PRODUCTS = {
+ "Empty Cavity - ECO4": {"id": 82733738177, "unit_price": 1000, "name": "Empty Cavity & Loft - ECO4"},
+ "Extract & Fill - ECO4": {"id": 100307905778, "unit_price": 500, "name": "Extract & Fill - ECO4"},
+ "Solar PV - ECO4": {"id": 82623589564, "unit_price": 1608, "name": "Solar PV - ECO4"},
+ "Solar PV + HHRSH - ECO4": {"id": 155529972924, "unit_price": 1608, "name": "Solar PV + HHRSH - ECO4"},
+ "Solar PV + Heating Upgrade - ECO4": {
+ "id": 109265426665, "unit_price": 1608, "name": "Solar PV + Heating Upgrade - ECO4"
+ },
+ }
+
def __init__(
self,
local_filepath,
@@ -1719,10 +1747,10 @@ class AssetList:
self.standardised_asset_list["cavity_reason"] = None
empty_cavity_map = {
- "non_intrusive_indicates_empty_cavity": "Non-Intrusive Data Shows Empty Cavity: ",
- "non_intrusive_indicates_empty_cavity_has_solar": "Non-Intrusive Data Shows Empty Cavity - property "
+ "non_intrusive_indicates_empty_cavity": self.EMPTY_CAVITY_NON_INTRUSIVE_PREFIX + ": ",
+ "non_intrusive_indicates_empty_cavity_has_solar": f"{self.EMPTY_CAVITY_NON_INTRUSIVE} - property "
"already has solar: ",
- "non_intrusive_indicates_empty_cavity_no_year_filter": f"Non-Intrusive Data Shows Empty Cavity, "
+ "non_intrusive_indicates_empty_cavity_no_year_filter": f"{self.EMPTY_CAVITY_NON_INTRUSIVE}, "
f"built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: ",
}
@@ -1747,7 +1775,7 @@ class AssetList:
)) &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity, inspections show retro drilled: " + self.standardised_asset_list[
+ f"{EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1759,7 +1787,7 @@ class AssetList:
self.standardised_asset_list['non_intrusive_indicates_cavity_extraction'] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity, inspections show filled or other: " + self.standardised_asset_list[
+ f"{self.EPC_EMPTY_INSPECTIONS_FILLED}: " + self.standardised_asset_list[
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1771,7 +1799,7 @@ class AssetList:
(self.standardised_asset_list['non-intrusives: Insulated'] == "RETRO DRILLED") &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity, inspections show retro drilled: " + self.standardised_asset_list[
+ f"{self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED}: " + self.standardised_asset_list[
"SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1783,8 +1811,7 @@ class AssetList:
(self.standardised_asset_list['non-intrusives: Insulated'] == "FILLED AT BUILD") &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity, inspections show filled at build: " + self.standardised_asset_list[
- "SAP Category"],
+ f"{self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD}: " + self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
else:
@@ -1794,7 +1821,7 @@ class AssetList:
~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity: " + self.standardised_asset_list["SAP Category"],
+ f"{self.EPC_EMPTY}: " + self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1804,10 +1831,12 @@ class AssetList:
~self.standardised_asset_list["non_intrusive_indicates_empty_cavity"] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "EPC Shows Empty Cavity, inspections show non-cavity build: " + self.standardised_asset_list[
- "SAP Category"],
+ f"{self.EPC_EMPTY_INSPECTIONS_NON_CAVITY}: " + self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
+
+ # Work type prefixes
+
# Landlord data: The landlord's data indicates that the wall is an uninsulated cavity wall, but EPC and
# inspections show filled
self.standardised_asset_list["cavity_reason"] = np.where(
@@ -1817,7 +1846,7 @@ class AssetList:
~self.standardised_asset_list["epc_indicates_empty_cavity"] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "Landlord Data Shows Empty Cavity, EPC & Inspections Shows Filled or Non-cavity: " +
+ f"{self.LANDLORD_EMPTY_INSPECTIONS_OTHER}: " +
self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1828,7 +1857,7 @@ class AssetList:
self.standardised_asset_list["non_intrusive_indicates_cavity_extraction"] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- "Non-Intrusive Data Shows Cavity Extraction: " + self.standardised_asset_list["SAP Category"],
+ f"{self.EXTRACTION_NON_INTRUSIVE}: " + self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1837,7 +1866,7 @@ class AssetList:
self.standardised_asset_list["non_intrusive_indicates_cavity_extraction_no_year_filter"] &
pd.isnull(self.standardised_asset_list["cavity_reason"])
),
- f"Non-Intrusive Data Shows Cavity Extraction, built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: " +
+ f"{self.EXTRACTION_NON_INTRUSIVE}, built after {self.EMPTY_CAVITY_YEAR_THRESHOLD}: " +
self.standardised_asset_list["SAP Category"],
self.standardised_asset_list["cavity_reason"]
)
@@ -1850,11 +1879,9 @@ class AssetList:
# Map of variables and fill values for the solar_reason variable
# ordering of this map is important, where we flag our prioritised work types first
solar_reason_map = {
- "solar_eligible": "Solar Eligible: ",
- "solar_eligible_solid_wall_uninsulated": "Solar Eligible, Solid Wall Uninsulated, EPC E or Below: ",
- "solar_eligible_needs_heating_upgrade": (
- "Solar Eligible, Needs Heating Upgrade: "
- )
+ "solar_eligible": f"{self.SOLAR_ELIGIBLE}: ",
+ "solar_eligible_solid_wall_uninsulated": f"{self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED}: ",
+ "solar_eligible_needs_heating_upgrade": f"{self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE}: "
}
for variable, reason in solar_reason_map.items():
@@ -2079,68 +2106,97 @@ class AssetList:
*contact_details[fullname_column].apply(self.split_full_name)
)
else:
- raise NotImplementedError("Implement me")
+ contact_details["title"] = None
self.contact_details = contact_details
- def prepare_for_crm(self, company_domain, crm_pipeline_name, first_dealstage, assigned_surveyors):
+ @classmethod
+ def load_standardised_asset_list(cls, filepath):
+ """
+ This function is designed to load the standardised asset list from a file
+ :return:
+ """
+ # This is a placeholder for now
+ # instantiate the class
+ instance = cls(
+ local_filepath=filepath,
+ sheet_name="Standardised Asset List",
+ address1_colname=cls.STANDARD_ADDRESS_1,
+ postcode_colname=cls.STANDARD_POSTCODE,
+ full_address_colname=cls.STANDARD_FULL_ADDRESS,
+ landlord_property_id=cls.STANDARD_LANDLORD_PROPERTY_ID,
+ full_address_cols_to_concat=[],
+ missing_postcodes_method=None,
+ address1_extraction_method=None,
+ landlord_year_built=None,
+ landlord_uprn=None,
+ landlord_property_type=None,
+ landlord_built_form=None,
+ landlord_wall_construction=None,
+ landlord_roof_construction=None,
+ landlord_heating_system=None,
+ landlord_existing_pv=None,
+ landlord_sap=None,
+ landlord_block_reference=None,
+ phase=False,
+ header=0
+ )
+ return instance
+
+ def prepare_for_crm(self, company_domain, installer_name):
"""
This function prepares the data for upload into Hubspot
:return:
"""
- # This is a placeholder for now
-
# This maps the opportunities as we reference them, to the product data as stored in Hubspot
- product_lookup_table = {
- "Non-Intrusive Data Showed Cavity Extraction": {
- "name": "Extract & Fill - ECO4", "id": 100307905778, "unit_price": 500
- },
- "Non-Intrusive Data Showed Empty Cavity": {
- "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
- },
- "Non-Intrusive Data Showed Empty Cavity but all SAP scores allowed": {
- "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
- },
- "Non-Intrusive Data Showed Cavity Extraction but all SAP scores allowed": {
- "name": "Extract & Fill - ECO4", "id": 100307905778, "unit_price": 500
- },
- "EPC Data Showed Empty Cavity": {
- "name": "Empty Cavity & Loft - ECO4", "id": 82733738177, "unit_price": 1000
- },
- "Solid Floor, Insulated, No Solar": {
- "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
- },
- "Solid Floor, Insulated, Needs Loft": {
- "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
- },
- "Other Floor, Insulated, No Solar": {
- "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
- },
- "Other Floor, Insulated, Needs Loft": {
- "name": "Solar PV - ECO4", "id": 82623589564, "unit_price": 1608
- }
+
+ prefixes_to_products = {
+ # Empty
+ self.EMPTY_CAVITY_NON_INTRUSIVE: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.EPC_EMPTY_INSPECTIONS_RETRO_DRILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.EPC_EMPTY_INSPECTIONS_FILLED: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.EPC_EMPTY_INSPECTIONS_FILLED_AT_BUILD: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.EPC_EMPTY_INSPECTIONS_NON_CAVITY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.EPC_EMPTY: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ self.LANDLORD_EMPTY_INSPECTIONS_OTHER: self.CRM_PRODUCTS["Empty Cavity - ECO4"],
+ # Extraction
+ self.EXTRACTION_NON_INTRUSIVE: self.CRM_PRODUCTS["Extract & Fill - ECO4"],
+ # Solar
+ self.SOLAR_ELIGIBLE: self.CRM_PRODUCTS["Solar PV - ECO4"],
+ self.SOLAR_ELIGIBLE_SOLID_WALL_UNINSULATED: self.CRM_PRODUCTS["Solar PV - ECO4"],
+ self.SOLAR_ELIGIBLE_NEEDS_HEATING_UPGRADE: self.CRM_PRODUCTS["Solar PV + Heating Upgrade - ECO4"],
}
+
# We check if all products are covered in the lookup table
- cavity_products = self.standardised_asset_list["cavity_reason"].unique()
- solar_products = self.standardised_asset_list["solar_reason"].unique()
- # Check if there any options not in out lookup table
- if (
- any(x for x in cavity_products if x not in product_lookup_table) or
- any(x for x in solar_products if x not in product_lookup_table)
- ):
- raise ValueError("We have products not referenced in the lookup table - check this")
+ cavity_products = self.standardised_asset_list["cavity_reason"].unique().tolist()
+ solar_products = self.standardised_asset_list["solar_reason"].unique().tolist()
+
+ product_map = {}
+ for identified_product in cavity_products + solar_products:
+ if pd.isnull(identified_product):
+ continue
+
+ matched_product = None
+ for product_prefix, crm_product in prefixes_to_products.items():
+ if identified_product.startswith(product_prefix):
+ matched_product = crm_product
+
+ product_map[identified_product] = matched_product
+
+ # For each cavity and solar product, we iterate through the prexies and map to the products
+
+ # # Check if there any options not in out lookup table
+ # if (
+ # any(x for x in cavity_products if x not in product_lookup_table) or
+ # any(x for x in solar_products if x not in product_lookup_table)
+ # ):
+ # raise ValueError("We have products not referenced in the lookup table - check this")
programme_data = self.standardised_asset_list.copy()
-
- # Exclusions - these are properties we won't treat for the moment
- product_exclusions = [
- "Other Floor, Insulated, No Solar",
- "Other Floor, Insulated, Needs Loft"
- ]
- if product_exclusions:
- logger.warning("Excluding products: %s", product_exclusions)
-
- programme_data = programme_data[programme_data["solar_reason"].isin(product_exclusions) == False]
+ # We take rows that have a survyor and a date for the survey
+ programme_data = programme_data[
+ ~pd.isnull(programme_data["survey_week"]) & ~pd.isnull(programme_data["surveyor"])
+ ]
# Merge on the contact details
programme_data = programme_data.merge(
@@ -2153,26 +2209,34 @@ class AssetList:
programme_data["Company Domain Name "] = company_domain
# Append the product data onto the programme data
programme_data["cavity_product"] = programme_data["cavity_reason"].map(
- lambda x: product_lookup_table.get(x, {"name": None})["name"]
+ lambda x: product_map.get(x, {"name": None})["name"]
)
programme_data["solar_product"] = programme_data["solar_reason"].map(
- lambda x: product_lookup_table.get(x, {"name": None})["name"]
+ lambda x: product_map.get(x, {"name": None})["name"]
)
- programme_data["domna_product"] = programme_data["solar_reason"].copy()
+ # We check if we have any missings
+ cavity_missing = pd.isnull(programme_data[~pd.isnull(programme_data["cavity_reason"])]["cavity_product"]).sum()
+ solar_missing = pd.isnull(programme_data[~pd.isnull(programme_data["solar_reason"])]["solar_product"]).sum()
+
+ if cavity_missing > 0 or solar_missing > 0:
+ raise ValueError(
+ f"We have {cavity_missing} cavity products and {solar_missing} solar products that are not "
+ "mapped to a product in the lookup table. Please check the mapping."
+ )
+
+ programme_data["domna_product"] = programme_data["solar_product"].copy()
programme_data["domna_product"] = np.where(
pd.isnull(programme_data["domna_product"]),
- programme_data["solar_product"],
+ programme_data["cavity_product"],
programme_data["domna_product"]
)
# We filter just on rows where we have a product
- programme_data = programme_data[
- ~pd.isnull(programme_data["domna_product"])
- ]
+ programme_data = programme_data[~pd.isnull(programme_data["domna_product"])]
programme_data = programme_data.drop(columns=["solar_product", "cavity_product"])
product_df = (
- pd.DataFrame(product_lookup_table).T[["name", "id", "unit_price"]]
+ pd.DataFrame(self.CRM_PRODUCTS).T[["name", "id", "unit_price"]]
.reset_index()
.rename(
columns={
@@ -2194,21 +2258,27 @@ class AssetList:
)
# Add in deal and pipeline information
- programme_data["dealname"] = programme_data[self.STANDARD_FULL_ADDRESS] + " : " + programme_data[
- "domna_product"]
- programme_data['Pipeline '] = crm_pipeline_name
- programme_data['Deal Stage '] = first_dealstage
+ programme_data["dealname"] = (
+ programme_data[self.STANDARD_FULL_ADDRESS] + " : " + programme_data["domna_product"]
+ )
+ programme_data['Pipeline '] = hubspot_config.CRM_PIPELINE_NAME
+ programme_data['Deal Stage '] = hubspot_config.CRM_PIPELINE_FIRST_STAGE_NAME
programme_data['Associations: Listing'] = "Property Owner"
- programme_data = programme_data.merge(
- assigned_surveyors.rename(
- columns={self.landlord_property_id: self.STANDARD_LANDLORD_PROPERTY_ID}
- ), how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID
+ # programme_data = programme_data.merge(
+ # assigned_surveyors.rename(
+ # columns={self.landlord_property_id: self.STANDARD_LANDLORD_PROPERTY_ID}
+ # ), how="left", on=self.STANDARD_LANDLORD_PROPERTY_ID
+ # )
+
+ # Add in some columns if we have them
+ date_of_inspections = (
+ "Non-Intrusives: Date of Inspection" if
+ "Non-Intrusives: Date of Inspection" in programme_data.columns else None
)
# This maps the hubspot schema to the template. Anything that is not covered in this will be flagged
schema_mappings = {
- 'Name ': self.DOMNA_PROPERTY_ID, # TODO: Maybe change this?
'Company Domain Name ': 'Company Domain Name ',
'Email ': (
self.contact_detail_fields["email"] if self.contact_detail_fields["email"] else None
@@ -2227,9 +2297,10 @@ class AssetList:
'Address 2 ': None, # TODO: Don't have this for the moment
'Postcode ': self.STANDARD_POSTCODE,
'Property Type ': self.STANDARD_PROPERTY_TYPE,
- 'Property Sub Type ': None, # TODO: Don't have this for the moment
+ 'Property Sub Type ': self.STANDARD_BUILT_FORM,
'Bedroom(s) ': None, # TODO: Don't have this for the moment
'Domna Property ID ': self.DOMNA_PROPERTY_ID,
+ # We populate this with the column that we have
'National UPRN ': (
self.STANDARD_UPRN if self.STANDARD_UPRN is not None else self.EPC_API_DATA_NAMES["uprn"]
),
@@ -2239,8 +2310,7 @@ class AssetList:
'Year Built ': self.STANDARD_YEAR_BUILT,
'Boiler Make ': None, # TODO: Don't have this for the moment
'Boiler Model ': None, # TODO: Don't have this for the moment
- 'Non-Intrusives: Date Checked ': None,
- # TODO: Don't have this for the moment
+ 'Non-Intrusives: Date Checked ': date_of_inspections,
'Non-Intrusives: Wall Type ': (
"non-intrusives: Construction" if self.non_intrusives_present else None
),
@@ -2283,16 +2353,22 @@ class AssetList:
'Last EPC: Age Band ': self.EPC_API_DATA_NAMES["construction-age-band"],
'Deal Stage ': 'Deal Stage ',
'Pipeline ': 'Pipeline ',
- 'Expected Commencement Date ': None, # TODO: Need to set this,
+ 'Expected Commencement Date ': "survey_week",
'Deal Name ': "dealname", # Need to create this,
'Product ID ': 'Product ID ',
'Name ': 'Name ',
'Unit price ': 'Unit price ',
'Quantity ': 'Quantity ',
'Deal Owner': 'surveyor_email',
- 'Amount ': 'Unit price ',
+ 'Project Code ': 'project_code',
+ 'Associations: Listing': 'Associations: Listing',
}
+ # We sometimes columns if the landlord never provided them
+ missed_mapping_cols = [c for c in schema_mappings.values() if c not in programme_data.columns if c is not None]
+ for c in missed_mapping_cols:
+ programme_data[c] = None
+
# We now create the finalised dataset to be uploaded into Hubspot
variables_required = list(schema_mappings.values())
variables_required = [v for v in variables_required if v is not None]
@@ -2307,6 +2383,22 @@ class AssetList:
columns={v: k for k, v in schema_mappings.items() if v is not None}
)
+ programme_data['Installer '] = installer_name
+ programme_data['Name '] = (
+ programme_data['Address 1 '] + " ," + programme_data['Postcode ']
+ )
+ # The listing owner email is the same as the surveyor email (deal owner), so they can see the listing
+ programme_data['Listing Owner Email '] = programme_data['Deal Owner']
+ programme_data['Amount '] = 0
+
+ # We make sure we have all of the columns that we need
+ missed_columns = [c for c in hubspot_config.CRM_UPLOAD_COLUMNS if c not in programme_data.columns]
+ if missed_columns:
+ raise ValueError(
+ f"We have the following columns that are not in the programme data: {missed_columns}. "
+ "Please check the mapping and ensure all required columns are present."
+ )
+
self.hubspot_data = programme_data
def flag_ecosurv(self, ecosurv_landlords=None, landlords_to_ignore=None):
diff --git a/asset_list/hubspot/config.py b/asset_list/hubspot/config.py
index 180bf0e0..6e16279a 100644
--- a/asset_list/hubspot/config.py
+++ b/asset_list/hubspot/config.py
@@ -1,5 +1,8 @@
from enum import IntEnum
+CRM_PIPELINE_NAME = 'Operations - Housing Associations'
+CRM_PIPELINE_FIRST_STAGE_NAME = 'READY TO BE SCHEDULED'
+
class HubspotProcessStatus(IntEnum):
def __new__(cls, value, label):
@@ -26,3 +29,43 @@ class HubspotProcessStatus(IntEnum):
LODGEMENT_COMPLETE = 7, "LODGEMENT COMPLETE"
# The property has been cancelled
INSTALLER_CANCELLED_FINALIZED = 8, "INSTALLER CANCELLED - FINALIZED"
+
+
+CRM_UPLOAD_COLUMNS = [
+ 'Name ', 'Associations: Listing', 'Company Domain Name ',
+ 'Email ', 'First Name ', 'Last Name ',
+ 'Phone ', 'Listing Owner Email ',
+ 'Full Address ', 'Address 1 ',
+ 'Address 2 ', 'Postcode ',
+ 'Property Type ', 'Property Sub Type ',
+ 'Bedroom(s) ', 'Domna Property ID ',
+ 'National UPRN ', 'Owner Property ID ',
+ 'Wall Construction ', 'Heating System ',
+ 'Year Built ', 'Boiler Make ',
+ 'Boiler Model ',
+ 'Non-Intrusives: Date Checked ',
+ 'Non-Intrusives: Wall Type ',
+ 'Non-intrusives: Insulation ',
+ 'Non-intrusives: Insulation Material ',
+ 'Non-Intrusives: CIGA Check Required ',
+ 'Non-Intrusives: PV Access Issues ',
+ 'Non-Intrusives: Roof Orientation ',
+ 'Non-Intrusives: Surveyor Notes ',
+ 'Non-Intrusives: Surveyor Name ',
+ 'CIGA: Date Requested ',
+ 'CIGA: Cavity Guarantee Found ',
+ 'Last EPC: Is Estimated ',
+ 'Last EPC: EPC Rating ',
+ 'Last EPC: SAP Rating ',
+ 'Last EPC: Main Heating Description ',
+ 'Last EPC: Heating Controls ',
+ 'Last EPC: Lodgement Date ',
+ 'Last EPC: Floor Area ', 'Last EPC: Wall ',
+ 'Last EPC: Roof ', 'Last EPC: Floor ',
+ 'Last EPC: Room Height ',
+ 'Last EPC: Age Band ', 'Deal Stage ',
+ 'Pipeline ', 'Expected Commencement Date ',
+ 'Deal Name ', 'Project Code ',
+ 'Product ID ', 'Name ', 'Unit price ',
+ 'Quantity ', 'Deal Owner', 'Amount ', 'Installer '
+]
diff --git a/asset_list/hubspot/prepare_for_hubspot.py b/asset_list/hubspot/prepare_for_hubspot.py
index 302d2673..8ed654f3 100644
--- a/asset_list/hubspot/prepare_for_hubspot.py
+++ b/asset_list/hubspot/prepare_for_hubspot.py
@@ -1,4 +1,5 @@
import pandas as pd
+from asset_list.AssetList import AssetList
def app():
@@ -9,10 +10,42 @@ def app():
cavity_reason and solar_reason are populated, as if we want to include historical surveys, this will remove
them
+
+ TODO: If we wish to upload deals in batches
+
:return:
"""
- filepath = ("/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Thrive "
- "Programme - reconciled.xlsx")
+ # inputs:
+ customer_domain = "https://thrivehomes.org.uk"
+ asset_list_filepath = (
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Hubspot Upload - "
+ "Sample.xlsx"
+ )
+ contact_details_filepath = (
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Testing Hubspot Upload/Sample contact "
+ "details.xlsx"
+ )
+ contacts_sheet_name = "Sheet1"
+ contacts_landlord_property_id = "landlord_property_id"
+ contacts_phone_number_column = "phone_number"
+ contacts_email_column = "email"
+ contacts_fullname_column = "fullname"
+ contacts_firstname_column = "firstname"
+ contacts_lastname_column = "lastname"
- standardised_asset_list = pd.read_excel(filepath, sheet_name="Standardised Asset List")
+ asset_list = AssetList.load_standardised_asset_list(asset_list_filepath)
+ asset_list.load_contact_details(
+ local_filepath=contact_details_filepath,
+ sheet_name=contacts_sheet_name,
+ landlord_property_id=contacts_landlord_property_id,
+ phone_number_column=contacts_phone_number_column,
+ email_column=contacts_email_column,
+ fullname_column=contacts_fullname_column,
+ firstname_column=contacts_firstname_column,
+ lastname_column=contacts_lastname_column
+ )
+
+ asset_list.prepare_for_crm(
+ company_domain=customer_domain
+ )
diff --git a/backend/Funding.py b/backend/Funding.py
index f5f85b9f..78440eac 100644
--- a/backend/Funding.py
+++ b/backend/Funding.py
@@ -411,3 +411,123 @@ class Funding:
self.gbis()
# self.eco4()
self.whlg()
+
+
+class Funding2:
+ """
+ New class to handle funding calculation
+ """
+
+ def __init__(self, tenure: HousingType):
+ self.tenure = tenure
+
+ @staticmethod
+ def get_sap_band(sap_score_number):
+ bands = [
+ ("High_A", 96, float("inf")),
+ ("Low_A", 92, 96),
+ ("High_B", 86, 92),
+ ("Low_B", 81, 86),
+ ("High_C", 74.5, 81),
+ ("Low_C", 69, 74.5),
+ ("High_D", 61.5, 69),
+ ("Low_D", 55, 61.5),
+ ("High_E", 46.5, 55),
+ ("Low_E", 39, 46.5),
+ ("High_F", 29.5, 39),
+ ("Low_F", 21, 29.5),
+ ("High_G", 10.5, 21),
+ ("Low_G", 1, 10.5),
+ ]
+
+ for band, lower, upper in bands:
+ if lower <= sap_score_number < upper:
+ return band
+
+ return None
+
+ def eco4_prs_eligibility(
+ self, starting_sap: int, measures: List, mainheat_description: str, heating_control_description: str
+ ):
+ """
+ Handles the eligibility criteria for private rental properties under eco
+ :return:
+ """
+
+ # Help to heat group
+ # 1) EPC E - G
+ # 2) Must receive one of SWI, FTCH, renewable heating or DHC
+ # 3) Tenant must be on benefits
+
+ # We don't consider the tenant being on benefits - we just notify the end user that this is a requirement
+
+ meets_epc = starting_sap <= 54
+ has_solid_wall = "internal_wall_insulation" in measures or "external_wall_insulation" in measures
+ # We check if the property has a heating system that means solar pv counts as a renewable heating system
+
+ has_eligible_electric_heating = any(x in mainheat_description for x in [
+ "air source heat pump", "ground source heat pump", "boiler and radiators, electric"
+ ]) | (("electric storage heaters" in mainheat_description) and
+ (heating_control_description.lower() == "controls for high heat retention storage heaters")
+ )
+
+ # Counts as renewable heating
+ solar_renweable_heating = has_eligible_electric_heating & ("solar_pv" in measures)
+ # Is a renewable heating
+ ashp = "air_source_heat_pump" in measures
+
+ if meets_epc & (solar_renweable_heating or ashp or has_solid_wall):
+ return True
+
+ return False
+
+ def check_funding(
+ self, measures: List,
+ starting_sap: int,
+ ending_sap: int,
+ mainheat_description: str,
+ heating_control_description: str
+ ):
+ """
+ Given a list of measures, this function will check if the package of measures is fundable
+ :param measures:
+ :param starting_sap:
+ :param ending_sap:
+ :return:
+ """
+
+ starting_band = self.get_sap_band(starting_sap)
+ ending_band = self.get_sap_band(ending_sap)
+
+ # For ECO4 eligibility, the property needs to end at a C if it starts at a D or E, otherwise should end at a
+ # D
+
+ if starting_band <= 38 & ending_band >= 55:
+ # F or G should get to D
+ raise NotImplementedError("Implement F or G to D eligibility")
+
+ ########################
+ # Private
+ ########################
+ # 1) ECO4
+ # 2) GBIS
+
+ if self.tenure == "Private":
+ is_eligible = self.eco4_prs_eligibility(
+ starting_sap=starting_sap,
+ measures=measures,
+ mainheat_description=mainheat_description,
+ heating_control_description=heating_control_description
+ )
+ pass
+
+ ########################
+ # Social
+ ########################
+ # 1) ECO4
+ # 2) GBIS
+
+ if self.tenure == "Social":
+ pass
+
+ raise NotImplementedError("Only implemented for Private or Social housing")
diff --git a/etl/customers/cambridge/surveys.py b/etl/customers/cambridge/surveys.py
new file mode 100644
index 00000000..2aa52d6f
--- /dev/null
+++ b/etl/customers/cambridge/surveys.py
@@ -0,0 +1,24 @@
+import pandas as pd
+from backend.ml_models.Valuation import PropertyValuation
+from backend.app.utils import sap_to_epc
+
+# Read in the survey data
+surveys = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Cambridge/Survey Data.xlsx",
+ sheet_name="Survey data",
+)
+
+increases = []
+for _, x in surveys.iterrows():
+ current_epc = sap_to_epc(x["Pre SAP"])
+ target_epc = sap_to_epc(x["Scenario 1 Post SAP"])
+ current_value = x["Valuation"]
+
+ val = PropertyValuation.estimate_valuation_improvement(
+ current_value,
+ current_epc,
+ target_epc,
+ total_cost=None
+ )
+ avg_increase = val["average_increase"]
+ increases.append(round(avg_increase))
diff --git a/etl/customers/places_for_people/abs.py b/etl/customers/places_for_people/abs.py
new file mode 100644
index 00000000..aa85a93f
--- /dev/null
+++ b/etl/customers/places_for_people/abs.py
@@ -0,0 +1,199 @@
+"""
+This script is to calculate the ABS for the Places for People London project
+"""
+
+import os
+import pandas as pd
+
+# London
+pfp_london_cav = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_areas_surrounding_london_reviewed_standardised_15052025.xlsx",
+ sheet_name="Cav Route",
+ header=1
+)
+pfp_london_cav = pfp_london_cav.rename(columns={"Route": "Route March"})
+pfp_london_pv = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_areas_surrounding_london_reviewed_standardised_15052025.xlsx",
+ sheet_name="PV Route",
+ header=1
+)
+pfp_london_pv = pfp_london_pv.rename(columns={"Route": "Route March"})
+pfp_london_cav["location"] = "London"
+pfp_london_pv["location"] = "London"
+# East
+pfp_east_cav = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_east_reviewed_standarised_15052025.xlsx",
+ sheet_name="Cav Route",
+ header=1
+)
+pfp_east_cav = pfp_east_cav.rename(columns={"Route": "Route March"})
+pfp_east_pv = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_east_reviewed_standarised_15052025.xlsx",
+ sheet_name="PV Route",
+ header=1
+)
+pfp_east_pv = pfp_east_pv.rename(columns={"Route": "Route March"})
+pfp_east_cav["location"] = "East"
+pfp_east_pv["location"] = "East"
+# North east
+pfp_north_east_cav = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_north_east_reviewed_standardised_15052025.xlsx",
+ sheet_name="Cav Route",
+ header=1
+)
+pfp_north_east_cav = pfp_north_east_cav.rename(columns={"Route": "Route March"})
+pfp_north_east_pv = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_north_east_reviewed_standardised_15052025.xlsx",
+ sheet_name="PV Route",
+ header=1
+)
+pfp_north_east_pv = pfp_north_east_pv.rename(columns={"Route": "Route March"})
+pfp_north_east_cav["location"] = "North East"
+pfp_north_east_pv["location"] = "North East"
+# North West
+pfp_north_west_cav = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_north_west_reviewed_standardised_15052025.xlsx",
+ sheet_name="Cav Route",
+ header=1
+)
+pfp_north_west_cav = pfp_north_west_cav.rename(columns={"Route": "Route March"})
+pfp_north_west_pv = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs "
+ "rates/PFP_north_west_reviewed_standardised_15052025.xlsx",
+ sheet_name="PV Route",
+ header=1
+)
+pfp_north_west_pv = pfp_north_west_pv.rename(columns={"Route": "Route March"})
+pfp_north_west_cav["location"] = "North West"
+pfp_north_west_pv["location"] = "North West"
+
+cav_route = pd.concat(
+ [
+ pfp_london_cav,
+ pfp_east_cav,
+ pfp_north_east_cav,
+ pfp_north_west_cav
+ ]
+)
+solar_route = pd.concat(
+ [
+ pfp_london_pv,
+ pfp_east_pv,
+ pfp_north_east_pv,
+ pfp_north_west_pv
+ ]
+)
+
+
+def get_band(sap_score_number):
+ bands = [
+ ("High_A", 96, float("inf")),
+ ("Low_A", 92, 96),
+ ("High_B", 86, 92),
+ ("Low_B", 81, 86),
+ ("High_C", 74.5, 81),
+ ("Low_C", 69, 74.5),
+ ("High_D", 61.5, 69),
+ ("Low_D", 55, 61.5),
+ ("High_E", 46.5, 55),
+ ("Low_E", 39, 46.5),
+ ("High_F", 29.5, 39),
+ ("Low_F", 21, 29.5),
+ ("High_G", 10.5, 21),
+ ("Low_G", 1, 10.5),
+ ]
+
+ for band, lower, upper in bands:
+ if lower <= sap_score_number < upper:
+ return band
+
+ return None
+
+
+def classify_floor_area(floor_area):
+ if floor_area <= 72:
+ return "0-72"
+
+ if floor_area <= 97:
+ return "73-97"
+
+ if floor_area <= 199:
+ return "98-199"
+
+ return "200+"
+
+
+# We classify the abs bounds
+solar_route["starting_abs_band"] = solar_route["epc_sap_score_on_register"].apply(get_band)
+solar_route["ending_abs_band_scenario1"] = "High_C"
+solar_route["ending_abs_band_scenario2"] = "Low_B"
+solar_route["epc_total_floor_area"] = solar_route["epc_total_floor_area"].fillna(90)
+solar_route["floor_area_band"] = solar_route["epc_total_floor_area"].apply(classify_floor_area)
+
+# We classify the abs bounds
+cav_route["epc_sap_score_on_register"] = cav_route["epc_sap_score_on_register"].fillna(68)
+cav_route["starting_abs_band"] = cav_route["epc_sap_score_on_register"].apply(get_band)
+cav_route["floor_area_band"] = cav_route["epc_total_floor_area"].apply(classify_floor_area)
+cav_route["ending_abs_band"] = "Low_C"
+
+abs_matrix = pd.read_csv(
+ "/Users/khalimconn-kowlessar/Downloads/ECO4 Full Project Scores Matrix.csv"
+)
+
+cav_route = cav_route.merge(
+ abs_matrix.rename(columns={"Cost Savings": "ABS Rate"}),
+ how="left",
+ left_on=["starting_abs_band", "ending_abs_band", "floor_area_band"],
+ right_on=["Starting Band", "Finishing Band", "Floor Area Segment"],
+)
+solar_route = solar_route.merge(
+ abs_matrix.rename(columns={"Cost Savings": "ABS Rate"}),
+ how="left",
+ left_on=["starting_abs_band", "ending_abs_band_scenario1", "floor_area_band"],
+ right_on=["Starting Band", "Finishing Band", "Floor Area Segment"],
+)
+cav_route["ABS Rate"] = cav_route["ABS Rate"].fillna(0)
+solar_route["ABS Rate"] = solar_route["ABS Rate"].fillna(0)
+
+cav_abs_agg = (
+ cav_route.groupby("Route March").agg(
+ {
+ "ABS Rate": "sum",
+ "landlord_property_id": "count",
+ }
+ ).reset_index()
+)
+cav_abs_agg["Week Number"] = cav_abs_agg["Route March"].str.extract(r"(\d+)").astype(int)
+cav_abs_agg = cav_abs_agg.sort_values("Week Number", ascending=True)
+cav_abs_agg = cav_abs_agg.rename(columns={"landlord_property_id": "Number of Properties"})
+
+solar_abs_agg = (
+ solar_route.groupby("Route March").agg(
+ {
+ "ABS Rate": "sum",
+ "landlord_property_id": "count",
+ }
+ ).reset_index()
+)
+solar_abs_agg["Week Number"] = solar_abs_agg["Route March"].str.extract(r"(\d+)").astype(int)
+solar_abs_agg = solar_abs_agg.rename(columns={"landlord_property_id": "Number of Properties"})
+solar_abs_agg = solar_abs_agg.sort_values("Week Number", ascending=True)
+
+# We store the data
+# Store as an excel
+filename = "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Places For People/abs rates/pfp programme rates.xlsx"
+# Store the data in two tabs. One for the asset list with the EPC data and the second with the flat data
+
+with pd.ExcelWriter(filename) as writer:
+ solar_abs_agg.to_excel(writer, sheet_name="Solar ABS", index=False)
+ cav_abs_agg.to_excel(writer, sheet_name="Cav ABS", index=False)
+
+ cav_route.to_excel(writer, sheet_name="Cavity data", index=False)
+ solar_route.to_excel(writer, sheet_name="Solar data", index=False)
diff --git a/etl/customers/thrive/Project codes.py b/etl/customers/thrive/Project codes.py
index 6235ebed..01a15497 100644
--- a/etl/customers/thrive/Project codes.py
+++ b/etl/customers/thrive/Project codes.py
@@ -38,10 +38,10 @@ PROJECT_CODE_MAP = {
'Phase 8': "THRIVE-008",
'Phase 9': "THRIVE-009",
'Phase 10': "THRIVE-010",
- "Week1": "THRIVE-WEEK-001",
- "Week2": "THRIVE-WEEK-002",
- "Week4": "THRIVE-WEEK-004",
- "Week7": "THRIVE-WEEK-007",
+ "Week 1": "THRIVE-WEEK-001",
+ "Week 2": "THRIVE-WEEK-002",
+ "Week 4": "THRIVE-WEEK-004",
+ "Week 7": "THRIVE-WEEK-007",
}
programme_codes["project_code"] = programme_codes["programme_reference"].map(PROJECT_CODE_MAP)
@@ -102,7 +102,29 @@ with pd.ExcelWriter(filename) as writer:
block_analysis.to_excel(writer, sheet_name="Block Analysis", index=False)
# If we have outcomes, we add a tab with the outcomes
outcomes.to_excel(writer, sheet_name="Outcomes", index=False)
-
unmatched_submissions.to_excel(writer, sheet_name="Unmatched Submissions", index=False)
-
unmatched_ecosurv.to_excel(writer, sheet_name="Unmatched Ecosurv", index=False)
+
+# A check, just comparing against the master tracker to make sure I have all of the installs
+asset_list = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Thrive Asset List - "
+ "Complete - Updated May 2025 - Standardised.xlsx",
+ sheet_name="Standardised Asset List",
+)
+
+master_tracker = pd.read_excel(
+ "/Users/khalimconn-kowlessar/Documents/hestia/Customers/Thrive/Programme Reconciliation/Master Tracker (Thrive - "
+ "Warmfront).xlsx",
+ sheet_name="Master Tracker",
+ header=1
+)
+
+df = asset_list[["landlord_property_id", "hubspot_status"]].merge(
+ master_tracker[~pd.isnull(master_tracker['Date Completed'])][["UPRN", "Date Completed"]],
+ how="inner",
+ left_on="landlord_property_id",
+ right_on="UPRN"
+)
+
+df["hubspot_status"].value_counts()
+df[df["hubspot_status"] == "SUBMITTED TO INSTALLER"]