diff --git a/backend/addresses/Addresses.py b/backend/addresses/Addresses.py index 4c046554..44083b86 100644 --- a/backend/addresses/Addresses.py +++ b/backend/addresses/Addresses.py @@ -110,6 +110,8 @@ class Addresses: landlord_multi_glaze_proportion=float(row["landlord_multi_glaze_proportion"]) if row.get( "landlord_multi_glaze_proportion") else None, landlord_construction_age_band=row.get("landlord_construction_age_band"), + lmk_key=None, + epc_certificate_number=None, ) @staticmethod diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py index fa508fbe..0ee58d54 100644 --- a/backend/app/db/models/hubspot_deal_data.py +++ b/backend/app/db/models/hubspot_deal_data.py @@ -67,6 +67,17 @@ class HubspotDealData(SQLModel, table=True): surveyed_date: Optional[datetime] = Field(default=None) design_type: Optional[str] = Field(default=None) + survey_type: Optional[str] = Field(default=None) + measures_for_pibi_ordered: Optional[str] = Field(default=None) + pibi_order_date: Optional[datetime] = Field(default=None) + pibi_completed_date: Optional[datetime] = Field(default=None) + property_halted_date: Optional[datetime] = Field(default=None) + property_halted_reason: Optional[str] = Field(default=None) + technical_approved_measures_for_install: Optional[str] = Field(default=None) + sent_to_installer_for_pricing: Optional[datetime] = Field(default=None) + domna_survey_required: Optional[bool] = Field(default=None) + domna_survey_date: Optional[datetime] = Field(default=None) + created_at: Optional[datetime] = Field( sa_column=Column( DateTime(timezone=True), diff --git a/backend/app/db/models/hubspot_user.py b/backend/app/db/models/hubspot_user.py new file mode 100644 index 00000000..424a0c17 --- /dev/null +++ b/backend/app/db/models/hubspot_user.py @@ -0,0 +1,13 @@ +from sqlmodel import SQLModel, Field +from datetime import datetime +from typing import Optional + + +class HubspotUser(SQLModel, table=True): + __tablename__ = "hubspot_users" + + hubspot_owner_id: str = Field(primary_key=True) + first_name: Optional[str] = Field(default=None) + last_name: Optional[str] = Field(default=None) + email: Optional[str] = Field(default=None) + updated_at: datetime diff --git a/backend/export/tests/test_export.py b/backend/export/tests/test_export.py index b00d1744..42177749 100644 --- a/backend/export/tests/test_export.py +++ b/backend/export/tests/test_export.py @@ -282,11 +282,6 @@ def test_default_export_integration(db_session): df["sap_points"].sum() ) - assert df.shape == ( - 10, - 100, - ), "Expected dataframe shape to be (10, 100), got {}".format(df.shape) - def test_solar_with_battery_example(db_session): test_portfolio_id = 1 @@ -337,7 +332,7 @@ def test_solar_with_battery_example(db_session): "creation_status": "PropertyCreationStatus.READY", "uprn": 100090438731, "landlord_property_id": "BARR052", - "building_reference_number": 3460742868.0, + "building_reference_number": 3460742868, "status": "PortfolioStatus.ASSESSMENT", "address": "52, Barrack Street", "postcode": "CO1 2LR", @@ -566,6 +561,8 @@ def test_solar_with_battery_example(db_session): creation_status=PropertyCreationStatus[row.creation_status.split(".")[-1]], status=PortfolioStatus[row.status.split(".")[-1]], uprn=row.uprn, + address=row.address, + postcode=row.postcode, property_type=row.property_type, current_sap_points=row.current_sap_points, current_epc_rating=Epc[row.current_epc_rating.split(".")[-1]], diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py index df28e4d6..92a6c7e1 100644 --- a/etl/hubspot/hubspotClient.py +++ b/etl/hubspot/hubspotClient.py @@ -255,13 +255,13 @@ class HubspotClient: "dampmould_growth", "damp_mould_and_repairs_comments", "pre_sap_score_dropdown", - "coordinator", + "coordinator_user", "mtp_completion_date", "mtp_re_model_completion_date", "ioe_v3_completion_date", "proposed_measures_dropdown", "approved_package", - "designer", + "designer_user", "design_completion_date", "actual_measures_installed", "installer", @@ -283,6 +283,16 @@ class HubspotClient: "ei_score__potential_", "epc_sap_score", "epc_sap_score__potential_", + "survey_type", + "measures_for_pibi_ordered", + "pibi_order_date", + "pibi_completed_date", + "property_halted_date", + "property_halted_reason", + "technical_approved_measures_for_install", + "sent_to_iw_for_pricing", + "osmosis_survey_required", + "osmosis_survey_date", ], ) ) @@ -290,6 +300,20 @@ class HubspotClient: deal_info: dict[str, str] = cast(dict[str, str], deal.properties) # type: ignore[reportUnknownMemberType] return deal_info + def get_owner_info(self, owner_id: str) -> Optional[dict[str, Optional[str]]]: + try: + owner = self._call_with_retry( + lambda: self.client.crm.owners.owners_api.get_by_id(owner_id) # type: ignore[reportUnknownMemberType] + ) + return { + "first_name": owner.first_name, # type: ignore[reportUnknownMemberType] + "last_name": owner.last_name, # type: ignore[reportUnknownMemberType] + "email": getattr(owner, "email", None), + } + except Exception: + self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}") + return None + def get_deal_and_company_and_listing( self, deal_id: str ) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]: diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index a2eb24c2..278fbe43 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -1,18 +1,18 @@ import os -from sqlmodel import select +from sqlmodel import select, Session from datetime import datetime, timezone from typing import Dict, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData +from backend.app.db.models.hubspot_user import HubspotUser from etl.hubspot.company_data import CompanyData from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.s3_uploader import S3Uploader from backend.app.db.connection import db_read_session from backend.app.db.models.organisation import Organisation -from etl.hubspot.utils import parse_hs_date +from etl.hubspot.utils import parse_hs_bool, parse_hs_date from utils.logger import setup_logger - logger = setup_logger() @@ -95,6 +95,9 @@ class HubspotDataToDb: with db_read_session() as session: deal_id = deal_data.get("hs_object_id") + self._sync_owner_to_db(deal_data.get("coordinator_user"), hubspot_client, session) + self._sync_owner_to_db(deal_data.get("designer_user"), hubspot_client, session) + statement = select(HubspotDealData).where( HubspotDealData.deal_id == deal_id ) @@ -125,6 +128,38 @@ class HubspotDataToDb: session.refresh(new_record) return new_record + def _sync_owner_to_db( + self, + owner_id: Optional[str], + hubspot_client: HubspotClient, + session: Session, + ) -> None: + if not owner_id: + return + + owner_info = hubspot_client.get_owner_info(owner_id) + if owner_info is None: + return + + now = datetime.now(timezone.utc) + existing: Optional[HubspotUser] = session.get(HubspotUser, owner_id) + if existing: + existing.first_name = owner_info["first_name"] + existing.last_name = owner_info["last_name"] + existing.email = owner_info["email"] + existing.updated_at = now + session.add(existing) + else: + session.add( + HubspotUser( + hubspot_owner_id=owner_id, + first_name=owner_info["first_name"], + last_name=owner_info["last_name"], + email=owner_info["email"], + updated_at=now, + ) + ) + def _update_existing_deal( self, existing: HubspotDealData, @@ -170,7 +205,7 @@ class HubspotDataToDb: "ei_score__potential_": deal_data.get("ei_score__potential_"), "epc_sap_score": deal_data.get("epc_sap_score"), "epc_sap_score__potential_": deal_data.get("epc_sap_score__potential_"), - "coordinator": deal_data.get("coordinator"), + "coordinator": deal_data.get("coordinator_user"), "mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")), "mtp_re_model_completion_date": parse_hs_date( deal_data.get("mtp_re_model_completion_date") @@ -180,7 +215,7 @@ class HubspotDataToDb: ), "proposed_measures": deal_data.get("proposed_measures_dropdown"), "approved_package": deal_data.get("approved_package"), - "designer": deal_data.get("designer"), + "designer": deal_data.get("designer_user"), "design_completion_date": parse_hs_date( deal_data.get("design_completion_date") ), @@ -202,6 +237,24 @@ class HubspotDataToDb: "confirmed_survey_time": deal_data.get("confirmed_survey_time"), "surveyed_date": parse_hs_date(deal_data.get("surveyed_date")), "design_type": deal_data.get("design_type"), + "survey_type": deal_data.get("survey_type"), + "measures_for_pibi_ordered": deal_data.get("measures_for_pibi_ordered"), + "pibi_order_date": parse_hs_date(deal_data.get("pibi_order_date")), + "pibi_completed_date": parse_hs_date(deal_data.get("pibi_completed_date")), + "property_halted_date": parse_hs_date( + deal_data.get("property_halted_date") + ), + "property_halted_reason": deal_data.get("property_halted_reason"), + "technical_approved_measures_for_install": deal_data.get( + "technical_approved_measures_for_install" + ), + "sent_to_installer_for_pricing": parse_hs_date( + deal_data.get("sent_to_iw_for_pricing") + ), + "domna_survey_required": parse_hs_bool( + deal_data.get("osmosis_survey_required") + ), + "domna_survey_date": parse_hs_date(deal_data.get("osmosis_survey_date")), }.items(): setattr(existing, attr, value or getattr(existing, attr)) @@ -249,7 +302,7 @@ class HubspotDataToDb: ei_score__potential_=deal_data.get("ei_score__potential_"), epc_sap_score=deal_data.get("epc_sap_score"), epc_sap_score__potential_=deal_data.get("epc_sap_score__potential_"), - coordinator=deal_data.get("coordinator"), + coordinator=deal_data.get("coordinator_user"), mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")), mtp_re_model_completion_date=parse_hs_date( deal_data.get("mtp_re_model_completion_date") @@ -259,7 +312,7 @@ class HubspotDataToDb: ), proposed_measures=deal_data.get("proposed_measures_dropdown"), approved_package=deal_data.get("approved_package"), - designer=deal_data.get("designer"), + designer=deal_data.get("designer_user"), design_completion_date=parse_hs_date( deal_data.get("design_completion_date") ), @@ -279,6 +332,22 @@ class HubspotDataToDb: confirmed_survey_time=deal_data.get("confirmed_survey_time"), surveyed_date=parse_hs_date(deal_data.get("surveyed_date")), design_type=deal_data.get("design_type"), + survey_type=deal_data.get("survey_type"), + measures_for_pibi_ordered=deal_data.get("measures_for_pibi_ordered"), + pibi_order_date=parse_hs_date(deal_data.get("pibi_order_date")), + pibi_completed_date=parse_hs_date(deal_data.get("pibi_completed_date")), + property_halted_date=parse_hs_date(deal_data.get("property_halted_date")), + property_halted_reason=deal_data.get("property_halted_reason"), + technical_approved_measures_for_install=deal_data.get( + "technical_approved_measures_for_install" + ), + sent_to_installer_for_pricing=parse_hs_date( + deal_data.get("sent_to_iw_for_pricing") + ), + domna_survey_required=parse_hs_bool( + deal_data.get("osmosis_survey_required") + ), + domna_survey_date=parse_hs_date(deal_data.get("osmosis_survey_date")), ) def _handle_existing_photo_upload( diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index fa5bbe42..9e7069fc 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -1,7 +1,7 @@ from typing import Dict, List, Optional from backend.app.db.models.hubspot_deal_data import HubspotDealData -from etl.hubspot.utils import parse_hs_date +from etl.hubspot.utils import parse_hs_bool, parse_hs_date class HubspotDealDiffer: @@ -71,10 +71,10 @@ class HubspotDealDiffer: "ei_score__potential_": "ei_score__potential_", "epc_sap_score": "epc_sap_score", "epc_sap_score__potential_": "epc_sap_score__potential_", - "coordinator": "coordinator", + "coordinator_user": "coordinator", "proposed_measures_dropdown": "proposed_measures", "approved_package": "approved_package", - "designer": "designer", + "designer_user": "designer", "actual_measures_installed": "actual_measures_installed", "installer": "installer", "installer_handover": "installer_handover", @@ -82,6 +82,10 @@ class HubspotDealDiffer: "design_type": "design_type", "surveyor": "surveyor", "confirmed_survey_time": "confirmed_survey_time", + "survey_type": "survey_type", + "measures_for_pibi_ordered": "measures_for_pibi_ordered", + "property_halted_reason": "property_halted_reason", + "technical_approved_measures_for_install": "technical_approved_measures_for_install", } for hs_field, db_field in FIELD_MAP.items(): @@ -102,6 +106,11 @@ class HubspotDealDiffer: ("expected_commencement_date", "expected_commencement_date"), ("confirmed_survey_date", "confirmed_survey_date"), ("surveyed_date", "surveyed_date"), + ("pibi_order_date", "pibi_order_date"), + ("pibi_completed_date", "pibi_completed_date"), + ("property_halted_date", "property_halted_date"), + ("sent_to_iw_for_pricing", "sent_to_installer_for_pricing"), + ("osmosis_survey_date", "domna_survey_date"), ] for hs_field, db_field in date_fields: @@ -111,6 +120,18 @@ class HubspotDealDiffer: if old_value != new_value: return True + # --- Boolean fields --- + bool_fields = [ + ("osmosis_survey_required", "domna_survey_required"), + ] + + for hs_field, db_field in bool_fields: + old_value = getattr(old_deal, db_field) + new_value = parse_hs_bool(new_deal.get(hs_field)) + + if old_value != new_value: + return True + # --- Time field --- if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"): return True diff --git a/etl/hubspot/utils.py b/etl/hubspot/utils.py index b7331f94..6077727e 100644 --- a/etl/hubspot/utils.py +++ b/etl/hubspot/utils.py @@ -14,3 +14,11 @@ def parse_hs_date(value: Optional[str]) -> Optional[datetime]: return dt.astimezone(timezone.utc) except ValueError: return None + + +def parse_hs_bool(value: Optional[str]) -> Optional[bool]: + if value is None or value == "": + return None + if isinstance(value, bool): + return value + return str(value).strip().lower() == "true" diff --git a/recommendations/LightingRecommendations.py b/recommendations/LightingRecommendations.py index 91db19c9..e5d2f863 100644 --- a/recommendations/LightingRecommendations.py +++ b/recommendations/LightingRecommendations.py @@ -100,7 +100,7 @@ class LightingRecommendations: :return: """ - if "sap05" in self.property.lighting["clean_description"].lower(): + if "sap05" in self.property.lighting.get("clean_description", "").lower(): return if self.property.lighting["low_energy_proportion"] >= 1: diff --git a/test.requirements.txt b/test.requirements.txt index 6c95f993..7fdd7dc4 100644 --- a/test.requirements.txt +++ b/test.requirements.txt @@ -7,4 +7,6 @@ psycopg[binary] pytest-postgresql hubspot-api-client fuzzywuzzy -pymupdf \ No newline at end of file +pymupdf +playwright==1.58.0 +msal \ No newline at end of file diff --git a/tox.ini b/tox.ini index 2f77a05d..354a74c6 100644 --- a/tox.ini +++ b/tox.ini @@ -9,4 +9,5 @@ deps = -rbackend/engine/requirements.txt -rbackend/app/requirements/requirements.txt -rtest.requirements.txt +commands_pre = playwright install --with-deps chromium commands = pytest {posargs}