junte/Model
1
0
Fork 0
mirror of https://github.com/Hestia-Homes/Model.git synced 2026-06-08 11:17:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1042 from Hestia-Homes/feature/khalim-additional-db-features

Browse source 
Feature/khalim additional db features
This commit is contained in:
KhalimCK 2026-05-01 10:54:28 +01:00 • committed by GitHub
commit b4ee59c82e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 168 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
backend/addresses/Addresses.py
View file

@ -110,6 +110,8 @@ class Addresses:
landlord_multi_glaze_proportion=float(row["landlord_multi_glaze_proportion"]) if row.get(
"landlord_multi_glaze_proportion") else None,
landlord_construction_age_band=row.get("landlord_construction_age_band"),
lmk_key=None,
epc_certificate_number=None,
)
@staticmethod

11
backend/app/db/models/hubspot_deal_data.py
View file

@ -67,6 +67,17 @@ class HubspotDealData(SQLModel, table=True):
surveyed_date: Optional[datetime] = Field(default=None)
design_type: Optional[str] = Field(default=None)
survey_type: Optional[str] = Field(default=None)
measures_for_pibi_ordered: Optional[str] = Field(default=None)
pibi_order_date: Optional[datetime] = Field(default=None)
pibi_completed_date: Optional[datetime] = Field(default=None)
property_halted_date: Optional[datetime] = Field(default=None)
property_halted_reason: Optional[str] = Field(default=None)
technical_approved_measures_for_install: Optional[str] = Field(default=None)
sent_to_installer_for_pricing: Optional[datetime] = Field(default=None)
domna_survey_required: Optional[bool] = Field(default=None)
domna_survey_date: Optional[datetime] = Field(default=None)
created_at: Optional[datetime] = Field(
sa_column=Column(
DateTime(timezone=True),

13
backend/app/db/models/hubspot_user.py Normal file
View file

@ -0,0 +1,13 @@
from sqlmodel import SQLModel, Field
from datetime import datetime
from typing import Optional
class HubspotUser(SQLModel, table=True):
__tablename__ = "hubspot_users"
hubspot_owner_id: str = Field(primary_key=True)
first_name: Optional[str] = Field(default=None)
last_name: Optional[str] = Field(default=None)
email: Optional[str] = Field(default=None)
updated_at: datetime

9
backend/export/tests/test_export.py
View file

@ -282,11 +282,6 @@ def test_default_export_integration(db_session):
df["sap_points"].sum()
)
assert df.shape == (
10,
100,
), "Expected dataframe shape to be (10, 100), got {}".format(df.shape)
def test_solar_with_battery_example(db_session):
test_portfolio_id = 1
@ -337,7 +332,7 @@ def test_solar_with_battery_example(db_session):
"creation_status": "PropertyCreationStatus.READY",
"uprn": 100090438731,
"landlord_property_id": "BARR052",
"building_reference_number": 3460742868.0,
"building_reference_number": 3460742868,
"status": "PortfolioStatus.ASSESSMENT",
"address": "52, Barrack Street",
"postcode": "CO1 2LR",
@ -566,6 +561,8 @@ def test_solar_with_battery_example(db_session):
creation_status=PropertyCreationStatus[row.creation_status.split(".")[-1]],
status=PortfolioStatus[row.status.split(".")[-1]],
uprn=row.uprn,
address=row.address,
postcode=row.postcode,
property_type=row.property_type,
current_sap_points=row.current_sap_points,
current_epc_rating=Epc[row.current_epc_rating.split(".")[-1]],

28
etl/hubspot/hubspotClient.py
View file

@ -255,13 +255,13 @@ class HubspotClient:
"dampmould_growth",
"damp_mould_and_repairs_comments",
"pre_sap_score_dropdown",
"coordinator",
"coordinator_user",
"mtp_completion_date",
"mtp_re_model_completion_date",
"ioe_v3_completion_date",
"proposed_measures_dropdown",
"approved_package",
"designer",
"designer_user",
"design_completion_date",
"actual_measures_installed",
"installer",
@ -283,6 +283,16 @@ class HubspotClient:
"ei_score__potential_",
"epc_sap_score",
"epc_sap_score__potential_",
"survey_type",
"measures_for_pibi_ordered",
"pibi_order_date",
"pibi_completed_date",
"property_halted_date",
"property_halted_reason",
"technical_approved_measures_for_install",
"sent_to_iw_for_pricing",
"osmosis_survey_required",
"osmosis_survey_date",
],
)
)
@ -290,6 +300,20 @@ class HubspotClient:
deal_info: dict[str, str] = cast(dict[str, str], deal.properties) # type: ignore[reportUnknownMemberType]
return deal_info
def get_owner_info(self, owner_id: str) -> Optional[dict[str, Optional[str]]]:
try:
owner = self._call_with_retry(
lambda: self.client.crm.owners.owners_api.get_by_id(owner_id) # type: ignore[reportUnknownMemberType]
)
return {
"first_name": owner.first_name, # type: ignore[reportUnknownMemberType]
"last_name": owner.last_name, # type: ignore[reportUnknownMemberType]
"email": getattr(owner, "email", None),
}
except Exception:
self.logger.warning(f"Failed to fetch HubSpot owner {owner_id}")
return None
def get_deal_and_company_and_listing(
self, deal_id: str
) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]:

83
etl/hubspot/hubspotDataTodB.py
View file

@ -1,18 +1,18 @@
import os
from sqlmodel import select
from sqlmodel import select, Session
from datetime import datetime, timezone
from typing import Dict, Optional
from backend.app.db.models.hubspot_deal_data import HubspotDealData
from backend.app.db.models.hubspot_user import HubspotUser
from etl.hubspot.company_data import CompanyData
from etl.hubspot.hubspotClient import HubspotClient
from etl.hubspot.s3_uploader import S3Uploader
from backend.app.db.connection import db_read_session
from backend.app.db.models.organisation import Organisation
from etl.hubspot.utils import parse_hs_date
from etl.hubspot.utils import parse_hs_bool, parse_hs_date
from utils.logger import setup_logger
logger = setup_logger()
@ -95,6 +95,9 @@ class HubspotDataToDb:
with db_read_session() as session:
deal_id = deal_data.get("hs_object_id")
self._sync_owner_to_db(deal_data.get("coordinator_user"), hubspot_client, session)
self._sync_owner_to_db(deal_data.get("designer_user"), hubspot_client, session)
statement = select(HubspotDealData).where(
HubspotDealData.deal_id == deal_id
)
@ -125,6 +128,38 @@ class HubspotDataToDb:
session.refresh(new_record)
return new_record
def _sync_owner_to_db(
self,
owner_id: Optional[str],
hubspot_client: HubspotClient,
session: Session,
) -> None:
if not owner_id:
return
owner_info = hubspot_client.get_owner_info(owner_id)
if owner_info is None:
return
now = datetime.now(timezone.utc)
existing: Optional[HubspotUser] = session.get(HubspotUser, owner_id)
if existing:
existing.first_name = owner_info["first_name"]
existing.last_name = owner_info["last_name"]
existing.email = owner_info["email"]
existing.updated_at = now
session.add(existing)
else:
session.add(
HubspotUser(
hubspot_owner_id=owner_id,
first_name=owner_info["first_name"],
last_name=owner_info["last_name"],
email=owner_info["email"],
updated_at=now,
)
)
def _update_existing_deal(
self,
existing: HubspotDealData,
@ -170,7 +205,7 @@ class HubspotDataToDb:
"ei_score__potential_": deal_data.get("ei_score__potential_"),
"epc_sap_score": deal_data.get("epc_sap_score"),
"epc_sap_score__potential_": deal_data.get("epc_sap_score__potential_"),
"coordinator": deal_data.get("coordinator"),
"coordinator": deal_data.get("coordinator_user"),
"mtp_completion_date": parse_hs_date(deal_data.get("mtp_completion_date")),
"mtp_re_model_completion_date": parse_hs_date(
deal_data.get("mtp_re_model_completion_date")
@ -180,7 +215,7 @@ class HubspotDataToDb:
),
"proposed_measures": deal_data.get("proposed_measures_dropdown"),
"approved_package": deal_data.get("approved_package"),
"designer": deal_data.get("designer"),
"designer": deal_data.get("designer_user"),
"design_completion_date": parse_hs_date(
deal_data.get("design_completion_date")
),
@ -202,6 +237,24 @@ class HubspotDataToDb:
"confirmed_survey_time": deal_data.get("confirmed_survey_time"),
"surveyed_date": parse_hs_date(deal_data.get("surveyed_date")),
"design_type": deal_data.get("design_type"),
"survey_type": deal_data.get("survey_type"),
"measures_for_pibi_ordered": deal_data.get("measures_for_pibi_ordered"),
"pibi_order_date": parse_hs_date(deal_data.get("pibi_order_date")),
"pibi_completed_date": parse_hs_date(deal_data.get("pibi_completed_date")),
"property_halted_date": parse_hs_date(
deal_data.get("property_halted_date")
),
"property_halted_reason": deal_data.get("property_halted_reason"),
"technical_approved_measures_for_install": deal_data.get(
"technical_approved_measures_for_install"
),
"sent_to_installer_for_pricing": parse_hs_date(
deal_data.get("sent_to_iw_for_pricing")
),
"domna_survey_required": parse_hs_bool(
deal_data.get("osmosis_survey_required")
),
"domna_survey_date": parse_hs_date(deal_data.get("osmosis_survey_date")),
}.items():
setattr(existing, attr, value or getattr(existing, attr))
@ -249,7 +302,7 @@ class HubspotDataToDb:
ei_score__potential_=deal_data.get("ei_score__potential_"),
epc_sap_score=deal_data.get("epc_sap_score"),
epc_sap_score__potential_=deal_data.get("epc_sap_score__potential_"),
coordinator=deal_data.get("coordinator"),
coordinator=deal_data.get("coordinator_user"),
mtp_completion_date=parse_hs_date(deal_data.get("mtp_completion_date")),
mtp_re_model_completion_date=parse_hs_date(
deal_data.get("mtp_re_model_completion_date")
@ -259,7 +312,7 @@ class HubspotDataToDb:
),
proposed_measures=deal_data.get("proposed_measures_dropdown"),
approved_package=deal_data.get("approved_package"),
designer=deal_data.get("designer"),
designer=deal_data.get("designer_user"),
design_completion_date=parse_hs_date(
deal_data.get("design_completion_date")
),
@ -279,6 +332,22 @@ class HubspotDataToDb:
confirmed_survey_time=deal_data.get("confirmed_survey_time"),
surveyed_date=parse_hs_date(deal_data.get("surveyed_date")),
design_type=deal_data.get("design_type"),
survey_type=deal_data.get("survey_type"),
measures_for_pibi_ordered=deal_data.get("measures_for_pibi_ordered"),
pibi_order_date=parse_hs_date(deal_data.get("pibi_order_date")),
pibi_completed_date=parse_hs_date(deal_data.get("pibi_completed_date")),
property_halted_date=parse_hs_date(deal_data.get("property_halted_date")),
property_halted_reason=deal_data.get("property_halted_reason"),
technical_approved_measures_for_install=deal_data.get(
"technical_approved_measures_for_install"
),
sent_to_installer_for_pricing=parse_hs_date(
deal_data.get("sent_to_iw_for_pricing")
),
domna_survey_required=parse_hs_bool(
deal_data.get("osmosis_survey_required")
),
domna_survey_date=parse_hs_date(deal_data.get("osmosis_survey_date")),
)
def _handle_existing_photo_upload(

27
etl/hubspot/hubspot_deal_differ.py
View file

@ -1,7 +1,7 @@
from typing import Dict, List, Optional
from backend.app.db.models.hubspot_deal_data import HubspotDealData
from etl.hubspot.utils import parse_hs_date
from etl.hubspot.utils import parse_hs_bool, parse_hs_date
class HubspotDealDiffer:
@ -71,10 +71,10 @@ class HubspotDealDiffer:
"ei_score__potential_": "ei_score__potential_",
"epc_sap_score": "epc_sap_score",
"epc_sap_score__potential_": "epc_sap_score__potential_",
"coordinator": "coordinator",
"coordinator_user": "coordinator",
"proposed_measures_dropdown": "proposed_measures",
"approved_package": "approved_package",
"designer": "designer",
"designer_user": "designer",
"actual_measures_installed": "actual_measures_installed",
"installer": "installer",
"installer_handover": "installer_handover",
@ -82,6 +82,10 @@ class HubspotDealDiffer:
"design_type": "design_type",
"surveyor": "surveyor",
"confirmed_survey_time": "confirmed_survey_time",
"survey_type": "survey_type",
"measures_for_pibi_ordered": "measures_for_pibi_ordered",
"property_halted_reason": "property_halted_reason",
"technical_approved_measures_for_install": "technical_approved_measures_for_install",
}
for hs_field, db_field in FIELD_MAP.items():
@ -102,6 +106,11 @@ class HubspotDealDiffer:
("expected_commencement_date", "expected_commencement_date"),
("confirmed_survey_date", "confirmed_survey_date"),
("surveyed_date", "surveyed_date"),
("pibi_order_date", "pibi_order_date"),
("pibi_completed_date", "pibi_completed_date"),
("property_halted_date", "property_halted_date"),
("sent_to_iw_for_pricing", "sent_to_installer_for_pricing"),
("osmosis_survey_date", "domna_survey_date"),
]
for hs_field, db_field in date_fields:
@ -111,6 +120,18 @@ class HubspotDealDiffer:
if old_value != new_value:
return True
# --- Boolean fields ---
bool_fields = [
("osmosis_survey_required", "domna_survey_required"),
]
for hs_field, db_field in bool_fields:
old_value = getattr(old_deal, db_field)
new_value = parse_hs_bool(new_deal.get(hs_field))
if old_value != new_value:
return True
# --- Time field ---
if old_deal.confirmed_survey_time != new_deal.get("confirmed_survey_time"):
return True

8
etl/hubspot/utils.py
View file

@ -14,3 +14,11 @@ def parse_hs_date(value: Optional[str]) -> Optional[datetime]:
return dt.astimezone(timezone.utc)
except ValueError:
return None
def parse_hs_bool(value: Optional[str]) -> Optional[bool]:
if value is None or value == "":
return None
if isinstance(value, bool):
return value
return str(value).strip().lower() == "true"

2
recommendations/LightingRecommendations.py
View file

@ -100,7 +100,7 @@ class LightingRecommendations:
:return:
"""
if "sap05" in self.property.lighting["clean_description"].lower():
if "sap05" in self.property.lighting.get("clean_description", "").lower():
return
if self.property.lighting["low_energy_proportion"] >= 1:

4
test.requirements.txt
View file

@ -7,4 +7,6 @@ psycopg[binary]
pytest-postgresql
hubspot-api-client
fuzzywuzzy
pymupdf
pymupdf
playwright==1.58.0
msal

1
tox.ini
View file

@ -9,4 +9,5 @@ deps =
-rbackend/engine/requirements.txt
-rbackend/app/requirements/requirements.txt
-rtest.requirements.txt
commands_pre = playwright install --with-deps chromium
commands = pytest {posargs}