diff --git a/etl/db/hubSpotLoad.py b/etl/db/hubSpotLoad.py index a893245..4adaa71 100644 --- a/etl/db/hubSpotLoad.py +++ b/etl/db/hubSpotLoad.py @@ -13,10 +13,10 @@ class HubspotTodb(): self.hubspot = HubSpotClient() self.deals_in_hubspot = None self.data_in_sharepoint = [] + self.sp = SurveyPrice() def get_all_deals(self): - sp = SurveyPrice() - self.deals_in_hubspot = sp.get_all_surveys_from_hubspot() + self.deals_in_hubspot = self.sp.get_all_surveys_from_hubspot() return self.deals_in_hubspot def get_sharepoint_path(self, url): diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index 0866d24..a05c309 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -4,6 +4,9 @@ from hubspot.crm.deals import PublicObjectSearchRequest from hubspot.crm.deals.models import SimplePublicObjectInput from etl.hubSpotClient.types import SubmissionInfoFromDeal import time +from pydantic import ValidationError +from etl.utils.logger import Logger +import logging @@ -12,11 +15,13 @@ class DealStage(Enum): SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915" CUSTOMER_CONTACTED = "888730834" SURVEYED_COMPLETED_SIGNED_OFF = "1617223916" + NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000" class HubSpotClient(): def __init__(self): self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6" self.client = hubspot.Client.create(access_token=self.access_token) + self.logger = Logger(name='HubSpotClient', level=logging.INFO).get_logger() def get_all_deals(self): return self.client.crm.deals.get_all() @@ -199,20 +204,29 @@ class HubSpotClient(): all_deals = [] for deal in found_deals: domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id) - all_deals.append(SubmissionInfoFromDeal( - deal_id= deal.properties["hs_object_id"], - deal_name=deal.properties["dealname"], - work_type=deal.properties["work_type"], - needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False, - post_sap_score=int(deal.properties["domna_survey_post_sap"]), - existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", - no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), - installer=deal.properties["installer"], - submission_folder_path = deal.properties["submission_folder"], - landlord_id = landlord_id, - domna_id = domna_id, - uprn = uprn, - )) + try: + all_deals.append(SubmissionInfoFromDeal( + deal_id= deal.properties["hs_object_id"], + deal_name=deal.properties["dealname"], + work_type=deal.properties["work_type"], + needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False, + post_sap_score=int(deal.properties["domna_survey_post_sap"]), + existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None", + no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]), + installer=deal.properties["installer"], + submission_folder_path = deal.properties["submission_folder"], + landlord_id = landlord_id, + domna_id = domna_id, + uprn = uprn, + )) + except Exception as e: + deal_id = deal.properties['hs_object_id'] + self.logger.info(f"Deal <{deal_id}> not valid") + self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) + + + + return all_deals @@ -235,4 +249,4 @@ class HubSpotClient(): deal_id, simple_public_object_input=deal_properties ) - print(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") + self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index f95603f..82c9cd3 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -2,6 +2,7 @@ from sqlmodel import Field, SQLModel from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID import uuid +from pydantic import Field, field_validator, ValidationError class BaseModel(SQLModel): id: uuid.UUID = Field( @@ -11,15 +12,22 @@ class BaseModel(SQLModel): class SubmissionInfoFromDeal(BaseModel): - deal_id: str - deal_name: str - work_type: str + deal_id: str = Field(..., min_length=1) + deal_name: str = Field(..., min_length=1) + work_type: str = Field(..., min_length=1) needs_trickle_ventilation: bool post_sap_score: int - existing_wall_insulation: str + existing_wall_insulation: str = Field(..., min_length=1) no_of_wet_rooms: int - installer: str - submission_folder_path: str - landlord_id: str - domna_id: str - uprn: str \ No newline at end of file + installer: str = Field(..., min_length=1) + submission_folder_path: str = Field(..., min_length=1) + landlord_id: str = Field(..., min_length=1) + domna_id: str = Field(..., min_length=1) + uprn: str = Field(..., min_length=1) + + @field_validator('post_sap_score', 'no_of_wet_rooms') + @classmethod + def must_be_non_negative(cls, v): + if v < 0: + raise ValidationError("Must be non-negative for Post Sap Score") + return v \ No newline at end of file diff --git a/etl/hubspot_to_invoice_rewrite.py b/etl/hubspot_to_invoice_rewrite.py new file mode 100644 index 0000000..7ca77dc --- /dev/null +++ b/etl/hubspot_to_invoice_rewrite.py @@ -0,0 +1,32 @@ +import os + + +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" +# Local development +os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" + +from etl.surveyPrice.surveyPrice import SurveyPrice +from etl.db.hubSpotLoad import HubspotTodb + + + +# Load to db +dbLoader = HubspotTodb() + +df = dbLoader.get_all_deals() + + + + +# For each deal +# if not Validate: +# move to a different stage, with option to add a note to this to state what the error was +# work out price and add to grand list +# load to db and move deal correctly to correct position +# +# Once grand list is finsihed for price +# uploaad to sharepoint diff --git a/etl/utils/logger.py b/etl/utils/logger.py index 62970c6..bb70887 100644 --- a/etl/utils/logger.py +++ b/etl/utils/logger.py @@ -3,21 +3,18 @@ import os class Logger: def __init__(self, name, level=logging.INFO): - # Create a custom logger self.logger = logging.getLogger(name) self.logger.setLevel(level) - # Create handlers - c_handler = logging.StreamHandler() - c_handler.setLevel(level) + # ✅ Prevent adding multiple handlers + if not self.logger.handlers: + c_handler = logging.StreamHandler() + c_handler.setLevel(level) - # Create formatters and add it to handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - c_handler.setFormatter(formatter) - - # Add handlers to the logger - self.logger.addHandler(c_handler) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + c_handler.setFormatter(formatter) + self.logger.addHandler(c_handler) def get_logger(self): return self.logger \ No newline at end of file