save working changes

etl/db/hubSpotLoad.py

@@ -13,10 +13,10 @@ class HubspotTodb():
         self.hubspot = HubSpotClient()
         self.deals_in_hubspot = None
         self.data_in_sharepoint = []
+        self.sp = SurveyPrice()
 
     def get_all_deals(self):
-        sp = SurveyPrice()
-        self.deals_in_hubspot = sp.get_all_surveys_from_hubspot()
+        self.deals_in_hubspot = self.sp.get_all_surveys_from_hubspot()
         return self.deals_in_hubspot
     
     def get_sharepoint_path(self, url):

etl/hubSpotClient/hubspot.py

@@ -4,6 +4,9 @@ from hubspot.crm.deals import PublicObjectSearchRequest
 from hubspot.crm.deals.models import SimplePublicObjectInput
 from etl.hubSpotClient.types import SubmissionInfoFromDeal
 import time
+from pydantic import ValidationError
+from etl.utils.logger import Logger
+import logging
 
 
 
@@ -12,11 +15,13 @@ class DealStage(Enum):
     SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
     CUSTOMER_CONTACTED = "888730834"
     SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
+    NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR = "1887736000"
 
 class HubSpotClient():
     def __init__(self):
         self.access_token = "pat-eu1-064f7f5c-a7d8-4d93-a9b2-b604da6164a6"
         self.client = hubspot.Client.create(access_token=self.access_token)
+        self.logger = Logger(name='HubSpotClient', level=logging.INFO).get_logger()
 
     def get_all_deals(self):
         return self.client.crm.deals.get_all()
@@ -199,20 +204,29 @@ class HubSpotClient():
         all_deals = []
         for deal in found_deals:
             domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
-            all_deals.append(SubmissionInfoFromDeal(
-                deal_id= deal.properties["hs_object_id"],
-                deal_name=deal.properties["dealname"],
-                work_type=deal.properties["work_type"],
-                needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False,
-                post_sap_score=int(deal.properties["domna_survey_post_sap"]),
-                existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
-                no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
-                installer=deal.properties["installer"],
-                submission_folder_path = deal.properties["submission_folder"],
-                landlord_id = landlord_id,
-                domna_id = domna_id,
-                uprn = uprn,
-            ))
+            try:
+                all_deals.append(SubmissionInfoFromDeal(
+                    deal_id= deal.properties["hs_object_id"],
+                    deal_name=deal.properties["dealname"],
+                    work_type=deal.properties["work_type"],
+                    needs_trickle_ventilation=True if deal.properties.get("property_needs_trickle_vents", "NO").upper() == "YES" else False,
+                    post_sap_score=int(deal.properties["domna_survey_post_sap"]),
+                    existing_wall_insulation=deal.properties.get("existing_wall_insulation") if deal.properties.get("existing_wall_insulation") else "None",
+                    no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
+                    installer=deal.properties["installer"],
+                    submission_folder_path = deal.properties["submission_folder"],
+                    landlord_id = landlord_id,
+                    domna_id = domna_id,
+                    uprn = uprn,
+                ))
+            except Exception as e:
+                deal_id = deal.properties['hs_object_id']
+                self.logger.info(f"Deal <{deal_id}> not valid")
+                self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)               
+                
+
+
+
 
         return all_deals
     
@@ -235,4 +249,4 @@ class HubSpotClient():
                 deal_id,
                 simple_public_object_input=deal_properties
             )
-            print(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")
+            self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")

etl/hubSpotClient/types.py

@@ -2,6 +2,7 @@ from sqlmodel import Field, SQLModel
 from sqlalchemy import Column
 from sqlalchemy.dialects.postgresql import UUID
 import uuid
+from pydantic import  Field, field_validator, ValidationError
 
 class BaseModel(SQLModel):
     id: uuid.UUID = Field(
@@ -11,15 +12,22 @@ class BaseModel(SQLModel):
 
 
 class SubmissionInfoFromDeal(BaseModel):
-    deal_id: str
-    deal_name: str
-    work_type: str
+    deal_id: str = Field(..., min_length=1)
+    deal_name: str = Field(..., min_length=1)
+    work_type: str = Field(..., min_length=1)
     needs_trickle_ventilation: bool
     post_sap_score: int
-    existing_wall_insulation: str
+    existing_wall_insulation: str = Field(..., min_length=1)
     no_of_wet_rooms: int
-    installer: str
-    submission_folder_path: str
-    landlord_id: str
-    domna_id: str
-    uprn: str
+    installer: str = Field(..., min_length=1)
+    submission_folder_path: str = Field(..., min_length=1)
+    landlord_id: str = Field(..., min_length=1)
+    domna_id: str = Field(..., min_length=1)
+    uprn: str = Field(..., min_length=1)
+
+    @field_validator('post_sap_score', 'no_of_wet_rooms')
+    @classmethod
+    def must_be_non_negative(cls, v):
+        if v < 0:
+            raise ValidationError("Must be non-negative for Post Sap Score")
+        return v

etl/hubspot_to_invoice_rewrite.py

@@ -0,0 +1,32 @@
+import os
+
+
+os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
+os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
+os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
+os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
+os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
+# Local development
+os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
+
+from etl.surveyPrice.surveyPrice import SurveyPrice
+from etl.db.hubSpotLoad import HubspotTodb
+
+
+
+# Load to db
+dbLoader = HubspotTodb()
+
+df = dbLoader.get_all_deals()
+
+
+
+
+# For each deal
+# if not Validate:
+#     move to a different stage, with option to add a note to this to state what the error was
+# work out price and add to grand list
+# load to db and move deal correctly to correct position
+# 
+# Once grand list is finsihed for price
+# uploaad to sharepoint

etl/utils/logger.py

@@ -3,21 +3,18 @@ import os
 
 class Logger:
     def __init__(self, name, level=logging.INFO):
-        # Create a custom logger
         self.logger = logging.getLogger(name)
         self.logger.setLevel(level)
 
-        # Create handlers
-        c_handler = logging.StreamHandler()
-        c_handler.setLevel(level)
+        # ✅ Prevent adding multiple handlers
+        if not self.logger.handlers:
+            c_handler = logging.StreamHandler()
+            c_handler.setLevel(level)
 
-        # Create formatters and add it to handlers
-        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-        c_handler.setFormatter(formatter)
-
-        # Add handlers to the logger
-        self.logger.addHandler(c_handler)
+            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+            c_handler.setFormatter(formatter)
 
+            self.logger.addHandler(c_handler)
 
     def get_logger(self):
         return self.logger