diff --git a/etl/hubSpotClient/hubspot.py b/etl/hubSpotClient/hubspot.py index a05c309..c2d6af6 100644 --- a/etl/hubSpotClient/hubspot.py +++ b/etl/hubSpotClient/hubspot.py @@ -7,7 +7,10 @@ import time from pydantic import ValidationError from etl.utils.logger import Logger import logging - +import traceback +from hubspot.crm.objects.notes import SimplePublicObjectInput as NoteInput +from hubspot.crm.associations import BatchInputPublicAssociation, PublicAssociation +import time class DealStage(Enum): @@ -41,6 +44,7 @@ class HubSpotClient(): except Exception as e: return "Unknown Deal" # Fallback if the deal name is not found + def get_listings_from_deals_id(self, deals_id): from hubspot.crm.objects import PublicObjectSearchRequest found_notes = [] @@ -205,6 +209,9 @@ class HubSpotClient(): for deal in found_deals: domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id) try: + deal_name = deal.properties['dealname'] + self.logger.info(f"Validating <{deal_name}>") + input all_deals.append(SubmissionInfoFromDeal( deal_id= deal.properties["hs_object_id"], deal_name=deal.properties["dealname"], @@ -220,13 +227,46 @@ class HubSpotClient(): uprn = uprn, )) except Exception as e: + def format_error_note(e): + note_text = "⚠️ Error occurred while verifying deal data:

" + + if hasattr(e, "errors") and callable(e.errors): + note_text += "❌ Validation Errors:
" + for error in e.errors(): + loc = error.get('loc', 'N/A') + msg = error.get('msg', 'N/A') + error_type = error.get('type', 'N/A') + error_input = error.get('input', 'N/A') + + note_text += ( + f"• Field: {loc}
" + f"  - Message: {msg}
" + f"  - Type: {error_type}
" + f"  - Input: {error_input}

" + ) + else: + note_text += ( + "❗ Non-validation error:
" + f"
{str(e)}

" + ) + + note_text += ( + "🛠️ Please review this error and take necessary actions.
" + "Contact Jun-te if help is needed: +44 7519 530 549 or via Teams." + ) + + return note_text + deal_id = deal.properties['hs_object_id'] - self.logger.info(f"Deal <{deal_id}> not valid") + if hasattr(e, "errors"): + for error in e.errors(): + self.add_note_to_deal(deal_id, format_error_note(e)) + else: + self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True) + + + self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'") self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value) - - - - return all_deals @@ -250,3 +290,40 @@ class HubSpotClient(): simple_public_object_input=deal_properties ) self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.") + + + + def add_note_to_deal(self, deal_id, note_text): + try: + # Generate current time in milliseconds since epoch + hs_timestamp = int(time.time() * 1000) + + # Step 1: Create the note with hs_timestamp + note = NoteInput( + properties={ + "hs_note_body": note_text, + "hs_timestamp": hs_timestamp # Required field in your HubSpot setup + } + ) + created_note = self.client.crm.objects.notes.basic_api.create(note) + note_id = created_note.id + + # Step 2: Associate the note to the deal + association = PublicAssociation( + _from=note_id, + to=deal_id, + type="note_to_deal" + ) + + self.client.crm.associations.batch_api.create( + 'notes', + 'deals', + batch_input_public_association=BatchInputPublicAssociation( + inputs=[association] + ) + ) + + self.logger.info(f"📝 Note added to deal {deal_id}: {note_text}") + + except Exception as e: + self.logger.error(f"❌ Failed to add note to deal {deal_id}: {e}", exc_info=True) diff --git a/etl/hubSpotClient/types.py b/etl/hubSpotClient/types.py index 22ff6ff..e77acde 100644 --- a/etl/hubSpotClient/types.py +++ b/etl/hubSpotClient/types.py @@ -2,10 +2,14 @@ from sqlmodel import Field, SQLModel from sqlalchemy import Column from sqlalchemy.dialects.postgresql import UUID import uuid -from pydantic import Field, field_validator, ValidationError, model_validator +from pydantic import Field, field_validator, model_validator from etl.utils.utils import get_sharepoint_path from etl.scraper.scraper import SharePointScraper, SharePointInstaller + + + + def string_to_installer(installer): if installer.upper() == "J & J CRUMP": return SharePointInstaller.JJC @@ -40,19 +44,24 @@ class SubmissionInfoFromDeal(BaseModel): @field_validator('post_sap_score', 'no_of_wet_rooms') @classmethod - def must_be_non_negative(cls, v): + def must_be_non_negative(cls, v, info): if v < 0: - raise ValidationError("Must be non-negative for Post Sap Score") + raise ValueError(f"{info.field_name} must be non-negative") return v @model_validator(mode="after") def check_submission_folder_path(self): - path = get_sharepoint_path(self.submission_folder_path) - installer = string_to_installer(self.installer) - sp = SharePointScraper(installer) - files = sp.get_folders_in_path(path) - if "value" in files: - if len(files["value"]) > 0: + errors = [] + + try: + path = get_sharepoint_path(self.submission_folder_path) + installer = string_to_installer(self.installer) + sp = SharePointScraper(installer) + files = sp.get_folders_in_path(path) + + if "value" in files and len(files["value"]) > 0: return self - - raise RuntimeError("Sharepoint URL invalid") + raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}") + + except Exception as e: + raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}") \ No newline at end of file diff --git a/etl/hubspot_verification_to_db_load.py b/etl/hubspot_verification_to_db_load.py index bb42922..5553196 100644 --- a/etl/hubspot_verification_to_db_load.py +++ b/etl/hubspot_verification_to_db_load.py @@ -1,18 +1,15 @@ import os -import pprint -from etl.hubSpotClient.hubspot import DealStage +from pprint import pprint -# Local development, comment this out to be production +os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf" +os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E" +os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f" +os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3" +os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284" + +from etl.hubSpotClient.hubspot import DealStage, HubSpotClient +# Local development os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres" -from etl.db.hubSpotLoad import HubspotTodb - - -hubspotClient = HubspotTodb() +hubspotClient = HubSpotClient() deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF) -pprint(deals) - - - - - diff --git a/etl/scraper/scraper.py b/etl/scraper/scraper.py index 6cbfd37..550e05d 100644 --- a/etl/scraper/scraper.py +++ b/etl/scraper/scraper.py @@ -13,7 +13,6 @@ from datetime import datetime, timedelta def previous_monday(): today = datetime.today() - print(f"Todays date is {today}") monday = today - timedelta(days=today.weekday()) # weekday() = 0 for Monday return f"W.C. {monday.strftime('%d.%m.%Y')}" diff --git a/etl/utils/utils.py b/etl/utils/utils.py index 286b645..584381a 100644 --- a/etl/utils/utils.py +++ b/etl/utils/utils.py @@ -1,23 +1,40 @@ from urllib.parse import unquote +class SharePointURLError(ValueError): + """Custom error for SharePoint URL parsing issues.""" + pass + + def get_sharepoint_path(url): - url_parts = url.split('/') - # Find the index of 'Forms' - forms_index = url_parts.index('Forms') - # Get the part after 'Forms' - after_forms = url_parts[forms_index + 1] - - # Find 'id=' and extract after it - if 'id=' in after_forms: - id_part = after_forms.split('id=')[1] - # Only keep the path before '&' (to ignore other parameters) - id_path = id_part.split('&')[0] - # Decode the path - decoded_path = unquote(id_path) - # Now, remove the leading '/sites/xxx/Shared Documents/' part - parts = decoded_path.split('Shared Documents') - if len(parts) > 1: - final_path = parts[1].strip('/') - return final_path + try: + url_parts = url.split('/') + + # Find the index of 'Forms' + forms_index = url_parts.index('Forms') + + # Get the part after 'Forms' + after_forms = url_parts[forms_index + 1] + + # Find 'id=' and extract after it + if 'id=' in after_forms: + id_part = after_forms.split('id=')[1] + + # Only keep the path before '&' (to ignore other parameters) + id_path = id_part.split('&')[0] + + # Decode the path + decoded_path = unquote(id_path) + + # Remove the leading '/sites/xxx/Shared Documents/' part + parts = decoded_path.split('Shared Documents') + + if len(parts) > 1: + final_path = parts[1].strip('/') + return final_path + else: + return decoded_path.strip('/') else: - return decoded_path.strip('/') \ No newline at end of file + raise SharePointURLError(f"The URL does not contain 'id=' parameter. URL: {url}") + + except (IndexError, ValueError) as e: + raise SharePointURLError(f"Error parsing SharePoint URL: {url}. Reason: {e}") \ No newline at end of file