mirror of
https://github.com/Hestia-Homes/survey-extraction.git
synced 2026-06-30 13:10:56 +00:00
save current progress
This commit is contained in:
Jun-te Kim
parent
4d455cdda3
commit
58e27915cb
5 changed files with 149 additions and 50 deletions
|
|
@ -7,7 +7,10 @@ import time
|
|||
from pydantic import ValidationError
|
||||
from etl.utils.logger import Logger
|
||||
import logging
|
||||
|
||||
import traceback
|
||||
from hubspot.crm.objects.notes import SimplePublicObjectInput as NoteInput
|
||||
from hubspot.crm.associations import BatchInputPublicAssociation, PublicAssociation
|
||||
import time
|
||||
|
||||
|
||||
class DealStage(Enum):
|
||||
|
|
@ -41,6 +44,7 @@ class HubSpotClient():
|
|||
except Exception as e:
|
||||
return "Unknown Deal" # Fallback if the deal name is not found
|
||||
|
||||
|
||||
def get_listings_from_deals_id(self, deals_id):
|
||||
from hubspot.crm.objects import PublicObjectSearchRequest
|
||||
found_notes = []
|
||||
|
|
@ -205,6 +209,9 @@ class HubSpotClient():
|
|||
for deal in found_deals:
|
||||
domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
|
||||
try:
|
||||
deal_name = deal.properties['dealname']
|
||||
self.logger.info(f"Validating <{deal_name}>")
|
||||
input
|
||||
all_deals.append(SubmissionInfoFromDeal(
|
||||
deal_id= deal.properties["hs_object_id"],
|
||||
deal_name=deal.properties["dealname"],
|
||||
|
|
@ -220,13 +227,46 @@ class HubSpotClient():
|
|||
uprn = uprn,
|
||||
))
|
||||
except Exception as e:
|
||||
def format_error_note(e):
|
||||
note_text = "⚠️ <b>Error occurred while verifying deal data:</b><br><br>"
|
||||
|
||||
if hasattr(e, "errors") and callable(e.errors):
|
||||
note_text += "❌ <b>Validation Errors:</b><br>"
|
||||
for error in e.errors():
|
||||
loc = error.get('loc', 'N/A')
|
||||
msg = error.get('msg', 'N/A')
|
||||
error_type = error.get('type', 'N/A')
|
||||
error_input = error.get('input', 'N/A')
|
||||
|
||||
note_text += (
|
||||
f"• <b>Field:</b> <code>{loc}</code><br>"
|
||||
f" - <b>Message:</b> {msg}<br>"
|
||||
f" - <b>Type:</b> {error_type}<br>"
|
||||
f" - <b>Input:</b> {error_input}<br><br>"
|
||||
)
|
||||
else:
|
||||
note_text += (
|
||||
"❗ <b>Non-validation error:</b><br>"
|
||||
f"<pre>{str(e)}</pre><br>"
|
||||
)
|
||||
|
||||
note_text += (
|
||||
"🛠️ Please review this error and take necessary actions.<br>"
|
||||
"Contact <b>Jun-te</b> if help is needed: <b>+44 7519 530 549</b> or via Teams."
|
||||
)
|
||||
|
||||
return note_text
|
||||
|
||||
deal_id = deal.properties['hs_object_id']
|
||||
self.logger.info(f"Deal <{deal_id}> not valid")
|
||||
if hasattr(e, "errors"):
|
||||
for error in e.errors():
|
||||
self.add_note_to_deal(deal_id, format_error_note(e))
|
||||
else:
|
||||
self.logger.error(f"Non-validation error occurred: {str(e)}", exc_info=True)
|
||||
|
||||
|
||||
self.logger.info(f"Deal name <{deal_name}> moving to 'needs additional information'")
|
||||
self.move_deals_to_different_stage([deal_id], DealStage.NEEDS_ADDITIONAL_INFORMATION_FROM_ASSESSOR.value)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
return all_deals
|
||||
|
||||
|
|
@ -250,3 +290,40 @@ class HubSpotClient():
|
|||
simple_public_object_input=deal_properties
|
||||
)
|
||||
self.logger.info(f"Deal {deal_id} moved to stage with ID {to_stage_id}.")
|
||||
|
||||
|
||||
|
||||
def add_note_to_deal(self, deal_id, note_text):
|
||||
try:
|
||||
# Generate current time in milliseconds since epoch
|
||||
hs_timestamp = int(time.time() * 1000)
|
||||
|
||||
# Step 1: Create the note with hs_timestamp
|
||||
note = NoteInput(
|
||||
properties={
|
||||
"hs_note_body": note_text,
|
||||
"hs_timestamp": hs_timestamp # Required field in your HubSpot setup
|
||||
}
|
||||
)
|
||||
created_note = self.client.crm.objects.notes.basic_api.create(note)
|
||||
note_id = created_note.id
|
||||
|
||||
# Step 2: Associate the note to the deal
|
||||
association = PublicAssociation(
|
||||
_from=note_id,
|
||||
to=deal_id,
|
||||
type="note_to_deal"
|
||||
)
|
||||
|
||||
self.client.crm.associations.batch_api.create(
|
||||
'notes',
|
||||
'deals',
|
||||
batch_input_public_association=BatchInputPublicAssociation(
|
||||
inputs=[association]
|
||||
)
|
||||
)
|
||||
|
||||
self.logger.info(f"📝 Note added to deal {deal_id}: {note_text}")
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error(f"❌ Failed to add note to deal {deal_id}: {e}", exc_info=True)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,14 @@ from sqlmodel import Field, SQLModel
|
|||
from sqlalchemy import Column
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
import uuid
|
||||
from pydantic import Field, field_validator, ValidationError, model_validator
|
||||
from pydantic import Field, field_validator, model_validator
|
||||
from etl.utils.utils import get_sharepoint_path
|
||||
from etl.scraper.scraper import SharePointScraper, SharePointInstaller
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def string_to_installer(installer):
|
||||
if installer.upper() == "J & J CRUMP":
|
||||
return SharePointInstaller.JJC
|
||||
|
|
@ -40,19 +44,24 @@ class SubmissionInfoFromDeal(BaseModel):
|
|||
|
||||
@field_validator('post_sap_score', 'no_of_wet_rooms')
|
||||
@classmethod
|
||||
def must_be_non_negative(cls, v):
|
||||
def must_be_non_negative(cls, v, info):
|
||||
if v < 0:
|
||||
raise ValidationError("Must be non-negative for Post Sap Score")
|
||||
raise ValueError(f"{info.field_name} must be non-negative")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_submission_folder_path(self):
|
||||
path = get_sharepoint_path(self.submission_folder_path)
|
||||
installer = string_to_installer(self.installer)
|
||||
sp = SharePointScraper(installer)
|
||||
files = sp.get_folders_in_path(path)
|
||||
if "value" in files:
|
||||
if len(files["value"]) > 0:
|
||||
errors = []
|
||||
|
||||
try:
|
||||
path = get_sharepoint_path(self.submission_folder_path)
|
||||
installer = string_to_installer(self.installer)
|
||||
sp = SharePointScraper(installer)
|
||||
files = sp.get_folders_in_path(path)
|
||||
|
||||
if "value" in files and len(files["value"]) > 0:
|
||||
return self
|
||||
|
||||
raise RuntimeError("Sharepoint URL invalid")
|
||||
raise ValueError(f"SharePoint folder is empty: {self.submission_folder_path}")
|
||||
|
||||
except Exception as e:
|
||||
raise ValueError(f"Error accessing SharePoint path: {self.submission_folder_path}. Error: {str(e)}")
|
||||
|
|
@ -1,18 +1,15 @@
|
|||
import os
|
||||
import pprint
|
||||
from etl.hubSpotClient.hubspot import DealStage
|
||||
from pprint import pprint
|
||||
|
||||
# Local development, comment this out to be production
|
||||
os.environ["SHAREPOINT_CLIENT_ID"] = "895e3b77-b1d7-43ec-b18f-dcfe07cdfeaf"
|
||||
os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a6E"
|
||||
os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
|
||||
os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
|
||||
os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
|
||||
|
||||
from etl.hubSpotClient.hubspot import DealStage, HubSpotClient
|
||||
# Local development
|
||||
os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
|
||||
|
||||
from etl.db.hubSpotLoad import HubspotTodb
|
||||
|
||||
|
||||
hubspotClient = HubspotTodb()
|
||||
hubspotClient = HubSpotClient()
|
||||
deals = hubspotClient.get_deals_from_deal_stage(DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF)
|
||||
pprint(deals)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ from datetime import datetime, timedelta
|
|||
|
||||
def previous_monday():
|
||||
today = datetime.today()
|
||||
print(f"Todays date is {today}")
|
||||
monday = today - timedelta(days=today.weekday()) # weekday() = 0 for Monday
|
||||
return f"W.C. {monday.strftime('%d.%m.%Y')}"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,23 +1,40 @@
|
|||
from urllib.parse import unquote
|
||||
|
||||
class SharePointURLError(ValueError):
|
||||
"""Custom error for SharePoint URL parsing issues."""
|
||||
pass
|
||||
|
||||
|
||||
def get_sharepoint_path(url):
|
||||
url_parts = url.split('/')
|
||||
# Find the index of 'Forms'
|
||||
forms_index = url_parts.index('Forms')
|
||||
# Get the part after 'Forms'
|
||||
after_forms = url_parts[forms_index + 1]
|
||||
|
||||
# Find 'id=' and extract after it
|
||||
if 'id=' in after_forms:
|
||||
id_part = after_forms.split('id=')[1]
|
||||
# Only keep the path before '&' (to ignore other parameters)
|
||||
id_path = id_part.split('&')[0]
|
||||
# Decode the path
|
||||
decoded_path = unquote(id_path)
|
||||
# Now, remove the leading '/sites/xxx/Shared Documents/' part
|
||||
parts = decoded_path.split('Shared Documents')
|
||||
if len(parts) > 1:
|
||||
final_path = parts[1].strip('/')
|
||||
return final_path
|
||||
try:
|
||||
url_parts = url.split('/')
|
||||
|
||||
# Find the index of 'Forms'
|
||||
forms_index = url_parts.index('Forms')
|
||||
|
||||
# Get the part after 'Forms'
|
||||
after_forms = url_parts[forms_index + 1]
|
||||
|
||||
# Find 'id=' and extract after it
|
||||
if 'id=' in after_forms:
|
||||
id_part = after_forms.split('id=')[1]
|
||||
|
||||
# Only keep the path before '&' (to ignore other parameters)
|
||||
id_path = id_part.split('&')[0]
|
||||
|
||||
# Decode the path
|
||||
decoded_path = unquote(id_path)
|
||||
|
||||
# Remove the leading '/sites/xxx/Shared Documents/' part
|
||||
parts = decoded_path.split('Shared Documents')
|
||||
|
||||
if len(parts) > 1:
|
||||
final_path = parts[1].strip('/')
|
||||
return final_path
|
||||
else:
|
||||
return decoded_path.strip('/')
|
||||
else:
|
||||
return decoded_path.strip('/')
|
||||
raise SharePointURLError(f"The URL does not contain 'id=' parameter. URL: {url}")
|
||||
|
||||
except (IndexError, ValueError) as e:
|
||||
raise SharePointURLError(f"Error parsing SharePoint URL: {url}. Reason: {e}")
|
||||
Loading…
Add table
Reference in a new issue