diff --git a/backend/app/db/models/hubspot_deal_data.py b/backend/app/db/models/hubspot_deal_data.py new file mode 100644 index 00000000..d5a51ace --- /dev/null +++ b/backend/app/db/models/hubspot_deal_data.py @@ -0,0 +1,77 @@ +import uuid +from sqlmodel import SQLModel, Field, Column, text +from datetime import datetime +from typing import Optional +from sqlalchemy import DateTime +from sqlalchemy.sql import func + + +class HubspotDealData(SQLModel, table=True): + __tablename__ = "hubspot_deal_data" + + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + + # HubSpot Deal identifiers + deal_id: str = Field(index=True, nullable=False) + dealname: Optional[str] = Field(default=None) + dealstage: Optional[str] = Field(default=None) + company_id: Optional[str] = Field(default=None) + project_code: Optional[str] = Field(default=None) + + # HubSpot custom properties + landlord_property_id: Optional[str] = Field(default=None) + uprn: Optional[str] = Field(default=None) + outcome: Optional[str] = Field(default=None) + outcome_notes: Optional[str] = Field(default=None) + + major_condition_issue_description: Optional[str] = Field(default=None) + major_condition_issue_photos: Optional[str] = Field(default=None) + major_condition_issue_evidence_s3_url: Optional[str] = Field(default=None) + + coordination_status: Optional[str] = Field(default=None) + coordination_comments: Optional[str] = Field(default=None) + design_status: Optional[str] = Field(default=None) + + listing_id: Optional[str] = Field(default=None) + pashub_link: Optional[str] = Field(default=None) + sharepoint_link: Optional[str] = Field(default=None) + dampmould_growth: Optional[str] = Field(default=None) + damp_mould_and_repairs_comments: Optional[str] = Field(default=None) + pre_sap: Optional[str] = Field(default=None) + coordinator: Optional[str] = Field(default=None) + mtp_completion_date: Optional[datetime] = Field(default=None) + mtp_re_model_completion_date: Optional[datetime] = Field(default=None) + ioe_v3_completion_date: Optional[datetime] = Field(default=None) + proposed_measures: Optional[str] = Field(default=None) + approved_package: Optional[str] = Field(default=None) + designer: Optional[str] = Field(default=None) + design_completion_date: Optional[datetime] = Field(default=None) + actual_measures_installed: Optional[str] = Field(default=None) + installer: Optional[str] = Field(default=None) + installer_handover: Optional[str] = Field(default=None) + lodgement_status: Optional[str] = Field(default=None) + measures_lodgement_date: Optional[datetime] = Field(default=None) + lodgement_date: Optional[datetime] = Field(default=None) + expected_commencement_date: Optional[datetime] = Field(default=None) + surveyor: Optional[str] = Field(default=None) + confirmed_survey_date: Optional[datetime] = Field(default=None) + confirmed_survey_time: Optional[str] = Field(default=None) + surveyed_date: Optional[datetime] = Field(default=None) + design_type: Optional[str] = Field(default=None) + + created_at: datetime = Field( + sa_column=Column( + DateTime(timezone=True), + server_default=text("(NOW() AT TIME ZONE 'utc')"), + nullable=False, + ) + ) + + updated_at: datetime = Field( + sa_column=Column( + DateTime(timezone=True), + server_default=text("(NOW() AT TIME ZONE 'utc')"), + onupdate=func.now(), + nullable=False, + ) + ) diff --git a/backend/app/db/models/organisation.py b/backend/app/db/models/organisation.py index 784cc4ad..8afc5d63 100644 --- a/backend/app/db/models/organisation.py +++ b/backend/app/db/models/organisation.py @@ -1,9 +1,7 @@ -from sqlmodel import SQLModel, Field, Column, text +import uuid +from sqlmodel import SQLModel, Field from datetime import datetime, timezone from typing import Optional -from sqlalchemy import DateTime -from sqlalchemy.sql import func -import uuid class Organisation(SQLModel, table=True): @@ -13,74 +11,3 @@ class Organisation(SQLModel, table=True): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) hubspot_company_id: Optional[str] = None name: Optional[str] = None - - -class HubspotDealData(SQLModel, table=True): - __tablename__ = "hubspot_deal_data" - - id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - - # HubSpot Deal identifiers - deal_id: str = Field(index=True, nullable=False) - dealname: Optional[str] = Field(default=None) - dealstage: Optional[str] = Field(default=None) - company_id: Optional[str] = Field(default=None) - project_code: Optional[str] = Field(default=None) - - # HubSpot custom properties - landlord_property_id: Optional[str] = Field(default=None) - uprn: Optional[str] = Field(default=None) - outcome: Optional[str] = Field(default=None) - outcome_notes: Optional[str] = Field(default=None) - - major_condition_issue_description: Optional[str] = Field(default=None) - major_condition_issue_photos: Optional[str] = Field(default=None) - major_condition_issue_evidence_s3_url: Optional[str] = Field(default=None) - - coordination_status: Optional[str] = Field(default=None) - coordination_comments: Optional[str] = Field(default=None) - design_status: Optional[str] = Field(default=None) - - listing_id: Optional[str] = Field(default=None) - pashub_link: Optional[str] = Field(default=None) - sharepoint_link: Optional[str] = Field(default=None) - dampmould_growth: Optional[str] = Field(default=None) - damp_mould_and_repairs_comments: Optional[str] = Field(default=None) - pre_sap: Optional[str] = Field(default=None) - coordinator: Optional[str] = Field(default=None) - mtp_completion_date: Optional[datetime] = Field(default=None) - mtp_re_model_completion_date: Optional[datetime] = Field(default=None) - ioe_v3_completion_date: Optional[datetime] = Field(default=None) - proposed_measures: Optional[str] = Field(default=None) - approved_package: Optional[str] = Field(default=None) - designer: Optional[str] = Field(default=None) - design_completion_date: Optional[datetime] = Field(default=None) - actual_measures_installed: Optional[str] = Field(default=None) - installer: Optional[str] = Field(default=None) - installer_handover: Optional[str] = Field(default=None) - lodgement_status: Optional[str] = Field(default=None) - measures_lodgement_date: Optional[datetime] = Field(default=None) - lodgement_date: Optional[datetime] = Field(default=None) - expected_commencement_date: Optional[datetime] = Field(default=None) - surveyor: Optional[str] = Field(default=None) - confirmed_survey_date: Optional[datetime] = Field(default=None) - confirmed_survey_time: Optional[str] = Field(default=None) - surveyed_date: Optional[datetime] = Field(default=None) - design_type: Optional[str] = Field(default=None) - - created_at: datetime = Field( - sa_column=Column( - DateTime(timezone=True), - server_default=text("(NOW() AT TIME ZONE 'utc')"), - nullable=False, - ) - ) - - updated_at: datetime = Field( - sa_column=Column( - DateTime(timezone=True), - server_default=text("(NOW() AT TIME ZONE 'utc')"), - onupdate=func.now(), - nullable=False, - ) - ) diff --git a/etl/hubspot/hubspotDataTodB.py b/etl/hubspot/hubspotDataTodB.py index 3c017f0e..5ebc8c73 100644 --- a/etl/hubspot/hubspotDataTodB.py +++ b/etl/hubspot/hubspotDataTodB.py @@ -1,13 +1,15 @@ -from backend.app.db.connection import db_read_session -from backend.app.db.models.organisation import Organisation, HubspotDealData +import hashlib +import os from sqlmodel import select from datetime import datetime, timezone from typing import Dict, Optional + +from backend.app.db.models.hubspot_deal_data import HubspotDealData from etl.hubspot.company_data import CompanyData from etl.hubspot.hubspotClient import HubspotClient from etl.hubspot.s3_uploader import S3Uploader -import hashlib -import os +from backend.app.db.connection import db_read_session +from backend.app.db.models.organisation import Organisation class HubspotDataToDb: diff --git a/etl/hubspot/hubspot_deal_differ.py b/etl/hubspot/hubspot_deal_differ.py index 1dd4ed51..dd992243 100644 --- a/etl/hubspot/hubspot_deal_differ.py +++ b/etl/hubspot/hubspot_deal_differ.py @@ -1,6 +1,6 @@ from typing import Dict, List, Optional -from backend.app.db.models.organisation import HubspotDealData +from backend.app.db.models.hubspot_deal_data import HubspotDealData class HubspotDealDiffer: diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index 768a86eb..826d7e05 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -1,14 +1,13 @@ -from backend.app.db.models.organisation import HubspotDealData -from etl.hubspot.hubspotClient import HubspotClient - -from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb -from backend.utils.subtasks import task_handler from typing import Any, Dict, Optional +from etl.hubspot.hubspotClient import HubspotClient +from etl.hubspot.hubspotDataTodB import CompanyData, HubspotDataToDb from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer from etl.hubspot.hubspot_trigger_orchestrator_trigger_request import ( HubspotTriggerOrchestratorTriggerRequest, ) +from backend.utils.subtasks import task_handler +from backend.app.db.models.hubspot_deal_data import HubspotDealData @task_handler() diff --git a/etl/hubspot/tests/test_hubspot_deal_differ.py b/etl/hubspot/tests/test_hubspot_deal_differ.py index 876fcab9..74d3f057 100644 --- a/etl/hubspot/tests/test_hubspot_deal_differ.py +++ b/etl/hubspot/tests/test_hubspot_deal_differ.py @@ -4,7 +4,7 @@ import uuid import pytest -from backend.app.db.models.organisation import HubspotDealData +from backend.app.db.models.hubspot_deal_data import HubspotDealData from etl.hubspot.hubspot_deal_differ import HubspotDealDiffer