db migration and hubspot deal@

alembic/env.py

@@ -6,6 +6,7 @@ from alembic import context
 from sqlmodel import SQLModel
 
 from etl.load.preSiteNoteTypes import *
+from etl.load.topLevel import *
 
 import os
 

etl/db/hubSpotLoad.py

@@ -81,7 +81,10 @@ class HubspotTodb():
             data_loc = self.create_files_locally(sp, path, row["HUBSPOT_DEAL_ADDRESS"])
 
             for add, file_loc in data_loc.items():
-                self.data_in_sharepoint.append(surveyedDataProcessor(add, file_loc))
+                sdp = surveyedDataProcessor(add, file_loc)
+                sdp.hubspot_deal_id = row["HUBSPOT_DEAL_ID"]
+                self.data_in_sharepoint.append(sdp)
+
     
     def load_all(self, fast=False):
         if fast is False:
@@ -102,4 +105,14 @@ class HubspotTodb():
             # Creates the a final pre site note table that links all information
             presitenote = surveyedData.create_pre_site_note_table(db_session, assessor, summary_info, property_description)
 
+            df = self.deals_in_hubspot
+            df = df[df["HUBSPOT_DEAL_ID"] == str(surveyedData.hubspot_deal_id)]
+            building_table = surveyedData.create_buildings_table(
+                db_session,
+                df["HUBSPOT_LANDLORD_ID"].values[0],
+                df["HUBSPOT_DOMNA_ID"].values[0],
+            )
+            documents = surveyedData.create_document_table_via_pre_site_note(db_session, presitenote, assessor, building_table)
+            # Create building table or find building table to add new pre_site_note
+
 

etl/hubSpotClient/hubspot.py

@@ -35,7 +35,45 @@ class HubSpotClient():
             return deal.properties.get("dealname", "No deal name")
         except Exception as e:
             return "Unknown Deal"  # Fallback if the deal name is not found
+        
+    def get_listings_from_deals_id(self, deals_id):
+        from hubspot.crm.objects import PublicObjectSearchRequest
+        found_notes = []
+        after = None
+        while True:
+            # Correct filter for notes associated with the given deal ID
+            search_request = PublicObjectSearchRequest(
+                filter_groups=[{
+                    "filters": [{
+                        "propertyName": "associations.deal",  # Filter by association to the deal
+                        "operator": "EQ",
+                        "value": deals_id,
+                    }]
+                }],
+                properties=["domna_property_id", "owner_property_id", 'national_uprn'],  # Properties of the note you need
+                limit=200,
+                after=after,
+            )
+            # Call the search API
+            response = self.client.crm.objects.search_api.do_search(object_type="0-420", public_object_search_request=search_request)
+            time.sleep(1)
+
+            # Add the results to the found_notes list
+            found_notes.extend(response.results)
+
+            # Handle pagination if more results are available
+            if not response.paging or not response.paging.next:
+                break
+            after = response.paging.next.after
+
+        if found_notes:
+            return found_notes[0]
+        return None
     
+    def get_domna_and_landlord_id(self, deals_id):
+        data = self.get_listings_from_deals_id(deals_id)
+        return data.properties['domna_property_id'], data.properties['owner_property_id'], data.properties['national_uprn']
+
     def get_notes_from_deals_id(self, deals_id):
         from hubspot.crm.objects import PublicObjectSearchRequest
         found_notes = []
@@ -115,6 +153,17 @@ class HubSpotClient():
                 "deal_owner": deal.properties.get("hubspot_owner_id"),
             })
         return all_deals
+    
+    def get_associations_for_deal(self, deal_id, to_object_type):
+        """
+        Returns a list of associated object IDs of type `to_object_type`
+        (e.g. "contacts", "companies", "notes", etc.)
+        """
+        assoc_resp = self.client.crm.deals.associations_api.get_all(
+            deal_id=deal_id,
+            to_object_type=to_object_type
+        )
+        return [assoc.id for assoc in assoc_resp.results]
 
     def get_deals_from_deal_stage(self, deal_stage: DealStage):
         found_deals = []
@@ -149,6 +198,7 @@ class HubSpotClient():
 
         all_deals = []
         for deal in found_deals:
+            domna_id, landlord_id, uprn = self.get_domna_and_landlord_id(deal.id)
             all_deals.append(SubmissionInfoFromDeal(
                 deal_id= deal.properties["hs_object_id"],
                 deal_name=deal.properties["dealname"],
@@ -159,7 +209,11 @@ class HubSpotClient():
                 no_of_wet_rooms=int(deal.properties["number_of_wet_rooms_needing_ventilation"]),
                 installer=deal.properties["installer"],
                 submission_folder_path = deal.properties["submission_folder"],
+                landlord_id = landlord_id,
+                domna_id = domna_id,
+                uprn = uprn,
             ))
+
         return all_deals
     
     def print_all_pipeline_ids(self):

etl/hubSpotClient/types.py

@@ -19,4 +19,7 @@ class SubmissionInfoFromDeal(BaseModel):
     existing_wall_insulation: str
     no_of_wet_rooms: int
     installer: str
-    submission_folder_path: str
+    submission_folder_path: str
+    landlord_id: str
+    domna_id: str
+    uprn: str

etl/hubspot_to_invoice.py

@@ -6,9 +6,13 @@ os.environ["SHAREPOINT_CLIENT_SECRET"] = "SOf8Q~-is4wdQiqvEEm9FlJQRAY9ELGaj5Qz-a
 os.environ["SHAREPOINT_TENANT_ID"] = "c3f7519c-2719-4547-af04-6da6cbfd8f8f"
 os.environ["SOUTH_COAST_INSULATION_SERVICE_SHAREPOINT_ID"] = "b5a51507-9427-4ee0-b03e-90ec7681e2d3"
 os.environ["JJC_SERVICE_SHAREPOINT_ID"] = "7fdd0485-bbf3-4b29-b30f-98c81c2a6284"
-os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
+# Local development
+# os.environ["DATABASE_URL"] = "postgresql://postgres:makingwarmhomes@db:5432/postgres"
 
 from etl.surveyPrice.surveyPrice import SurveyPrice
+from etl.db.hubSpotLoad import HubspotTodb
+
+
 
 
 sp = SurveyPrice()
@@ -32,48 +36,10 @@ sp.upload_to_sharepoint(sp.get_master_rate_card_path(), "COPY_OF_RATE_CARD_USED.
 
 deal_ids = df["HUBSPOT_DEAL_ID"].tolist()
 
+
+# Load to db
+dbLoader = HubspotTodb()
+dbLoader.load_all()
+
 # Commented out as i don't want to sync up hubspot_to_db just yet
 sp.move_deals_to_completed(deal_ids)
-
-"""
- TODO: 
- Tuesday 
- P1) - Get read for demo, 3 examples of solar ( JJC AND SCIS), 3 examples of cavity wall ( SCIS and JJC) 12 in total
- P2) Review deem score with last weeks deem score values to ensure accuracy
-
-
- P3) Figure out what to do if I see an address that isn't registered but surveyrod
- P3) Write documentation for tech demos from Khalims demo - Handed off to cyrus
-"""
-
-# Look for
-# JJC
-
-# 3 examples of Solar
-# No solar example in april deem scroe
-# 3 examples Cavity Wall, FOAM, Empty and General ideally
-# (in hubspot )111 Duddell Road General ( fibre) - 500, 2 wet rooms
-# Empty
-# ( in hubspot ) 29  Lower King ( empty ) - 500 - 400
-# Foam
-# ( in hubspot ) 6 STOKESAY STREET (foam) - 400
-
-# SCIS
-# 3 examples of Solar
-# ( in hubspot ) 12 short hedges - Solar 1608
-# ( in hubspot ) 18 short hedge - Solar 1608
-# ( in hubspot) 6 forety road -Solar 1608
-
-# 3 examples Cavity Wall, FOAM, Empty and General ideally
-# ( in hubspot ) 319 Muirfield Road, (Empty Cavity) - 1000
-# (  hubspot ) 2 queensway, (Fibre) - 500
-# ( in hubspot )56 Aughton Crescent -(foam) -  To be worked out by Lewis but lets use this as an oppurtunity -
-
-# Compare value with what I should get and in the deem score. Keep tabs below so I can check easily
-
-# Change w.c. date to a weird one to speed up automation
-
-
-# Observation:
-
-

etl/load/preSiteNoteTypes.py

@@ -5,14 +5,10 @@ from datetime import datetime
 from pydantic import EmailStr
 from sqlalchemy import Column
 from sqlalchemy.dialects.postgresql import UUID
-
-
-class BaseModel(SQLModel):
-    id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
+from etl.load.topLevel import BaseModel, Documents
 
 
 class PreSiteNote(BaseModel, table=True):
-# One class to rule them all
     summary_info_id: uuid.UUID = Field(
         foreign_key="presitenotessummaryinfo.id",
         nullable=False
@@ -295,7 +291,7 @@ class AssessorInfo(BaseModel, table=True):
     company: Optional["CompanyInfo"] = Relationship(back_populates="assessors")
 
     pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="assessor")
-
+    documents: List["Documents"] = Relationship(back_populates="author")
 
 
 class PreSiteNotesSummaryInfo(BaseModel, table=True):
@@ -321,6 +317,7 @@ class PreSiteNotesSummaryInfo(BaseModel, table=True):
     pre_site_notes: List["PreSiteNote"] = Relationship(back_populates="summary_info")
 
 class CompanyInfo(BaseModel, table=True):
+    address: str
     trading_name: str
     post_code: str
     fax_number: Optional[str] = None
@@ -336,3 +333,4 @@ class Insulation(BaseModel, table=True):
 
 
 PreSiteNote.update_forward_refs()
+AssessorInfo.update_forward_refs()

etl/load/topLevel.py

@@ -0,0 +1,37 @@
+from sqlmodel import Field, SQLModel, Relationship
+import uuid
+from typing import Optional, List
+from datetime import datetime
+from pydantic import EmailStr
+from sqlalchemy import Column
+from sqlalchemy.dialects.postgresql import UUID
+from etl.pdfReader.reportType import ReportType
+
+class BaseModel(SQLModel):
+    id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True)
+
+class Buildings(BaseModel, table=True):
+    address: str
+    postcode: str
+    UPRN: str
+    landlord_id: str
+    domna_id: str
+
+    documents: List["Documents"] = Relationship(back_populates="building")
+
+class Documents(BaseModel, table=True):
+    assessor_id: uuid.UUID = Field(
+        foreign_key="assessorinfo.id",
+        nullable=False
+    )
+    author: Optional["AssessorInfo"] = Relationship(back_populates="documents")
+    created_at: datetime
+    document_type: ReportType
+
+    building_id: uuid.UUID = Field(foreign_key="buildings.id", nullable=False)
+    building: Optional["Buildings"] = Relationship(back_populates="documents")
+
+    target_table: str
+    target_id: uuid.UUID
+
+Documents.update_forward_refs()

etl/osmosis_google_maps_.py

@@ -5,7 +5,7 @@ import time
 from tqdm import tqdm
 
 
-board_id = "8829428746"
+board_id = "3584401309"
 monday_key = "eyJhbGciOiJIUzI1NiJ9.eyJ0aWQiOjQ5ODc2ODQxOCwiYWFpIjoxMSwidWlkIjozNjE3ODAzNCwiaWFkIjoiMjAyNS0wNC0xMVQxMToyMzoxNy40NjdaIiwicGVyIjoibWU6d3JpdGUiLCJhY3RpZCI6MTM5OTc4MjMsInJnbiI6InVzZTEifQ.-2Lit4s46ZF6AXuMW9t0TxIaFLkHqD4Yo-PyM9i2XZY"
 monday = MondayClient(monday_key)
 

etl/pdfReader/reportType.py

@@ -2,8 +2,8 @@ from enum import Enum
 
 
 class ReportType(Enum):
-    QUIDOS_PRESITE_NOTE = 1
-    CHARTED_SURVEYOR_REPORT = 2
-    ENERGY_PERFORMANCE_REPORT = 3
-    U_VALUE_CALCULATOR_REPORT = 4
-    OVERWRITING_U_VALUE_DECLARATION_FORM = 5
+    QUIDOS_PRESITE_NOTE = "quidos_presite_note"
+    CHARTED_SURVEYOR_REPORT = "charted_surveyor_report"
+    ENERGY_PERFORMANCE_REPORT = "energy_performance_report"
+    U_VALUE_CALCULATOR_REPORT = "u_value_calculator_report"
+    OVERWRITING_U_VALUE_DECLARATION_FORM = "overwriting_u_value_declaration_form"

etl/pdfReader/sitenotes.py

@@ -145,6 +145,7 @@ class QuidosSiteNotesExtractor(SiteNotesExtractor):
 
 
         self.company_information = CompanyInfo(
+            address=self.raw_data[self.get_x_occurance(self.raw_data,'Address', 1) + 1],
             trading_name = get_value('Company name/trading name'),
             post_code = get_value('POST CODE'),
             fax_number = get_value('Fax number'),

etl/surveyPrice/surveyPrice.py

@@ -149,6 +149,9 @@ class SurveyPrice():
                 "HUBSPOT_INSTALLER": deal.installer,
                 "HUBSPOT_WETROOMS": deal.no_of_wet_rooms,
                 "HUBSPOT_SHAREPOINT_PATH": deal.submission_folder_path,
+                "HUBSPOT_LANDLORD_ID": deal.landlord_id,
+                "HUBSPOT_DOMNA_ID": deal.domna_id,
+                "HUBSPOT_UPRN": deal.uprn,
             })
 
         self.all_hubspot_submissions = pd.DataFrame(all_deals)

etl/surveyedData/surveryedData.py

@@ -10,6 +10,9 @@ from etl.load.preSiteNoteTypes import (
     SolarWaterHeating, HotWaterCylinder, WaterHeating, Lighting, VentilationAndCooling,
     Door, Walls, Roofs, Floors, PropertyDetail, Windows
 )
+from etl.load.topLevel import(
+    Buildings, Documents
+)
 import uuid
 
 class surveyedDataProcessor():
@@ -19,6 +22,7 @@ class surveyedDataProcessor():
         self.pre_site_note = None
         self.csr = None
         self.identify_files()
+        self.hubspot_deal_id = None
 
 
     def identify_files(self):
@@ -40,6 +44,20 @@ class surveyedDataProcessor():
             lookup_field="reference_number"
         )
     
+    def create_building_table(self, db_session):
+        return self.upsert_record(
+            db_session=db_session,
+            model_class=Buildings,
+            data_dict={
+                "address":"foo",
+                "potcode": "foobar",
+                "UPRN": self.pre_site_note.survey_information.uprn,
+                "landlord_id": "landlord_id",
+                "domna_id": "landlord_id",
+            },
+            lookup_field="UPRN",
+        )
+    
     def get_attribute_and_load(self, obj, attr_string, pydanticModel, db_session):
         found = getattr(obj, attr_string, None)
         if found:
@@ -299,7 +317,6 @@ class surveyedDataProcessor():
                 "ex4_property_id": ex4_property.id if ex4_property else None,
             }
         )
-        print(f"property_description {property_description}")
         return property_description
             
 
@@ -313,6 +330,46 @@ class surveyedDataProcessor():
             lookup_field="trading_name"
         )
     
+
+    def create_document_table_via_pre_site_note(self, db_session, pre_site_note, assessor, building):
+        data = {
+            "assessor_id": assessor.id,
+            "created_at": self.pre_site_note.survey_information.inspection_date,
+            "document_type": ReportType.QUIDOS_PRESITE_NOTE,
+            "building_id": building.id,
+            "target_table": "pre_site_note",
+            "target_id": pre_site_note.id
+        }
+        return self.upsert_record(
+            db_session=db_session,
+            model_class=Documents,
+            data_dict=data,
+            lookup_field=None, 
+        )
+
+    def create_buildings_table(
+        self,
+        db_session,
+        landlord_id,
+        domna_id,
+    ):
+        
+        data = {
+            "address": self.pre_site_note.survey_information.address,
+            "postcode": self.pre_site_note.survey_information.postcode,
+            "UPRN": self.pre_site_note.survey_information.uprn,
+            "landlord_id": landlord_id,
+            "domna_id": domna_id
+        }
+        building = self.upsert_record(
+            db_session=db_session,
+            model_class=Buildings,
+            data_dict=data,
+            lookup_field="UPRN",
+        )
+
+        return building
+
     
     def create_pre_site_note_table(
         self, 

etl/transform/preSiteNoteTypes.py

@@ -1,4 +1,4 @@
-from sqlmodel import Field, SQLModel
+from sqlmodel import Field, SQLModel, Relationship
 import uuid
 from datetime import datetime
 from pydantic import field_validator, EmailStr
@@ -16,6 +16,7 @@ class Dimension(BaseModel):
     party_wall_length_m: float
 
 class CompanyInfo(BaseModel):
+    address: str
     trading_name: str
     post_code: str
     fax_number: Optional[str] = None
@@ -82,6 +83,7 @@ class AssessorInfo(BaseModel):
     phone_number: Optional[str] = None
     email_address: Optional[EmailStr] = None
 
+
 class VentilationAndCooling(BaseModel):
     no_of_open_fireplaces: int
     ventilation_type: str

migration_db.sh

@@ -1,3 +1,4 @@
-#poetry run alembic revision --autogenerate -m "mistake on foreign key"
+#poetry run alembic revision --autogenerate -m "Add address in company"
+
 
 poetry run alembic upgrade head

pyproject.toml

@@ -19,10 +19,10 @@ dependencies = [
     "pydantic-settings (>=2.8.1,<3.0.0)",
     "alembic (>=1.15.1,<2.0.0)",
     "pytest (>=8.3.5,<9.0.0)",
-    "hubspot-api-client (>=11.1.0,<12.0.0)",
     "monday (>=2.0.1,<3.0.0)",
     "beautifulsoup4 (>=4.13.4,<5.0.0)",
     "tqdm (>=4.67.1,<5.0.0)",
+    "hubspot-api-client (>=12.0.0,<13.0.0)",
 ]
 
 [tool.poetry]