From e9db66b6b423f455697193af36922a3ace130da9 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Thu, 12 Mar 2026 11:58:46 +0000
Subject: [PATCH 01/14] added hubspot dependency to backend

---
 .devcontainer/backend/Dockerfile | 3 ++-
 etl/hubspot/hubspotClient.py     | 5 +++++
 etl/hubspot/requirements.txt     | 1 +
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 etl/hubspot/hubspotClient.py
 create mode 100644 etl/hubspot/requirements.txt

diff --git a/.devcontainer/backend/Dockerfile b/.devcontainer/backend/Dockerfile
index 662f53b0..6a1cc120 100644
--- a/.devcontainer/backend/Dockerfile
+++ b/.devcontainer/backend/Dockerfile
@@ -35,7 +35,8 @@ ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
 ADD backend/engine/requirements.txt requirements1.txt
 ADD backend/app/requirements/requirements.txt requirements2.txt
 ADD .devcontainer/backend/requirements.txt requirements3.txt
-RUN cat requirements1.txt requirements2.txt requirements3.txt > requirements.txt
+ADD etl/hubspot/requirements.txt requirements4.txt
+RUN cat requirements1.txt requirements2.txt requirements3.txt requirements4.txt > requirements.txt
 RUN pip install -r requirements.txt
 
 # 5) Workdir
diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
new file mode 100644
index 00000000..39cea6a1
--- /dev/null
+++ b/etl/hubspot/hubspotClient.py
@@ -0,0 +1,5 @@
+import hubspot
+
+class HubspotClient():
+
+    def
\ No newline at end of file
diff --git a/etl/hubspot/requirements.txt b/etl/hubspot/requirements.txt
new file mode 100644
index 00000000..105cba07
--- /dev/null
+++ b/etl/hubspot/requirements.txt
@@ -0,0 +1 @@
+hubspot
\ No newline at end of file

From 76dbde602b1ff2d5cb29d4a946411283b951b7e2 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 13:27:52 +0000
Subject: [PATCH 02/14] added tests and hubspot client

---
 .devcontainer/backend/requirements.txt        |   2 +-
 backend/app/config.py                         |   2 +
 conftest.py                                   |   1 +
 etl/hubspot/hubspotClient.py                  | 442 +++++++++++++++++-
 etl/hubspot/requirements.txt                  |   2 +-
 etl/hubspot/tests/__init__.py                 |   0
 .../tests/test_hubspot_client_integration.py  | 117 +++++
 pyrightconfig.json                            |   2 +-
 pytest.ini                                    |   2 +-
 9 files changed, 563 insertions(+), 7 deletions(-)
 create mode 100644 etl/hubspot/tests/__init__.py
 create mode 100644 etl/hubspot/tests/test_hubspot_client_integration.py

diff --git a/.devcontainer/backend/requirements.txt b/.devcontainer/backend/requirements.txt
index 5cd40ced..f6e1f665 100644
--- a/.devcontainer/backend/requirements.txt
+++ b/.devcontainer/backend/requirements.txt
@@ -23,4 +23,4 @@ psycopg[binary]
 pytest-postgresql
 # Formatting
 black==26.1.0
-boto3-stubs
\ No newline at end of file
+boto3-stubs
diff --git a/backend/app/config.py b/backend/app/config.py
index 6604fec9..46301e30 100644
--- a/backend/app/config.py
+++ b/backend/app/config.py
@@ -65,6 +65,8 @@ class Settings(BaseSettings):
 
     ORDNANCE_SURVEY_API_KEY: str = "changeme"
 
+    HUBSPOT_API_KEY: Optional[str] = None
+
     # Optional AWS creds (only required in local)
     AWS_ACCESS_KEY_ID: Optional[str] = None
     AWS_SECRET_KEY_ID: Optional[str] = None
diff --git a/conftest.py b/conftest.py
index d93f0023..2ea20ebb 100644
--- a/conftest.py
+++ b/conftest.py
@@ -30,6 +30,7 @@ DEFAULT_ENV = {
     "HEATING_KWH_PREDICTIONS_BUCKET": "test",
     "HOTWATER_KWH_PREDICTIONS_BUCKET": "test",
     "ENERGY_ASSESSMENTS_BUCKET": "test",
+    "HUBSPOT_API_KEY": "changeme",
 }
 
 # runs immediately when pytest starts, BEFORE collection
diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
index 39cea6a1..9c1cd31e 100644
--- a/etl/hubspot/hubspotClient.py
+++ b/etl/hubspot/hubspotClient.py
@@ -1,5 +1,441 @@
-import hubspot
+import os
+from enum import Enum
+from typing import Optional, cast
 
-class HubspotClient():
+from hubspot.client import Client  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations import ApiException  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects import SimplePublicObjectInput  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.products.api.basic_api import BasicApi as ProductsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.line_items.api.basic_api import BasicApi as LineItemsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.models import (  # type: ignore[reportMissingTypeStubs]
+    CollectionResponsePipelineNoPaging as PipelinesResponse,
+)
+from hubspot.crm.pipelines.models import Pipeline as HubspotPipeline  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.models import PipelineStage as HubspotPipelineStage  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects.models import SimplePublicObject as HubspotObject  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4 import AssociationSpec  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4.api.basic_api import BasicApi as AssociationsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4.models import (  # type: ignore[reportMissingTypeStubs]
+    CollectionResponseMultiAssociatedObjectWithLabelForwardPaging as AssociationsPageResponse,
+    MultiAssociatedObjectWithLabel as AssociationsResult,
+    ForwardPaging as AssociationsPaging,
+    NextPage as AssociationsPagingNext,
+)
 
-    def
\ No newline at end of file
+
+from backend.app.config import get_settings
+from utils.logger import setup_logger
+
+import mimetypes
+import requests
+
+
+class Companies(Enum):
+    ABRI = "237615001799"
+    SOUTHERN_HOUSING_GROUP = "109343619305"
+    LIVEWEST = "86205872354"
+    SURESERVE = "301745289413"
+    HOMEGROUP = "94946071794"
+    APPLE = "184769046716"
+    THE_GUINESS_PARTNERSHIP = "86970043613"
+
+
+class DealStage(Enum):
+    SURVEYED_COMPLETE_NEEDS_SIGN_OFF = "1617223914"
+    SURVEYED_NO_ACCESS_NEED_SIGN_OFF = "1617223915"
+    CUSTOMER_CONTACTED = "888730834"
+    SURVEYED_COMPLETED_SIGNED_OFF = "1617223916"
+    FILES_MISSING_FROM_ASSESSOR = "1887736000"
+
+
+class Pipeline(Enum):
+    OPERATIONS_SOCIAL_HOUSING = "1167582403"
+
+
+# TODO get guiness working from here
+
+
+class HubspotClient:
+
+    def __init__(self):
+        """
+        Hey Tech Team, Hubspot Library doesn't do type hitting.
+        We have type hinted stuff but pylance never becomes happy.
+        However, because I added the type hinting to the best of ability
+        and you'll still get sensible ide suggestions.
+        """
+        settings = get_settings()
+        access_token = settings.HUBSPOT_API_KEY
+        if access_token is None:
+            raise RuntimeError("Missing HUBSPOT_API_KEY in env")
+        self.access_token: str = access_token
+        self.logger = setup_logger()
+        self.client: Client = Client.create(access_token=self.access_token)  # type: ignore[reportUnknownMemberType]
+        # [Developer Only]
+        # Add a dot in front of client and see the wonders of ide suggestions
+        # This wouldn't work if we didn't add ': Client' to self.client.
+        # Sorry - not sorry but enjoy, Past Junte 13/03/2026
+        # self.client
+
+    def get_deal_ids_from_company(self, company_id: str) -> list[str]:
+        associations_api: AssociationsBasicApi = (  # type: ignore[reportUnknownMemberType]
+            self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
+        )
+
+        deal_ids: list[str] = []
+        after: Optional[str] = None
+
+        while True:
+            response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+                object_type="companies",
+                object_id=company_id,
+                to_object_type="deals",
+                limit=100,
+                after=after,
+            )
+
+            results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
+            for assoc in results:
+                assoc: AssociationsResult
+                object_id: str = cast(str, assoc.to_object_id)  # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
+                deal_ids.append(object_id)
+
+            paging: Optional[AssociationsPaging] = cast(Optional[AssociationsPaging], response.paging)  # type: ignore[reportUnknownMemberType]
+            if not paging:
+                break
+
+            paging_next: Optional[AssociationsPagingNext] = cast(Optional[AssociationsPagingNext], paging.next)  # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            if not paging_next:
+                break
+
+            after = cast(str, paging_next.after)  # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
+
+        return deal_ids
+
+    def from_deal_id_get_associated_company_id(self, deal_id: str) -> Optional[str]:
+        """
+        Get the associated company ID from a given deal ID.
+        Returns the associated company ID, or None if not found.
+        """
+        try:
+            associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
+
+            # Fetch associations for this specific deal only
+            response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+                object_type="deals",
+                object_id=deal_id,
+                to_object_type="companies",
+                limit=1,  # Expect only one associated company
+            )
+
+            results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
+            if not results:
+                self.logger.info(f"No company association found for deal {deal_id}")
+                return None
+
+            first: AssociationsResult = results[0]
+            company_id: str = cast(str, first.to_object_id)  # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
+            self.logger.info(f"Associated company ID for deal {deal_id}: {company_id}")
+            return company_id
+
+        except ApiException as e:
+            self.logger.error(
+                f"Error fetching associated company for deal {deal_id}: {e}"
+            )
+            return None
+
+    def from_deal_id_get_associated_listing(
+        self, deal_id: str
+    ) -> Optional[dict[str, str]]:
+        """
+        Get the associated listing information for a given deal.
+        Returns a dictionary of listing properties, or None if not found.
+        """
+        associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
+        listings_api: ObjectsBasicApi = self.client.crm.objects.basic_api  # type: ignore[reportUnknownMemberType]  # works for custom objects like "listing"
+
+        # Fetch associated listing(s)
+        response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+            object_type="deals",
+            object_id=deal_id,
+            to_object_type="0-420",  # <-- use your exact custom object name slug here
+            limit=1,
+        )
+
+        results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
+        if not results:
+            self.logger.info(f"No listing association found for deal {deal_id}")
+            return None
+
+        first: AssociationsResult = results[0]
+        listing_id: str = cast(str, first.to_object_id)  # type: ignore[reportUnknownMemberType, reportUnknownVariableType]
+        self.logger.info(f"Associated listing ID for deal {deal_id}: {listing_id}")
+
+        # Fetch listing details (the "listing information")
+        listing: HubspotObject = listings_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+            object_type="0-420",  # again, must match your HubSpot object name
+            object_id=listing_id,
+            properties=[
+                "national_uprn",
+                "domna_property_id",
+                "owner_property_id",
+            ],
+        )
+
+        listing_info: dict[str, str] = cast(dict[str, str], listing.properties)  # type: ignore[reportUnknownMemberType]
+        self.logger.info(f"Listing info for deal {deal_id}: {listing_info}")
+        return listing_info
+
+    def from_deal_id_get_info(self, deal_id: str) -> dict[str, str]:
+        deals_api: DealsBasicApi = self.client.crm.deals.basic_api  # type: ignore[reportUnknownMemberType]
+
+        deal: HubspotObject = deals_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+            deal_id,
+            properties=[
+                "dealname",
+                "dealstage",
+                "pipeline",
+                "outcome",  # outcome,
+                "outcome_notes",  # outcome notes
+                "project_code",
+                "major_condition_issue_description",
+                "major_condition_issue_photos",
+                "coordination_status__stage_1_",  # Coordiantion Status (Stage 1),
+                "retrofit_design_status",  # Retrofit Design Status
+            ],
+        )
+
+        deal_info: dict[str, str] = cast(dict[str, str], deal.properties)  # type: ignore[reportUnknownMemberType]
+        return deal_info
+
+    def get_deal_info_for_db(
+        self, deal_id: str
+    ) -> tuple[dict[str, str], Optional[str], Optional[dict[str, str]]]:
+        deal: dict[str, str] = self.from_deal_id_get_info(deal_id)
+        company: Optional[str] = self.from_deal_id_get_associated_company_id(deal_id)
+        listing: Optional[dict[str, str]] = self.from_deal_id_get_associated_listing(
+            deal_id
+        )
+
+        return deal, company, listing
+
+    def get_company_information(self, company_id: str) -> dict[str, str]:
+        companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api  # type: ignore[reportUnknownMemberType]
+
+        company: HubspotObject = companies_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+            company_id,
+            properties=[
+                "name",
+            ],
+        )
+
+        company_info: dict[str, str] = cast(dict[str, str], company.properties)  # type: ignore[reportUnknownMemberType]
+        return company_info
+
+    def get_all_pipelines(self) -> list[dict[str, str]]:
+        """
+        Retrieve all pipelines for deals, returning a list of dicts with pipeline names and IDs.
+        """
+        try:
+            pipelines_api: PipelinesApi = self.client.crm.pipelines.pipelines_api  # type: ignore[reportUnknownMemberType]
+            response: PipelinesResponse = pipelines_api.get_all(object_type="deals")  # type: ignore[reportUnknownMemberType]
+
+            results: list[HubspotPipeline] = cast(list[HubspotPipeline], response.results)  # type: ignore[reportUnknownMemberType]
+            pipelines: list[dict[str, str]] = []
+            for pipeline in results:
+                pipeline: HubspotPipeline
+                pipelines.append(
+                    {
+                        "name": cast(str, pipeline.label),  # type: ignore[reportUnknownMemberType]
+                        "id": cast(str, pipeline.id),  # type: ignore[reportUnknownMemberType]
+                    }
+                )
+
+            self.logger.info(f"Retrieved {len(pipelines)} pipelines.")
+            return pipelines
+
+        except Exception as e:
+            self.logger.error(f"Error retrieving pipelines: {e}")
+            return []
+
+    def get_deal_stages_from_pipeline_id(
+        self, pipeline_id: Optional[str] = None
+    ) -> list[dict[str, str]]:
+        """
+        Retrieve all deal stages for a given pipeline.
+        If no pipeline_id is provided, retrieves all stages for all pipelines.
+        Returns a list of dicts with pipeline name, stage name, and stage ID.
+        """
+        try:
+            pipelines_api: PipelinesApi = self.client.crm.pipelines.pipelines_api  # type: ignore[reportUnknownMemberType]
+            response: PipelinesResponse = pipelines_api.get_all(object_type="deals")  # type: ignore[reportUnknownMemberType]
+
+            all_stages: list[dict[str, str]] = []
+
+            for pipeline in cast(list[HubspotPipeline], response.results):  # type: ignore[reportUnknownMemberType]
+                pipeline: HubspotPipeline
+                # Skip other pipelines if a specific one is requested
+                pipeline_id_str: str = cast(str, pipeline.id)  # type: ignore[reportUnknownMemberType]
+                if pipeline_id and pipeline_id_str != str(pipeline_id):
+                    continue
+
+                for stage in cast(list[HubspotPipelineStage], pipeline.stages):  # type: ignore[reportUnknownMemberType]
+                    stage: HubspotPipelineStage
+                    all_stages.append(
+                        {
+                            "pipeline_name": cast(str, pipeline.label),  # type: ignore[reportUnknownMemberType]
+                            "pipeline_id": pipeline_id_str,
+                            "stage_name": cast(str, stage.label),  # type: ignore[reportUnknownMemberType]
+                            "stage_id": cast(str, stage.id),  # type: ignore[reportUnknownMemberType]
+                        }
+                    )
+
+            if not all_stages:
+                self.logger.info(
+                    f"No deal stages found for pipeline {pipeline_id if pipeline_id else 'ALL'}"
+                )
+            else:
+                self.logger.info(f"Retrieved {len(all_stages)} deal stages.")
+
+            return all_stages
+
+        except Exception as e:
+            self.logger.error(f"Error retrieving deal stages: {e}")
+            return []
+
+    def download_file_from_url(
+        self, download_url: str, save_path: Optional[str] = None
+    ) -> str:
+        """
+        Download a file from a HubSpot file URL (public or private), keeping its original file type.
+        """
+
+        try:
+            headers: dict[str, str] = {}
+            if "hubspotusercontent" not in download_url:
+                headers["Authorization"] = f"Bearer {self.access_token}"
+
+            self.logger.info(f"Downloading HubSpot file: {download_url}")
+            response = requests.get(
+                download_url, headers=headers, stream=True, allow_redirects=True
+            )
+            response.raise_for_status()
+
+            # Try to infer filename from Content-Disposition header
+            content_disposition = response.headers.get("content-disposition")
+            if content_disposition and "filename=" in content_disposition:
+                filename = content_disposition.split("filename=")[1].strip('"')
+            else:
+                # fallback: extract from URL or content-type
+                filename = (
+                    os.path.basename(download_url.split("?")[0]) or "hubspot_download"
+                )
+                if "." not in filename:
+                    content_type = response.headers.get("content-type")
+                    ext = (
+                        mimetypes.guess_extension(content_type.split(";")[0])
+                        if content_type
+                        else None
+                    )
+                    if ext:
+                        filename += ext
+
+            # Make sure save_path is valid
+            if save_path is None:
+                save_path = os.path.abspath(filename)
+            elif os.path.isdir(save_path):
+                save_path = os.path.join(save_path, filename)
+            else:
+                # if user passes a file path directly, leave it
+                save_path = os.path.abspath(save_path)
+
+            with open(save_path, "wb") as f:
+                for chunk in response.iter_content(chunk_size=8192):
+                    f.write(chunk)
+
+            self.logger.info(f"File downloaded successfully → {save_path}")
+            return save_path
+
+        except requests.exceptions.RequestException as e:
+            self.logger.error(f"Failed to download file from HubSpot: {e}")
+            raise
+
+    def create_line_item_from_product(self, product_id: str, quantity: int = 1) -> str:
+        products_api: ProductsBasicApi = self.client.crm.products.basic_api  # type: ignore[reportUnknownMemberType]
+
+        # Fetch product mapping
+        product: HubspotObject = products_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+            product_id, properties=["name", "price", "hs_price"]
+        )
+        product_properties: dict[str, str] = cast(dict[str, str], product.properties)  # type: ignore[reportUnknownMemberType]
+
+        name: Optional[str] = product_properties.get("name")
+        price: str = product_properties.get("price") or product_properties.get("hs_price") or "0"
+
+        # Build line item payload
+        line_item_input = SimplePublicObjectInput(
+            properties={
+                "hs_product_id": product_id,
+                "name": name,
+                "quantity": str(quantity),
+                "price": price,
+                "amount": str(float(price) * quantity),
+                "invoiced": "Outstanding",
+            }
+        )
+
+        line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
+
+        # Create line item
+        line_item: HubspotObject = line_items_api.create(line_item_input)  # type: ignore[reportUnknownMemberType]
+        return cast(str, line_item.id)  # type: ignore[reportUnknownMemberType]
+
+    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str) -> None:
+        self.logger.info(f"Associating line item {line_item_id} → deal {deal_id}")
+
+        association_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
+
+        association_api.create(  # type: ignore[reportUnknownMemberType]
+            "0-3",  # to object type
+            deal_id,  # to object id
+            "line_items",  # from object type
+            line_item_id,  # from object id
+            [
+                AssociationSpec(
+                    association_category="HUBSPOT_DEFINED",
+                    association_type_id=19,  # line_item → deal
+                )
+            ],
+        )
+
+    def add_product_line_item_to_deal(
+        self, deal_id: str, product_id: str, quantity: int = 1
+    ) -> str:
+        # Step 1: Create the line item from product mapping
+        line_item_id: str = self.create_line_item_from_product(product_id, quantity)
+
+        # Step 2: Associate the created line item to the deal
+        self.associate_line_item_to_deal(line_item_id, deal_id)
+
+        return line_item_id
+
+    def delete_line_item(self, line_item_id: str) -> bool:
+        """
+        Delete (archive) a line item in HubSpot by its ID.
+        """
+        try:
+            self.logger.info(f"Deleting line item {line_item_id}...")
+
+            line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
+            line_items_api.archive(line_item_id)  # type: ignore[reportUnknownMemberType]
+
+            self.logger.info(f"Line item {line_item_id} deleted successfully.")
+            return True
+
+        except ApiException as e:
+            self.logger.error(f"Failed to delete line item {line_item_id}: {e}")
+            return False
diff --git a/etl/hubspot/requirements.txt b/etl/hubspot/requirements.txt
index 105cba07..ef8e3ebc 100644
--- a/etl/hubspot/requirements.txt
+++ b/etl/hubspot/requirements.txt
@@ -1 +1 @@
-hubspot
\ No newline at end of file
+hubspot-api-client
\ No newline at end of file
diff --git a/etl/hubspot/tests/__init__.py b/etl/hubspot/tests/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/etl/hubspot/tests/test_hubspot_client_integration.py b/etl/hubspot/tests/test_hubspot_client_integration.py
new file mode 100644
index 00000000..d7cf46fd
--- /dev/null
+++ b/etl/hubspot/tests/test_hubspot_client_integration.py
@@ -0,0 +1,117 @@
+import os
+from typing import Optional
+
+import pytest
+from etl.hubspot.hubspotClient import HubspotClient, Companies, Pipeline, DealStage
+
+
+class TestHubspotClientIntegration:
+    """Integration tests using real HubSpot API calls."""
+
+    @pytest.fixture
+    def client(self):
+        """Initialize HubSpot client with env variables."""
+        return HubspotClient()
+
+    def test_client_initialization(self, client: HubspotClient):
+        """Test that client initializes successfully with API key."""
+        assert client.access_token is not None
+        assert client.client is not None
+        assert client.logger is not None
+
+    def test_get_deal_ids_from_company(self, client: HubspotClient):
+        """Test getting deal IDs from Apple company includes expected deal."""
+        company_id: str = Companies.APPLE.value
+
+        deal_ids: list[str] = client.get_deal_ids_from_company(company_id)
+
+        # https://app-eu1.hubspot.com/contacts/145275138/record/0-3/263490768079
+        assert "263490768079" in deal_ids
+
+    def test_get_company_id_from_deal_id(self, client: HubspotClient):
+        deal_id: str = "263490768079"
+
+        company_id: Optional[str] = client.from_deal_id_get_associated_company_id(
+            deal_id
+        )
+        # https://app-eu1.hubspot.com/contacts/145275138/record/0-3/263490768079
+        assert company_id == Companies.APPLE.value
+
+    def test_from_deal_id_get_associated_listing(self, client: HubspotClient):
+        deal_id: str = "263490768079"
+
+        listing_info: Optional[dict[str, str]] = (
+            client.from_deal_id_get_associated_listing(deal_id)
+        )
+
+        assert listing_info is not None
+        assert "hs_object_id" in listing_info
+        assert "national_uprn" in listing_info
+        assert "owner_property_id" in listing_info
+        assert "domna_property_id" in listing_info
+
+    def test_from_deal_id_get_info(self, client: HubspotClient):
+        deal_id: str = "263490768079"
+
+        deal_info: dict[str, str] = client.from_deal_id_get_info(deal_id)
+
+        assert "dealname" in deal_info
+        assert "dealstage" in deal_info
+        assert "pipeline" in deal_info
+        assert "outcome" in deal_info  # outcome
+        assert "outcome_notes" in deal_info  # outcome notes
+        assert "project_code" in deal_info
+        assert "major_condition_issue_description" in deal_info
+        assert "major_condition_issue_photos" in deal_info
+        assert (
+            "coordination_status__stage_1_" in deal_info
+        )  # Coordiantion Status (Stage 1)
+        assert "retrofit_design_status" in deal_info  # Retrofit Design Status
+
+    def test_get_deal_info_for_db(self, client: HubspotClient):
+        deal_id: str = "263490768079"
+
+        deal, company, listing = client.get_deal_info_for_db(deal_id)
+
+        assert "dealname" in deal
+        assert "dealstage" in deal
+        assert "pipeline" in deal
+
+        assert company == Companies.APPLE.value
+
+        assert listing is None or "hs_object_id" in listing
+
+    def test_get_company_information(self, client: HubspotClient):
+        company_id: str = Companies.APPLE.value
+
+        company_info: dict[str, str] = client.get_company_information(company_id)
+
+        assert "name" in company_info
+        assert company_info["name"].lower() == "Apple".lower()
+
+    def test_get_all_pipelines(self, client: HubspotClient):
+        pipelines: list[dict[str, str]] = client.get_all_pipelines()
+
+        assert len(pipelines) > 0
+        pipeline_ids: list[str] = [p["id"] for p in pipelines]
+        assert Pipeline.OPERATIONS_SOCIAL_HOUSING.value in pipeline_ids
+
+    def test_get_deal_stages_from_pipeline_id(self, client: HubspotClient):
+        stages: list[dict[str, str]] = client.get_deal_stages_from_pipeline_id(
+            Pipeline.OPERATIONS_SOCIAL_HOUSING.value
+        )
+
+        assert len(stages) > 0
+        stage_ids: list[str] = [s["stage_id"] for s in stages]
+        assert DealStage.SURVEYED_COMPLETE_NEEDS_SIGN_OFF.value in stage_ids
+
+    def test_download_file_from_url(
+        self, client: HubspotClient, tmp_path: Optional[str]
+    ):
+        deal_info: dict[str, str] = client.from_deal_id_get_info("254427203793")
+        download_url: str = deal_info["major_condition_issue_photos"]
+
+        save_path: str = client.download_file_from_url(download_url, str(tmp_path))
+
+        assert os.path.exists(save_path)
+        assert os.path.getsize(save_path) > 0
diff --git a/pyrightconfig.json b/pyrightconfig.json
index d4e0e2a4..18f578a5 100644
--- a/pyrightconfig.json
+++ b/pyrightconfig.json
@@ -2,7 +2,7 @@
   "typeCheckingMode": "strict",
   "venvPath": "/Users/khalimconn-kowlessar/opt/anaconda3/envs/",
   "venv": "Fastapi-backend",
-  "include": [
+"include": [
     "."
   ]
 }
\ No newline at end of file
diff --git a/pytest.ini b/pytest.ini
index 608d5e0c..c9dd8ca8 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -3,4 +3,4 @@ pythonpath = .
 log_cli = true
 log_cli_level = INFO
 addopts = --cov-report term-missing --cov=etl/epc --cov=recommendations --cov=backend --cov=etl/epc_clean --cov=etl/spatial
-testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests
+testpaths = recommendations/tests backend/tests etl/epc/tests etl/epc_clean/tests etl/spatial/tests backend/condition/tests backend/address2UPRN/tests backend/onboarders/tests backend/categorisation/tests backend/export/tests etl/hubspot/tests

From 3970d70518f1432bc68f2af2532eec63308e5ff4 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 14:36:53 +0000
Subject: [PATCH 03/14] its now perfect

---
 etl/hubspot/hubspotClient.py | 64 ++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 35 deletions(-)

diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
index 9c1cd31e..b41d71f8 100644
--- a/etl/hubspot/hubspotClient.py
+++ b/etl/hubspot/hubspotClient.py
@@ -2,24 +2,22 @@ import os
 from enum import Enum
 from typing import Optional, cast
 
-from hubspot.client import Client  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.associations import ApiException  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.objects import SimplePublicObjectInput  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.products.api.basic_api import BasicApi as ProductsBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.line_items.api.basic_api import BasicApi as LineItemsBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.pipelines.models import (  # type: ignore[reportMissingTypeStubs]
+from hubspot.client import Client
+from hubspot.crm.associations import ApiException
+from hubspot.crm.objects import SimplePublicObjectInput
+from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi
+from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi
+from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi
+from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi
+from hubspot.crm.pipelines.models import (
     CollectionResponsePipelineNoPaging as PipelinesResponse,
 )
-from hubspot.crm.pipelines.models import Pipeline as HubspotPipeline  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.pipelines.models import PipelineStage as HubspotPipelineStage  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.objects.models import SimplePublicObject as HubspotObject  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.associations.v4 import AssociationSpec  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.associations.v4.api.basic_api import BasicApi as AssociationsBasicApi  # type: ignore[reportMissingTypeStubs]
-from hubspot.crm.associations.v4.models import (  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.models import Pipeline as HubspotPipeline
+from hubspot.crm.pipelines.models import PipelineStage as HubspotPipelineStage
+from hubspot.crm.objects.models import SimplePublicObject as HubspotObject
+from hubspot.crm.associations.v4 import AssociationSpec
+from hubspot.crm.associations.v4.api.basic_api import BasicApi as AssociationsBasicApi
+from hubspot.crm.associations.v4.models import (
     CollectionResponseMultiAssociatedObjectWithLabelForwardPaging as AssociationsPageResponse,
     MultiAssociatedObjectWithLabel as AssociationsResult,
     ForwardPaging as AssociationsPaging,
@@ -364,17 +362,16 @@ class HubspotClient:
             self.logger.error(f"Failed to download file from HubSpot: {e}")
             raise
 
-    def create_line_item_from_product(self, product_id: str, quantity: int = 1) -> str:
-        products_api: ProductsBasicApi = self.client.crm.products.basic_api  # type: ignore[reportUnknownMemberType]
-
+    def create_line_item_from_product(self, product_id: str, quantity: int = 1):
         # Fetch product mapping
-        product: HubspotObject = products_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+        product = self.client.crm.products.basic_api.get_by_id(
             product_id, properties=["name", "price", "hs_price"]
         )
-        product_properties: dict[str, str] = cast(dict[str, str], product.properties)  # type: ignore[reportUnknownMemberType]
 
-        name: Optional[str] = product_properties.get("name")
-        price: str = product_properties.get("price") or product_properties.get("hs_price") or "0"
+        name = product.properties.get("name")
+        price = (
+            product.properties.get("price") or product.properties.get("hs_price") or "0"
+        )
 
         # Build line item payload
         line_item_input = SimplePublicObjectInput(
@@ -388,18 +385,16 @@ class HubspotClient:
             }
         )
 
-        line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
-
         # Create line item
-        line_item: HubspotObject = line_items_api.create(line_item_input)  # type: ignore[reportUnknownMemberType]
-        return cast(str, line_item.id)  # type: ignore[reportUnknownMemberType]
+        line_item = self.client.crm.line_items.basic_api.create(line_item_input)
+        return line_item.id
 
-    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str) -> None:
+    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str):
         self.logger.info(f"Associating line item {line_item_id} → deal {deal_id}")
 
-        association_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
+        association_api = self.client.crm.associations.v4.basic_api
 
-        association_api.create(  # type: ignore[reportUnknownMemberType]
+        association_api.create(
             "0-3",  # to object type
             deal_id,  # to object id
             "line_items",  # from object type
@@ -414,24 +409,23 @@ class HubspotClient:
 
     def add_product_line_item_to_deal(
         self, deal_id: str, product_id: str, quantity: int = 1
-    ) -> str:
+    ):
         # Step 1: Create the line item from product mapping
-        line_item_id: str = self.create_line_item_from_product(product_id, quantity)
+        line_item_id = self.create_line_item_from_product(product_id, quantity)
 
         # Step 2: Associate the created line item to the deal
         self.associate_line_item_to_deal(line_item_id, deal_id)
 
         return line_item_id
 
-    def delete_line_item(self, line_item_id: str) -> bool:
+    def delete_line_item(self, line_item_id: str):
         """
         Delete (archive) a line item in HubSpot by its ID.
         """
         try:
             self.logger.info(f"Deleting line item {line_item_id}...")
 
-            line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
-            line_items_api.archive(line_item_id)  # type: ignore[reportUnknownMemberType]
+            self.client.crm.line_items.basic_api.archive(line_item_id)
 
             self.logger.info(f"Line item {line_item_id} deleted successfully.")
             return True

From cca72928d91ebb03d4b5fc5aa92715f264221c5b Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 14:38:10 +0000
Subject: [PATCH 04/14] its now perfect

---
 etl/hubspot/hubspotClient.py | 60 +++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 28 deletions(-)

diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
index b41d71f8..1946bcdf 100644
--- a/etl/hubspot/hubspotClient.py
+++ b/etl/hubspot/hubspotClient.py
@@ -2,22 +2,22 @@ import os
 from enum import Enum
 from typing import Optional, cast
 
-from hubspot.client import Client
-from hubspot.crm.associations import ApiException
-from hubspot.crm.objects import SimplePublicObjectInput
-from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi
-from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi
-from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi
-from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi
-from hubspot.crm.pipelines.models import (
+from hubspot.client import Client  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations import ApiException  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects import SimplePublicObjectInput  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.models import (  # type: ignore[reportMissingTypeStubs]
     CollectionResponsePipelineNoPaging as PipelinesResponse,
 )
-from hubspot.crm.pipelines.models import Pipeline as HubspotPipeline
-from hubspot.crm.pipelines.models import PipelineStage as HubspotPipelineStage
-from hubspot.crm.objects.models import SimplePublicObject as HubspotObject
-from hubspot.crm.associations.v4 import AssociationSpec
-from hubspot.crm.associations.v4.api.basic_api import BasicApi as AssociationsBasicApi
-from hubspot.crm.associations.v4.models import (
+from hubspot.crm.pipelines.models import Pipeline as HubspotPipeline  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.pipelines.models import PipelineStage as HubspotPipelineStage  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.objects.models import SimplePublicObject as HubspotObject  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4 import AssociationSpec  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4.api.basic_api import BasicApi as AssociationsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.associations.v4.models import (  # type: ignore[reportMissingTypeStubs]
     CollectionResponseMultiAssociatedObjectWithLabelForwardPaging as AssociationsPageResponse,
     MultiAssociatedObjectWithLabel as AssociationsResult,
     ForwardPaging as AssociationsPaging,
@@ -362,15 +362,17 @@ class HubspotClient:
             self.logger.error(f"Failed to download file from HubSpot: {e}")
             raise
 
-    def create_line_item_from_product(self, product_id: str, quantity: int = 1):
+    def create_line_item_from_product(self, product_id: str, quantity: int = 1) -> str:
         # Fetch product mapping
-        product = self.client.crm.products.basic_api.get_by_id(
+        products_api: ProductsBasicApi = self.client.crm.products.basic_api  # type: ignore[reportUnknownMemberType]
+        product: HubspotObject = products_api.get_by_id(  # type: ignore[reportUnknownMemberType]
             product_id, properties=["name", "price", "hs_price"]
         )
+        properties: dict[str, str] = cast(dict[str, str], product.properties)  # type: ignore[reportUnknownMemberType]
 
-        name = product.properties.get("name")
-        price = (
-            product.properties.get("price") or product.properties.get("hs_price") or "0"
+        name: str = properties.get("name") or ""
+        price: str = (
+            properties.get("price") or properties.get("hs_price") or "0"
         )
 
         # Build line item payload
@@ -386,15 +388,16 @@ class HubspotClient:
         )
 
         # Create line item
-        line_item = self.client.crm.line_items.basic_api.create(line_item_input)
-        return line_item.id
+        line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
+        line_item: HubspotObject = line_items_api.create(line_item_input)  # type: ignore[reportUnknownMemberType]
+        return cast(str, line_item.id)  # type: ignore[reportUnknownMemberType]
 
-    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str):
+    def associate_line_item_to_deal(self, line_item_id: str, deal_id: str) -> None:
         self.logger.info(f"Associating line item {line_item_id} → deal {deal_id}")
 
-        association_api = self.client.crm.associations.v4.basic_api
+        association_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
 
-        association_api.create(
+        association_api.create(  # type: ignore[reportUnknownMemberType]
             "0-3",  # to object type
             deal_id,  # to object id
             "line_items",  # from object type
@@ -409,23 +412,24 @@ class HubspotClient:
 
     def add_product_line_item_to_deal(
         self, deal_id: str, product_id: str, quantity: int = 1
-    ):
+    ) -> str:
         # Step 1: Create the line item from product mapping
-        line_item_id = self.create_line_item_from_product(product_id, quantity)
+        line_item_id: str = self.create_line_item_from_product(product_id, quantity)
 
         # Step 2: Associate the created line item to the deal
         self.associate_line_item_to_deal(line_item_id, deal_id)
 
         return line_item_id
 
-    def delete_line_item(self, line_item_id: str):
+    def delete_line_item(self, line_item_id: str) -> bool:
         """
         Delete (archive) a line item in HubSpot by its ID.
         """
         try:
             self.logger.info(f"Deleting line item {line_item_id}...")
 
-            self.client.crm.line_items.basic_api.archive(line_item_id)
+            line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
+            line_items_api.archive(line_item_id)  # type: ignore[reportUnknownMemberType]
 
             self.logger.info(f"Line item {line_item_id} deleted successfully.")
             return True

From 2349eba89e7239f7768d021171480c6e06e1cdfd Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 14:38:32 +0000
Subject: [PATCH 05/14] its now perfect

---
 etl/hubspot/hubspotClient.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etl/hubspot/hubspotClient.py b/etl/hubspot/hubspotClient.py
index 1946bcdf..f93a736c 100644
--- a/etl/hubspot/hubspotClient.py
+++ b/etl/hubspot/hubspotClient.py
@@ -8,6 +8,8 @@ from hubspot.crm.objects import SimplePublicObjectInput  # type: ignore[reportMi
 from hubspot.crm.objects.api.basic_api import BasicApi as ObjectsBasicApi  # type: ignore[reportMissingTypeStubs]
 from hubspot.crm.deals.api.basic_api import BasicApi as DealsBasicApi  # type: ignore[reportMissingTypeStubs]
 from hubspot.crm.companies.api.basic_api import BasicApi as CompaniesBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.products.api.basic_api import BasicApi as ProductsBasicApi  # type: ignore[reportMissingTypeStubs]
+from hubspot.crm.line_items.api.basic_api import BasicApi as LineItemsBasicApi  # type: ignore[reportMissingTypeStubs]
 from hubspot.crm.pipelines.api.pipelines_api import PipelinesApi  # type: ignore[reportMissingTypeStubs]
 from hubspot.crm.pipelines.models import (  # type: ignore[reportMissingTypeStubs]
     CollectionResponsePipelineNoPaging as PipelinesResponse,

From f8187634058d05fa7f2961b9980f2ec720911b84 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 14:43:55 +0000
Subject: [PATCH 06/14] make tests work

---
 test.requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/test.requirements.txt b/test.requirements.txt
index d8b8b777..4bd89caa 100644
--- a/test.requirements.txt
+++ b/test.requirements.txt
@@ -4,4 +4,5 @@ pytest-cov
 pytest-mock
 dotenv
 psycopg[binary]
-pytest-postgresql
\ No newline at end of file
+pytest-postgresql
+hubspot-api-client
\ No newline at end of file

From 6e8f29afc8dbd385c9e526f70b43cb4ec9613b04 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 14:52:41 +0000
Subject: [PATCH 07/14] added to rerun

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 9268ba25..b470e12c 100644
--- a/README.md
+++ b/README.md
@@ -39,3 +39,4 @@ pytest --cov-config=model_data/.coveragerc --cov=model_data
 
 This will produce the test results and coverage reports
 
+

From 8294a80fdfd05be346ffcef8d38331ba5744b2a4 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:12:43 +0000
Subject: [PATCH 08/14] change the way the tests are ran as i don't like
 makefile

---
 .github/workflows/unit_tests.yml | 18 ++++++++----------
 Makefile                         | 30 ------------------------------
 2 files changed, 8 insertions(+), 40 deletions(-)
 delete mode 100644 Makefile

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index cc6431b8..91ca7e26 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -14,17 +14,15 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
+      - name: Build test image
+        run: docker build -f Dockerfile.test -t model-test .
 
-      - name: Install tox via Makefile
-        run: |
-          make setup
-
-      - name: Run tests with tox via Makefile
+      - name: Run tests
         env:
           EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
+          
         run: |
-          make test
\ No newline at end of file
+          docker run --rm \
+            -e EPC_AUTH_TOKEN=${{ secrets.DEV_EPC_AUTH_TOKEN }} \
+            -e HUBSPOT_API_KEY=${{ secrets.HUBSPOT_API_KEY }} \
+            model-test pytest
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 00942acd..00000000
--- a/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-# Project Makefile
-
-PYTHON = python
-
-.PHONY: setup test lint typecheck check clean
-
-# Install dev dependencies + tox
-setup:
-	$(PYTHON) -m pip install --upgrade pip
-	$(PYTHON) -m pip install tox black ruff mypy
-
-# Run tests (pass ARGS="..." for specific tests)
-test:
-	tox -- $(ARGS)
-
-# Code formatting check + linting
-lint:
-	ruff .
-	black --check .
-
-# Static type checks
-typecheck:
-	mypy .
-
-# Full quality check (all checks + tests)
-check: lint typecheck test
-
-# Clean up tox environments
-clean:
-	rm -rf .tox

From 81d84368cfd88232239ec3c92d8e77e6fc5d8417 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:15:14 +0000
Subject: [PATCH 09/14] we are going to use docker instead

---
 Dockerfile.test | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 Dockerfile.test

diff --git a/Dockerfile.test b/Dockerfile.test
new file mode 100644
index 00000000..d566c435
--- /dev/null
+++ b/Dockerfile.test
@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+# Install PostgreSQL binaries — required by pytest-postgresql to spawn ephemeral test databases
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends postgresql \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy requirements first so Docker can cache the install layer
+COPY backend/engine/requirements.txt backend/engine/requirements.txt
+COPY backend/app/requirements/requirements.txt backend/app/requirements/requirements.txt
+COPY test.requirements.txt test.requirements.txt
+
+RUN pip install --no-cache-dir \
+    -r backend/engine/requirements.txt \
+    -r backend/app/requirements/requirements.txt \
+    -r test.requirements.txt
+
+# Copy source
+COPY . .
+
+CMD ["pytest"]

From 7fb8ee9202fc3c739942b96269a657e519a22d13 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:20:15 +0000
Subject: [PATCH 10/14]  re run

---
 Dockerfile.test | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile.test b/Dockerfile.test
index d566c435..debbfa8b 100644
--- a/Dockerfile.test
+++ b/Dockerfile.test
@@ -6,6 +6,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
+ENV PYTHONPATH=/app
 
 # Copy requirements first so Docker can cache the install layer
 COPY backend/engine/requirements.txt backend/engine/requirements.txt

From 6f6aa62efee423692dcf6eb332a636c4d7bc6bff Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:26:44 +0000
Subject: [PATCH 11/14] add more requirements

---
 Dockerfile.test       | 2 ++
 test.requirements.txt | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/Dockerfile.test b/Dockerfile.test
index debbfa8b..6091aa50 100644
--- a/Dockerfile.test
+++ b/Dockerfile.test
@@ -11,11 +11,13 @@ ENV PYTHONPATH=/app
 # Copy requirements first so Docker can cache the install layer
 COPY backend/engine/requirements.txt backend/engine/requirements.txt
 COPY backend/app/requirements/requirements.txt backend/app/requirements/requirements.txt
+COPY asset_list/requirements.txt asset_list/requirements.txt
 COPY test.requirements.txt test.requirements.txt
 
 RUN pip install --no-cache-dir \
     -r backend/engine/requirements.txt \
     -r backend/app/requirements/requirements.txt \
+    -r asset_list/requirements.txt \
     -r test.requirements.txt
 
 # Copy source
diff --git a/test.requirements.txt b/test.requirements.txt
index 4bd89caa..936e2f7d 100644
--- a/test.requirements.txt
+++ b/test.requirements.txt
@@ -5,4 +5,5 @@ pytest-mock
 dotenv
 psycopg[binary]
 pytest-postgresql
-hubspot-api-client
\ No newline at end of file
+hubspot-api-client
+fuzzywuzzy
\ No newline at end of file

From 27f17563d46ecf05a901e092d74b5d6654706179 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:41:43 +0000
Subject: [PATCH 12/14] pytest ini

---
 .github/workflows/unit_tests.yml |  1 +
 Dockerfile.test                  |  2 --
 Dockerfile.test.dockerignore     | 11 +++++++++++
 3 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 Dockerfile.test.dockerignore

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
index 91ca7e26..116bc265 100644
--- a/.github/workflows/unit_tests.yml
+++ b/.github/workflows/unit_tests.yml
@@ -20,6 +20,7 @@ jobs:
       - name: Run tests
         env:
           EPC_AUTH_TOKEN: ${{ secrets.DEV_EPC_AUTH_TOKEN }}
+          HUBSPOT_API_KEY: ${{ secrets.HUBSPOT_API_KEY }}
           
         run: |
           docker run --rm \
diff --git a/Dockerfile.test b/Dockerfile.test
index 6091aa50..debbfa8b 100644
--- a/Dockerfile.test
+++ b/Dockerfile.test
@@ -11,13 +11,11 @@ ENV PYTHONPATH=/app
 # Copy requirements first so Docker can cache the install layer
 COPY backend/engine/requirements.txt backend/engine/requirements.txt
 COPY backend/app/requirements/requirements.txt backend/app/requirements/requirements.txt
-COPY asset_list/requirements.txt asset_list/requirements.txt
 COPY test.requirements.txt test.requirements.txt
 
 RUN pip install --no-cache-dir \
     -r backend/engine/requirements.txt \
     -r backend/app/requirements/requirements.txt \
-    -r asset_list/requirements.txt \
     -r test.requirements.txt
 
 # Copy source
diff --git a/Dockerfile.test.dockerignore b/Dockerfile.test.dockerignore
new file mode 100644
index 00000000..8a846047
--- /dev/null
+++ b/Dockerfile.test.dockerignore
@@ -0,0 +1,11 @@
+# We need this file otherwise it'll use .dockerignore
+# Exclude large/irrelevant directories that are not needed for testing
+model_data/local_data/
+backend/node_modules/
+backend/.idea/
+infrastructure/
+data_collection/
+node_modules/
+conservation_areas/
+open_uprn/
+land_registry/

From 08478b17fb838584cc3a63641700da6586d3cfa5 Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:44:16 +0000
Subject: [PATCH 13/14] run tests

---
 Dockerfile.test.dockerignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Dockerfile.test.dockerignore b/Dockerfile.test.dockerignore
index 8a846047..4f79c6ee 100644
--- a/Dockerfile.test.dockerignore
+++ b/Dockerfile.test.dockerignore
@@ -3,6 +3,7 @@
 model_data/local_data/
 backend/node_modules/
 backend/.idea/
+backend/.env
 infrastructure/
 data_collection/
 node_modules/

From ad189b4cacf56f2944b3a519cec4fff17b27c7fc Mon Sep 17 00:00:00 2001
From: Jun-te Kim <juntekim@googlemail.com>
Date: Fri, 13 Mar 2026 15:56:13 +0000
Subject: [PATCH 14/14] post gres can't be ran as root

---
 Dockerfile.test | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/Dockerfile.test b/Dockerfile.test
index debbfa8b..802eb3a4 100644
--- a/Dockerfile.test
+++ b/Dockerfile.test
@@ -21,4 +21,8 @@ RUN pip install --no-cache-dir \
 # Copy source
 COPY . .
 
+# pg_ctl refuses to run as root — create an unprivileged user
+RUN useradd -m testuser && chown -R testuser /app
+USER testuser
+
 CMD ["pytest"]