update spreadsheet with properties that have already been processed

add change and put it to production

MEMORY.md

@@ -0,0 +1,22 @@
+# Project Memory
+
+## HubSpot New Field Addition Process
+
+When adding a new field from HubSpot to the deal table, touch these 4 locations in order:
+
+1. **DB model** — `backend/app/db/models/organisation.py`
+   Add `field_name: Optional[str] = Field(default=None)` to `HubspotDealData`, near related fields.
+
+2. **HubSpot API fetch** — `etl/hubspot/hubspotClient.py`, `from_deal_id_get_info()`
+   Add the HubSpot internal property name string to the `properties=[...]` list.
+
+3. **Soft check** — `etl/hubspot/hubspotDataTodB.py`, `update_deal_with_checks()`
+   Add a `soft_assert(deal_in_db.field_name == hs_deal.get("hs_property_name"), "field_name mismatch")` entry to the `checks` list.
+
+4. **Upsert** — `etl/hubspot/hubspotDataTodB.py`, `upsert_deal()`
+   - **Update branch**: add `"field_name": deal_data.get("hs_property_name")` to the attr dict
+   - **Insert branch**: add `field_name=deal_data.get("hs_property_name")` to the `HubspotDealData(...)` constructor
+
+**Notes:**
+- Date fields: wrap with `self._parse_hs_date(deal_data.get(...))` in steps 3 & 4.
+- If HubSpot property name differs from DB column name (e.g. `coordination_status__stage_1_` → `coordination_status`), use the HS name in `.get()` and the DB name as the key/attr.

backend/address2UPRN/main.py

@@ -462,6 +462,15 @@ def handler(event, context, local=False):
                 # Validate postcode before processing
                 if not AddressMatch.is_valid_postcode(postcode):
                     logger.warning(f"Postcode {postcode} is invalid, skipping")
+                    for row in postcode_rows:
+                        results_data.append(
+                            {
+                                **row,
+                                "address2uprn_uprn": "invalid postcode",
+                                "address2uprn_address": "invalid postcode",
+                                "address2uprn_lexiscore": "invalid postcode",
+                            }
+                        )
                     continue
 
                 # Fetch EPC data once per postcode

backend/app/db/models/organisation.py

@@ -44,6 +44,7 @@ class HubspotDealData(SQLModel, table=True):
     pashub_link: Optional[str] = Field(default=None)
     sharepoint_link: Optional[str] = Field(default=None)
     dampmould_growth: Optional[str] = Field(default=None)
+    damp_mould_and_repairs_comments: Optional[str] = Field(default=None)
     pre_sap: Optional[str] = Field(default=None)
     coordinator: Optional[str] = Field(default=None)
     mtp_completion_date: Optional[datetime] = Field(default=None)

backend/ecmk_fetcher/address_list.py

@@ -0,0 +1,117 @@
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, Optional, cast
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
+from openpyxl.cell.cell import Cell
+
+
+@dataclass
+class PropertyRow:
+    row_index: int
+    address: str
+    processed: bool
+
+
+def extract_addresses_from_spreadsheet(
+    filepath: str,
+) -> Dict[str, PropertyRow]:
+    wb: Workbook = load_workbook(filepath, data_only=True)
+    ws: Worksheet = wb["Southern RA-Lite Programme 3103"]
+
+    header_row: int = 1
+    id_col: Optional[int] = None
+    deal_name_col: Optional[int] = None
+    processed_col: Optional[int] = None
+
+    # find columns
+    for col in range(1, ws.max_column + 1):
+        raw_value: Any = ws.cell(row=header_row, column=col).value
+        value: str = str(raw_value).strip().lower() if raw_value else ""
+
+        if value == "id":
+            id_col = col
+        elif value == "deal name":
+            deal_name_col = col
+        elif value == "processed":
+            processed_col = col
+
+    if id_col is None or deal_name_col is None:
+        raise Exception("Missing required columns")
+
+    # create processed column if missing
+    if processed_col is None:
+        processed_col = ws.max_column + 1
+        cast(Cell, ws.cell(row=header_row, column=processed_col)).value = "processed"
+
+    properties: Dict[str, PropertyRow] = {}
+
+    for row in range(2, ws.max_row + 1):
+        id_val: Any = ws.cell(row=row, column=id_col).value
+        deal_name: Any = ws.cell(row=row, column=deal_name_col).value
+
+        if not id_val or not deal_name:
+            continue
+
+        processed_val: Any = ws.cell(row=row, column=processed_col).value
+        processed: bool = str(processed_val).lower() == "true"
+
+        property_id: str = str(id_val).strip()
+
+        properties[property_id] = PropertyRow(
+            row_index=row,
+            address=extract_succinct_address(str(deal_name)),
+            processed=processed,
+        )
+
+    return properties
+
+
+def mark_properties_as_processed(
+    filepath: str,
+    property_map: Dict[str, PropertyRow],
+) -> None:
+    wb: Workbook = load_workbook(filepath)
+    ws: Worksheet = wb["Southern RA-Lite Programme 3103"]
+
+    header_row: int = 1
+
+    # find processed column
+    processed_col: int | None = None
+
+    for col in range(1, ws.max_column + 1):
+        value = ws.cell(row=header_row, column=col).value
+        if value and str(value).strip().lower() == "processed":
+            processed_col = col
+            break
+
+    if processed_col is None:
+        raise Exception("Processed column not found")
+
+    # update rows
+    for property_row in property_map.values():
+        if property_row.processed:
+            cast(
+                Cell,
+                ws.cell(
+                    row=property_row.row_index,
+                    column=processed_col,
+                ),
+            ).value = True
+
+    wb.save(filepath)
+
+
+def extract_succinct_address(deal_name: str) -> str:
+    left_part = deal_name.split("|")[0].strip()
+
+    postcode_match: Optional[re.Match[str]] = re.search(
+        r"\b([A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2})\b",
+        left_part,
+        re.IGNORECASE,
+    )
+
+    postcode = postcode_match.group(1).upper() if postcode_match else None
+    first_part = left_part.split(",")[0].strip()
+
+    return f"{first_part} {postcode}" if postcode else first_part

backend/ecmk_fetcher/browser.py

@@ -0,0 +1,98 @@
+import os
+from typing import Optional
+from playwright.sync_api import Page, Locator, Response
+from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+
+from backend.ecmk_fetcher.reports import build_report_selector
+from utils.logger import setup_logger
+
+# from .reports import build_report_selector
+
+logger = setup_logger()
+
+
+def attach_debug_listeners(page: Page) -> None:
+    def handle_response(response: Response) -> None:
+        if "download" in response.url or "report" in response.url:
+            logger.info(f"[RESPONSE] {response.status} {response.url}")
+
+    page.on("response", handle_response)
+
+
+def login(page: Page, username: str, password: str) -> None:
+    page.goto("https://assessorhub.net/", timeout=30000)
+
+    page.locator("#Username").fill(username)
+    page.locator("#Password").fill(password)
+
+    with page.expect_navigation():
+        page.click("button[type='submit']")
+
+    if "login" in page.url.lower():
+        raise Exception("Login failed")
+
+    logger.info("Login successful")
+
+
+def go_to_assessments(page: Page) -> None:
+    page.goto("https://assessorhub.net/Companies/Assessments")
+    page.wait_for_selector("#assessmentDatatable tbody tr")
+
+
+def go_to_assessment_details(page: Page, row: Locator) -> None:
+    row.locator("a").click()
+    page.wait_for_load_state("networkidle")
+    page.wait_for_selector("a.download-report-btn")
+
+
+def get_first_row_signature(page: Page) -> str:
+    return page.locator("#assessmentDatatable tbody tr").first.inner_text()
+
+
+def go_to_next_page(page: Page) -> bool:
+    before = get_first_row_signature(page)
+
+    page.locator("#assessmentDatatable_next a").click()
+    page.wait_for_timeout(2000)
+
+    after = get_first_row_signature(page)
+
+    return before != after
+
+
+def download_report_by_selector(page: Page, selector: str) -> Optional[str]:
+    try:
+        element = page.locator(selector)
+        element.wait_for(state="visible", timeout=10000)
+
+        if not element.is_enabled():
+            return None
+
+        element.scroll_into_view_if_needed()
+
+        with page.expect_download(timeout=15000) as download_info:
+            element.click()
+
+        download = download_info.value
+        filename = download.suggested_filename
+
+        save_path = os.path.join(os.getcwd(), filename)
+        download.save_as(save_path)
+
+        return save_path
+
+    except PlaywrightTimeoutError:
+        logger.error(f"Download failed for {selector}")
+        return None
+
+
+def download_with_retry(page: Page, report_type: int) -> Optional[str]:
+    selector: str = build_report_selector(report_type)
+
+    for _ in range(3):
+        file_path = download_report_by_selector(page, selector)
+        if file_path:
+            return file_path
+        page.wait_for_timeout(1500)
+
+    return None

backend/ecmk_fetcher/handler/handler.py

@@ -1,337 +1,10 @@
-from datetime import datetime, timezone
-import os
-from enum import Enum
-import re
-from typing import Any, Dict, List, Mapping, Optional
-from openpyxl import load_workbook
-from playwright.sync_api import (
-    Locator,
-    Page,
-    sync_playwright,
-    TimeoutError as PlaywrightTimeoutError,
-)
+from typing import Any, Mapping
 
-from backend.app.db.connection import db_session
-from backend.app.db.models.uploaded_file import FileSourceEnum, UploadedFile
-from utils.logger import setup_logger
-from utils.s3 import upload_file_to_s3
-from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
-from utils.sharepoint.domna_sites import DomnaSites
-
-logger = setup_logger()
-
-
-class file_download_button_types(Enum):
-    ASSESSOR_HUB_SITENOTE_REPORT = 11
-    CERTIFICATE = 9
-    SITENOTE_REPORT = 8
-    RAW_XML = 7
-    SAP_WORK_SHEET = 15
-
-
-def extract_addresses_from_spreadsheet(filepath: str) -> Dict[str, str]:
-    wb = load_workbook(filepath, data_only=True)
-    ws = wb["Southern RA-Lite Programme 3103"]
-
-    properties: Dict[str, str] = {}
-
-    header_row = 1
-    id_col_index = None
-    deal_name_col_index = None
-
-    for col in range(1, ws.max_column + 1):
-        cell_value = ws.cell(row=header_row, column=col).value
-
-        if cell_value and str(cell_value).strip().lower() == "id":
-            id_col_index = col
-
-        if cell_value and str(cell_value).strip().lower() == "deal name":
-            deal_name_col_index = col
-            break
-
-    if id_col_index is None:
-        raise Exception("ID column not found in spreadsheet")
-
-    if deal_name_col_index is None:
-        raise Exception("Deal Name column not found in spreadsheet")
-
-    for row in range(2, ws.max_row + 1):
-        id_cell_value = ws.cell(row=row, column=id_col_index).value
-        deal_name_cell_value = ws.cell(row=row, column=deal_name_col_index).value
-
-        if id_cell_value is None or deal_name_cell_value is None:
-            continue
-
-        id_str = str(id_cell_value).strip()
-        deal_name_str = str(deal_name_cell_value).strip()
-
-        if not id_str:
-            continue
-
-        sharepoint_address = extract_succinct_address(deal_name_str)
-
-        properties[id_str] = sharepoint_address
-
-    return properties
-
-
-def extract_succinct_address(deal_name: str) -> str:
-    """
-    Input:
-      '1 My Random Close, Town, AB12 3DC | Retrofit Assessment'
-
-    Output:
-      '1 My Random Close AB12 3DC'
-    """
-    left_part = deal_name.split("|")[0].strip()
-
-    postcode_match: Optional[re.Match[str]] = re.search(
-        r"\b([A-Z]{1,2}\d[A-Z\d]?\s*\d[A-Z]{2})\b",
-        left_part,
-        re.IGNORECASE,
-    )
-
-    postcode = None
-    if postcode_match:
-        postcode = postcode_match.group(1).upper()
-
-    first_part = left_part.split(",")[0].strip()
-
-    if postcode:
-        return f"{first_part} {postcode}"
-    else:
-        return first_part
-
-
-def build_property_id(address: str, postcode: str) -> str:
-    """
-    Extract number from address and concat with postcode
-    Example:
-    '9 Random Close', 'AB1 2YZ' → '9AB12YZ'
-    """
-    number = address.split(" ")[0]
-
-    postcode_clean = postcode.replace(" ", "").upper()
-
-    return f"{number}{postcode_clean}"
-
-
-def login(page: Page, username: str, password: str) -> None:
-    page.goto("https://assessorhub.net/", timeout=30000)
-
-    username_input: Locator = page.locator("#Username")
-    password_input: Locator = page.locator("#Password")
-
-    username_input.wait_for(state="visible", timeout=10000)
-    username_input.fill(username)
-
-    password_input.wait_for(state="visible", timeout=10000)
-    password_input.fill(password)
-
-    with page.expect_navigation(timeout=15000):
-        page.click("button[type='submit']")
-
-    if "login" in page.url.lower():
-        raise Exception("Login failed")
-
-    logger.info("Login successful")
-
-
-def go_to_assessments(page: Page) -> None:
-    page.goto("https://assessorhub.net/Companies/Assessments", timeout=30000)
-    page.wait_for_selector("#assessmentDatatable tbody tr", timeout=20000)
-
-
-def go_to_assessment_details(page: Page, row: Locator) -> None:
-    account_link: Locator = row.locator("a")
-    with page.expect_navigation():
-        account_link.click()
-
-
-def go_to_next_page(page: Page) -> bool:
-    next_button: Locator = page.locator("#assessmentDatatable_next a")
-
-    class_attr: Optional[str] = next_button.get_attribute("class") or ""
-
-    if "disabled" in class_attr:
-        logger.info("No more pages")
-        return False
-
-    next_button.scroll_into_view_if_needed()
-    next_button.click()
-
-    page.wait_for_timeout(2000)
-    return True
-
-
-def build_report_selector(report_type: int) -> str:
-    return f"a.download-report-btn[data-report-type='{report_type}']"
-
-
-def download_report_by_selector(page: Page, selector: str) -> str:
-    page.wait_for_selector(selector, timeout=10000)
-
-    with page.expect_download() as download_info:
-        page.click(selector)
-
-    download = download_info.value
-    filename: str = download.suggested_filename
-
-    save_path: str = os.path.join(os.getcwd(), filename)
-    download.save_as(save_path)
-
-    logger.info(f"Downloaded: {filename}")
-
-    return save_path
-
-
-def upload_job_to_s3_and_update_db(job_files: List[str], uprn: str) -> None:
-    bucket = "retrofit-energy-assessments-dev"
-
-    base_path = f"documents/uprn/{uprn}"
-
-    uploaded_files: List[UploadedFile] = []
-
-    for file_path in job_files:
-        filename = os.path.basename(file_path)
-        file_key = f"{base_path}/{filename}"
-
-        upload_file_to_s3(file_path, bucket, file_key)
-
-        # load row to db
-        uploaded_files.append(
-            UploadedFile(
-                s3_file_bucket=bucket,
-                s3_file_key=file_key,
-                s3_upload_timestamp=datetime.now(timezone.utc),
-                uprn=int(uprn),
-                file_source=FileSourceEnum.ECMK.value,
-            )
-        )
-
-    with db_session() as session:
-        session.add_all(uploaded_files)
-        session.commit()
-
-    pass
-
-
-def download_report() -> None:
-    username: str = ""
-    password: str = ""
-
-    property_list_file: str = (
-        "hubspot-crm-exports-southern-ra-lite-programme-3103-2026-03-31-2.xlsx"
-    )
-
-    BASE_DIR: str = os.path.dirname(os.path.dirname(__file__))
-    filepath: str = os.path.join(BASE_DIR, property_list_file)
-
-    property_id_to_address_map: Dict[str, str] = extract_addresses_from_spreadsheet(
-        filepath
-    )
-    property_ids: List[str] = list(property_id_to_address_map.keys())
-
-    matching_properties: List[str] = []
-
-    sharepoint_client = DomnaSharepointClient(
-        sharepoint_location=DomnaSites.PRIVATE_PAY
-    )
-    sharepoint_base_path = "/Projects/Southern Housing/SH-SURV-26-001/Assessments"
-
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-
-        context = browser.new_context()
-        page = context.new_page()
-
-        try:
-            login(page, username, password)
-            print("Login successful:", page.url)
-
-            go_to_assessments(page)
-
-            while True:
-                rows: Locator = page.locator("#assessmentDatatable tbody tr")
-                row_count: int = rows.count()
-
-                logger.info(f"Processing {row_count} rows on current page")
-
-                for i in range(row_count):
-                    row: Locator = rows.nth(i)
-
-                    try:
-                        cells: Locator = row.locator("td")
-
-                        first_name: str = cells.nth(1).inner_text().strip()
-                        last_name: str = cells.nth(2).inner_text().strip()
-                        address: str = cells.nth(5).inner_text().strip()
-                        postcode: str = cells.nth(7).inner_text().strip()
-                        uprn: str = cells.nth(8).inner_text().strip()
-                        status: str = cells.nth(9).inner_text().strip()
-
-                        if first_name == "Oliver" and last_name == "Stephens":
-                            continue
-
-                        if status != "Submitted (not Lodged)":
-                            continue
-
-                        property_id: str = build_property_id(address, postcode)
-
-                        if property_id not in property_ids:
-                            continue
-
-                        logger.info(f"MATCH FOUND: {property_id}")
-                        matching_properties.append(property_id)
-
-                        sharepoint_address: str = property_id_to_address_map[
-                            property_id
-                        ]
-                        go_to_assessment_details(page, row)
-
-                        report_types: List[int] = [
-                            file_download_button_types.ASSESSOR_HUB_SITENOTE_REPORT.value,
-                            file_download_button_types.SITENOTE_REPORT.value,
-                        ]
-
-                        for report_type in report_types:
-                            selector: str = build_report_selector(report_type)
-                            file_path: str = download_report_by_selector(page, selector)
-                            try:
-                                sharepoint_client.upload_file(
-                                    file_path=file_path,
-                                    sharepoint_path=f"{sharepoint_base_path}/{sharepoint_address}/1. Retrofit Assessment/A. Assessment",
-                                    file_name=os.path.basename(file_path),
-                                )
-                                # TODO: could s3 load happen for all files at once to reduce db roundtrips?
-                                if uprn:
-                                    upload_job_to_s3_and_update_db([file_path], uprn)
-                            finally:
-                                if os.path.exists(file_path):
-                                    os.remove(file_path)
-                                    logger.info(f"Deleted local file: {file_path}")
-
-                        page.go_back()
-                        page.wait_for_selector(
-                            "#assessmentDatatable tbody tr", timeout=15000
-                        )
-
-                    except PlaywrightTimeoutError as e:
-                        raise Exception(f"Timeout occurred: {str(e)}")
-
-                if not go_to_next_page(page):
-                    break
-
-        except PlaywrightTimeoutError as e:
-            raise Exception(f"Timeout occurred: {str(e)}")
-
-        finally:
-            context.close()
-            browser.close()
+from backend.ecmk_fetcher.processor import run_job
 
 
 def handler(event: Mapping[str, Any], context: Any) -> None:
-    download_report()
+    run_job()
 
 
 if __name__ == "__main__":

backend/ecmk_fetcher/processor.py

@@ -0,0 +1,139 @@
+import os
+from typing import Dict
+from playwright.sync_api import (
+    sync_playwright,
+    Locator,
+    Page,
+    Browser,
+    BrowserContext,
+)
+
+from backend.ecmk_fetcher.address_list import (
+    PropertyRow,
+    extract_addresses_from_spreadsheet,
+    mark_properties_as_processed,
+)
+from backend.ecmk_fetcher.browser import (
+    attach_debug_listeners,
+    download_with_retry,
+    go_to_assessment_details,
+    go_to_assessments,
+    go_to_next_page,
+    login,
+)
+from backend.ecmk_fetcher.reports import REPORT_TYPES, build_property_id
+from backend.ecmk_fetcher.sharepoint import upload_file_to_sharepoint
+from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
+from utils.sharepoint.domna_sites import DomnaSites
+
+
+def run_job() -> None:
+    username: str = ""
+    password: str = ""
+
+    property_list_file: str = (
+        "hubspot-crm-exports-southern-ra-lite-programme-3103-2026-03-31-2.xlsx"
+    )
+
+    BASE_DIR: str = os.path.dirname(__file__)
+    filepath: str = os.path.join(BASE_DIR, property_list_file)
+
+    property_map: Dict[str, PropertyRow] = extract_addresses_from_spreadsheet(filepath)
+
+    sharepoint_client: DomnaSharepointClient = DomnaSharepointClient(
+        sharepoint_location=DomnaSites.PRIVATE_PAY
+    )
+
+    sharepoint_base_path: str = "/Projects/Southern Housing/SH-SURV-26-001/Assessments"
+
+    with sync_playwright() as p:
+        browser: Browser = p.chromium.launch(headless=True)
+        context: BrowserContext = browser.new_context()
+        page: Page = context.new_page()
+
+        attach_debug_listeners(page)
+
+        try:
+            login(page, username, password)
+            go_to_assessments(page)
+
+            while True:
+                rows: Locator = page.locator("#assessmentDatatable tbody tr")
+                row_count: int = rows.count()
+
+                for i in range(row_count):
+                    row: Locator = rows.nth(i)
+
+                    try:
+                        cells: Locator = row.locator("td")
+
+                        first_name: str = cells.nth(1).inner_text().strip()
+                        last_name: str = cells.nth(2).inner_text().strip()
+                        address: str = cells.nth(5).inner_text().strip()
+                        postcode: str = cells.nth(7).inner_text().strip()
+                        status: str = cells.nth(9).inner_text().strip()
+
+                        if first_name == "Oliver" and last_name == "Stephens":
+                            continue
+
+                        if status != "Submitted (not Lodged)":
+                            continue
+
+                        property_id: str = build_property_id(address, postcode)
+
+                        property_row: PropertyRow | None = property_map.get(property_id)
+
+                        if not property_row:
+                            continue
+
+                        if property_row.processed:
+                            continue
+
+                        sharepoint_address: str = property_row.address
+
+                        go_to_assessment_details(page, row)
+
+                        all_uploaded: bool = True
+
+                        for report_type in REPORT_TYPES:
+                            file_path: str | None = download_with_retry(
+                                page, report_type
+                            )
+
+                            if not file_path:
+                                all_uploaded = False
+                                continue
+
+                            try:
+                                upload_file_to_sharepoint(
+                                    client=sharepoint_client,
+                                    file_path=file_path,
+                                    base_path=sharepoint_base_path,
+                                    subpath=sharepoint_address,
+                                )
+                            except Exception:
+                                all_uploaded = False
+                                raise
+                            finally:
+                                if os.path.exists(file_path):
+                                    os.remove(file_path)
+
+                        if all_uploaded:
+                            property_row.processed = True
+
+                        page.go_back()
+                        page.wait_for_selector(
+                            "#assessmentDatatable tbody tr", timeout=15000
+                        )
+
+                    except Exception as e:
+                        raise Exception(f"Row processing failed: {str(e)}") from e
+
+                if not go_to_next_page(page):
+                    break
+
+        finally:
+            context.close()
+            browser.close()
+
+        mark_properties_as_processed(filepath, property_map)

backend/ecmk_fetcher/reports.py

@@ -0,0 +1,25 @@
+from enum import Enum
+
+
+class FileDownloadButtonType(Enum):
+    ASSESSOR_HUB_SITENOTE_REPORT = 11
+    CERTIFICATE = 9
+    SITENOTE_REPORT = 8
+    RAW_XML = 7
+    SAP_WORK_SHEET = 15
+
+
+REPORT_TYPES = [
+    FileDownloadButtonType.ASSESSOR_HUB_SITENOTE_REPORT.value,
+    FileDownloadButtonType.SITENOTE_REPORT.value,
+]
+
+
+def build_report_selector(report_type: int) -> str:
+    return f"a.download-report-btn[data-report-type='{report_type}']"
+
+
+def build_property_id(address: str, postcode: str) -> str:
+    number = address.split(" ")[0]
+    postcode_clean = postcode.replace(" ", "").upper()
+    return f"{number}{postcode_clean}"

backend/ecmk_fetcher/sharepoint.py

@@ -0,0 +1,20 @@
+import os
+
+from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
+
+
+def upload_file_to_sharepoint(
+    client: DomnaSharepointClient,
+    file_path: str,
+    base_path: str,
+    subpath: str,
+) -> None:
+    filename = os.path.basename(file_path)
+
+    full_path = f"{base_path}/{subpath}/1. Retrofit Assessment/A. Assessment"
+
+    client.upload_file(
+        file_path=file_path,
+        sharepoint_path=full_path,
+        file_name=filename,
+    )

etl/hubspot/hubspotClient.py

@@ -1,6 +1,7 @@
 import os
+import time
 from enum import Enum
-from typing import Optional, cast
+from typing import Optional, cast, Callable, Any
 
 from hubspot.client import Client  # type: ignore[reportMissingTypeStubs]
 from hubspot.crm.associations import ApiException  # type: ignore[reportMissingTypeStubs]
@@ -83,6 +84,34 @@ class HubspotClient:
         # Sorry - not sorry but enjoy, Past Junte 13/03/2026
         # self.client
 
+    def _call_with_retry(self, fn: Callable[[], Any], max_retries: int = 2) -> Any:
+        """
+        Call fn(), retrying up to max_retries times on 429 rate-limit errors.
+        Waits the minimal amount: the remaining interval window reported by HubSpot headers.
+        Falls back to the full interval (10s) if headers are absent.
+
+        Note: each HubSpot sub-module (deals, companies, etc.) ships its own ApiException
+        class with no shared base beyond Exception, so we detect 429s via duck-typing.
+        """
+        for attempt in range(max_retries + 1):
+            try:
+                return fn()
+            except Exception as e:
+                status = getattr(e, "status", None)
+                if status != 429 or attempt == max_retries:
+                    raise
+                headers = getattr(e, "headers", None) or {}
+                interval_ms = int(
+                    headers.get("x-hubspot-ratelimit-interval-milliseconds", 10000)
+                )
+                wait_s = interval_ms / 1000.0
+                self.logger.warning(
+                    f"HubSpot 429 (attempt {attempt + 1}/{max_retries}), "
+                    f"waiting {wait_s:.1f}s before retry."
+                )
+                time.sleep(wait_s)
+        raise RuntimeError("Unreachable")  # pragma: no cover
+
     def get_deal_ids_from_company(self, company_id: str) -> list[str]:
         associations_api: AssociationsBasicApi = (  # type: ignore[reportUnknownMemberType]
             self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
@@ -92,12 +121,14 @@ class HubspotClient:
         after: Optional[str] = None
 
         while True:
-            response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
-                object_type="companies",
-                object_id=company_id,
-                to_object_type="deals",
-                limit=100,
-                after=after,
+            response: AssociationsPageResponse = self._call_with_retry(
+                lambda: associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+                    object_type="companies",
+                    object_id=company_id,
+                    to_object_type="deals",
+                    limit=100,
+                    after=after,
+                )
             )
 
             results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
@@ -127,11 +158,13 @@ class HubspotClient:
             associations_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
 
             # Fetch associations for this specific deal only
-            response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
-                object_type="deals",
-                object_id=deal_id,
-                to_object_type="companies",
-                limit=1,  # Expect only one associated company
+            response: AssociationsPageResponse = self._call_with_retry(
+                lambda: associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+                    object_type="deals",
+                    object_id=deal_id,
+                    to_object_type="companies",
+                    limit=1,
+                )
             )
 
             results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
@@ -161,11 +194,13 @@ class HubspotClient:
         listings_api: ObjectsBasicApi = self.client.crm.objects.basic_api  # type: ignore[reportUnknownMemberType]  # works for custom objects like "listing"
 
         # Fetch associated listing(s)
-        response: AssociationsPageResponse = associations_api.get_page(  # type: ignore[reportUnknownMemberType]
-            object_type="deals",
-            object_id=deal_id,
-            to_object_type="0-420",  # <-- to get an listing object
-            limit=1,
+        response: AssociationsPageResponse = self._call_with_retry(
+            lambda: associations_api.get_page(  # type: ignore[reportUnknownMemberType]
+                object_type="deals",
+                object_id=deal_id,
+                to_object_type="0-420",
+                limit=1,
+            )
         )
 
         results: list[AssociationsResult] = cast(list[AssociationsResult], response.results)  # type: ignore[reportUnknownMemberType]
@@ -178,14 +213,16 @@ class HubspotClient:
         self.logger.info(f"Associated listing ID for deal {deal_id}: {listing_id}")
 
         # Fetch listing details (the "listing information")
-        listing: HubspotObject = listings_api.get_by_id(  # type: ignore[reportUnknownMemberType]
-            object_type="0-420",  # again, must match your HubSpot object name
-            object_id=listing_id,
-            properties=[
-                "national_uprn",
-                "domna_property_id",
-                "owner_property_id",
-            ],
+        listing: HubspotObject = self._call_with_retry(
+            lambda: listings_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+                object_type="0-420",
+                object_id=listing_id,
+                properties=[
+                    "national_uprn",
+                    "domna_property_id",
+                    "owner_property_id",
+                ],
+            )
         )
 
         listing_info: dict[str, str] = cast(dict[str, str], listing.properties)  # type: ignore[reportUnknownMemberType]
@@ -196,44 +233,47 @@ class HubspotClient:
     def from_deal_id_get_info(self, deal_id: str) -> dict[str, str]:
         deals_api: DealsBasicApi = self.client.crm.deals.basic_api  # type: ignore[reportUnknownMemberType]
 
-        deal: HubspotObject = deals_api.get_by_id(  # type: ignore[reportUnknownMemberType]
-            deal_id,
-            properties=[
-                "dealname",
-                "dealstage",
-                "pipeline",
-                "outcome",
-                "outcome_notes",
-                "project_code",
-                "major_condition_issue_description",
-                "major_condition_issue_photos",
-                "coordination_status__stage_1_",
-                "retrofit_design_status",
-                "pashub_link",
-                "sharepoint_link",
-                "dampmould_growth",
-                "pre_sap",
-                "coordinator",
-                "mtp_completion_date",
-                "mtp_re_model_completion_date",
-                "ioe_v3_completion_date",
-                "proposed_measures",
-                "approved_package",
-                "designer",
-                "design_completion_date",
-                "actual_measures_installed",
-                "installer",
-                "installer_handover",
-                "lodgement_status",
-                "measures_lodgement_date",
-                "lodgement_date",
-                "expected_commencement_date",
-                "surveyor",
-                "confirmed_survey_date",
-                "confirmed_survey_time",
-                "surveyed_date",
-                "design_type",
-            ],
+        deal: HubspotObject = self._call_with_retry(
+            lambda: deals_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+                deal_id,
+                properties=[
+                    "dealname",
+                    "dealstage",
+                    "pipeline",
+                    "outcome",
+                    "outcome_notes",
+                    "project_code",
+                    "major_condition_issue_description",
+                    "major_condition_issue_photos",
+                    "coordination_status__stage_1_",
+                    "retrofit_design_status",
+                    "pashub_link",
+                    "sharepoint_link",
+                    "dampmould_growth",
+                    "damp_mould_and_repairs_comments",
+                    "pre_sap",
+                    "coordinator",
+                    "mtp_completion_date",
+                    "mtp_re_model_completion_date",
+                    "ioe_v3_completion_date",
+                    "proposed_measures",
+                    "approved_package",
+                    "designer",
+                    "design_completion_date",
+                    "actual_measures_installed",
+                    "installer",
+                    "installer_handover",
+                    "lodgement_status",
+                    "measures_lodgement_date",
+                    "lodgement_date",
+                    "expected_commencement_date",
+                    "surveyor",
+                    "confirmed_survey_date",
+                    "confirmed_survey_time",
+                    "surveyed_date",
+                    "design_type",
+                ],
+            )
         )
 
         deal_info: dict[str, str] = cast(dict[str, str], deal.properties)  # type: ignore[reportUnknownMemberType]
@@ -260,11 +300,11 @@ class HubspotClient:
     def get_company_information(self, company_id: str) -> CompanyData:
         companies_api: CompaniesBasicApi = self.client.crm.companies.basic_api  # type: ignore[reportUnknownMemberType]
 
-        company: HubspotObject = companies_api.get_by_id(  # type: ignore[reportUnknownMemberType]
-            company_id,
-            properties=[
-                "name",
-            ],
+        company: HubspotObject = self._call_with_retry(
+            lambda: companies_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+                company_id,
+                properties=["name"],
+            )
         )
 
         company_info: CompanyData = company.properties  # type: ignore[reportUnknownMemberType]
@@ -401,8 +441,10 @@ class HubspotClient:
     def create_line_item_from_product(self, product_id: str, quantity: int = 1) -> str:
         # Fetch product mapping
         products_api: ProductsBasicApi = self.client.crm.products.basic_api  # type: ignore[reportUnknownMemberType]
-        product: HubspotObject = products_api.get_by_id(  # type: ignore[reportUnknownMemberType]
-            product_id, properties=["name", "price", "hs_price"]
+        product: HubspotObject = self._call_with_retry(
+            lambda: products_api.get_by_id(  # type: ignore[reportUnknownMemberType]
+                product_id, properties=["name", "price", "hs_price"]
+            )
         )
         properties: dict[str, str] = cast(dict[str, str], product.properties)  # type: ignore[reportUnknownMemberType]
 
@@ -423,7 +465,9 @@ class HubspotClient:
 
         # Create line item
         line_items_api: LineItemsBasicApi = self.client.crm.line_items.basic_api  # type: ignore[reportUnknownMemberType]
-        line_item: HubspotObject = line_items_api.create(line_item_input)  # type: ignore[reportUnknownMemberType]
+        line_item: HubspotObject = self._call_with_retry(
+            lambda: line_items_api.create(line_item_input)  # type: ignore[reportUnknownMemberType]
+        )
         return cast(str, line_item.id)  # type: ignore[reportUnknownMemberType]
 
     def associate_line_item_to_deal(self, line_item_id: str, deal_id: str) -> None:
@@ -431,17 +475,19 @@ class HubspotClient:
 
         association_api: AssociationsBasicApi = self.client.crm.associations.v4.basic_api  # type: ignore[reportUnknownMemberType]
 
-        association_api.create(  # type: ignore[reportUnknownMemberType]
-            "0-3",  # to object type
-            deal_id,  # to object id
-            "line_items",  # from object type
-            line_item_id,  # from object id
-            [
-                AssociationSpec(
-                    association_category="HUBSPOT_DEFINED",
-                    association_type_id=19,  # line_item → deal
-                )
-            ],
+        self._call_with_retry(
+            lambda: association_api.create(  # type: ignore[reportUnknownMemberType]
+                "0-3",
+                deal_id,
+                "line_items",
+                line_item_id,
+                [
+                    AssociationSpec(
+                        association_category="HUBSPOT_DEFINED",
+                        association_type_id=19,
+                    )
+                ],
+            )
         )
 
     def add_product_line_item_to_deal(

etl/hubspot/hubspotDataTodB.py

@@ -181,6 +181,11 @@ class HubspotDataToDb:
                 deal_in_db.dampmould_growth == hs_deal.get("dampmould_growth"),
                 "dampmould_growth mismatch",
             ),
+            soft_assert(
+                deal_in_db.damp_mould_and_repairs_comments
+                == hs_deal.get("damp_mould_and_repairs_comments"),
+                "damp_mould_and_repairs_comments mismatch",
+            ),
             soft_assert(
                 deal_in_db.pre_sap == hs_deal.get("pre_sap"),
                 "pre_sap mismatch",
@@ -190,15 +195,18 @@ class HubspotDataToDb:
                 "coordinator mismatch",
             ),
             soft_assert(
-                deal_in_db.mtp_completion_date == self._parse_hs_date(hs_deal.get("mtp_completion_date")),
+                deal_in_db.mtp_completion_date
+                == self._parse_hs_date(hs_deal.get("mtp_completion_date")),
                 "mtp_completion_date mismatch",
             ),
             soft_assert(
-                deal_in_db.mtp_re_model_completion_date == self._parse_hs_date(hs_deal.get("mtp_re_model_completion_date")),
+                deal_in_db.mtp_re_model_completion_date
+                == self._parse_hs_date(hs_deal.get("mtp_re_model_completion_date")),
                 "mtp_re_model_completion_date mismatch",
             ),
             soft_assert(
-                deal_in_db.ioe_v3_completion_date == self._parse_hs_date(hs_deal.get("ioe_v3_completion_date")),
+                deal_in_db.ioe_v3_completion_date
+                == self._parse_hs_date(hs_deal.get("ioe_v3_completion_date")),
                 "ioe_v3_completion_date mismatch",
             ),
             soft_assert(
@@ -214,11 +222,13 @@ class HubspotDataToDb:
                 "designer mismatch",
             ),
             soft_assert(
-                deal_in_db.design_completion_date == self._parse_hs_date(hs_deal.get("design_completion_date")),
+                deal_in_db.design_completion_date
+                == self._parse_hs_date(hs_deal.get("design_completion_date")),
                 "design_completion_date mismatch",
             ),
             soft_assert(
-                deal_in_db.actual_measures_installed == hs_deal.get("actual_measures_installed"),
+                deal_in_db.actual_measures_installed
+                == hs_deal.get("actual_measures_installed"),
                 "actual_measures_installed mismatch",
             ),
             soft_assert(
@@ -234,15 +244,18 @@ class HubspotDataToDb:
                 "lodgement_status mismatch",
             ),
             soft_assert(
-                deal_in_db.measures_lodgement_date == self._parse_hs_date(hs_deal.get("measures_lodgement_date")),
+                deal_in_db.measures_lodgement_date
+                == self._parse_hs_date(hs_deal.get("measures_lodgement_date")),
                 "measures_lodgement_date mismatch",
             ),
             soft_assert(
-                deal_in_db.lodgement_date == self._parse_hs_date(hs_deal.get("lodgement_date")),
+                deal_in_db.lodgement_date
+                == self._parse_hs_date(hs_deal.get("lodgement_date")),
                 "lodgement_date mismatch",
             ),
             soft_assert(
-                deal_in_db.expected_commencement_date == self._parse_hs_date(hs_deal.get("expected_commencement_date")),
+                deal_in_db.expected_commencement_date
+                == self._parse_hs_date(hs_deal.get("expected_commencement_date")),
                 "expected_commencement_date mismatch",
             ),
             soft_assert(
@@ -250,15 +263,18 @@ class HubspotDataToDb:
                 "surveyor mismatch",
             ),
             soft_assert(
-                deal_in_db.confirmed_survey_date == self._parse_hs_date(hs_deal.get("confirmed_survey_date")),
+                deal_in_db.confirmed_survey_date
+                == self._parse_hs_date(hs_deal.get("confirmed_survey_date")),
                 "confirmed_survey_date mismatch",
             ),
             soft_assert(
-                deal_in_db.confirmed_survey_time == hs_deal.get("confirmed_survey_time"),
+                deal_in_db.confirmed_survey_time
+                == hs_deal.get("confirmed_survey_time"),
                 "confirmed_survey_time mismatch",
             ),
             soft_assert(
-                deal_in_db.surveyed_date == self._parse_hs_date(hs_deal.get("surveyed_date")),
+                deal_in_db.surveyed_date
+                == self._parse_hs_date(hs_deal.get("surveyed_date")),
                 "surveyed_date mismatch",
             ),
             soft_assert(
@@ -349,9 +365,11 @@ class HubspotDataToDb:
                 for attr, value in {
                     "dealname": deal_data.get("dealname"),
                     "dealstage": deal_data.get("dealstage"),
-                    "listing_id": listing.get("listing_id"),
-                    "landlord_property_id": listing.get("owner_property_id"),
-                    "uprn": listing.get("national_uprn"),
+                    "listing_id": listing.get("listing_id", None) if listing else None,
+                    "landlord_property_id": (
+                        listing.get("owner_property_id", None) if listing else None
+                    ),
+                    "uprn": listing.get("national_uprn", None) if listing else None,
                     "outcome": deal_data.get("outcome"),
                     "outcome_notes": deal_data.get("outcome_notes"),
                     "project_code": deal_data.get("project_code"),
@@ -369,26 +387,49 @@ class HubspotDataToDb:
                     "pashub_link": deal_data.get("pashub_link"),
                     "sharepoint_link": deal_data.get("sharepoint_link"),
                     "dampmould_growth": deal_data.get("dampmould_growth"),
+                    "damp_mould_and_repairs_comments": deal_data.get(
+                        "damp_mould_and_repairs_comments"
+                    ),
                     "pre_sap": deal_data.get("pre_sap"),
                     "coordinator": deal_data.get("coordinator"),
-                    "mtp_completion_date": self._parse_hs_date(deal_data.get("mtp_completion_date")),
-                    "mtp_re_model_completion_date": self._parse_hs_date(deal_data.get("mtp_re_model_completion_date")),
-                    "ioe_v3_completion_date": self._parse_hs_date(deal_data.get("ioe_v3_completion_date")),
+                    "mtp_completion_date": self._parse_hs_date(
+                        deal_data.get("mtp_completion_date")
+                    ),
+                    "mtp_re_model_completion_date": self._parse_hs_date(
+                        deal_data.get("mtp_re_model_completion_date")
+                    ),
+                    "ioe_v3_completion_date": self._parse_hs_date(
+                        deal_data.get("ioe_v3_completion_date")
+                    ),
                     "proposed_measures": deal_data.get("proposed_measures"),
                     "approved_package": deal_data.get("approved_package"),
                     "designer": deal_data.get("designer"),
-                    "design_completion_date": self._parse_hs_date(deal_data.get("design_completion_date")),
-                    "actual_measures_installed": deal_data.get("actual_measures_installed"),
+                    "design_completion_date": self._parse_hs_date(
+                        deal_data.get("design_completion_date")
+                    ),
+                    "actual_measures_installed": deal_data.get(
+                        "actual_measures_installed"
+                    ),
                     "installer": deal_data.get("installer"),
                     "installer_handover": deal_data.get("installer_handover"),
                     "lodgement_status": deal_data.get("lodgement_status"),
-                    "measures_lodgement_date": self._parse_hs_date(deal_data.get("measures_lodgement_date")),
-                    "lodgement_date": self._parse_hs_date(deal_data.get("lodgement_date")),
-                    "expected_commencement_date": self._parse_hs_date(deal_data.get("expected_commencement_date")),
+                    "measures_lodgement_date": self._parse_hs_date(
+                        deal_data.get("measures_lodgement_date")
+                    ),
+                    "lodgement_date": self._parse_hs_date(
+                        deal_data.get("lodgement_date")
+                    ),
+                    "expected_commencement_date": self._parse_hs_date(
+                        deal_data.get("expected_commencement_date")
+                    ),
                     "surveyor": deal_data.get("surveyor"),
-                    "confirmed_survey_date": self._parse_hs_date(deal_data.get("confirmed_survey_date")),
+                    "confirmed_survey_date": self._parse_hs_date(
+                        deal_data.get("confirmed_survey_date")
+                    ),
                     "confirmed_survey_time": deal_data.get("confirmed_survey_time"),
-                    "surveyed_date": self._parse_hs_date(deal_data.get("surveyed_date")),
+                    "surveyed_date": self._parse_hs_date(
+                        deal_data.get("surveyed_date")
+                    ),
                     "design_type": deal_data.get("design_type"),
                 }.items():
                     setattr(existing, attr, value or getattr(existing, attr))
@@ -435,9 +476,9 @@ class HubspotDataToDb:
                     deal_id=deal_id,
                     dealname=deal_data.get("dealname"),
                     dealstage=deal_data.get("dealstage"),
-                    listing_id=listing.get("listing_id"),
-                    landlord_property_id=listing.get("owner_property_id"),
-                    uprn=listing.get("national_uprn"),
+                    listing_id=listing.get("listing_id", None) if listing else None,
+                    landlord_property_id=listing.get("owner_property_id") if listing else None,
+                    uprn=listing.get("national_uprn") if listing else None,
                     outcome=deal_data.get("outcome"),
                     outcome_notes=deal_data.get("outcome_notes"),
                     project_code=deal_data.get("project_code"),
@@ -453,24 +494,43 @@ class HubspotDataToDb:
                     pashub_link=deal_data.get("pashub_link"),
                     sharepoint_link=deal_data.get("sharepoint_link"),
                     dampmould_growth=deal_data.get("dampmould_growth"),
+                    damp_mould_and_repairs_comments=deal_data.get(
+                        "damp_mould_and_repairs_comments"
+                    ),
                     pre_sap=deal_data.get("pre_sap"),
                     coordinator=deal_data.get("coordinator"),
-                    mtp_completion_date=self._parse_hs_date(deal_data.get("mtp_completion_date")),
-                    mtp_re_model_completion_date=self._parse_hs_date(deal_data.get("mtp_re_model_completion_date")),
-                    ioe_v3_completion_date=self._parse_hs_date(deal_data.get("ioe_v3_completion_date")),
+                    mtp_completion_date=self._parse_hs_date(
+                        deal_data.get("mtp_completion_date")
+                    ),
+                    mtp_re_model_completion_date=self._parse_hs_date(
+                        deal_data.get("mtp_re_model_completion_date")
+                    ),
+                    ioe_v3_completion_date=self._parse_hs_date(
+                        deal_data.get("ioe_v3_completion_date")
+                    ),
                     proposed_measures=deal_data.get("proposed_measures"),
                     approved_package=deal_data.get("approved_package"),
                     designer=deal_data.get("designer"),
-                    design_completion_date=self._parse_hs_date(deal_data.get("design_completion_date")),
-                    actual_measures_installed=deal_data.get("actual_measures_installed"),
+                    design_completion_date=self._parse_hs_date(
+                        deal_data.get("design_completion_date")
+                    ),
+                    actual_measures_installed=deal_data.get(
+                        "actual_measures_installed"
+                    ),
                     installer=deal_data.get("installer"),
                     installer_handover=deal_data.get("installer_handover"),
                     lodgement_status=deal_data.get("lodgement_status"),
-                    measures_lodgement_date=self._parse_hs_date(deal_data.get("measures_lodgement_date")),
+                    measures_lodgement_date=self._parse_hs_date(
+                        deal_data.get("measures_lodgement_date")
+                    ),
                     lodgement_date=self._parse_hs_date(deal_data.get("lodgement_date")),
-                    expected_commencement_date=self._parse_hs_date(deal_data.get("expected_commencement_date")),
+                    expected_commencement_date=self._parse_hs_date(
+                        deal_data.get("expected_commencement_date")
+                    ),
                     surveyor=deal_data.get("surveyor"),
-                    confirmed_survey_date=self._parse_hs_date(deal_data.get("confirmed_survey_date")),
+                    confirmed_survey_date=self._parse_hs_date(
+                        deal_data.get("confirmed_survey_date")
+                    ),
                     confirmed_survey_time=deal_data.get("confirmed_survey_time"),
                     surveyed_date=self._parse_hs_date(deal_data.get("surveyed_date")),
                     design_type=deal_data.get("design_type"),

etl/hubspot/scripts/scraper/bulk_load.py

@@ -6,7 +6,13 @@ import json
 
 PIPELINE_ID = Pipeline.OPERATIONS_SOCIAL_HOUSING.value
 
-companies = list([Companies.THE_GUINESS_PARTNERSHIP, Companies.SOUTHERN_HOUSING_GROUP])
+companies = list(
+    [
+        # Companies.THE_GUINESS_PARTNERSHIP,
+        # Companies.SOUTHERN_HOUSING_GROUP,
+        Companies.CALICO_HOMES,
+    ]
+)
 
 
 def bulk_load(companies: list[Companies] | None = None) -> None:
@@ -17,20 +23,24 @@ def bulk_load(companies: list[Companies] | None = None) -> None:
     hubspot = HubspotClient()
     targets = companies or list(Companies)
 
-    for company in tqdm(targets, desc="Companies", unit="co"):
+    for company in tqdm(targets, desc="Companies", unit="co", leave=False):
         company_id = company.value
         deal_ids = hubspot.get_deal_ids_from_company(company_id)
 
         processed = 0
-        with tqdm(deal_ids, desc=company.name, unit="deal", leave=False) as deal_bar:
+        with tqdm(
+            deal_ids, desc=company.name, unit="deal", leave=True, position=0
+        ) as deal_bar:
             for deal_id in deal_bar:
                 deal_data = hubspot.from_deal_id_get_info(deal_id)
                 if deal_data.get("pipeline") != PIPELINE_ID:
                     deal_bar.set_postfix({"status": "skip", "deal": deal_id})
                     continue
-
                 deal_bar.set_postfix({"status": "uploading", "deal": deal_id})
-                handler({"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]}, context=None)
+                handler(
+                    {"Records": [{"body": json.dumps({"hubspot_deal_id": deal_id})}]},
+                    context=None,
+                )
                 processed += 1
                 deal_bar.set_postfix({"status": "done", "deal": deal_id})
 

etl/hubspot/scripts/scraper/main.py

@@ -1,17 +1,7 @@
-"""
-1) [completed]Get hubspot deal properties from one deal
-2) Put it in some class
-3) [completed] Load the db and check if upsert it into the table
-4) [completed]Getting working on a AWS lambda
-5) [completed] subtask and tasks history
-6) [completed]The new sexy deal properties, move it over
-"""
-
 from etl.hubspot.hubspotClient import HubspotClient
 from etl.hubspot.hubspotDataTodB import HubspotDataToDb
 from backend.utils.subtasks import task_handler
 from typing import Any
-import json
 
 
 @task_handler()
@@ -22,12 +12,11 @@ def handler(body: dict[str, Any], context: Any) -> None:
         raise RuntimeError(
             "Missing Hubspot Deal ID in SQS body request, 'hubspot_deal_id'"
         )
+        hubspot_deal_id = "327170793707"
 
     hubspot: HubspotClient = HubspotClient()
     dbloader: HubspotDataToDb = HubspotDataToDb()
-
     deal = dbloader.find_deal_with_deal_id(hubspot_deal_id)
-
     if deal:
         dbloader.update_deal_with_checks(deal, hubspot)
     else: