Merge branch 'main' into feature/epc-dataclasses

asset_list/app.py

@@ -74,23 +74,23 @@ def app():
     """
 
     data_folder = "/workspaces/model/asset_list"
-    data_filename = "Calico ARA Upload Review.xlsx"
+    data_filename = "Waverley UPRN Match.xlsx"
-    sheet_name = "Sheet1"
+    sheet_name = "in"
-    postcode_column = "Postcode"
+    postcode_column = "postcode_clean"
-    address1_column = "Units"
+    address1_column = "domna_found_address"
     address1_method = None
-    fulladdress_column = "Units"
+    fulladdress_column = "domna_found_address"
-    address_cols_to_concat = ["Units"]
+    address_cols_to_concat = []
     missing_postcodes_method = None
     landlord_year_built = None
-    landlord_os_uprn = None
+    landlord_os_uprn = "domna_found_uprn"
-    landlord_property_type = None  # Good to include if landlord gave
+    landlord_property_type = "Property Type 1"  # Good to include if landlord gave
     landlord_built_form = None  # Good to include if landlord gave
     landlord_wall_construction = None
     landlord_roof_construction = None
     landlord_heating_system = None
     landlord_existing_pv = None
-    landlord_property_id = "llid"
+    landlord_property_id = "WBC Ref"
     landlord_sap = None
     outcomes_filename = None
     outcomes_sheetname = None

backend/address2UPRN/main.py

@@ -351,9 +351,9 @@ def handler(event, context, local=False):
                 {
                     "body": json.dumps(
                         {
-                            "task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917",
+                            "sub_task_id": "d7363c83-2ef7-4474-b30f-980fd587350c",
-                            "sub_task_id": "6a427b6e-1ece-4983-b1e5-9bffccc53d1d",
+                            "task_id": "a042af13-8b57-4709-ad22-ecac1ccca4bd",
-                            "s3_uri": "s3://retrofit-data-dev/ara_postcode_splitter_batches/e31f2f21-175b-4a91-a3ec-a6baa325e917/8673913b-1a88-42d7-8578-0449123d94b0/2026-02-18T11:47:00.822579_f95467f5.csv",
+                            "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/essex/Copy of EPC register Essex(August 2025)(in) (2).csv",
                         }
                     )
                 }

backend/pashub_fetcher/handler/handler.py

@@ -20,6 +20,7 @@ from backend.pashub_fetcher.pashub_to_ara_trigger_request import (
 )
 from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
 from backend.pashub_fetcher.token_getter import get_token_from_local_storage
+from backend.utils.subtasks import task_handler
 from utils.logger import setup_logger
 from utils.s3 import upload_file_to_s3
 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
@@ -165,6 +166,7 @@ def process_job(
     return job_files
 
 
+@task_handler()
 def handler(event: Mapping[str, Any], context: Any) -> None:
     logger.info("Received message")
     logger.info(f"Number of events: {len(event.get('Records', []))}")

etl/hubspot/scripts/scraper/main.py

@@ -22,7 +22,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
     hubspot_client = HubspotClient()
 
     sqs_client = boto3.client("sqs")
-    PASHUB_TRIGGER_QUEUE_URL = get_settings().PASHUB_TO_ARA_SQS_URL
 
     payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body)
     hubspot_deal_id: str = payload.hubspot_deal_id
@@ -41,7 +40,6 @@ def handler(body: dict[str, Any], context: Any) -> None:
 
     deal_changed = False
     if not db_deal:
-        # New hubspot deal, no diffing to do
         logger.info(f"New HubSpot deal of ID {hubspot_deal_id}. Loading to database...")
         if company:
             company_data: CompanyData = hubspot_client.get_company_information(company)
@@ -49,6 +47,15 @@ def handler(body: dict[str, Any], context: Any) -> None:
             db_client.upsert_organisation(company_data)
 
         db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client)
+
+        # ==============================
+        # Orchestration of other lambdas
+        # ==============================
+        if hubspot_deal["pashub_link"]:
+            logger.info(
+                f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
+            )
+            _trigger_pashub_fetcher(sqs_client, hubspot_deal)
     else:
         # Deal already in db, check whether anything has changed
         logger.info(
@@ -84,22 +91,7 @@ def handler(body: dict[str, Any], context: Any) -> None:
             logger.info(
                 f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}"
             )
-            message_body: Dict[str, Optional[str]] = {
+            _trigger_pashub_fetcher(sqs_client, hubspot_deal)
-                "pashub_link": hubspot_deal["pashub_link"],
-                "address": None,  # potentially available from Listing, leave as None for now
-                "sharepoint_link": hubspot_deal["sharepoint_link"],
-                "uprn": hubspot_deal["national_uprn"],
-                "landlord_property_id": hubspot_deal["owner_property_id"],
-                "deal_stage": hubspot_deal["deal_stage"],
-            }
-
-            response = sqs_client.send_message(
-                QueueUrl=PASHUB_TRIGGER_QUEUE_URL, MessageBody=json.dumps(message_body)
-            )
-
-            logger.info(
-                f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
-            )
         else:
             logger.info(
                 f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}"
@@ -108,6 +100,26 @@ def handler(body: dict[str, Any], context: Any) -> None:
     print("done")
 
 
+def _trigger_pashub_fetcher(sqs_client: Any, hubspot_deal: Dict[str, str]) -> None:
+    message_body: Dict[str, Optional[str]] = {
+        "pashub_link": hubspot_deal["pashub_link"],
+        "address": None,  # potentially available from Listing, leave as None for now
+        "sharepoint_link": hubspot_deal["sharepoint_link"],
+        "uprn": hubspot_deal["national_uprn"],
+        "landlord_property_id": hubspot_deal["owner_property_id"],
+        "deal_stage": hubspot_deal["deal_stage"],
+    }
+
+    response = sqs_client.send_message(
+        QueueUrl=get_settings().PASHUB_TO_ARA_SQS_URL,
+        MessageBody=json.dumps(message_body),
+    )
+
+    logger.info(
+        f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}"
+    )
+
+
 if __name__ == "__main__":
     handler({"hubspot_deal_id": "371470706915"}, "")
     print("beep")

utils/s3.py

@@ -330,7 +330,7 @@ def read_csv_from_s3(bucket_name: str, filepath: str) -> list[dict[str, str]]:
     body = s3_object["Body"].read()
 
     # Use StringIO to create a file-like object from the string
-    csv_data = StringIO(body.decode("utf-8"))
+    csv_data = StringIO(body.decode("utf-8-sig"))
 
     # Use csv library to read it into a list of dictionaries
     reader = csv.DictReader(csv_data)