diff --git a/asset_list/app.py b/asset_list/app.py index 5794eaf3..b0030667 100644 --- a/asset_list/app.py +++ b/asset_list/app.py @@ -74,23 +74,23 @@ def app(): """ data_folder = "/workspaces/model/asset_list" - data_filename = "Calico ARA Upload Review.xlsx" - sheet_name = "Sheet1" - postcode_column = "Postcode" - address1_column = "Units" + data_filename = "Waverley UPRN Match.xlsx" + sheet_name = "in" + postcode_column = "postcode_clean" + address1_column = "domna_found_address" address1_method = None - fulladdress_column = "Units" - address_cols_to_concat = ["Units"] + fulladdress_column = "domna_found_address" + address_cols_to_concat = [] missing_postcodes_method = None landlord_year_built = None - landlord_os_uprn = None - landlord_property_type = None # Good to include if landlord gave + landlord_os_uprn = "domna_found_uprn" + landlord_property_type = "Property Type 1" # Good to include if landlord gave landlord_built_form = None # Good to include if landlord gave landlord_wall_construction = None landlord_roof_construction = None landlord_heating_system = None landlord_existing_pv = None - landlord_property_id = "llid" + landlord_property_id = "WBC Ref" landlord_sap = None outcomes_filename = None outcomes_sheetname = None diff --git a/backend/address2UPRN/main.py b/backend/address2UPRN/main.py index 97e2037a..79c0de69 100644 --- a/backend/address2UPRN/main.py +++ b/backend/address2UPRN/main.py @@ -351,9 +351,9 @@ def handler(event, context, local=False): { "body": json.dumps( { - "task_id": "e31f2f21-175b-4a91-a3ec-a6baa325e917", - "sub_task_id": "6a427b6e-1ece-4983-b1e5-9bffccc53d1d", - "s3_uri": "s3://retrofit-data-dev/ara_postcode_splitter_batches/e31f2f21-175b-4a91-a3ec-a6baa325e917/8673913b-1a88-42d7-8578-0449123d94b0/2026-02-18T11:47:00.822579_f95467f5.csv", + "sub_task_id": "d7363c83-2ef7-4474-b30f-980fd587350c", + "task_id": "a042af13-8b57-4709-ad22-ecac1ccca4bd", + "s3_uri": "s3://retrofit-data-dev/ara_raw_inputs/essex/Copy of EPC register Essex(August 2025)(in) (2).csv", } ) } diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 793b9d70..3689efe9 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -20,6 +20,7 @@ from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( ) from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from backend.utils.subtasks import task_handler from utils.logger import setup_logger from utils.s3 import upload_file_to_s3 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient @@ -165,6 +166,7 @@ def process_job( return job_files +@task_handler() def handler(event: Mapping[str, Any], context: Any) -> None: logger.info("Received message") logger.info(f"Number of events: {len(event.get('Records', []))}") diff --git a/etl/hubspot/scripts/scraper/main.py b/etl/hubspot/scripts/scraper/main.py index d754cbb1..e6c788ea 100644 --- a/etl/hubspot/scripts/scraper/main.py +++ b/etl/hubspot/scripts/scraper/main.py @@ -22,7 +22,6 @@ def handler(body: dict[str, Any], context: Any) -> None: hubspot_client = HubspotClient() sqs_client = boto3.client("sqs") - PASHUB_TRIGGER_QUEUE_URL = get_settings().PASHUB_TO_ARA_SQS_URL payload = HubspotTriggerOrchestratorTriggerRequest.model_validate(body) hubspot_deal_id: str = payload.hubspot_deal_id @@ -41,7 +40,6 @@ def handler(body: dict[str, Any], context: Any) -> None: deal_changed = False if not db_deal: - # New hubspot deal, no diffing to do logger.info(f"New HubSpot deal of ID {hubspot_deal_id}. Loading to database...") if company: company_data: CompanyData = hubspot_client.get_company_information(company) @@ -49,6 +47,15 @@ def handler(body: dict[str, Any], context: Any) -> None: db_client.upsert_organisation(company_data) db_client.upsert_deal(hubspot_deal, company, listing, hubspot_client) + + # ============================== + # Orchestration of other lambdas + # ============================== + if hubspot_deal["pashub_link"]: + logger.info( + f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" + ) + _trigger_pashub_fetcher(sqs_client, hubspot_deal) else: # Deal already in db, check whether anything has changed logger.info( @@ -84,22 +91,7 @@ def handler(body: dict[str, Any], context: Any) -> None: logger.info( f"Triggering Pas Hub file fetcher for HubSpot deal ID {hubspot_deal_id}" ) - message_body: Dict[str, Optional[str]] = { - "pashub_link": hubspot_deal["pashub_link"], - "address": None, # potentially available from Listing, leave as None for now - "sharepoint_link": hubspot_deal["sharepoint_link"], - "uprn": hubspot_deal["national_uprn"], - "landlord_property_id": hubspot_deal["owner_property_id"], - "deal_stage": hubspot_deal["deal_stage"], - } - - response = sqs_client.send_message( - QueueUrl=PASHUB_TRIGGER_QUEUE_URL, MessageBody=json.dumps(message_body) - ) - - logger.info( - f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}" - ) + _trigger_pashub_fetcher(sqs_client, hubspot_deal) else: logger.info( f"Not Triggering PasHub file fetcher for HubSpot deal ID {hubspot_deal_id}" @@ -108,6 +100,26 @@ def handler(body: dict[str, Any], context: Any) -> None: print("done") +def _trigger_pashub_fetcher(sqs_client: Any, hubspot_deal: Dict[str, str]) -> None: + message_body: Dict[str, Optional[str]] = { + "pashub_link": hubspot_deal["pashub_link"], + "address": None, # potentially available from Listing, leave as None for now + "sharepoint_link": hubspot_deal["sharepoint_link"], + "uprn": hubspot_deal["national_uprn"], + "landlord_property_id": hubspot_deal["owner_property_id"], + "deal_stage": hubspot_deal["deal_stage"], + } + + response = sqs_client.send_message( + QueueUrl=get_settings().PASHUB_TO_ARA_SQS_URL, + MessageBody=json.dumps(message_body), + ) + + logger.info( + f"Sent message to Pashub To Ara queue. MessageId: {response['MessageId']}" + ) + + if __name__ == "__main__": handler({"hubspot_deal_id": "371470706915"}, "") print("beep") diff --git a/utils/s3.py b/utils/s3.py index 242e0db5..930e2e15 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -330,7 +330,7 @@ def read_csv_from_s3(bucket_name: str, filepath: str) -> list[dict[str, str]]: body = s3_object["Body"].read() # Use StringIO to create a file-like object from the string - csv_data = StringIO(body.decode("utf-8")) + csv_data = StringIO(body.decode("utf-8-sig")) # Use csv library to read it into a list of dictionaries reader = csv.DictReader(csv_data)