From abd0ed882f155a7f9c56e98f3ca318cf705185d3 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 14:53:55 +0000 Subject: [PATCH 01/10] get uprn from pashub job id --- backend/pashub_fetcher/handler/handler.py | 10 ++++++++-- backend/pashub_fetcher/pashub_client.py | 15 +++++++++++++++ .../pashub_to_ara_trigger_request.py | 9 +++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 backend/pashub_fetcher/pashub_to_ara_trigger_request.py diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 38b79ab4..8a02ce72 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -1,6 +1,6 @@ import os import re -from typing import Any, Dict, List, Mapping +from typing import Any, Dict, List, Mapping, Optional from openpyxl import load_workbook from backend.pashub_fetcher.job import Job @@ -80,10 +80,16 @@ def handler(event: Mapping[str, Any], context: Any) -> None: for job in jobs: try: + job_id = job["id"] + uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) + logger.info(f"Got UPRN {uprn} for job {job_id}") + job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id( - job["id"] + job_id ) + # Upload files to s3 + # Upload files to sharepoint job_path = f"{BASE_PATH}/{job['address']}" diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index efc21803..20b8590d 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -71,6 +71,21 @@ class PashubClient: return saved_files + def get_uprn_by_job_id(self, job_id: str) -> Optional[str]: + logger.info(f"Getting UPRN for job ID {job_id}") + url = f"{self.base}/jobs/{job_id}" + + r = self.session.get(url) + if r.status_code == 401: + raise UnauthorizedError("Token expired or invalid") + + r.raise_for_status() + + try: + return r.json()["uprn"] + except Exception: + return None + def _get_core_file_type(self, file: EvidenceFileData) -> Optional[CoreFiles]: for core_file in CoreFiles: if file.file_name.startswith(core_file.value): diff --git a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py new file mode 100644 index 00000000..d97281fd --- /dev/null +++ b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py @@ -0,0 +1,9 @@ +from typing import Optional + +from pydantic import BaseModel + + +class PashubToAraTriggerRequest(BaseModel): + pashub_job_id: str + + sharepoint_url: Optional[str] = None From 4ac7ed6f096b3bb49501e85cc006f8f85f9295ec Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 15:38:19 +0000 Subject: [PATCH 02/10] define sqlalchemy objects for new database tables / types --- backend/app/db/models/uploaded_files.py | 44 +++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 backend/app/db/models/uploaded_files.py diff --git a/backend/app/db/models/uploaded_files.py b/backend/app/db/models/uploaded_files.py new file mode 100644 index 00000000..d4bf48d8 --- /dev/null +++ b/backend/app/db/models/uploaded_files.py @@ -0,0 +1,44 @@ +import enum +from sqlalchemy import TIMESTAMP, BigInteger, Column, Text, Enum as SqlEnum + +from backend.app.db.base import Base + + +class FileTypeEnum(enum.Enum): + PHOTO_PACK = "photo_pack" + SITE_NOTE = "site_note" + RD_SAP_SITE_NOTE = "rd_sap_site_note" + PAS_2023_VENTILATION = "pas_2023_ventilation" + PAS_2023_CONDITION = "pas_2023_condition" + PAS_SIGNIFICANCE = "pas_significance" + PAR_PHOTO_PACK = "par_photo_pack" + PAS_2023_PROPERTY = "pas_2023_property" + PAS_2023_OCCUPANCY = "pas_2023_occupancy" + + +class FileSourceEnum(enum.Enum): + PAS_HUB = "pas hub" + SHAREPOINT = "sharepoint" + HUBSPOT = "hubspot" + + +class UploadedFiles(Base): + __tablename__ = "uploaded_files" + + id = Column(BigInteger, primary_key=True) + + s3_file_bucket = Column(Text, nullable=False) + s3_file_key = Column(Text, nullable=False) + s3_upload_timestamp = Column(TIMESTAMP(timezone=True), nullable=False) + + landlord_property_id = Column(Text, nullable=True) + uprn = Column(BigInteger, nullable=True) + hubspot_listing_id = Column(BigInteger, nullable=True) + + file_type = Column( + SqlEnum(FileTypeEnum, name="file_type", create_type=False), nullable=True + ) + + source = Column( + SqlEnum(FileSourceEnum, name="file_source", create_type=False), nullable=True + ) From 1a01ad94acb22628486c1c561bbdc3f6d9e50dfa Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 16:07:06 +0000 Subject: [PATCH 03/10] refactor handler --- backend/pashub_fetcher/handler/handler.py | 152 ++++++++++++++-------- 1 file changed, 97 insertions(+), 55 deletions(-) diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 8a02ce72..f8539201 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -39,94 +39,136 @@ def extract_jobs(filepath: str) -> List[Job]: if not name or not link: continue - link = str(link) - - match = re.search(r"/jobs/([0-9a-fA-F\-]+)/", link) + match = re.search(r"/jobs/([0-9a-fA-F\-]+)/", str(link)) if not match: continue - job_id = match.group(1) - - jobs.append({"id": job_id, "address": str(name)}) + jobs.append( + { + "id": match.group(1), + "address": str(name), + } + ) return jobs +def get_pashub_client(email: str, password: str) -> PashubClient: + token = get_token_from_local_storage(email, password) + logger.info("Token extracted successfully") + return PashubClient(token=token) + + +def upload_job_to_sharepoint( + sharepoint_client: DomnaSharepointClient, + base_path: str, + job: Job, + job_files: List[str], +) -> None: + job_path = f"{base_path}/{job['address']}" + + # Create main job folder + sharepoint_client.makedir(job["address"], base_path) + + # Create subfolders + for folder in SharepointSubfolders: + sharepoint_client.makedir(folder.value, job_path) + + # Upload into assessment folder + assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}" + + for file_path in job_files: + filename = file_path.split("/")[-1] + + sharepoint_client.upload_file( + file_path, + assessment_path, + filename, + ) + + +def upload_job_to_s3( + job: Job, + job_files: List[str], +) -> None: + # Example: + # for file_path in job_files: + # s3_client.upload_file(...) + pass + + +def process_job( + job: Job, + pashub_client: PashubClient, + sharepoint_client: DomnaSharepointClient, + base_path: str, +) -> List[str]: + job_id = job["id"] + + uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) + logger.info(f"Got UPRN {uprn} for job {job_id}") + + job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(job_id) + + upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) + upload_job_to_s3(job, job_files) + + return job_files + + def handler(event: Mapping[str, Any], context: Any) -> None: BASE_DIR = os.path.dirname(os.path.dirname(__file__)) filepath = os.path.join(BASE_DIR, "Watford_Warm_Homes_Wave_3_RA Downloads .xlsx") jobs: List[Job] = extract_jobs(filepath) - logger.info("Successfully loaded jobs from spreadsheet") - pas_hub_email = "random@test.com" - pas_hub_password = "my_fake_password" + # pas_hub_email = "random@test.com" + # pas_hub_password = "my_fake_password" - try: - token: str = get_token_from_local_storage(pas_hub_email, pas_hub_password) - logger.info(f"Token extracted successfully") - except: - logger.error("Error getting auth token from Pas Hub") - raise + pas_hub_email = "sebastian@osmosis-acd.com" + pas_hub_password = "Osmosis2025!" + + pashub_client = get_pashub_client(pas_hub_email, pas_hub_password) - pashub_client = PashubClient(token=token) sharepoint_client = DomnaSharepointClient( sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3 ) + BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)" + saved_file_paths: List[str] = [] - BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)" # TODO: get from request body for job in jobs: try: - job_id = job["id"] - uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) - logger.info(f"Got UPRN {uprn} for job {job_id}") - - job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id( - job_id + files = process_job( + job, + pashub_client, + sharepoint_client, + BASE_PATH, ) - - # Upload files to s3 - - # Upload files to sharepoint - job_path = f"{BASE_PATH}/{job['address']}" - - sharepoint_client.makedir(job["address"], BASE_PATH) - - for folder in SharepointSubfolders: - sharepoint_client.makedir(folder.value, job_path) - - assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}" - - for file_path in job_files: - filename = file_path.split("/")[-1] - - sharepoint_client.upload_file( - file_path, - assessment_path, - filename, - ) - - saved_file_paths.extend(job_files) + saved_file_paths.extend(files) except UnauthorizedError: logger.warning("Token expired - refreshing") - token = get_token_from_local_storage(pas_hub_email, pas_hub_password) - - pashub_client = PashubClient(token=token) - - # retry once - saved_file_paths.extend( - pashub_client.get_core_evidence_files_by_job_id(job["id"]) + pashub_client = get_pashub_client( + pas_hub_email, + pas_hub_password, ) - print(f"saved {len(saved_file_paths)} files") + # retry once + files = process_job( + job, + pashub_client, + sharepoint_client, + BASE_PATH, + ) + saved_file_paths.extend(files) + + logger.info(f"Saved {len(saved_file_paths)} files") if __name__ == "__main__": event = {"Records": [{"body": "{}"}]} - handler(event, None) From 3ee80cde100edafeb0978bc81ea2f8eb25ee4aa2 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 16:10:26 +0000 Subject: [PATCH 04/10] dummy credentials --- backend/pashub_fetcher/handler/handler.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index f8539201..c728661f 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -123,11 +123,8 @@ def handler(event: Mapping[str, Any], context: Any) -> None: jobs: List[Job] = extract_jobs(filepath) logger.info("Successfully loaded jobs from spreadsheet") - # pas_hub_email = "random@test.com" - # pas_hub_password = "my_fake_password" - - pas_hub_email = "sebastian@osmosis-acd.com" - pas_hub_password = "Osmosis2025!" + pas_hub_email = "random@test.com" + pas_hub_password = "my_fake_password" pashub_client = get_pashub_client(pas_hub_email, pas_hub_password) From 4a4099901ffef9dc01202c053af984a75e71dd08 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 16:33:35 +0000 Subject: [PATCH 05/10] upload files to s3 --- backend/pashub_fetcher/handler/handler.py | 28 +++++++++++++++-------- utils/s3.py | 11 +++++++++ 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index c728661f..45ca74e3 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -3,11 +3,13 @@ import re from typing import Any, Dict, List, Mapping, Optional from openpyxl import load_workbook +from backend.app.config import get_settings from backend.pashub_fetcher.job import Job from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from backend.pashub_fetcher.token_getter import get_token_from_local_storage from utils.logger import setup_logger +from utils.s3 import upload_file_to_s3 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient from utils.sharepoint.domna_sites import DomnaSites @@ -87,13 +89,17 @@ def upload_job_to_sharepoint( ) -def upload_job_to_s3( - job: Job, - job_files: List[str], -) -> None: - # Example: - # for file_path in job_files: - # s3_client.upload_file(...) +def upload_job_to_s3(job_files: List[str], uprn: str) -> None: + bucket = get_settings().DATA_BUCKET + + base_path = f"uprn/{uprn}" + + for file_path in job_files: + filename = os.path.basename(file_path) + file_key = f"{base_path}/{filename}" + + upload_file_to_s3(file_path, bucket, file_key) + pass @@ -106,12 +112,16 @@ def process_job( job_id = job["id"] uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) - logger.info(f"Got UPRN {uprn} for job {job_id}") + if uprn: + logger.info(f"Got UPRN {uprn} for job {job_id}") + else: + logger.info(f"No UPRN found for job {job_id}") job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(job_id) upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) - upload_job_to_s3(job, job_files) + if uprn: + upload_job_to_s3(job_files, uprn) return job_files diff --git a/utils/s3.py b/utils/s3.py index 6aa3f44e..242e0db5 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -472,3 +472,14 @@ def list_xmls_in_s3_folder(bucket_name, folder_name): f"Failed to list XML files in folder {folder_name} in bucket {bucket_name}: {str(e)}" ) return [] + + +def upload_file_to_s3(file_path: str, bucket_name: str, file_key: str) -> None: + try: + s3 = boto3.resource("s3") + bucket = s3.Bucket(bucket_name) + bucket.upload_file(file_path, file_key) + logger.info(f"Uploaded {file_path} to s3://{bucket_name}/{file_key}") + except Exception as e: + logger.error(f"Failed to upload {file_path} to S3: {e}") + raise From a989700c9e7aeb869f607d296ddea73a457449f2 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 17:16:08 +0000 Subject: [PATCH 06/10] corrections to get data into s3 --- backend/app/config.py | 9 +++++++++ backend/pashub_fetcher/handler/handler.py | 9 +++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 6604fec9..8a9260c0 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -65,6 +65,15 @@ class Settings(BaseSettings): ORDNANCE_SURVEY_API_KEY: str = "changeme" + # Sharepoint + SHAREPOINT_CLIENT_ID: str = "changeme" + SHAREPOINT_CLIENT_SECRET: str = "changeme" + SHAREPOINT_TENANT_ID: str = "changeme" + DOMNA_SHAREPOINT_ID: str = "changeme" + OSMOSIS_ACD_SHAREPOINT_ID: str = "changeme" + PRIVATE_PAY_SHAREPOINT_ID: str = "changeme" + SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID: str = "changeme" + # Optional AWS creds (only required in local) AWS_ACCESS_KEY_ID: Optional[str] = None AWS_SECRET_KEY_ID: Optional[str] = None diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 45ca74e3..6fa2a909 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -3,7 +3,6 @@ import re from typing import Any, Dict, List, Mapping, Optional from openpyxl import load_workbook -from backend.app.config import get_settings from backend.pashub_fetcher.job import Job from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders @@ -90,9 +89,9 @@ def upload_job_to_sharepoint( def upload_job_to_s3(job_files: List[str], uprn: str) -> None: - bucket = get_settings().DATA_BUCKET + bucket = "retrofit-energy-assessments-dev" # TODO: create new bucket - base_path = f"uprn/{uprn}" + base_path = f"documents/uprn/{uprn}" for file_path in job_files: filename = os.path.basename(file_path) @@ -119,10 +118,12 @@ def process_job( job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(job_id) - upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) if uprn: + logger.info("Uploading files to s3") upload_job_to_s3(job_files, uprn) + upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) + return job_files From 68ecbe11095ff8a97d453a1ef98164595c88d146 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 27 Mar 2026 09:49:18 +0000 Subject: [PATCH 07/10] update db after loading file to s3 --- .../{uploaded_files.py => uploaded_file.py} | 6 ++-- backend/pashub_fetcher/core_files.py | 23 +++++++++++++ backend/pashub_fetcher/handler/handler.py | 33 +++++++++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) rename backend/app/db/models/{uploaded_files.py => uploaded_file.py} (91%) diff --git a/backend/app/db/models/uploaded_files.py b/backend/app/db/models/uploaded_file.py similarity index 91% rename from backend/app/db/models/uploaded_files.py rename to backend/app/db/models/uploaded_file.py index d4bf48d8..726ed0a3 100644 --- a/backend/app/db/models/uploaded_files.py +++ b/backend/app/db/models/uploaded_file.py @@ -22,10 +22,10 @@ class FileSourceEnum(enum.Enum): HUBSPOT = "hubspot" -class UploadedFiles(Base): +class UploadedFile(Base): __tablename__ = "uploaded_files" - id = Column(BigInteger, primary_key=True) + id = Column(BigInteger, primary_key=True, autoincrement=True) s3_file_bucket = Column(Text, nullable=False) s3_file_key = Column(Text, nullable=False) @@ -39,6 +39,6 @@ class UploadedFiles(Base): SqlEnum(FileTypeEnum, name="file_type", create_type=False), nullable=True ) - source = Column( + file_source = Column( SqlEnum(FileSourceEnum, name="file_source", create_type=False), nullable=True ) diff --git a/backend/pashub_fetcher/core_files.py b/backend/pashub_fetcher/core_files.py index 82637f70..4da10661 100644 --- a/backend/pashub_fetcher/core_files.py +++ b/backend/pashub_fetcher/core_files.py @@ -1,4 +1,7 @@ from enum import Enum +from typing import Optional + +from backend.app.db.models.uploaded_file import FileTypeEnum class CoreFiles(Enum): @@ -11,3 +14,23 @@ class CoreFiles(Enum): PAR_PHOTOPACK = "PAR Photo Pack" PAS2023_PROPERTY = "PAS 2023 Property Assessment Report" PAS2023_OCCUPANCY = "PAS 2023 Occupancy Assessment Report" + + +CORE_TO_FILETYPE_MAP = { + CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value, + CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value, + CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value, + CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value, + CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value, + CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value, + CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value, + CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value, + CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value, +} + + +def infer_file_type(filename: str) -> Optional[str]: + for core_file, file_type in CORE_TO_FILETYPE_MAP.items(): + if filename.startswith(core_file.value): + return file_type + return None diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 6fa2a909..34f44589 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -1,8 +1,15 @@ +from datetime import datetime, timezone import os import re from typing import Any, Dict, List, Mapping, Optional from openpyxl import load_workbook +from backend.app.db.connection import db_session +from backend.app.db.models.uploaded_file import ( + FileSourceEnum, + UploadedFile, +) +from backend.pashub_fetcher.core_files import infer_file_type from backend.pashub_fetcher.job import Job from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders @@ -88,17 +95,35 @@ def upload_job_to_sharepoint( ) -def upload_job_to_s3(job_files: List[str], uprn: str) -> None: - bucket = "retrofit-energy-assessments-dev" # TODO: create new bucket +def upload_job_to_s3_and_update_db(job_files: List[str], uprn: str) -> None: + bucket = "retrofit-energy-assessments-dev" base_path = f"documents/uprn/{uprn}" + uploaded_files: List[UploadedFile] = [] + for file_path in job_files: filename = os.path.basename(file_path) file_key = f"{base_path}/{filename}" upload_file_to_s3(file_path, bucket, file_key) + # load row to db + uploaded_files.append( + UploadedFile( + s3_file_bucket=bucket, + s3_file_key=file_key, + s3_upload_timestamp=datetime.now(timezone.utc), + uprn=int(uprn), + file_source=FileSourceEnum.PAS_HUB.value, + file_type=infer_file_type(filename), + ) + ) + + with db_session() as session: + session.add_all(uploaded_files) + session.commit() + pass @@ -120,7 +145,9 @@ def process_job( if uprn: logger.info("Uploading files to s3") - upload_job_to_s3(job_files, uprn) + upload_job_to_s3_and_update_db(job_files, uprn) + + # add record of new file in db upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) From 43e9466e05c76363c05f7e7401061cf207f18210 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 27 Mar 2026 16:23:21 +0000 Subject: [PATCH 08/10] log object better --- .../pashub_fetcher/handler/test_handler.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/pashub_fetcher/handler/test_handler.py b/backend/pashub_fetcher/handler/test_handler.py index 996835a2..07f651f4 100644 --- a/backend/pashub_fetcher/handler/test_handler.py +++ b/backend/pashub_fetcher/handler/test_handler.py @@ -1,7 +1,23 @@ from typing import Any, Mapping import json +from utils.logger import setup_logger + +logger = setup_logger() + def handler(event: Mapping[str, Any], context: Any) -> None: - print("Received event:") - print(json.dumps(event, indent=2)) + logger.info("Received event:") + + for record in event.get("Records", []): + body_str = record.get("body", "") + + try: + body_obj = json.loads(body_str) + except json.JSONDecodeError: + logger.error("Failed to parse body as JSON") + logger.info(body_str) + continue + + logger.info("Parsed message body:") + logger.info(json.dumps(body_obj, indent=2)) From b9e9c9edb5a904927ddfbce870071da08ef4f7fc Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 27 Mar 2026 16:32:41 +0000 Subject: [PATCH 09/10] get pashub email and password from environment vars --- backend/app/config.py | 4 ++++ backend/pashub_fetcher/handler/handler.py | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/app/config.py b/backend/app/config.py index 8a9260c0..e137429f 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -74,6 +74,10 @@ class Settings(BaseSettings): PRIVATE_PAY_SHAREPOINT_ID: str = "changeme" SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID: str = "changeme" + # Pas Hub + PASHUB_EMAIL: str = "changeme" + PASHUB_PASSWORD: str = "changeme" + # Optional AWS creds (only required in local) AWS_ACCESS_KEY_ID: Optional[str] = None AWS_SECRET_KEY_ID: Optional[str] = None diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 34f44589..be8131bf 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -4,6 +4,7 @@ import re from typing import Any, Dict, List, Mapping, Optional from openpyxl import load_workbook +from backend.app.config import get_settings from backend.app.db.connection import db_session from backend.app.db.models.uploaded_file import ( FileSourceEnum, @@ -155,14 +156,16 @@ def process_job( def handler(event: Mapping[str, Any], context: Any) -> None: + settings = get_settings() + BASE_DIR = os.path.dirname(os.path.dirname(__file__)) filepath = os.path.join(BASE_DIR, "Watford_Warm_Homes_Wave_3_RA Downloads .xlsx") jobs: List[Job] = extract_jobs(filepath) logger.info("Successfully loaded jobs from spreadsheet") - pas_hub_email = "random@test.com" - pas_hub_password = "my_fake_password" + pas_hub_email = settings.PASHUB_EMAIL + pas_hub_password = settings.PASHUB_PASSWORD pashub_client = get_pashub_client(pas_hub_email, pas_hub_password) From 540cafc62a5168b5f7f1d4d2edbfff139af5117d Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Fri, 27 Mar 2026 16:37:45 +0000 Subject: [PATCH 10/10] update trigger request object --- backend/pashub_fetcher/pashub_to_ara_trigger_request.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py index d97281fd..5ef41883 100644 --- a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py +++ b/backend/pashub_fetcher/pashub_to_ara_trigger_request.py @@ -6,4 +6,7 @@ from pydantic import BaseModel class PashubToAraTriggerRequest(BaseModel): pashub_job_id: str - sharepoint_url: Optional[str] = None + address: Optional[str] = None + sharepoint_link: Optional[str] = None + uprn: Optional[str] = None + landlord_property_id: Optional[str] = None