From 1a01ad94acb22628486c1c561bbdc3f6d9e50dfa Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 16:07:06 +0000 Subject: [PATCH] refactor handler --- backend/pashub_fetcher/handler/handler.py | 152 ++++++++++++++-------- 1 file changed, 97 insertions(+), 55 deletions(-) diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index 8a02ce72..f8539201 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -39,94 +39,136 @@ def extract_jobs(filepath: str) -> List[Job]: if not name or not link: continue - link = str(link) - - match = re.search(r"/jobs/([0-9a-fA-F\-]+)/", link) + match = re.search(r"/jobs/([0-9a-fA-F\-]+)/", str(link)) if not match: continue - job_id = match.group(1) - - jobs.append({"id": job_id, "address": str(name)}) + jobs.append( + { + "id": match.group(1), + "address": str(name), + } + ) return jobs +def get_pashub_client(email: str, password: str) -> PashubClient: + token = get_token_from_local_storage(email, password) + logger.info("Token extracted successfully") + return PashubClient(token=token) + + +def upload_job_to_sharepoint( + sharepoint_client: DomnaSharepointClient, + base_path: str, + job: Job, + job_files: List[str], +) -> None: + job_path = f"{base_path}/{job['address']}" + + # Create main job folder + sharepoint_client.makedir(job["address"], base_path) + + # Create subfolders + for folder in SharepointSubfolders: + sharepoint_client.makedir(folder.value, job_path) + + # Upload into assessment folder + assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}" + + for file_path in job_files: + filename = file_path.split("/")[-1] + + sharepoint_client.upload_file( + file_path, + assessment_path, + filename, + ) + + +def upload_job_to_s3( + job: Job, + job_files: List[str], +) -> None: + # Example: + # for file_path in job_files: + # s3_client.upload_file(...) + pass + + +def process_job( + job: Job, + pashub_client: PashubClient, + sharepoint_client: DomnaSharepointClient, + base_path: str, +) -> List[str]: + job_id = job["id"] + + uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) + logger.info(f"Got UPRN {uprn} for job {job_id}") + + job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(job_id) + + upload_job_to_sharepoint(sharepoint_client, base_path, job, job_files) + upload_job_to_s3(job, job_files) + + return job_files + + def handler(event: Mapping[str, Any], context: Any) -> None: BASE_DIR = os.path.dirname(os.path.dirname(__file__)) filepath = os.path.join(BASE_DIR, "Watford_Warm_Homes_Wave_3_RA Downloads .xlsx") jobs: List[Job] = extract_jobs(filepath) - logger.info("Successfully loaded jobs from spreadsheet") - pas_hub_email = "random@test.com" - pas_hub_password = "my_fake_password" + # pas_hub_email = "random@test.com" + # pas_hub_password = "my_fake_password" - try: - token: str = get_token_from_local_storage(pas_hub_email, pas_hub_password) - logger.info(f"Token extracted successfully") - except: - logger.error("Error getting auth token from Pas Hub") - raise + pas_hub_email = "sebastian@osmosis-acd.com" + pas_hub_password = "Osmosis2025!" + + pashub_client = get_pashub_client(pas_hub_email, pas_hub_password) - pashub_client = PashubClient(token=token) sharepoint_client = DomnaSharepointClient( sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3 ) + BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)" + saved_file_paths: List[str] = [] - BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)" # TODO: get from request body for job in jobs: try: - job_id = job["id"] - uprn: Optional[str] = pashub_client.get_uprn_by_job_id(job_id) - logger.info(f"Got UPRN {uprn} for job {job_id}") - - job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id( - job_id + files = process_job( + job, + pashub_client, + sharepoint_client, + BASE_PATH, ) - - # Upload files to s3 - - # Upload files to sharepoint - job_path = f"{BASE_PATH}/{job['address']}" - - sharepoint_client.makedir(job["address"], BASE_PATH) - - for folder in SharepointSubfolders: - sharepoint_client.makedir(folder.value, job_path) - - assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}" - - for file_path in job_files: - filename = file_path.split("/")[-1] - - sharepoint_client.upload_file( - file_path, - assessment_path, - filename, - ) - - saved_file_paths.extend(job_files) + saved_file_paths.extend(files) except UnauthorizedError: logger.warning("Token expired - refreshing") - token = get_token_from_local_storage(pas_hub_email, pas_hub_password) - - pashub_client = PashubClient(token=token) - - # retry once - saved_file_paths.extend( - pashub_client.get_core_evidence_files_by_job_id(job["id"]) + pashub_client = get_pashub_client( + pas_hub_email, + pas_hub_password, ) - print(f"saved {len(saved_file_paths)} files") + # retry once + files = process_job( + job, + pashub_client, + sharepoint_client, + BASE_PATH, + ) + saved_file_paths.extend(files) + + logger.info(f"Saved {len(saved_file_paths)} files") if __name__ == "__main__": event = {"Records": [{"body": "{}"}]} - handler(event, None)