From 9b274bbc157f331c19f52fc16cd303a2972abe64 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Tue, 24 Mar 2026 16:03:35 +0000 Subject: [PATCH] upload files to sharepoint --- backend/pashub_fetcher/handler/Dockerfile | 4 +- backend/pashub_fetcher/handler/handler.py | 46 ++++++++++++++++++----- backend/pashub_fetcher/job.py | 6 +++ backend/pashub_fetcher/pashub_client.py | 19 +++++----- 4 files changed, 54 insertions(+), 21 deletions(-) create mode 100644 backend/pashub_fetcher/job.py diff --git a/backend/pashub_fetcher/handler/Dockerfile b/backend/pashub_fetcher/handler/Dockerfile index cbd3c228..fd66f53e 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/backend/pashub_fetcher/handler/Dockerfile @@ -5,12 +5,12 @@ ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest RUN chmod +x /usr/local/bin/aws-lambda-rie # Install Lambda runtime client -RUN pip install awslambdaric playwright==1.58.0 requests +RUN pip install awslambdaric playwright==1.58.0 requests msal # Set working directory (Lambda task root) WORKDIR /var/task -COPY backend/.env.test backend/.env +COPY .env backend/.env COPY utils/ utils/ COPY backend/pashub_fetcher/ backend/pashub_fetcher/ diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index a4ee5339..f99dd688 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -1,9 +1,12 @@ import time from typing import Any, List, Mapping +from backend.pashub_fetcher.job import Job from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError from backend.pashub_fetcher.token_getter import get_token_from_local_storage from utils.logger import setup_logger +from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient +from utils.sharepoint.domna_sites import DomnaSites logger = setup_logger() @@ -21,29 +24,52 @@ def handler(event: Mapping[str, Any], context: Any) -> None: raise pashub_client = PashubClient(token=token) + sharepoint_client = DomnaSharepointClient( + sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3 + ) - jobs = [ - "5abf6e27-e4c4-4ba8-b69d-9e34939e0002", - "047f4455-85e2-4293-97b1-6b460137d33e", - ] # TODO: get these from request body + jobs: List[Job] = [ + { + "id": "5abf6e27-e4c4-4ba8-b69d-9e34939e0002", + "address": "FLAT 11 Abbey View, Garsmouth Way, Watford, WD25 9DY", + }, + { + "id": "047f4455-85e2-4293-97b1-6b460137d33e", + "address": "FLAT 14 Abbey View, Garsmouth Way, Watford, WD25 9DY", + }, + ] # TODO: get these from request body or spreadsheet - saved_files: List[str] = [] - for job_id in jobs: + sharepoint_client.makedir("Watford Test", "/JTK Test Folder") + + saved_file_paths: List[str] = [] + for job in jobs: try: - saved_files.extend(pashub_client.get_core_envidence_files_by_job_id(job_id)) + job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id( + job["id"] + ) # Upload files to sharepoint + sharepoint_client.makedir(job["address"], "/JTK Test Folder/Watford Test") + for file_path in job_files: + sharepoint_client.upload_file( + file_path, + f"/JTK Test Folder/Watford Test/{job['address']}", + file_path.split("/")[-1], + ) + saved_file_paths.extend(job_files) except UnauthorizedError: - logger.warning("Token expired — refreshing") + logger.warning("Token expired - refreshing") token = get_token_from_local_storage(pas_hub_email, pas_hub_password) pashub_client = PashubClient(token=token) # retry once - saved_files.extend(pashub_client.get_core_envidence_files_by_job_id(job_id)) + saved_file_paths.extend( + pashub_client.get_core_evidence_files_by_job_id(job["id"]) + ) time.sleep(10) # Simulate manual download - print(f"saved {len(saved_files)} files") + print(f"saved {len(saved_file_paths)} files") diff --git a/backend/pashub_fetcher/job.py b/backend/pashub_fetcher/job.py new file mode 100644 index 00000000..959ca137 --- /dev/null +++ b/backend/pashub_fetcher/job.py @@ -0,0 +1,6 @@ +from typing import TypedDict + + +class Job(TypedDict): + id: str + address: str diff --git a/backend/pashub_fetcher/pashub_client.py b/backend/pashub_fetcher/pashub_client.py index bed7395f..efc21803 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/backend/pashub_fetcher/pashub_client.py @@ -1,4 +1,5 @@ from collections import defaultdict +import os from typing import Dict, List, Optional from datetime import datetime @@ -33,7 +34,7 @@ class PashubClient: ) logger.info("Finished initialising CotalityClient") - def get_core_envidence_files_by_job_id(self, job_id: str) -> List[str]: + def get_core_evidence_files_by_job_id(self, job_id: str) -> List[str]: logger.info(f"Getting Core Evidence Files for job ID {job_id}") evidence_list: List[EvidenceFileData] = self._get_evidence_list(job_id) logger.info(f"Found {len(evidence_list)} Evidence files to get") @@ -59,11 +60,14 @@ class PashubClient: ) download_url: str = self._build_download_url(metadata, evidence.file_id) - file_name = evidence.file_name + output_dir: str = "/tmp" - self._download_file(download_url, file_name) + file_name: str = evidence.file_name + file_path: str = os.path.join(output_dir, file_name) + + self._download_file(download_url, file_path) logger.info("Successfully downloaded file") - saved_files.append(file_name) + saved_files.append(file_path) return saved_files @@ -126,12 +130,9 @@ class PashubClient: return f"{base}{container}/{file_id}?{sas}" - def _download_file(self, url: str, file_name: str) -> None: + def _download_file(self, url: str, file_path: str) -> None: r = requests.get(url) - if r.status_code == 401: - raise UnauthorizedError() - r.raise_for_status() - with open(file_name, "wb") as f: + with open(file_path, "wb") as f: f.write(r.content)