From a71e7e30d84d7817b3c8e2207c30ad11b9c299a8 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Thu, 26 Mar 2026 10:33:49 +0000 Subject: [PATCH] correct sharepoint file structure and playwright videos for debugging --- .gitignore | 5 +++- backend/pashub_fetcher/handler/Dockerfile | 3 +- backend/pashub_fetcher/handler/handler.py | 26 ++++++++++++----- .../pashub_fetcher/sharepoint_subfolders.py | 9 ++++++ backend/pashub_fetcher/token_getter.py | 28 +++++++++++++++++-- utils/sharepoint/sharepoint_client.py | 4 +++ 6 files changed, 63 insertions(+), 12 deletions(-) create mode 100644 backend/pashub_fetcher/sharepoint_subfolders.py diff --git a/.gitignore b/.gitignore index 15050bdd..299e03d4 100644 --- a/.gitignore +++ b/.gitignore @@ -282,4 +282,7 @@ cache/ local_data* # pyright local config -pyrightconfig.json \ No newline at end of file +pyrightconfig.json + +# playwright output +*/pashub_fetcher/videos/* \ No newline at end of file diff --git a/backend/pashub_fetcher/handler/Dockerfile b/backend/pashub_fetcher/handler/Dockerfile index 9398a93d..d045becd 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/backend/pashub_fetcher/handler/Dockerfile @@ -22,4 +22,5 @@ ENTRYPOINT ["python", "-m", "awslambdaric"] # ----------------------------- # Lambda handler # ----------------------------- -CMD ["backend.pashub_fetcher.handler.test_handler.handler"] \ No newline at end of file +CMD ["backend.pashub_fetcher.handler.test_handler.handler"] +# CMD ["backend.pashub_fetcher.handler.handler.handler"] \ No newline at end of file diff --git a/backend/pashub_fetcher/handler/handler.py b/backend/pashub_fetcher/handler/handler.py index fa48487e..6bc06e77 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/backend/pashub_fetcher/handler/handler.py @@ -1,12 +1,11 @@ import os import re - from typing import Any, Dict, List, Mapping - from openpyxl import load_workbook from backend.pashub_fetcher.job import Job from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError +from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from backend.pashub_fetcher.token_getter import get_token_from_local_storage from utils.logger import setup_logger from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient @@ -76,9 +75,9 @@ def handler(event: Mapping[str, Any], context: Any) -> None: sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3 ) - sharepoint_client.makedir("Watford Test", "/JTK Test Folder") - saved_file_paths: List[str] = [] + BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)" # TODO: get from request body + for job in jobs: try: job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id( @@ -86,15 +85,28 @@ def handler(event: Mapping[str, Any], context: Any) -> None: ) # Upload files to sharepoint - sharepoint_client.makedir(job["address"], "/JTK Test Folder/Watford Test") + job_path = f"{BASE_PATH}/{job['address']}" + + # Create main job folder + sharepoint_client.makedir(job["address"], BASE_PATH) + + # Create subfolders + for folder in SharepointSubfolders: + sharepoint_client.makedir(folder.value, job_path) + + assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}" + for file_path in job_files: + filename = file_path.split("/")[-1] + sharepoint_client.upload_file( file_path, - f"/JTK Test Folder/Watford Test/{job['address']}", - file_path.split("/")[-1], + assessment_path, + filename, ) saved_file_paths.extend(job_files) + except UnauthorizedError: logger.warning("Token expired - refreshing") diff --git a/backend/pashub_fetcher/sharepoint_subfolders.py b/backend/pashub_fetcher/sharepoint_subfolders.py new file mode 100644 index 00000000..6644098b --- /dev/null +++ b/backend/pashub_fetcher/sharepoint_subfolders.py @@ -0,0 +1,9 @@ +from enum import Enum + + +class SharepointSubfolders(str, Enum): + ASSESSMENT = "1. Retrofit Assessment" + COORDINATION = "2. Retrofit Coordination" + DESIGN = "3. Retrofit Design" + INSTALLER = "4. Installer Information" + TRUSTMARK = "5. TrustMark Lodgement" diff --git a/backend/pashub_fetcher/token_getter.py b/backend/pashub_fetcher/token_getter.py index 5954feec..5534d114 100644 --- a/backend/pashub_fetcher/token_getter.py +++ b/backend/pashub_fetcher/token_getter.py @@ -1,3 +1,5 @@ +import os + from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError from utils.logger import setup_logger @@ -8,20 +10,36 @@ logger = setup_logger() def get_token_from_local_storage(email: str, password: str) -> str: logger.info("Starting Playwright flow") + # For local testing / debugging, we save videos of the flow + video_dir = os.path.join(os.path.dirname(__file__), "videos") + os.makedirs(video_dir, exist_ok=True) + with sync_playwright() as p: browser = p.chromium.launch( headless=True, args=["--no-sandbox", "--disable-dev-shm-usage"], ) - page = browser.new_page() + + context = browser.new_context( + record_video_dir=video_dir, + record_video_size={"width": 1280, "height": 720}, + ) + page = context.new_page() try: logger.info("Navigating to site...") page.goto("https://pashub.net/", timeout=30000) logger.info("Filling login form...") - page.fill("#email", email) - page.fill("#password", password) + + email_input = page.locator("#email") + email_input.wait_for(state="visible", timeout=10000) + email_input.fill(email) + + password_input = page.locator("#password") + password_input.wait_for(state="visible", timeout=10000) + password_input.focus() + password_input.fill(password) logger.info("Submitting login...") page.wait_for_selector("#btn-login", state="visible", timeout=10000) @@ -53,4 +71,8 @@ def get_token_from_local_storage(email: str, password: str) -> str: raise Exception(f"Unexpected error: {str(e)}") finally: + logger.info("Closing browser context (saving video)...") + context.close() browser.close() + + logger.info(f"Video(s) saved in: {video_dir}") diff --git a/utils/sharepoint/sharepoint_client.py b/utils/sharepoint/sharepoint_client.py index 67c4315c..71f82b68 100644 --- a/utils/sharepoint/sharepoint_client.py +++ b/utils/sharepoint/sharepoint_client.py @@ -14,8 +14,12 @@ from functools import wraps import time from io import BytesIO +from utils.logger import setup_logger + # Api Documentation: https://learn.microsoft.com/en-us/graph/api/drive-get?view=graph-rest-1.0&tabs=http +logger = setup_logger() + def handle_error(response): """