correct sharepoint file structure and playwright videos for debugging

.gitignore

@@ -282,4 +282,7 @@ cache/
 local_data*
 
 # pyright local config
-pyrightconfig.json
+pyrightconfig.json
+
+# playwright output
+*/pashub_fetcher/videos/*

backend/pashub_fetcher/handler/Dockerfile

@@ -22,4 +22,5 @@ ENTRYPOINT ["python", "-m", "awslambdaric"]
 # -----------------------------
 # Lambda handler
 # -----------------------------
-CMD ["backend.pashub_fetcher.handler.test_handler.handler"]
+CMD ["backend.pashub_fetcher.handler.test_handler.handler"]
+# CMD ["backend.pashub_fetcher.handler.handler.handler"]

backend/pashub_fetcher/handler/handler.py

@@ -1,12 +1,11 @@
 import os
 import re
-
 from typing import Any, Dict, List, Mapping
-
 from openpyxl import load_workbook
 
 from backend.pashub_fetcher.job import Job
 from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError
+from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
 from backend.pashub_fetcher.token_getter import get_token_from_local_storage
 from utils.logger import setup_logger
 from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
@@ -76,9 +75,9 @@ def handler(event: Mapping[str, Any], context: Any) -> None:
         sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3
     )
 
-    sharepoint_client.makedir("Watford Test", "/JTK Test Folder")
-
     saved_file_paths: List[str] = []
+    BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)"  # TODO: get from request body
+
     for job in jobs:
         try:
             job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(
@@ -86,15 +85,28 @@ def handler(event: Mapping[str, Any], context: Any) -> None:
             )
 
             # Upload files to sharepoint
-            sharepoint_client.makedir(job["address"], "/JTK Test Folder/Watford Test")
+            job_path = f"{BASE_PATH}/{job['address']}"
+
+            # Create main job folder
+            sharepoint_client.makedir(job["address"], BASE_PATH)
+
+            # Create subfolders
+            for folder in SharepointSubfolders:
+                sharepoint_client.makedir(folder.value, job_path)
+
+            assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}"
+
             for file_path in job_files:
+                filename = file_path.split("/")[-1]
+
                 sharepoint_client.upload_file(
                     file_path,
-                    f"/JTK Test Folder/Watford Test/{job['address']}",
-                    file_path.split("/")[-1],
+                    assessment_path,
+                    filename,
                 )
 
             saved_file_paths.extend(job_files)
+
         except UnauthorizedError:
             logger.warning("Token expired - refreshing")
 

backend/pashub_fetcher/sharepoint_subfolders.py

@@ -0,0 +1,9 @@
+from enum import Enum
+
+
+class SharepointSubfolders(str, Enum):
+    ASSESSMENT = "1. Retrofit Assessment"
+    COORDINATION = "2. Retrofit Coordination"
+    DESIGN = "3. Retrofit Design"
+    INSTALLER = "4. Installer Information"
+    TRUSTMARK = "5. TrustMark Lodgement"

backend/pashub_fetcher/token_getter.py

@@ -1,3 +1,5 @@
+import os
+
 from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
 
 from utils.logger import setup_logger
@@ -8,20 +10,36 @@ logger = setup_logger()
 def get_token_from_local_storage(email: str, password: str) -> str:
     logger.info("Starting Playwright flow")
 
+    # For local testing / debugging, we save videos of the flow
+    video_dir = os.path.join(os.path.dirname(__file__), "videos")
+    os.makedirs(video_dir, exist_ok=True)
+
     with sync_playwright() as p:
         browser = p.chromium.launch(
             headless=True,
             args=["--no-sandbox", "--disable-dev-shm-usage"],
         )
-        page = browser.new_page()
+
+        context = browser.new_context(
+            record_video_dir=video_dir,
+            record_video_size={"width": 1280, "height": 720},
+        )
+        page = context.new_page()
 
         try:
             logger.info("Navigating to site...")
             page.goto("https://pashub.net/", timeout=30000)
 
             logger.info("Filling login form...")
-            page.fill("#email", email)
-            page.fill("#password", password)
+
+            email_input = page.locator("#email")
+            email_input.wait_for(state="visible", timeout=10000)
+            email_input.fill(email)
+
+            password_input = page.locator("#password")
+            password_input.wait_for(state="visible", timeout=10000)
+            password_input.focus()
+            password_input.fill(password)
 
             logger.info("Submitting login...")
             page.wait_for_selector("#btn-login", state="visible", timeout=10000)
@@ -53,4 +71,8 @@ def get_token_from_local_storage(email: str, password: str) -> str:
             raise Exception(f"Unexpected error: {str(e)}")
 
         finally:
+            logger.info("Closing browser context (saving video)...")
+            context.close()
             browser.close()
+
+            logger.info(f"Video(s) saved in: {video_dir}")

utils/sharepoint/sharepoint_client.py

@@ -14,8 +14,12 @@ from functools import wraps
 import time
 from io import BytesIO
 
+from utils.logger import setup_logger
+
 # Api Documentation: https://learn.microsoft.com/en-us/graph/api/drive-get?view=graph-rest-1.0&tabs=http
 
+logger = setup_logger()
+
 
 def handle_error(response):
     """