.devcontainer/backend/requirements.txt

@@ -6,14 +6,12 @@ psycopg2-binary==2.9.10
 python-jose==3.3.0
 cryptography==43.0.3
 mangum==0.19.0
-playwright==1.58.0
 # AWS
 boto3==1.35.44
 # Data
 openpyxl==3.1.5
 # Basic
 pytz
-msal
 uvicorn[standard]
 sqlmodel
 # Testing

.github/workflows/_deploy_lambda.yml

@@ -62,24 +62,7 @@ on:
         required: false
       TF_VAR_ordnance_survey_api_key:
         required: false
-      TF_VAR_sharepoint_client_id:
-        required: false
-      TF_VAR_sharepoint_client_secret:
-        required: false
-      TF_VAR_sharepoint_tenant_id:
-        required: false
-      TF_VAR_domna_sharepoint_id:
-        required: false
-      TF_VAR_osmosis_acd_sharepoint_id:
-        required: false
-      TF_VAR_private_pay_sharepoint_id:
-        required: false
-      TF_VAR_social_housing_wave_3_sharepoint_id:
-        required: false
-      TF_VAR_pashub_email:
-        required: false
-      TF_VAR_pashub_password:
-        required: false
+
 jobs:
   deploy:
     runs-on: ubuntu-latest
@@ -137,15 +120,6 @@ jobs:
           TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
           TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
           TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }}
-          TF_VAR_sharepoint_client_id: ${{ secrets.TF_VAR_sharepoint_client_id }}
-          TF_VAR_sharepoint_client_secret: ${{ secrets.TF_VAR_sharepoint_client_secret }}
-          TF_VAR_sharepoint_tenant_id: ${{ secrets.TF_VAR_sharepoint_tenant_id }}
-          TF_VAR_domna_sharepoint_id: ${{ secrets.TF_VAR_domna_sharepoint_id }}
-          TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }}
-          TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }}
-          TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }}
-          TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }}
-          TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }}
         run: |
           ECR_REPO_URL_VAR=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then
@@ -182,15 +156,6 @@ jobs:
           TF_VAR_epc_auth_token: ${{ secrets.TF_VAR_epc_auth_token }}
           TF_VAR_google_solar_api_key: ${{ secrets.TF_VAR_google_solar_api_key }}
           TF_VAR_ordnance_survey_api_key: ${{ secrets.TF_VAR_ordnance_survey_api_key }}
-          TF_VAR_sharepoint_client_id: ${{ secrets.TF_VAR_sharepoint_client_id }}
-          TF_VAR_sharepoint_client_secret: ${{ secrets.TF_VAR_sharepoint_client_secret }}
-          TF_VAR_sharepoint_tenant_id: ${{ secrets.TF_VAR_sharepoint_tenant_id }}
-          TF_VAR_domna_sharepoint_id: ${{ secrets.TF_VAR_domna_sharepoint_id }}
-          TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.TF_VAR_osmosis_acd_sharepoint_id }}
-          TF_VAR_private_pay_sharepoint_id: ${{ secrets.TF_VAR_private_pay_sharepoint_id }}
-          TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.TF_VAR_social_housing_wave_3_sharepoint_id }}
-          TF_VAR_pashub_email: ${{ secrets.TF_VAR_pashub_email }}
-          TF_VAR_pashub_password: ${{ secrets.TF_VAR_pashub_password }}
         run: |
           EXTRA_VARS=""
           if [[ -n "${{ inputs.ecr_repo }}" ]]; then

.github/workflows/deploy_terraform.yml

@@ -322,57 +322,6 @@ jobs:
       AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
       TF_VAR_ORDNANCE_SURVEY_API_KEY: ${{ secrets.ORDNANCE_SURVEY_API_KEY }}
 
-  # ============================================================
-  # Build Pas Hub to Ara image and Push
-  # ============================================================
-  pashub_to_ara_image:
-    needs: [determine_stage, shared_terraform]
-    uses: ./.github/workflows/_build_image.yml
-    with:
-      ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }}
-      dockerfile_path: backend/pashub_fetcher/handler/Dockerfile
-      build_context: .
-      build_args: |
-        DEV_DB_HOST=$DEV_DB_HOST
-        DEV_DB_PORT=$DEV_DB_PORT
-        DEV_DB_NAME=$DEV_DB_NAME
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
-      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
-      DEV_DB_HOST: ${{ secrets.DEV_DB_HOST }}
-      DEV_DB_PORT: ${{ secrets.DEV_DB_PORT }}
-      DEV_DB_NAME: ${{ secrets.DEV_DB_NAME }}
-
-
-  # ============================================================
-  # Deploy Pas Hub to Ara Lambda
-  # ============================================================
-  pashub_to_ara_lambda:
-    needs: [pashub_to_ara_image, determine_stage]
-    uses: ./.github/workflows/_deploy_lambda.yml
-    with:
-      lambda_name: pashub_to_ara
-      lambda_path: infrastructure/terraform/lambda/pashub_to_ara
-      stage: ${{ needs.determine_stage.outputs.stage }}
-      ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }}
-      image_digest: ${{ needs.pashub_to_ara_image.outputs.image_digest }}
-      terraform_apply: ${{ needs.determine_stage.outputs.terraform_apply }}
-    secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY }}
-      AWS_REGION: ${{ secrets.DEV_AWS_REGION }}
-      TF_VAR_sharepoint_client_id: ${{ secrets.SHAREPOINT_CLIENT_ID }}
-      TF_VAR_sharepoint_client_secret: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
-      TF_VAR_sharepoint_tenant_id: ${{ secrets.SHAREPOINT_TENANT_ID }}
-      TF_VAR_domna_sharepoint_id: ${{ secrets.DOMNA_SHAREPOINT_ID }}
-      TF_VAR_osmosis_acd_sharepoint_id: ${{ secrets.OSMOSIS_ACD_SHAREPOINT_ID }}
-      TF_VAR_private_pay_sharepoint_id: ${{ secrets.PRIVATE_PAY_SHAREPOINT_ID }}
-      TF_VAR_social_housing_wave_3_sharepoint_id: ${{ secrets.SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID }}
-      TF_VAR_pashub_email: ${{ secrets.PASHUB_EMAIL }}
-      TF_VAR_pashub_password: ${{ secrets.PASHUB_PASSWORD }}
-
-
   # ============================================================
   # Deploy FastAPI Lambda
   # ============================================================

.gitignore

@@ -282,7 +282,4 @@ cache/
 local_data*
 
 # pyright local config
-pyrightconfig.json
-
-# playwright output
-*/pashub_fetcher/videos/*
+pyrightconfig.json

backend/app/plan/router.py

@@ -174,8 +174,6 @@ async def trigger_plan_entrypoint(body: PlanTriggerRequest):
                 service="plan_engine",
                 inputs=data,
                 task_only=True,
-                source=SourceEnum.PORTFOLIO,
-                source_id=str(body.portfolio_id),
             )
 
             subtask_interface = SubTaskInterface()
@@ -224,8 +222,6 @@ async def trigger_plan_entrypoint(body: PlanTriggerRequest):
                 service="plan_engine",
                 inputs=data,
                 task_only=False,
-                source=SourceEnum.PORTFOLIO,
-                source_id=str(body.portfolio_id),
             )
             data["task_id"] = str(task_id)
             data["subtask_id"] = str(subtask_id)

backend/docker-compose-local-lambdas.yml

@@ -0,0 +1,11 @@
+version: "3.9"
+
+services:
+  categorisation-lambda:
+    build:
+      context: ../
+      dockerfile: backend/categorisation/handler/Dockerfile
+    ports:
+      - "9000:8080"
+    env_file:
+      - ../.env

backend/engine/engine.py

@@ -32,6 +32,7 @@ from backend.ml_models.api import ModelApi
 from backend.ml_models.Valuation import PropertyValuation
 from backend.Property import Property
 from backend.apis.GoogleSolarApi import GoogleSolarApi
+from backend.addresses.Addresses import Addresses
 
 from recommendations.optimiser.CostOptimiser import CostOptimiser
 from recommendations.optimiser.GainOptimiser import GainOptimiser
@@ -641,9 +642,7 @@ async def model_engine(body: PlanTriggerRequest):
 
             epc_records = patch_epc(patch, epc_records)
 
-            prepared_epc = EPCRecord(
-                epc_records=epc_records, run_mode="newdata", cleaning_data=cleaning_data, address_metadata=addr
-            )
+            prepared_epc = EPCRecord(epc_records=epc_records, run_mode="newdata", cleaning_data=cleaning_data)
 
             input_properties.append(
                 Property(

backend/export/property_scenarios/main.py

@@ -134,18 +134,10 @@ def handler(event: Mapping[str, Any], context: Optional[Any]) -> Mapping[str, Un
     body_dict = {
         "task_id": "test",
         "subtask_id": "test",
-        "portfolio_id": 655,
+        "portfolio_id": 569,
         "scenario_ids": [],
         "default_plans_only": True,
     }
-
-    body_dict = {
-        "task_id": "test",
-        "subtask_id": "test",
-        "portfolio_id": 655,
-        "scenario_ids": [1174],
-        "default_plans_only": False,
-    }
     :param event: Lambda event containing export request details
     :param context: Lambda context (not used in this handler but included for completeness)
     :return: HTTP response indicating success or failure of the export operation

backend/pashub_fetcher/core_files.py

@@ -1,13 +0,0 @@
-from enum import Enum
-
-
-class CoreFiles(Enum):
-    PHOTOPACK = "Photopack"
-    SITENOTE = "SiteNote"
-    RDSAP_SITENOTE = "RdSAP_SiteNote"
-    PAS2023_VENTILATION = "PAS 2023 Ventilation Assessment Report"
-    PAS2023_CONDITION = "PAS 2023 Condition Report"
-    PAS_SIGNIFICANCE = "PAS Significance"
-    PAR_PHOTOPACK = "PAR Photo Pack"
-    PAS2023_PROPERTY = "PAS 2023 Property Assessment Report"
-    PAS2023_OCCUPANCY = "PAS 2023 Occupancy Assessment Report"

backend/pashub_fetcher/evidence_file_data.py

@@ -1,25 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from typing import Any, Dict, Optional
-
-
-@dataclass
-class EvidenceFileData:
-    file_id: str
-    file_name: str
-    created_utc: str
-    file_size: int
-    file_extension: str
-
-    evidence_category: Optional[str] = None
-
-    @classmethod
-    def from_api(cls, data: Dict[str, Any]) -> EvidenceFileData:
-        return cls(
-            file_id=data["fileId"],
-            file_name=data["fileName"],
-            created_utc=data["createdUtc"],
-            file_size=data["fileSize"],
-            file_extension=data["fileExtension"],
-            evidence_category=data.get("evidenceCategory"),
-        )

backend/pashub_fetcher/evidence_metadata.py

@@ -1,16 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from typing import Any, Dict
-
-
-@dataclass
-class EvidenceMetadata:
-    container_name: str
-    blob_uri: str
-
-    @classmethod
-    def from_api(cls, data: Dict[str, Any]) -> EvidenceMetadata:
-        return cls(
-            container_name=data["containerName"],
-            blob_uri=data["blobUri"],
-        )

backend/pashub_fetcher/handler/Dockerfile

@@ -1,26 +0,0 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-jammy
-
-# Install AWS Lambda RIE
-ADD https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie /usr/local/bin/aws-lambda-rie
-RUN chmod +x /usr/local/bin/aws-lambda-rie
-
-# Set working directory (Lambda task root)
-WORKDIR /var/task
-
-COPY utils/ utils/
-COPY backend/pashub_fetcher/ backend/pashub_fetcher/
-
-COPY backend/pashub_fetcher/handler/requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-# Local lambda entrypoint 
-# ENTRYPOINT ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"]
-
-#AWS lambda entrypoint
-ENTRYPOINT ["python", "-m", "awslambdaric"]
-
-# -----------------------------
-# Lambda handler
-# -----------------------------
-CMD ["backend.pashub_fetcher.handler.test_handler.handler"]
-# CMD ["backend.pashub_fetcher.handler.handler.handler"]

backend/pashub_fetcher/handler/handler.py

@@ -1,126 +0,0 @@
-import os
-import re
-from typing import Any, Dict, List, Mapping
-from openpyxl import load_workbook
-
-from backend.pashub_fetcher.job import Job
-from backend.pashub_fetcher.pashub_client import PashubClient, UnauthorizedError
-from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders
-from backend.pashub_fetcher.token_getter import get_token_from_local_storage
-from utils.logger import setup_logger
-from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient
-from utils.sharepoint.domna_sites import DomnaSites
-
-
-logger = setup_logger()
-
-
-def extract_jobs(filepath: str) -> List[Job]:
-    wb = load_workbook(filepath, data_only=True)
-    ws = wb["watford warm homes (wave 3) mai"]
-
-    HEADER_ROW = 3
-
-    headers: Dict[str, int] = {}
-    for col in range(1, ws.max_column + 1):
-        value = str(ws.cell(row=HEADER_ROW, column=col).value)
-        if value:
-            headers[value.strip()] = col
-
-    name_col = headers["Name"]
-    link_col = headers["Pashub Link"]
-
-    jobs: List[Job] = []
-
-    for row in range(HEADER_ROW + 1, ws.max_row + 1):
-        name = ws.cell(row=row, column=name_col).value
-        link = ws.cell(row=row, column=link_col).value
-
-        if not name or not link:
-            continue
-
-        link = str(link)
-
-        match = re.search(r"/jobs/([0-9a-fA-F\-]+)/", link)
-        if not match:
-            continue
-
-        job_id = match.group(1)
-
-        jobs.append({"id": job_id, "address": str(name)})
-
-    return jobs
-
-
-def handler(event: Mapping[str, Any], context: Any) -> None:
-    BASE_DIR = os.path.dirname(os.path.dirname(__file__))
-    filepath = os.path.join(BASE_DIR, "Watford_Warm_Homes_Wave_3_RA Downloads .xlsx")
-
-    jobs: List[Job] = extract_jobs(filepath)
-
-    logger.info("Successfully loaded jobs from spreadsheet")
-
-    pas_hub_email = "random@test.com"
-    pas_hub_password = "my_fake_password"
-
-    try:
-        token: str = get_token_from_local_storage(pas_hub_email, pas_hub_password)
-        logger.info(f"Token extracted successfully")
-    except:
-        logger.error("Error getting auth token from Pas Hub")
-        raise
-
-    pashub_client = PashubClient(token=token)
-    sharepoint_client = DomnaSharepointClient(
-        sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3
-    )
-
-    saved_file_paths: List[str] = []
-    BASE_PATH = "/Osmosis-ACD Projects/Watford Warm Homes/Watford Property Folders (Shared with Client)"  # TODO: get from request body
-
-    for job in jobs:
-        try:
-            job_files: List[str] = pashub_client.get_core_evidence_files_by_job_id(
-                job["id"]
-            )
-
-            # Upload files to sharepoint
-            job_path = f"{BASE_PATH}/{job['address']}"
-
-            sharepoint_client.makedir(job["address"], BASE_PATH)
-
-            for folder in SharepointSubfolders:
-                sharepoint_client.makedir(folder.value, job_path)
-
-            assessment_path = f"{job_path}/{SharepointSubfolders.ASSESSMENT.value}"
-
-            for file_path in job_files:
-                filename = file_path.split("/")[-1]
-
-                sharepoint_client.upload_file(
-                    file_path,
-                    assessment_path,
-                    filename,
-                )
-
-            saved_file_paths.extend(job_files)
-
-        except UnauthorizedError:
-            logger.warning("Token expired - refreshing")
-
-            token = get_token_from_local_storage(pas_hub_email, pas_hub_password)
-
-            pashub_client = PashubClient(token=token)
-
-            # retry once
-            saved_file_paths.extend(
-                pashub_client.get_core_evidence_files_by_job_id(job["id"])
-            )
-
-    print(f"saved {len(saved_file_paths)} files")
-
-
-if __name__ == "__main__":
-    event = {"Records": [{"body": "{}"}]}
-
-    handler(event, None)

backend/pashub_fetcher/handler/requirements.txt

@@ -1,5 +0,0 @@
-awslambdaric
-playwright==1.58.0
-requests
-msal
-openpyxl

backend/pashub_fetcher/handler/test_handler.py

@@ -1,7 +0,0 @@
-from typing import Any, Mapping
-import json
-
-
-def handler(event: Mapping[str, Any], context: Any) -> None:
-    print("Received event:")
-    print(json.dumps(event, indent=2))

backend/pashub_fetcher/job.py

@@ -1,6 +0,0 @@
-from typing import TypedDict
-
-
-class Job(TypedDict):
-    id: str
-    address: str

backend/pashub_fetcher/local_handler/docker-compose.yml

@@ -1,11 +0,0 @@
-version: "3.9"
-
-services:
-  pashub-fetcher-lambda:
-    build:
-      context: ../../../
-      dockerfile: backend/pashub_fetcher/handler/Dockerfile
-    ports:
-      - "9000:8080"
-    env_file:
-      - ../../../.env

backend/pashub_fetcher/local_handler/invoke_local_lambda.py

@@ -1,26 +0,0 @@
-#!/usr/bin/env python3
-import json
-import requests
-
-HOST = "localhost"
-PORT = "9000"
-
-LAMBDA_URL = f"http://{HOST}:{PORT}/2015-03-31/functions/function/invocations"
-
-payload = {
-    "Records": [
-        {
-            "body": json.dumps(
-                {
-                    "uprn": 123456,
-                }
-            )
-        }
-    ]
-}
-
-response = requests.post(LAMBDA_URL, json=payload)
-
-print("Status code:", response.status_code)
-print("Response:")
-print(response.text)

backend/pashub_fetcher/pashub_client.py

@@ -1,138 +0,0 @@
-from collections import defaultdict
-import os
-from typing import Dict, List, Optional
-from datetime import datetime
-
-import requests
-
-from backend.pashub_fetcher.core_files import CoreFiles
-from backend.pashub_fetcher.evidence_file_data import EvidenceFileData
-from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata
-from utils.logger import setup_logger
-
-
-logger = setup_logger()
-
-
-class UnauthorizedError(Exception):
-    pass
-
-
-class PashubClient:
-    def __init__(self, token: str):
-
-        self.token = token
-        self.company_id = "cb5249e2-8f31-4ef4-aefd-08ddaccb1fa2"
-        self.base = "https://pashub.net/api"
-
-        self.session = requests.Session()
-        self.session.headers.update(
-            {
-                "Authorization": f"Bearer {self.token}",
-                "Accept": "application/json",
-            }
-        )
-        logger.info("Finished initialising CotalityClient")
-
-    def get_core_evidence_files_by_job_id(self, job_id: str) -> List[str]:
-        logger.info(f"Getting Core Evidence Files for job ID {job_id}")
-        evidence_list: List[EvidenceFileData] = self._get_evidence_list(job_id)
-        logger.info(f"Found {len(evidence_list)} Evidence files to get")
-        if not evidence_list:
-            return []
-
-        saved_files: List[str] = []
-
-        core_files: Dict[CoreFiles, EvidenceFileData] = self._select_latest_core_files(
-            evidence_list
-        )
-
-        logger.info(f"Number of core files to download is {len(core_files)}")
-
-        for _, evidence in core_files.items():
-            evidence_id = evidence.file_id
-            if not evidence_id:
-                continue
-
-            logger.info(f"Getting metadata for file {evidence.file_name}")
-            metadata: EvidenceMetadata = self._get_evidence_metadata(
-                job_id, evidence_id
-            )
-
-            download_url: str = self._build_download_url(metadata, evidence.file_id)
-            output_dir: str = "/tmp"
-
-            file_name: str = evidence.file_name
-            file_path: str = os.path.join(output_dir, file_name)
-
-            self._download_file(download_url, file_path)
-            logger.info("Successfully downloaded file")
-            saved_files.append(file_path)
-
-        return saved_files
-
-    def _get_core_file_type(self, file: EvidenceFileData) -> Optional[CoreFiles]:
-        for core_file in CoreFiles:
-            if file.file_name.startswith(core_file.value):
-                return core_file
-        return None
-
-    def _select_latest_core_files(
-        self,
-        files: List[EvidenceFileData],
-    ) -> Dict[CoreFiles, EvidenceFileData]:
-        grouped: Dict[CoreFiles, List[EvidenceFileData]] = defaultdict(list)
-
-        for file in files:
-            core_type = self._get_core_file_type(file)
-            if not core_type:
-                continue
-            grouped[core_type].append(file)
-
-        latest_files: Dict[CoreFiles, EvidenceFileData] = {}
-
-        for core_type, group in grouped.items():
-            latest = max(group, key=lambda f: datetime.fromisoformat(f.created_utc))
-            latest_files[core_type] = latest
-
-        return latest_files
-
-    def _get_evidence_list(self, job_id: str) -> List[EvidenceFileData]:
-        url = f"{self.base}/jobs/{job_id}/evidence"
-
-        r = self.session.get(url)
-        if r.status_code == 401:
-            raise UnauthorizedError("Token expired or invalid")
-
-        r.raise_for_status()
-
-        results = r.json().get("results", [])
-
-        return [EvidenceFileData.from_api(item) for item in results]
-
-    def _get_evidence_metadata(self, job_id: str, evidence_id: str) -> EvidenceMetadata:
-        url = f"{self.base}/jobs/{job_id}/evidenceMetadata"
-        params = {"evidenceIds": evidence_id}
-
-        r = self.session.get(url, params=params)
-        if r.status_code == 401:
-            raise UnauthorizedError()
-
-        r.raise_for_status()
-
-        return EvidenceMetadata.from_api(r.json())
-
-    def _build_download_url(self, metadata: EvidenceMetadata, file_id: str) -> str:
-        container = metadata.container_name
-        blob_uri = metadata.blob_uri
-
-        base, sas = blob_uri.split("?", 1)
-
-        return f"{base}{container}/{file_id}?{sas}"
-
-    def _download_file(self, url: str, file_path: str) -> None:
-        r = requests.get(url)
-        r.raise_for_status()
-
-        with open(file_path, "wb") as f:
-            f.write(r.content)

backend/pashub_fetcher/sharepoint_subfolders.py

@@ -1,9 +0,0 @@
-from enum import Enum
-
-
-class SharepointSubfolders(str, Enum):
-    ASSESSMENT = "1. Retrofit Assessment"
-    COORDINATION = "2. Retrofit Coordination"
-    DESIGN = "3. Retrofit Design"
-    INSTALLER = "4. Installer Information"
-    TRUSTMARK = "5. TrustMark Lodgement"

backend/pashub_fetcher/token_getter.py

@@ -1,78 +0,0 @@
-import os
-
-from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
-
-from utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def get_token_from_local_storage(email: str, password: str) -> str:
-    logger.info("Starting Playwright flow")
-
-    # For local testing / debugging, we save videos of the flow
-    video_dir = os.path.join(os.path.dirname(__file__), "videos")
-    os.makedirs(video_dir, exist_ok=True)
-
-    with sync_playwright() as p:
-        browser = p.chromium.launch(
-            headless=True,
-            args=["--no-sandbox", "--disable-dev-shm-usage"],
-        )
-
-        context = browser.new_context(
-            record_video_dir=video_dir,
-            record_video_size={"width": 1280, "height": 720},
-        )
-        page = context.new_page()
-
-        try:
-            logger.info("Navigating to site...")
-            page.goto("https://pashub.net/", timeout=30000)
-
-            logger.info("Filling login form...")
-
-            email_input = page.locator("#email")
-            email_input.wait_for(state="visible", timeout=10000)
-            email_input.fill(email)
-
-            password_input = page.locator("#password")
-            password_input.wait_for(state="visible", timeout=10000)
-            password_input.focus()
-            password_input.fill(password)
-
-            logger.info("Submitting login...")
-            page.wait_for_selector("#btn-login", state="visible", timeout=10000)
-            with page.expect_navigation(timeout=15000):
-                page.click("#btn-login")
-
-            page.wait_for_timeout(3000)
-
-            if "login" in page.url.lower():
-                raise Exception("Login failed (still on login page)")
-
-            logger.info(f"Login likely successful. URL: {page.url}")
-
-            token = page.evaluate(
-                """() => {
-                    return localStorage.getItem('token');
-                }"""
-            )
-
-            if not token:
-                raise Exception("Login succeeded but no token found")
-
-            return token
-
-        except PlaywrightTimeoutError as e:
-            raise Exception(f"Timeout during login flow: {str(e)}")
-
-        except Exception as e:
-            raise Exception(f"Unexpected error: {str(e)}")
-
-        finally:
-            logger.info("Closing browser context (saving video)...")
-            context.close()
-            browser.close()
-
-            logger.info(f"Video(s) saved in: {video_dir}")

backend/tests/test_rebaselining_pipeline.py

@@ -2,10 +2,6 @@ import os
 import pickle
 import pandas as pd
 import pytest
-from datetime import datetime
-from backend.ml_models.api import ModelApi
-from backend.app.utils import sap_to_epc
-from backend.app.config import get_prediction_buckets
 
 
 def load_sample_certificates():
@@ -64,6 +60,12 @@ def load_cleaning_data():
 
 @pytest.mark.integration
 def test_rebaselining_pipeline_with_real_data():
+    import pandas as pd
+    from datetime import datetime
+    from backend.ml_models.api import ModelApi
+    from backend.app.utils import sap_to_epc
+    from backend.app.config import get_prediction_buckets
+
     df = load_sample_certificates()
     cleaning_data = load_cleaning_data()
     input_properties = [make_property_from_row(row, cleaning_data=cleaning_data) for _, row in df.iterrows()]

etl/epc/Record.py

@@ -566,7 +566,6 @@ class EPCRecord:
             "multi_glaze_proportion": addr.landlord_multi_glaze_proportion,
             "construction_age_band": addr.landlord_construction_age_band,
         }
-        landlord_remapping = {k: v for k, v in landlord_remapping.items() if v is not None}
 
         # Sanity check - ensure valid keys
         if any(k not in self._prepared_epc for k in landlord_remapping):
@@ -574,7 +573,6 @@ class EPCRecord:
 
         self.landlord_differences = {}  # Anything actaully changed
         for k, v in landlord_remapping.items():
-
             if k == "total_floor_area":
                 existing = self._prepared_epc.get(k)
                 if existing is not None and v is not None and abs(existing - v) > 1:  # 1m tolerance

infrastructure/terraform/lambda/pashub_to_ara/main.tf

@@ -1,27 +0,0 @@
-data "terraform_remote_state" "shared" {
-  backend = "s3"
-  config = {
-    bucket = "assessment-model-terraform-state"
-    key = "env:/${var.stage}/terraform.tfstate"
-    region = "eu-west-2"
-  }
-}
-
-module "lambda" {
-  source = "../../modules/lambda_with_sqs"
-
-  name  = "pashub_to_ara"
-  stage = var.stage
-
-  image_uri = local.image_uri
-
-  # Optional: Set maximum_concurrency to limit concurrent SQS-triggered invocations (2-1000)
-  maximum_concurrency = var.maximum_concurrency
-
-  batch_size = var.batch_size
-
-  environment = {
-    STAGE = var.stage
-    LOG_LEVEL = "info"
-  }
-}

infrastructure/terraform/lambda/pashub_to_ara/provider.tf

@@ -1,16 +0,0 @@
-terraform {
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = ">= 5.0"
-    }
-  }
-
-  backend "s3" {
-    bucket = "pashub-to-ara-terraform-state"
-    key    = "terraform.tfstate"
-    region = "eu-west-2"
-  }
-
-  required_version = ">= 1.2.0"
-}

infrastructure/terraform/lambda/pashub_to_ara/variables.tf

@@ -1,37 +0,0 @@
-variable "lambda_name" {
-  type        = string
-  description = "Logical name of the lambda (e.g. address2uprn)"
-}
-
-variable "stage" {
-  description = "Deployment stage (e.g. dev, prod)"
-  type        = string
-}
-variable "ecr_repo_url" {
-  type        = string
-  description = "ECR repository URL (no tag, no digest)"
-}
-
-variable "image_digest" {
-  type        = string
-  description = "Image digest (sha256:...)"
-}
-
-variable "maximum_concurrency" {
-  type        = number
-  default     = null
-  description = "Maximum number of concurrent Lambda invocations from SQS (2-1000). null = no limit."
-}
-
-variable "batch_size" {
-  type    = number
-  default = 1
-}
-
-locals {
-  image_uri = "${var.ecr_repo_url}@${var.image_digest}"
-}
-
-output "resolved_image_uri" {
-  value = local.image_uri
-}

infrastructure/terraform/shared/main.tf

@@ -524,20 +524,6 @@ output "ordnance_s3_read_and_write_arn" {
   value = module.ordnance_s3_read_and_write.policy_arn
 }
 
-################################################
-# Pas Hub to Ara – Lambda
-################################################
-module "pashub_to_ara_state_bucket" {
-  source      = "../modules/tf_state_bucket"
-  bucket_name = "pashub-to-ara-terraform-state"
-}
-
-module "pashub_to_ara_registry" {
-  source = "../modules/container_registry"
-  name   = "pashub_to_ara"
-  stage  = var.stage
-}
-
 ################################################
 # Engine – Lambda ECR
 ################################################

recommendations/Costs.py

@@ -209,7 +209,7 @@ class Costs:
                 # Try and get the region after converting the keys to lower
                 self.region = {
                     k.lower(): v for k, v in county_to_region_map.items()
-                }.get(self.property.epc_record.local_authority_label.lower(), None)
+                }.get(self.property.property.epc_record.local_authority_label.lower(), None)
 
             if self.region is None:
                 logger.warning("No region found for county %s, defaulting to South East England",

recommendations/HeatingRecommender.py

@@ -1062,8 +1062,8 @@ class HeatingRecommender:
             **hot_water_simulation_config
         }
         # This upgrade will only take the heating system to average energy efficiency
-        if self.property.epc_record.mainheat_energy_eff in ["Very Poor", "Poor", "Average"] and not self.dual_heating:
-            heating_simulation_config["mainheat_energy_eff_ending"] = "Good"
+        if self.property.epc_record.mainheat_energy_eff in ["Very Poor", "Poor"] and not self.dual_heating:
+            heating_simulation_config["mainheat_energy_eff_ending"] = "Average"
         else:
             heating_simulation_config["mainheat_energy_eff_ending"] = self.property.epc_record.mainheat_energy_eff
 

scripts/download_cotality_evidence.py

@@ -1,7 +1,7 @@
 import requests
 import json
 
-TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik1EUTRNRU5GUTBVNU9FUXpOelk1TVRFME0wUkdOMFpFUkRoR1JVVkJNVGMxT1RFNFJERXlPQSJ9.eyJodHRwOi8vZW1haWwiOiJzZWJhc3RpYW5Ab3Ntb3Npcy1hY2QuY29tIiwiaHR0cDovL2NsdWsudG9rZW4vbGFzdFBhc3N3b3JkQ2hhbmdlIjoiMjAyNS0wOC0yNlQwOTo1NDoyNi4zMjZaIiwiaHR0cDovL2NsdWsudG9rZW4vY29ubmVjdGlvbiI6ImVUZWNoSUQiLCJodHRwOi8vY2x1ay50b2tlbi9zdHJhdGVneSI6ImF1dGgwIiwiaHR0cDovL2NsdWsudG9rZW4vc3RyYXRlZ3lUeXBlIjoiZGF0YWJhc2UiLCJpc3MiOiJodHRwczovL2V0ZWNoaWQuZXUuYXV0aDAuY29tLyIsInN1YiI6ImF1dGgwfDY4YWQ4NDUyZDI2YzI1ZmMyMzkwZmYxYSIsImF1ZCI6WyJodHRwczovL3Bhc2h1Yi5hcGkuZXRlY2gubmV0IiwiaHR0cHM6Ly9ldGVjaGlkLmV1LmF1dGgwLmNvbS91c2VyaW5mbyJdLCJpYXQiOjE3NzQyODczOTMsImV4cCI6MTc3NDI5NDU5Mywic2NvcGUiOiJvcGVuaWQiLCJhenAiOiJEaVp6d3VVaTVkVmozOXR3NG00bWZ6emZvRm5MdmVLZyJ9.NHh21XfnRofsFkRkc-28Dz-vQAdY70lXkEmh-Mzz7Fg6gjDbZeMu7PnBwgbDP_U8r6R0mI_pDIUc1MzJe1Rf5SF2-RV36TcGzmVzb3ek9wPsy3lxST5WL-vn-qUJ7GsZiGOeQ-jDLLFn8b8tjFrD7BGv8uphrfYAbPDm0atznkdbUSQQy-rfRJWhisnDtHf99j96TuJz3dV4bfI6VGrin-jezbg6BCvUYWQtttUs7knQKEWO0sGGDxtS29sbn4MX8Jqz4-hf6N2XSlgv52aIDwTVX-lyMWzfoeuIGhvCKuDiJeVw2c0r2UZFpHqjnfhXcb0_aacukXe8z-srj8-Rdw"
+TOKEN = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCIsImtpZCI6Ik1EUTRNRU5GUTBVNU9FUXpOelk1TVRFME0wUkdOMFpFUkRoR1JVVkJNVGMxT1RFNFJERXlPQSJ9.eyJodHRwOi8vZW1haWwiOiJzZWJhc3RpYW5Ab3Ntb3Npcy1hY2QuY29tIiwiaHR0cDovL2NsdWsudG9rZW4vbGFzdFBhc3N3b3JkQ2hhbmdlIjoiMjAyNS0wOC0yNlQwOTo1NDoyNi4zMjZaIiwiaHR0cDovL2NsdWsudG9rZW4vY29ubmVjdGlvbiI6ImVUZWNoSUQiLCJodHRwOi8vY2x1ay50b2tlbi9zdHJhdGVneSI6ImF1dGgwIiwiaHR0cDovL2NsdWsudG9rZW4vc3RyYXRlZ3lUeXBlIjoiZGF0YWJhc2UiLCJpc3MiOiJodHRwczovL2V0ZWNoaWQuZXUuYXV0aDAuY29tLyIsInN1YiI6ImF1dGgwfDY4YWQ4NDUyZDI2YzI1ZmMyMzkwZmYxYSIsImF1ZCI6WyJodHRwczovL3Bhc2h1Yi5hcGkuZXRlY2gubmV0IiwiaHR0cHM6Ly9ldGVjaGlkLmV1LmF1dGgwLmNvbS91c2VyaW5mbyJdLCJpYXQiOjE3NzMyMzc4MjQsImV4cCI6MTc3MzI0NTAyNCwic2NvcGUiOiJvcGVuaWQiLCJhenAiOiJEaVp6d3VVaTVkVmozOXR3NG00bWZ6emZvRm5MdmVLZyJ9.mkkxeZiD_ByHY4TJKpLQ-trmeGs15s0ekL6u1n-ek9j-EzNyf6qalEHCyHf8gzdNhU_vay96bIOMRHp4vXFaLqSANwKZayIS3EoA_b9-u2FAZpooxEvReAMNJGoZ6WLD01AQXWv-l7ww1ZqAnQzw0moL_Oma6hVmA5oa-RJKJ3MerS7e0Wei97Db48E140-EAbQf2iPcKYYtCNRA4il6n8DFiqGeoUMGo99jkR1ceZAvMpOAj8RhKX-4qSiDfX6yXUS2G96U5m7S_GWI-DEj5TazkN10Af3TyOY3EVjmZoJcRpiAR4cFmlfcTydjrShU03DWmPZm1QItf2McxfCpNA"
 
 base = "https://pashub.net/api"
 

utils/sharepoint/domna_sharepoint_client.py

@@ -1,105 +0,0 @@
-import os
-from typing import Any, Dict, Optional
-from io import BytesIO
-
-from utils.logger import setup_logger
-from utils.sharepoint.domna_sites import DomnaSites
-from utils.sharepoint.sharepoint_client import SharePointClient
-
-
-class DomnaSharepointClient:
-    """
-    A simple scraper to get the contents of a sharepoint and validatate inputs so I can manually change
-    """
-
-    def __init__(self, sharepoint_location: DomnaSites):
-        self.logger = setup_logger()
-        self.sharepoint_client_id = os.getenv("SHAREPOINT_CLIENT_ID", None)
-        self.sharepoint_client_secret = os.getenv("SHAREPOINT_CLIENT_SECRET", None)
-        self.sharepoint_tenant_id = os.getenv("SHAREPOINT_TENANT_ID", None)
-        self.sharepoint_drive = sharepoint_location
-
-        assert (
-            self.sharepoint_client_id is not None
-        ), "Please assign SHAREPOINT_CLIENT_ID env variable"
-        assert (
-            self.sharepoint_client_secret is not None
-        ), "Please assign SHAREPOINT_CLIENT_SECRET env variable"
-        assert (
-            self.sharepoint_tenant_id is not None
-        ), "Please assign SHAREPOINT_TENANT_ID env variable"
-
-    def get_folders_in_path(self, path: str) -> Dict[str, Any]:
-        sharepoint_client = SharePointClient(
-            tenant_id=self.sharepoint_tenant_id,
-            client_id=self.sharepoint_client_id,
-            client_secret=self.sharepoint_client_secret,
-            site_id=self.sharepoint_drive.value,
-        )
-
-        return sharepoint_client.list_folder_contents(path)
-
-    def does_folder_exists_at(self, file_name: str, file_path: str):
-        folders: Dict[str, Any] = self.get_folders_in_path(file_path)
-        if "value" in folders:
-            for folder in folders["value"]:
-                if file_name.upper() in folder["name"].upper():
-                    return True
-        return False
-
-    def create_dir(self, dir_name: str, at_path: str = "/") -> str:
-        sharepoint_client = SharePointClient(
-            tenant_id=self.sharepoint_tenant_id,
-            client_id=self.sharepoint_client_id,
-            client_secret=self.sharepoint_client_secret,
-            site_id=self.sharepoint_drive.value,
-        )
-
-        folders: Dict[str, Any] = self.get_folders_in_path(at_path)
-
-        # Check if folder already exists (case-insensitive match)
-        if "value" in folders:
-            for folder in folders["value"]:
-                if "name" in folder and folder["name"].lower() == dir_name.lower():
-                    self.logger.info(f"Folder already exists: {dir_name} at {at_path}")
-                    return folder["webUrl"]  # ✅ return existing folder
-
-        # Folder does NOT exist → create it
-        self.logger.info(f"Creating folder: {dir_name} at {at_path}")
-        created: Dict[str, Any] = sharepoint_client.create_folder(dir_name, at_path)
-
-        return created["webUrl"]
-
-    def makedir(self, dir_name: str, at_path: str = "/") -> str:
-        return self.create_dir(dir_name, at_path)
-
-    def upload_file(
-        self, file_path: str, sharepoint_path: str, file_name: str
-    ) -> Optional[Dict[str, Any]]:
-        sharepoint_client = SharePointClient(
-            tenant_id=self.sharepoint_tenant_id,
-            client_id=self.sharepoint_client_id,
-            client_secret=self.sharepoint_client_secret,
-            site_id=self.sharepoint_drive.value,
-        )
-
-        def get_file_stream(file_path: str):
-            return open(file_path, "rb")
-
-        sharepoint_client.upload_file(
-            file_name, get_file_stream(file_path), sharepoint_path
-        )
-
-    def create_temp_file(self, content: BytesIO, path: str):
-        # Ensure the path is under /tmp/
-        new_path = os.path.join("/tmp/sharepoint", path)
-
-        # Ensure the parent directory exists
-        os.makedirs(os.path.dirname(new_path), exist_ok=True)
-
-        # Write content to the specified file
-        with open(new_path, "wb+") as temp_file:
-            temp_file.write(content.getvalue())
-
-        self.logger.debug(f"Temporary file created at: {new_path}")
-        return new_path

utils/sharepoint/domna_sites.py

@@ -1,11 +0,0 @@
-from enum import Enum
-import os
-
-
-class DomnaSites(Enum):
-    # https//{tenant}.sharepoint.com/sites/{site}/_api/site/id
-    # TODO: Add these to github secrets!!!
-    DOMNA = os.getenv("DOMNA_SHAREPOINT_ID")
-    OSMOSIS_ACD = os.getenv("OSMOSIS_ACD_SHAREPOINT_ID")
-    PRIVATE_PAY = os.getenv("PRIVATE_PAY_SHAREPOINT_ID")
-    SOCIAL_HOUSING_WAVE_3 = os.getenv("SOCIAL_HOUSING_WAVE_3_SHAREPOINT_ID")

utils/sharepoint/main.py

@@ -1,25 +0,0 @@
-# This is small script to see if Domna Sharepoint Client works
-# for basic functionality
-
-# Can we import it?
-from io import BytesIO
-
-from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient, DomnaSites
-
-
-# can we initliase it
-client = DomnaSharepointClient(sharepoint_location=DomnaSites.SOCIAL_HOUSING_WAVE_3)
-
-# can we get an example of root path?
-
-client.get_folders_in_path("/")
-client.get_folders_in_path("/JTK Test Folder")
-
-# can we make a folder appear in JTK Test Folder?
-client.makedir("Dan is the best", "/JTK Test Folder")
-
-content = BytesIO(b"Hello, this is some file content!")
-path = client.create_temp_file(content, "some/place/over/the/rainbow")
-client.upload_file(
-    path, "/JTK Test Folder/Dan is the best", "junte_is_the_worst_at_python.txt"
-)

utils/sharepoint/sharepoint_client.py

@@ -1,346 +0,0 @@
-"""
-This file contains the functions which enable interaction with SharePoint via the API.
-
-Documentation to get api_id:
-https://answers.microsoft.com/en-us/msoffice/forum/all/what-is-the-best-way-to-findout-the-share-point/7b2d4183-4188-4cd5-8441-dd93207c5a01
-"""
-
-from typing import Any, BinaryIO, Dict, Optional
-
-from msal import ConfidentialClientApplication
-from datetime import datetime, timedelta
-import requests
-from functools import wraps
-import time
-from io import BytesIO
-
-from utils.logger import setup_logger
-
-# Api Documentation: https://learn.microsoft.com/en-us/graph/api/drive-get?view=graph-rest-1.0&tabs=http
-
-logger = setup_logger()
-
-
-def handle_error(response):
-    """
-    Handle errors based on HTTP status codes and log detailed information.
-    """
-    try:
-        error_json = response.json().get("error", {})
-    except ValueError:
-        error_json = {}
-
-    error_code = error_json.get("code", "unknownError")
-    error_message = error_json.get("message", "No detailed error message provided.")
-    inner_error = error_json.get("innererror", {})
-    details = error_json.get("details", [])
-
-    logger.error(f"Error Code: {error_code}")
-    logger.error(f"Error Message: {error_message}")
-    if inner_error:
-        logger.error(f"Inner Error: {inner_error}")
-    if details:
-        logger.error(f"Error Details: {details}")
-
-    if response.status_code == 401:
-        logger.error("Unauthorized. Token might be invalid.")
-    elif response.status_code == 403:
-        logger.error("Forbidden. Access denied to the requested resource.")
-    elif response.status_code == 404:
-        logger.error("Not Found. The requested resource doesn’t exist.")
-    elif response.status_code == 429:
-        retry_after = int(
-            response.headers.get("Retry-After", 5)
-        )  # Default to 5 seconds if not provided
-        logger.warning(f"Too Many Requests. Retrying after {retry_after} seconds...")
-        time.sleep(retry_after)
-        return "retry"
-    elif response.status_code in (500, 503):
-        retry_after = int(
-            response.headers.get("Retry-After", 5)
-        )  # Default to 5 seconds if not provided
-        logger.error(f"Server error. Retrying after {retry_after} seconds...")
-        time.sleep(retry_after)
-        return "retry"
-    else:
-        raise ValueError(
-            f"API request failed with status code {response.status_code} - {error_message}"
-        )
-
-    raise ValueError(
-        f"API request failed with status code {response.status_code} - {error_message}"
-    )
-
-
-def api_call_decorator(func):
-    """
-    Handles various aspects of the API call, including refreshing the access token if needed and handling pagination.
-    :param func: The function to be decorated.
-    :return: The wrapped function.
-    """
-
-    @wraps(func)
-    def wrapper(self, *args, **kwargs):
-        try:
-            # Check and refresh the access token if needed
-            if self.is_access_token_expired():
-                self.retrieve_access_token()
-                logger.debug("Access token refreshed.")
-
-            # Get the HTTP method, URL, and optionally data from the function
-            http_method, url, data = func(self, *args, **kwargs)
-
-            # Initialize the results list and handle pagination if page_size is provided
-            results = []
-            page_size = kwargs.get("page_size", None)
-            response_data = {}
-
-            while url:
-                response = requests.request(
-                    http_method, url, headers=self.headers, json=data
-                )
-
-                # Handle the response
-                if response.status_code == 200 or response.status_code == 201:
-                    response_json = response.json()  # Store the response JSON
-                    if page_size:
-                        results.extend(response_json.get("value", []))
-                        url = response_json.get("@odata.nextLink", None)
-                    else:
-                        response_data = (
-                            response_json  # Capture the full response for consistency
-                        )
-                        break
-                else:
-                    retry = handle_error(response)
-                    if retry == "retry":
-                        continue
-
-            if page_size:
-                response_data = {"value": results}
-
-            return response_data
-
-        except Exception as e:
-            logger.exception("An error occurred during the API call.")
-            raise e
-
-    return wrapper
-
-
-class SharePointClient:
-    access_token = None
-    access_token_request_timestamp = None
-    access_token_expiry = None
-    headers = None
-
-    TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
-
-    def __init__(
-        self,
-        tenant_id,
-        client_id,
-        client_secret,
-        site_id,
-        access_token=None,
-        access_token_expiration_details=None,
-    ):
-        """
-        Initializes the SharePointClient with necessary credentials and site information.
-        :param tenant_id: The tenant ID.
-        :param client_id: The client ID.
-        :param client_secret: The client secret.
-        :param site_id: The site ID.
-        :param access_token: The access token (optional)
-        :param access_token_expiration_details: The access token expiration details (optional)
-        """
-        self.tenant_id = tenant_id
-        self.client_id = client_id
-        self.client_secret = client_secret
-
-        if access_token:
-            if not access_token_expiration_details:
-                raise ValueError("Access token expiration details must be provided.")
-            self.access_token = access_token
-            self.set_access_token_expiration_details(access_token_expiration_details)
-            self.headers = {
-                "Authorization": f"Bearer {self.access_token['access_token']}"
-            }
-        else:
-            self.retrieve_access_token()
-
-        # Retrieve static identifiers
-        self.site_id = site_id
-        self.document_drive = self.get_documents_drive()
-        self.document_drive_id = self.document_drive["id"]
-
-    def get_token_expiration_details(self):
-        """
-        Returns the access token expiration details. Converts the datetime objects to strings for serialization.
-        :return:
-        """
-        return {
-            "access_token_request_timestamp": datetime.strftime(
-                self.access_token_request_timestamp, self.TIMESTAMP_FORMAT
-            ),
-            "access_token_expiry": datetime.strftime(
-                self.access_token_expiry, self.TIMESTAMP_FORMAT
-            ),
-        }
-
-    def set_access_token_expiration_details(self, access_token_expiration_details):
-        """
-        Sets the access token expiration details from a serialized dictionary.
-        :param access_token_expiration_details: The serialized access token expiration details.
-        :return:
-        """
-        self.access_token_request_timestamp = datetime.strptime(
-            access_token_expiration_details["access_token_request_timestamp"],
-            self.TIMESTAMP_FORMAT,
-        )
-        self.access_token_expiry = datetime.strptime(
-            access_token_expiration_details["access_token_expiry"],
-            self.TIMESTAMP_FORMAT,
-        )
-
-    def is_access_token_expired(self):
-        """
-        Checks if the access token has expired. If it has, a new access token is retrieved.
-        :return: True if expired, False otherwise.
-        """
-        return datetime.now() >= self.access_token_expiry
-
-    def retrieve_access_token(self, refresh=False):
-        """
-        Implements authentication using MSAL.
-        :param refresh: If True, force a refresh of the access token.
-        :return: None
-        """
-        app = ConfidentialClientApplication(
-            self.client_id,
-            authority=f"https://login.microsoftonline.com/{self.tenant_id}",
-            client_credential=self.client_secret,
-        )
-
-        scope = ["https://graph.microsoft.com/.default"]
-
-        access_token_request_timestamp = datetime.now()
-
-        if refresh:
-            logger.debug("Forcing refresh of access token.")
-            token = app.acquire_token_for_client(scopes=scope)
-        else:
-            # Check if a token is already cached
-            token = app.acquire_token_silent(scope, account=None)
-
-            if not token:
-                token = app.acquire_token_for_client(scopes=scope)
-
-        if "access_token" not in token:
-            logger.error("Authentication failed.")
-            raise ValueError("Authentication failed")
-
-        access_token_expiry = access_token_request_timestamp + timedelta(
-            seconds=token["expires_in"] - 20
-        )
-
-        self.access_token = token
-        self.access_token_request_timestamp = access_token_request_timestamp
-        self.access_token_expiry = access_token_expiry
-        self.headers = {"Authorization": f"Bearer {self.access_token['access_token']}"}
-
-        # logger.debug("Access token retrieved successfully.")
-
-    @api_call_decorator
-    def get_documents_drive(self):
-        """
-        Get the document drive of the SharePoint site.
-        :return: Tuple containing HTTP method, URL, and None for data.
-        """
-        url = f"https://graph.microsoft.com/v1.0/sites/{self.site_id}/drive"
-        # logger.debug(f"Getting document drive from URL: {url}")
-        return "GET", url, None
-
-    @api_call_decorator
-    def list_folder_contents(
-        self, folder_path: str, page_size: int = 100
-    ) -> Dict[str, Any]:
-        """
-        GET drive/root/children
-
-        This function will list the contents of a folder in SharePoint.
-        :param drive_id: The ID of the drive.
-        :param folder_path: The path of the folder.
-        :param page_size: The number of items per page (default is 100).
-        :return: Tuple containing HTTP method, URL, and None for data.
-        """
-        url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children?$top={page_size}"
-        # logger.debug(f"Listing folder contents from URL: {url}")
-        return "GET", url, None
-
-    @api_call_decorator
-    def create_folder(self, file_name: str, folder_path: str) -> Dict[str, Any]:
-        """
-        POST https://graph.microsoft.com/v1.0/me/drive/root/children
-        Content-Type: application/json
-        {
-            "name": "New Folder",
-            "folder": { },
-            "@microsoft.graph.conflictBehavior": "rename"
-        }
-
-        """
-        data: Dict[str, Any] = {
-            "name": file_name,
-            "folder": {},
-            "@microsoft.graph.conflictBehavior": "rename",
-        }
-        url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{folder_path}:/children"
-
-        return "POST", url, data
-
-    def upload_file(
-        self, file_name: str, file_stream: BinaryIO, sharepoint_parent_id: str
-    ) -> Optional[Dict[str, Any]]:
-        """
-        Uploads a file to SharePoint using the Graph API.
-        PUT /drives/{drive-id}/root:/{path-to-file}:/content
-
-        :param file_name: Name of the file to upload
-        :param sharepoint_path: Path within the SharePoint site (folder path)
-        :param file_stream: File content as a binary stream (e.g., BytesIO or open(file, 'rb'))
-        :return: Response JSON from the API
-        """
-        url = f"https://graph.microsoft.com/v1.0/drives/{self.document_drive_id}/root:/{sharepoint_parent_id}/{file_name}:/content"
-        # logger.debug(f"Uploading file to URL: {url}")
-
-        response = requests.put(url, headers=self.headers, data=file_stream)
-
-        if response.status_code in (200, 201):
-            # logger.info(f"File '{file_name}' uploaded successfully.")
-            return response.json()
-        else:
-            retry = handle_error(response)
-            if retry == "retry":
-                return self.upload_file(file_name, sharepoint_parent_id, file_stream)
-
-    @staticmethod
-    def download_sharepoint_file(download_url):
-        """
-        Downloads a file from the given URL and returns its content.
-
-        :param download_url: The URL to download the file from.
-        :return: The content of the downloaded file.
-        """
-        response = requests.get(download_url, stream=True)
-        response.raise_for_status()  # Check if the request was successful
-
-        file_content = BytesIO()
-
-        # Read the file content into memory
-        for chunk in response.iter_content(chunk_size=8192):
-            file_content.write(chunk)
-
-        file_content.seek(0)  # Reset the file pointer to the beginning
-
-        return file_content