diff --git a/.devcontainer/backend/devcontainer.json b/.devcontainer/backend/devcontainer.json index a96935020..77f3a1acd 100644 --- a/.devcontainer/backend/devcontainer.json +++ b/.devcontainer/backend/devcontainer.json @@ -5,7 +5,7 @@ "remoteUser": "vscode", "workspaceFolder": "/workspaces/model", "initializeCommand": "docker network create shared-dev 2>/dev/null || true; test -d \"$HOME/.config/gh\" || test -n \"$GITHUB_TOKEN\" || { echo >&2 'error: no GitHub auth found. Run `gh auth login && gh auth setup-git` on the host, or export GITHUB_TOKEN, then retry.'; exit 1; }", - "postCreateCommand": "gh repo clone Hestia-Homes/agentic-toolkit /tmp/agentic-toolkit -- --branch 0.0.7 --depth 1 && bash /tmp/agentic-toolkit/setup.sh", + "postCreateCommand": "gh repo clone Hestia-Homes/agentic-toolkit /tmp/agentic-toolkit -- --branch 0.0.10 --depth 1 && bash /tmp/agentic-toolkit/setup.sh", "postStartCommand": "bash .devcontainer/backend/post-install.sh", "mounts": [ "source=${localEnv:HOME},target=/workspaces/home,type=bind", diff --git a/.github/workflows/deploy_terraform.yml b/.github/workflows/deploy_terraform.yml index 87bb264a3..48158e4b3 100644 --- a/.github/workflows/deploy_terraform.yml +++ b/.github/workflows/deploy_terraform.yml @@ -453,7 +453,7 @@ jobs: uses: ./.github/workflows/_build_image.yml with: ecr_repo: pashub_to_ara-${{ needs.determine_stage.outputs.stage }} - dockerfile_path: backend/pashub_fetcher/handler/Dockerfile + dockerfile_path: applications/pashub_fetcher/handler/Dockerfile build_context: . secrets: AWS_ACCESS_KEY_ID: ${{ secrets.DEV_AWS_ACCESS_KEY_ID }} diff --git a/.github/workflows/lambda_smoke_tests.yml b/.github/workflows/lambda_smoke_tests.yml index d958abcb9..d408c46d8 100644 --- a/.github/workflows/lambda_smoke_tests.yml +++ b/.github/workflows/lambda_smoke_tests.yml @@ -109,7 +109,7 @@ # pashub_smoke_test: # uses: ./.github/workflows/_smoke_test_lambda.yml # with: -# dockerfile_path: backend/pashub_fetcher/handler/Dockerfile +# dockerfile_path: applications/pashub_fetcher/handler/Dockerfile # build_context: . # service_name: pashub diff --git a/backend/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx b/applications/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx similarity index 100% rename from backend/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx rename to applications/pashub_fetcher/The_Guinness_Partnership_AtkinsR_alis_Coordination_Design_Board_1774881298.xlsx diff --git a/backend/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx b/applications/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx similarity index 100% rename from backend/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx rename to applications/pashub_fetcher/Watford_Warm_Homes_Wave_3_RA Downloads .xlsx diff --git a/applications/pashub_fetcher/__init__.py b/applications/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/handler/handler.py b/applications/pashub_fetcher/handler.py similarity index 88% rename from backend/pashub_fetcher/handler/handler.py rename to applications/pashub_fetcher/handler.py index 00f0ddea9..967945436 100644 --- a/backend/pashub_fetcher/handler/handler.py +++ b/applications/pashub_fetcher/handler.py @@ -1,12 +1,12 @@ from typing import Any, Callable, Dict, List, Optional from backend.app.config import get_settings -from backend.pashub_fetcher.pashub_client import PashubClient -from backend.pashub_fetcher.pashub_service import PashubService -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from infrastructure.pashub_fetcher.pashub_client import PashubClient +from orchestration.pashub_fetcher_orchestrator import PashubFetcherOrchestrator +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) -from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage from backend.app.db.models.tasks import SourceEnum from backend.utils.subtasks import task_handler from utils.logger import setup_logger @@ -63,7 +63,7 @@ def handler(body: Dict[str, Any], context: Any) -> List[str]: f"Unrecognised sharepoint_site '{payload.sharepoint_site}'; skipping SharePoint upload" ) - service = PashubService( + service = PashubFetcherOrchestrator( pashub_client=get_pashub_client(pashub_email, pashub_password), sharepoint_client=sharepoint_client, s3_bucket=S3_BUCKET, diff --git a/backend/pashub_fetcher/handler/Dockerfile b/applications/pashub_fetcher/handler/Dockerfile similarity index 69% rename from backend/pashub_fetcher/handler/Dockerfile rename to applications/pashub_fetcher/handler/Dockerfile index d20d00d0c..b3c79a1f0 100644 --- a/backend/pashub_fetcher/handler/Dockerfile +++ b/applications/pashub_fetcher/handler/Dockerfile @@ -1,6 +1,6 @@ # jammy (Ubuntu 22.04) ships Python 3.10, which lacks enum.StrEnum — used by # domain/modelling/measure_type.py, which the handler pulls in transitively -# (handler -> pashub_service -> documents_parser.parser -> ... -> domain). +# (handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> ... -> domain). # noble (Ubuntu 24.04) ships Python 3.12, matching the project's 3.11+ standard. FROM mcr.microsoft.com/playwright/python:v1.58.0-noble @@ -14,14 +14,18 @@ WORKDIR /var/task COPY utils/ utils/ COPY backend/ backend/ COPY datatypes/ datatypes/ -# handler -> pashub_service -> documents_parser.parser -> datatypes.epc.domain.mapper +# handler -> pashub_fetcher_orchestrator -> documents_parser.parser -> datatypes.epc.domain.mapper # -> domain.sap10_calculator, and -> db_writer -> epc_property model # -> infrastructure.postgres. Without these the lambda fails at init with # "No module named 'domain'" / "'infrastructure'". COPY domain/ domain/ COPY infrastructure/ infrastructure/ +# The service now lives across the DDD layers: the handler is in applications/, +# PashubFetcherOrchestrator in orchestration/, the client in infrastructure/. +COPY applications/ applications/ +COPY orchestration/ orchestration/ -COPY backend/pashub_fetcher/handler/requirements.txt . +COPY applications/pashub_fetcher/handler/requirements.txt . RUN pip install --no-cache-dir -r requirements.txt # Local lambda entrypoint @@ -33,4 +37,4 @@ ENTRYPOINT ["python", "-m", "awslambdaric"] # ----------------------------- # Lambda handler # ----------------------------- -CMD ["backend.pashub_fetcher.handler.handler.handler"] \ No newline at end of file +CMD ["applications.pashub_fetcher.handler.handler"] \ No newline at end of file diff --git a/backend/pashub_fetcher/handler/requirements.txt b/applications/pashub_fetcher/handler/requirements.txt similarity index 100% rename from backend/pashub_fetcher/handler/requirements.txt rename to applications/pashub_fetcher/handler/requirements.txt diff --git a/backend/pashub_fetcher/handler/test_handler.py b/applications/pashub_fetcher/handler/test_handler.py similarity index 100% rename from backend/pashub_fetcher/handler/test_handler.py rename to applications/pashub_fetcher/handler/test_handler.py diff --git a/backend/pashub_fetcher/local_handler/docker-compose.yml b/applications/pashub_fetcher/local_handler/docker-compose.yml similarity index 65% rename from backend/pashub_fetcher/local_handler/docker-compose.yml rename to applications/pashub_fetcher/local_handler/docker-compose.yml index 8b183d804..0bb32123e 100644 --- a/backend/pashub_fetcher/local_handler/docker-compose.yml +++ b/applications/pashub_fetcher/local_handler/docker-compose.yml @@ -4,9 +4,9 @@ services: pashub-fetcher-lambda: build: context: ../../../ - dockerfile: backend/pashub_fetcher/handler/Dockerfile + dockerfile: applications/pashub_fetcher/handler/Dockerfile entrypoint: ["/usr/local/bin/aws-lambda-rie", "python", "-m", "awslambdaric"] - command: ["backend.pashub_fetcher.handler.handler.handler"] + command: ["applications.pashub_fetcher.handler.handler"] ports: - "9000:8080" env_file: diff --git a/backend/pashub_fetcher/local_handler/invoke_local_lambda.py b/applications/pashub_fetcher/local_handler/invoke_local_lambda.py similarity index 100% rename from backend/pashub_fetcher/local_handler/invoke_local_lambda.py rename to applications/pashub_fetcher/local_handler/invoke_local_lambda.py diff --git a/backend/pashub_fetcher/pashub_to_ara_trigger_request.py b/applications/pashub_fetcher/pashub_to_ara_trigger_request.py similarity index 100% rename from backend/pashub_fetcher/pashub_to_ara_trigger_request.py rename to applications/pashub_fetcher/pashub_to_ara_trigger_request.py diff --git a/backend/pashub_fetcher/trigger_local_lambda_from_file.py b/applications/pashub_fetcher/trigger_local_lambda_from_file.py similarity index 93% rename from backend/pashub_fetcher/trigger_local_lambda_from_file.py rename to applications/pashub_fetcher/trigger_local_lambda_from_file.py index ad3097da1..cfc8ad697 100644 --- a/backend/pashub_fetcher/trigger_local_lambda_from_file.py +++ b/applications/pashub_fetcher/trigger_local_lambda_from_file.py @@ -5,10 +5,10 @@ from typing import Any, Dict, List from openpyxl import load_workbook -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) -from backend.pashub_fetcher.handler.handler import handler +from applications.pashub_fetcher.handler import handler if __name__ == "__main__": BASE_DIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py b/applications/pashub_fetcher/trigger_pashub_sqs_from_file.py similarity index 98% rename from backend/pashub_fetcher/trigger_pashub_sqs_from_file.py rename to applications/pashub_fetcher/trigger_pashub_sqs_from_file.py index b647c6a76..aa36aa7a5 100644 --- a/backend/pashub_fetcher/trigger_pashub_sqs_from_file.py +++ b/applications/pashub_fetcher/trigger_pashub_sqs_from_file.py @@ -7,7 +7,7 @@ import boto3 from openpyxl import load_workbook from backend.app.config import get_settings -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) diff --git a/applications/sharepoint_renamer/handler/Dockerfile b/applications/sharepoint_renamer/handler/Dockerfile index 6a0a28b46..40373e7b0 100644 --- a/applications/sharepoint_renamer/handler/Dockerfile +++ b/applications/sharepoint_renamer/handler/Dockerfile @@ -8,7 +8,8 @@ RUN pip install --no-cache-dir -r requirements.txt COPY utils/ utils/ COPY utilities/ utilities/ COPY backend/__init__.py backend/__init__.py -COPY backend/pashub_fetcher/ backend/pashub_fetcher/ +# SharepointSubfolders only — the rest of the PasHub service is not needed here. +COPY domain/pashub_fetcher/ domain/pashub_fetcher/ COPY orchestration/ orchestration/ COPY applications/sharepoint_renamer/ applications/sharepoint_renamer/ CMD ["applications.sharepoint_renamer.handler.handler"] diff --git a/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md index e13f81c84..f79faf188 100644 --- a/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md +++ b/docs/HANDOVER_PASHUB_838_PROBLEM_PROPERTIES.md @@ -29,8 +29,9 @@ PDF) parses of 754881 are identical field-for-field — so the harness fixtures faithful and the ~2.7 MAE is a true calculator gap, not a fixture artifact. `parse_site_notes_pdf` never extracts the SAP rating from the PDF at all (`energy_rating_current` is populated by Dan's separate pashub-API fetch, -`pashub_service.py:45` ← `preSapRating`), which is why the rating is `None` on a -raw parse but present in the DB — unrelated to stripping. +`orchestration/pashub_fetcher_orchestrator.py` ← `preSapRating`), which is why +the rating is `None` on a raw parse but present in the DB — unrelated to +stripping. ## Properties that did not run (need PasHub re-extraction — data, not code) diff --git a/domain/pashub_fetcher/__init__.py b/domain/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/core_files.py b/domain/pashub_fetcher/core_files.py similarity index 60% rename from backend/pashub_fetcher/core_files.py rename to domain/pashub_fetcher/core_files.py index c6392e861..0f0bea34a 100644 --- a/backend/pashub_fetcher/core_files.py +++ b/domain/pashub_fetcher/core_files.py @@ -1,8 +1,6 @@ from enum import Enum from typing import Optional -from infrastructure.postgres.uploaded_file_table import FileTypeEnum - class CoreFiles(Enum): PHOTOPACK = "Photopack" @@ -20,23 +18,6 @@ class CoreFiles(Enum): MCS_COMPLIANCE_CERTIFICATE = "MCS Compliance Certificate" -_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = { - CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value, - CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value, - CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value, - CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value, - CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value, - CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value, - CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value, - CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value, - CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value, - CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value, - CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value, - CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value, - CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value, -} - - def get_core_file_type( filename: str, evidence_category: Optional[str] = None ) -> Optional[CoreFiles]: @@ -75,14 +56,3 @@ def get_core_file_type( return core_file return None - - -def get_file_type_string( - filename: str, evidence_category: Optional[str] = None -) -> Optional[str]: - core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category) - - if core_file is None: - return None - - return _CORE_FILE_TO_FILE_TYPE[core_file] diff --git a/backend/pashub_fetcher/evidence_categories.py b/domain/pashub_fetcher/evidence_categories.py similarity index 100% rename from backend/pashub_fetcher/evidence_categories.py rename to domain/pashub_fetcher/evidence_categories.py diff --git a/backend/pashub_fetcher/job.py b/domain/pashub_fetcher/job.py similarity index 100% rename from backend/pashub_fetcher/job.py rename to domain/pashub_fetcher/job.py diff --git a/backend/pashub_fetcher/sharepoint_subfolders.py b/domain/pashub_fetcher/sharepoint_subfolders.py similarity index 100% rename from backend/pashub_fetcher/sharepoint_subfolders.py rename to domain/pashub_fetcher/sharepoint_subfolders.py diff --git a/infrastructure/pashub_fetcher/__init__.py b/infrastructure/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/infrastructure/pashub_fetcher/core_file_types.py b/infrastructure/pashub_fetcher/core_file_types.py new file mode 100644 index 000000000..9168dadec --- /dev/null +++ b/infrastructure/pashub_fetcher/core_file_types.py @@ -0,0 +1,39 @@ +"""Maps the domain's CoreFiles onto the persisted ``uploaded_file.file_type``. + +The classification rules — which filename/evidence category means which core +file — are domain policy and live in ``domain.pashub_fetcher.core_files``. Only +the translation to the storage enum belongs here, so the domain stays free of +any import of the persistence layer. +""" + +from typing import Optional + +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type +from infrastructure.postgres.uploaded_file_table import FileTypeEnum + +_CORE_FILE_TO_FILE_TYPE: dict[CoreFiles, str] = { + CoreFiles.PHOTOPACK: FileTypeEnum.PHOTO_PACK.value, + CoreFiles.SITENOTE: FileTypeEnum.SITE_NOTE.value, + CoreFiles.RDSAP_SITENOTE: FileTypeEnum.RD_SAP_SITE_NOTE.value, + CoreFiles.PAS2023_VENTILATION: FileTypeEnum.PAS_2023_VENTILATION.value, + CoreFiles.PAS2023_CONDITION: FileTypeEnum.PAS_2023_CONDITION.value, + CoreFiles.PAS_SIGNIFICANCE: FileTypeEnum.PAS_SIGNIFICANCE.value, + CoreFiles.PAR_PHOTOPACK: FileTypeEnum.PAR_PHOTO_PACK.value, + CoreFiles.PAS2023_PROPERTY: FileTypeEnum.PAS_2023_PROPERTY.value, + CoreFiles.PAS2023_OCCUPANCY: FileTypeEnum.PAS_2023_OCCUPANCY.value, + CoreFiles.IMPROVEMENT_OPTION_EVALUATION: FileTypeEnum.IMPROVEMENT_OPTION_EVALUATION.value, + CoreFiles.MEDIUM_TERM_IMPROVEMENT_PLAN: FileTypeEnum.MEDIUM_TERM_IMPROVEMENT_PLAN.value, + CoreFiles.RETROFIT_DESIGN_DOC: FileTypeEnum.RETROFIT_DESIGN_DOC.value, + CoreFiles.MCS_COMPLIANCE_CERTIFICATE: FileTypeEnum.MCS_COMPLIANCE_CERTIFICATE.value, +} + + +def get_file_type_string( + filename: str, evidence_category: Optional[str] = None +) -> Optional[str]: + core_file: Optional[CoreFiles] = get_core_file_type(filename, evidence_category) + + if core_file is None: + return None + + return _CORE_FILE_TO_FILE_TYPE[core_file] diff --git a/backend/pashub_fetcher/evidence_file_data.py b/infrastructure/pashub_fetcher/evidence_file_data.py similarity index 100% rename from backend/pashub_fetcher/evidence_file_data.py rename to infrastructure/pashub_fetcher/evidence_file_data.py diff --git a/backend/pashub_fetcher/evidence_metadata.py b/infrastructure/pashub_fetcher/evidence_metadata.py similarity index 100% rename from backend/pashub_fetcher/evidence_metadata.py rename to infrastructure/pashub_fetcher/evidence_metadata.py diff --git a/backend/pashub_fetcher/pashub_client.py b/infrastructure/pashub_fetcher/pashub_client.py similarity index 87% rename from backend/pashub_fetcher/pashub_client.py rename to infrastructure/pashub_fetcher/pashub_client.py index 3d581293f..667756bd4 100644 --- a/backend/pashub_fetcher/pashub_client.py +++ b/infrastructure/pashub_fetcher/pashub_client.py @@ -5,13 +5,15 @@ from datetime import datetime import requests -from backend.pashub_fetcher.core_files import CoreFiles, get_core_file_type -from backend.pashub_fetcher.evidence_file_data import EvidenceFileData -from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type +from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData +from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata from utils.logger import setup_logger logger = setup_logger() +_EVIDENCE_PAGE_SIZE = 100 + class DownloadedFile(NamedTuple): file_path: str @@ -39,6 +41,10 @@ class UnauthorizedError(Exception): pass +class TruncatedEvidenceListError(Exception): + pass + + class PashubClient: def __init__(self, token: str): @@ -178,14 +184,25 @@ class PashubClient: def _get_evidence_list(self, job_id: str) -> List[EvidenceFileData]: url = f"{self.base}/jobs/{job_id}/evidence" + params = {"pageIndex": 0, "pageSize": _EVIDENCE_PAGE_SIZE} - r = self.session.get(url) + r = self.session.get(url, params=params) if r.status_code == 401: raise UnauthorizedError("Token expired or invalid") r.raise_for_status() - results = r.json().get("results", []) + payload: Dict[str, Any] = r.json() + results: List[Dict[str, Any]] = payload.get("results", []) + total: int = payload.get("totalItemCount", len(results)) + + if len(results) < total: + raise TruncatedEvidenceListError( + f"PasHub returned {len(results)} of {total} evidence files for job " + f"{job_id} - the response was truncated and the evidence set is " + f"incomplete. Increase _EVIDENCE_PAGE_SIZE in pashub_client.py " + f"above {total} and re-run." + ) return [EvidenceFileData.from_api(item) for item in results] diff --git a/backend/pashub_fetcher/spreadsheet.py b/infrastructure/pashub_fetcher/spreadsheet.py similarity index 95% rename from backend/pashub_fetcher/spreadsheet.py rename to infrastructure/pashub_fetcher/spreadsheet.py index 5f8f74f6d..172fefc9a 100644 --- a/backend/pashub_fetcher/spreadsheet.py +++ b/infrastructure/pashub_fetcher/spreadsheet.py @@ -2,7 +2,7 @@ import re from typing import Dict, List from openpyxl import load_workbook -from backend.pashub_fetcher.job import Job +from domain.pashub_fetcher.job import Job def extract_jobs(filepath: str) -> List[Job]: diff --git a/backend/pashub_fetcher/token_getter.py b/infrastructure/pashub_fetcher/token_getter.py similarity index 100% rename from backend/pashub_fetcher/token_getter.py rename to infrastructure/pashub_fetcher/token_getter.py diff --git a/backend/pashub_fetcher/pashub_service.py b/orchestration/pashub_fetcher_orchestrator.py similarity index 97% rename from backend/pashub_fetcher/pashub_service.py rename to orchestration/pashub_fetcher_orchestrator.py index 18caab333..92c9d0140 100644 --- a/backend/pashub_fetcher/pashub_service.py +++ b/orchestration/pashub_fetcher_orchestrator.py @@ -11,15 +11,15 @@ from infrastructure.postgres.uploaded_file_table import ( ) from backend.documents_parser.db_writer import save_epc_property_data from backend.documents_parser.parser import parse_site_notes_pdf -from backend.pashub_fetcher.core_files import get_file_type_string -from backend.pashub_fetcher.pashub_client import ( +from infrastructure.pashub_fetcher.core_file_types import get_file_type_string +from infrastructure.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, RdSapSummary, UnauthorizedError, ) -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) from datatypes.epc.domain.epc import Epc @@ -63,7 +63,7 @@ class _FileUploadRecord(NamedTuple): uprn: Optional[int] -class PashubService: +class PashubFetcherOrchestrator: def __init__( self, pashub_client: PashubClient, diff --git a/orchestration/sharepoint_renamer_orchestrator.py b/orchestration/sharepoint_renamer_orchestrator.py index 3c7d3e8d3..a73373188 100644 --- a/orchestration/sharepoint_renamer_orchestrator.py +++ b/orchestration/sharepoint_renamer_orchestrator.py @@ -2,7 +2,7 @@ import csv import os from typing import Optional -from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders +from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from utilities.logger import setup_logger from utils.sharepoint.domna_sharepoint_client import DomnaSharepointClient diff --git a/pytest.ini b/pytest.ini index 2bcd61785..7cc9a255c 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,7 +15,6 @@ testpaths = backend/ecmk_fetcher/tests backend/export/tests backend/onboarders/tests - backend/pashub_fetcher/tests datatypes/epc/domain/tests datatypes/epc/schema/tests datatypes/epc/surveys/tests diff --git a/tests/applications/pashub_fetcher/__init__.py b/tests/applications/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py b/tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py similarity index 94% rename from backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py rename to tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py index 56187350b..d1c2d91ea 100644 --- a/backend/pashub_fetcher/tests/test_pashub_to_ara_trigger_request.py +++ b/tests/applications/pashub_fetcher/test_pashub_to_ara_trigger_request.py @@ -1,6 +1,6 @@ import pytest -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) diff --git a/tests/domain/pashub_fetcher/__init__.py b/tests/domain/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/pashub_fetcher/tests/test_core_files.py b/tests/domain/pashub_fetcher/test_core_files.py similarity index 51% rename from backend/pashub_fetcher/tests/test_core_files.py rename to tests/domain/pashub_fetcher/test_core_files.py index a2047ecee..e29cf1997 100644 --- a/backend/pashub_fetcher/tests/test_core_files.py +++ b/tests/domain/pashub_fetcher/test_core_files.py @@ -1,96 +1,4 @@ -from backend.pashub_fetcher.core_files import ( - CoreFiles, - get_core_file_type, - get_file_type_string, -) - - -def test_file_type_for_photopack(): - assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack" - - -def test_file_type_for_sitenote(): - assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note" - - -def test_file_type_for_rdsap_sitenote(): - assert ( - get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf") - == "rd_sap_site_note" - ) - - -def test_file_type_for_pas2023_ventilation(): - assert ( - get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf") - == "pas_2023_ventilation" - ) - - -def test_file_type_for_pas2023_condition(): - assert ( - get_file_type_string("PAS 2023 Condition Report_123456.pdf") - == "pas_2023_condition" - ) - - -def test_file_type_for_pas_significance(): - assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance" - - -def test_file_type_for_par_photopack(): - assert ( - get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf") - == "par_photo_pack" - ) - - -def test_file_type_for_pas2023_property(): - assert ( - get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf") - == "pas_2023_property" - ) - - -def test_file_type_for_pas2023_occupancy(): - assert ( - get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf") - == "pas_2023_occupancy" - ) - - -def test_file_type_for_improvement_option_evaluation(): - # filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf" - assert ( - get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf") - == "improvement_option_evaluation" - ) - - -def test_file_type_for_medium_term_improvement_plan(): - # filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf" - assert ( - get_file_type_string( - "60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf" - ) - == "medium_term_improvement_plan" - ) - - -def test_file_type_for_retrofit_design_doc(): - assert ( - get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf") - == "retrofit_design_doc" - ) - assert ( - get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf") - == "retrofit_design_doc" - ) - - -# --------------------------------------------------------------------------- -# core_file_for -# --------------------------------------------------------------------------- +from domain.pashub_fetcher.core_files import CoreFiles, get_core_file_type def test_core_file_for_evidence_category_match_is_case_insensitive() -> None: @@ -209,18 +117,3 @@ def test_core_file_for_mcs_compliance_certificate_is_case_insensitive() -> None: # Assert assert result == CoreFiles.MCS_COMPLIANCE_CERTIFICATE - - -def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> ( - None -): - # Arrange - filename = "some_cert.pdf" - - # Act - result = get_file_type_string( - filename, evidence_category="MCS Compliance Certificate" - ) - - # Assert - assert result == "mcs_compliance_certificate" diff --git a/tests/infrastructure/pashub_fetcher/__init__.py b/tests/infrastructure/pashub_fetcher/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/infrastructure/pashub_fetcher/test_core_file_types.py b/tests/infrastructure/pashub_fetcher/test_core_file_types.py new file mode 100644 index 000000000..9b95c9b3e --- /dev/null +++ b/tests/infrastructure/pashub_fetcher/test_core_file_types.py @@ -0,0 +1,98 @@ +from infrastructure.pashub_fetcher.core_file_types import get_file_type_string + + +def test_file_type_for_photopack(): + assert get_file_type_string("Photopack_123456_V1.pdf") == "photo_pack" + + +def test_file_type_for_sitenote(): + assert get_file_type_string("SiteNote_123456_V1.pdf") == "site_note" + + +def test_file_type_for_rdsap_sitenote(): + assert ( + get_file_type_string("RdSAP_SiteNote_9510890_V1_Assessmet.pdf") + == "rd_sap_site_note" + ) + + +def test_file_type_for_pas2023_ventilation(): + assert ( + get_file_type_string("PAS 2023 Ventilation Assessment Report_123456.pdf") + == "pas_2023_ventilation" + ) + + +def test_file_type_for_pas2023_condition(): + assert ( + get_file_type_string("PAS 2023 Condition Report_123456.pdf") + == "pas_2023_condition" + ) + + +def test_file_type_for_pas_significance(): + assert get_file_type_string("PAS Significance_123456.pdf") == "pas_significance" + + +def test_file_type_for_par_photopack(): + assert ( + get_file_type_string("PAR Photo Pack_95101890_V2_Assessment.pdf") + == "par_photo_pack" + ) + + +def test_file_type_for_pas2023_property(): + assert ( + get_file_type_string("PAS 2023 Property Assessment Report_123456.pdf") + == "pas_2023_property" + ) + + +def test_file_type_for_pas2023_occupancy(): + assert ( + get_file_type_string("PAS 2023 Occupancy Assessment Report_123456.pdf") + == "pas_2023_occupancy" + ) + + +def test_file_type_for_improvement_option_evaluation(): + # filename: "{job_id} - {postcode} - Improvement Option Evaluation.pdf" + assert ( + get_file_type_string("6000802 - NG4 4HD - Improvement Option Evaluation.pdf") + == "improvement_option_evaluation" + ) + + +def test_file_type_for_medium_term_improvement_plan(): + # filename: "{job_id} - {postcode} - Medium Term Improvement Plan IOE.pdf" + assert ( + get_file_type_string( + "60800802 - NG4 4HD - Medium Term Improvement Plan IOE.pdf" + ) + == "medium_term_improvement_plan" + ) + + +def test_file_type_for_retrofit_design_doc(): + assert ( + get_file_type_string("2512-OSM-H21M900-XX-DR-N-A_Lord Nelson Street 018.pdf") + == "retrofit_design_doc" + ) + assert ( + get_file_type_string("2603-OSM-B06M901-XX-DR-N-A_Alvaston Walk 022.pdf") + == "retrofit_design_doc" + ) + +def test_get_file_type_string_with_mcs_evidence_category_returns_mcs_compliance_certificate() -> ( + None +): + # Arrange + filename = "some_cert.pdf" + + # Act + result = get_file_type_string( + filename, evidence_category="MCS Compliance Certificate" + ) + + # Assert + assert result == "mcs_compliance_certificate" diff --git a/backend/pashub_fetcher/tests/test_pashub_client.py b/tests/infrastructure/pashub_fetcher/test_pashub_client.py similarity index 78% rename from backend/pashub_fetcher/tests/test_pashub_client.py rename to tests/infrastructure/pashub_fetcher/test_pashub_client.py index 67b2ebac5..c1ea1ee29 100644 --- a/backend/pashub_fetcher/tests/test_pashub_client.py +++ b/tests/infrastructure/pashub_fetcher/test_pashub_client.py @@ -5,13 +5,14 @@ from unittest.mock import MagicMock, patch import pytest import requests -from backend.pashub_fetcher.core_files import CoreFiles -from backend.pashub_fetcher.evidence_file_data import EvidenceFileData -from backend.pashub_fetcher.evidence_metadata import EvidenceMetadata -from backend.pashub_fetcher.pashub_client import ( +from domain.pashub_fetcher.core_files import CoreFiles +from infrastructure.pashub_fetcher.evidence_file_data import EvidenceFileData +from infrastructure.pashub_fetcher.evidence_metadata import EvidenceMetadata +from infrastructure.pashub_fetcher.pashub_client import ( DownloadedFiles, PashubClient, RdSapSummary, + TruncatedEvidenceListError, UnauthorizedError, ) @@ -346,3 +347,87 @@ def test_get_rdsap_summary_raises_on_http_error() -> None: ) with pytest.raises(requests.HTTPError): client.get_rdsap_summary_by_job_id("job-1") + + +# --------------------------------------------------------------------------- +# _get_evidence_list +# --------------------------------------------------------------------------- + + +def make_evidence_item(file_id: str = "id-1") -> dict[str, Any]: + return { + "fileId": file_id, + "fileName": "unknown.pdf", + "createdUtc": "2024-01-01T00:00:00", + "fileSize": 1024, + "fileExtension": "pdf", + } + + +def test_get_evidence_list_requests_a_single_large_page() -> None: + # Arrange + client = make_client() + payload: dict[str, Any] = {"results": [], "totalItemCount": 0} + client.session.get = MagicMock(return_value=make_response(payload=payload)) + + # Act + client._get_evidence_list("job-1") + + # Assert + assert client.session.get.call_args.kwargs["params"] == { + "pageIndex": 0, + "pageSize": 100, + } + + +def test_get_evidence_list_raises_when_response_is_truncated() -> None: + # Arrange + client = make_client() + payload = { + "results": [make_evidence_item(f"id-{i}") for i in range(100)], + "totalItemCount": 250, + } + client.session.get = MagicMock(return_value=make_response(payload=payload)) + + # Act / Assert + with pytest.raises(TruncatedEvidenceListError, match=r"100 of 250"): + client._get_evidence_list("job-1") + + +def test_get_evidence_list_returns_all_files_when_count_matches_total() -> None: + # Arrange + client = make_client() + payload = { + "results": [make_evidence_item(f"id-{i}") for i in range(3)], + "totalItemCount": 3, + } + client.session.get = MagicMock(return_value=make_response(payload=payload)) + + # Act + result = client._get_evidence_list("job-1") + + # Assert + assert [f.file_id for f in result] == ["id-0", "id-1", "id-2"] + + +def test_get_evidence_list_assumes_complete_when_total_item_count_absent() -> None: + # Arrange — a PasHub schema change that drops the field must not fail the fetch + client = make_client() + payload = {"results": [make_evidence_item()]} + client.session.get = MagicMock(return_value=make_response(payload=payload)) + + # Act + result = client._get_evidence_list("job-1") + + # Assert + assert len(result) == 1 + + +def test_get_evidence_list_raises_unauthorized_on_401() -> None: + # Arrange + client = make_client() + client.session.get = MagicMock(return_value=make_response(status_code=401)) + + # Act / Assert + with pytest.raises(UnauthorizedError): + client._get_evidence_list("job-1") diff --git a/backend/pashub_fetcher/tests/test_token_getter.py b/tests/infrastructure/pashub_fetcher/test_token_getter.py similarity index 84% rename from backend/pashub_fetcher/tests/test_token_getter.py rename to tests/infrastructure/pashub_fetcher/test_token_getter.py index 2c847d167..8e8b0c447 100644 --- a/backend/pashub_fetcher/tests/test_token_getter.py +++ b/tests/infrastructure/pashub_fetcher/test_token_getter.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock, patch -from backend.pashub_fetcher.token_getter import get_token_from_local_storage +from infrastructure.pashub_fetcher.token_getter import get_token_from_local_storage def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None: @@ -20,9 +20,9 @@ def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None: mock_sync_playwright.return_value.__enter__.return_value = mock_p -@patch("backend.pashub_fetcher.token_getter.shutil.rmtree") -@patch("backend.pashub_fetcher.token_getter.glob.glob") -@patch("backend.pashub_fetcher.token_getter.sync_playwright") +@patch("infrastructure.pashub_fetcher.token_getter.shutil.rmtree") +@patch("infrastructure.pashub_fetcher.token_getter.glob.glob") +@patch("infrastructure.pashub_fetcher.token_getter.sync_playwright") def test_playwright_tmp_dirs_are_cleaned_up_after_browser_close( mock_sync_playwright: MagicMock, mock_glob: MagicMock, diff --git a/backend/pashub_fetcher/tests/test_pashub_service.py b/tests/orchestration/test_pashub_fetcher_orchestrator.py similarity index 81% rename from backend/pashub_fetcher/tests/test_pashub_service.py rename to tests/orchestration/test_pashub_fetcher_orchestrator.py index 1acb583d3..57f478d5f 100644 --- a/backend/pashub_fetcher/tests/test_pashub_service.py +++ b/tests/orchestration/test_pashub_fetcher_orchestrator.py @@ -5,15 +5,15 @@ from unittest.mock import MagicMock, call, patch from infrastructure.postgres.uploaded_file_table import FileSourceEnum, FileTypeEnum -from backend.pashub_fetcher.pashub_client import ( +from infrastructure.pashub_fetcher.pashub_client import ( DownloadedFile, DownloadedFiles, PashubClient, RdSapSummary, UnauthorizedError, ) -from backend.pashub_fetcher.pashub_service import PashubService -from backend.pashub_fetcher.pashub_to_ara_trigger_request import ( +from orchestration.pashub_fetcher_orchestrator import PashubFetcherOrchestrator +from applications.pashub_fetcher.pashub_to_ara_trigger_request import ( PashubToAraTriggerRequest, ) from datatypes.epc.domain.epc import Epc @@ -101,8 +101,8 @@ def make_service( sharepoint_client: Optional[DomnaSharepointClient] = None, s3_bucket: str = "test-bucket", coordination_client_factory: Optional[Callable[[], PashubClient]] = None, -) -> PashubService: - return PashubService( +) -> PashubFetcherOrchestrator: + return PashubFetcherOrchestrator( pashub_client=pashub_client or MagicMock(spec=PashubClient), sharepoint_client=sharepoint_client or MagicMock(spec=DomnaSharepointClient), s3_bucket=s3_bucket, @@ -134,7 +134,7 @@ def test_run_returns_file_paths() -> None: service = make_service(pashub_client=mock_client) - with patch("backend.pashub_fetcher.pashub_service.os.remove"): + with patch("orchestration.pashub_fetcher_orchestrator.os.remove"): result = service.run(make_request()) assert result == ["/tmp/a.pdf", "/tmp/b.pdf"] @@ -157,7 +157,7 @@ def test_run_returns_core_and_other_file_paths() -> None: service = make_service(pashub_client=mock_client) # Act - with patch("backend.pashub_fetcher.pashub_service.os.remove"): + with patch("orchestration.pashub_fetcher_orchestrator.os.remove"): result = service.run(make_request(get_other_files=True)) # Assert @@ -179,8 +179,8 @@ def test_run_skips_upload_when_no_uprn_and_no_deal_id() -> None: service = make_service(pashub_client=mock_client) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3") as mock_s3, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn=None, hubspot_deal_id=None)) @@ -202,9 +202,9 @@ def test_run_uploads_files_to_s3_using_uprn_path() -> None: service = make_service(pashub_client=mock_client, s3_bucket="my-bucket") with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3") as mock_s3, + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345")) @@ -241,9 +241,9 @@ def test_run_persists_uploaded_file_records_to_db() -> None: service = make_service(pashub_client=mock_client) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345")) @@ -271,9 +271,9 @@ def test_run_uses_hubspot_deal_id_path_when_no_uprn() -> None: service = make_service(pashub_client=mock_client, s3_bucket="my-bucket") with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3") as mock_s3, + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn=None, hubspot_deal_id="deal-abc")) @@ -305,16 +305,16 @@ def test_run_parses_and_saves_site_notes_for_rd_sap_site_note_file() -> None: service = make_service(pashub_client=mock_client) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), patch( - "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + "orchestration.pashub_fetcher_orchestrator.parse_site_notes_pdf", return_value=parsed, ) as mock_parse, patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): fake_session.add_all = MagicMock( side_effect=lambda files: setattr(files[0], "id", fake_uploaded_file_id) @@ -355,9 +355,9 @@ def test_run_uses_coordination_client_when_pas_401_on_uprn_lookup() -> None: ) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): result = service.run(make_request()) @@ -383,9 +383,9 @@ def test_run_uses_coordination_client_when_pas_401_on_file_listing() -> None: ) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): result = service.run(make_request(uprn="12345")) @@ -441,9 +441,9 @@ def test_run_persists_coordination_hub_file_source_when_pas_401_on_uprn_lookup() ) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): mock_db.return_value.__enter__.return_value = fake_session service.run(make_request()) @@ -472,9 +472,9 @@ def test_run_persists_coordination_hub_file_source_when_pas_401_on_file_listing( ) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345")) @@ -501,7 +501,7 @@ def test_run_deletes_other_temp_files_when_get_other_files_true() -> None: service = make_service(pashub_client=mock_client) # Act - with patch("backend.pashub_fetcher.pashub_service.os.remove") as mock_remove: + with patch("orchestration.pashub_fetcher_orchestrator.os.remove") as mock_remove: service.run(make_request(get_other_files=True)) # Assert @@ -527,9 +527,9 @@ def test_run_uploads_other_files_to_s3_when_get_other_files_true() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3") as mock_s3, + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345", get_other_files=True)) @@ -560,9 +560,9 @@ def test_run_persists_other_files_with_other_file_type() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345", get_other_files=True)) @@ -591,9 +591,9 @@ def test_run_persists_mcs_cert_with_mcs_compliance_certificate_file_type() -> No # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session") as mock_db, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session") as mock_db, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): mock_db.return_value.__enter__.return_value = fake_session service.run(make_request(uprn="12345")) @@ -626,7 +626,7 @@ def test_sharepoint_uploads_all_files_to_property_folder() -> None: service = make_service(pashub_client=mock_client, sharepoint_client=mock_sharepoint) # Act - with patch("backend.pashub_fetcher.pashub_service.os.remove"): + with patch("orchestration.pashub_fetcher_orchestrator.os.remove"): service.run( make_request( sharepoint_link="Retrofit/Properties", @@ -661,8 +661,8 @@ def test_sharepoint_skips_upload_when_folder_not_found() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.os.remove"), - patch("backend.pashub_fetcher.pashub_service.logger") as mock_logger, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.logger") as mock_logger, ): service.run( make_request( @@ -684,14 +684,14 @@ def test_sharepoint_skips_upload_when_sharepoint_client_is_none() -> None: core=["/tmp/core.pdf"] ) - service = PashubService( + service = PashubFetcherOrchestrator( pashub_client=mock_client, sharepoint_client=None, s3_bucket="test-bucket", ) # Act — should not raise AttributeError on None._sharepoint_client - with patch("backend.pashub_fetcher.pashub_service.os.remove"): + with patch("orchestration.pashub_fetcher_orchestrator.os.remove"): result = service.run( make_request( sharepoint_link="Retrofit/Properties", @@ -713,17 +713,17 @@ def test_run_warns_and_continues_when_site_notes_parsing_fails() -> None: service = make_service(pashub_client=mock_client) with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), patch( - "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + "orchestration.pashub_fetcher_orchestrator.parse_site_notes_pdf", side_effect=ValueError("corrupt pdf"), ), patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.logger") as mock_logger, - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.logger") as mock_logger, + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): result = service.run(make_request(uprn="12345")) @@ -764,16 +764,16 @@ def test_run_overlays_full_lodged_performance_onto_saved_site_notes() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), patch( - "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + "orchestration.pashub_fetcher_orchestrator.parse_site_notes_pdf", return_value=parsed, ), patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345")) @@ -799,16 +799,16 @@ def test_run_overlays_only_present_fields_when_summary_partial() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), patch( - "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + "orchestration.pashub_fetcher_orchestrator.parse_site_notes_pdf", return_value=parsed, ), patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345")) @@ -828,12 +828,12 @@ def test_run_raises_and_skips_persistence_when_summary_fetch_fails() -> None: # Act / Assert with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3") as mock_s3, + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3") as mock_s3, patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): with pytest.raises(RuntimeError): service.run(make_request(uprn="12345")) @@ -849,9 +849,9 @@ def test_run_does_not_fetch_summary_for_evidence_only_job() -> None: # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345")) @@ -879,16 +879,16 @@ def test_run_fetches_summary_from_coordination_client_on_primary_auth_failure() # Act with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), patch( - "backend.pashub_fetcher.pashub_service.parse_site_notes_pdf", + "orchestration.pashub_fetcher_orchestrator.parse_site_notes_pdf", return_value=parsed, ), patch( - "backend.pashub_fetcher.pashub_service.save_epc_property_data" + "orchestration.pashub_fetcher_orchestrator.save_epc_property_data" ) as mock_save, - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): service.run(make_request(uprn="12345")) @@ -911,9 +911,9 @@ def test_run_uploads_to_sharepoint_even_when_summary_fetch_fails() -> None: # Act / Assert with ( - patch("backend.pashub_fetcher.pashub_service.upload_file_to_s3"), - patch("backend.pashub_fetcher.pashub_service.db_session"), - patch("backend.pashub_fetcher.pashub_service.os.remove"), + patch("orchestration.pashub_fetcher_orchestrator.upload_file_to_s3"), + patch("orchestration.pashub_fetcher_orchestrator.db_session"), + patch("orchestration.pashub_fetcher_orchestrator.os.remove"), ): with pytest.raises(RuntimeError): service.run( diff --git a/tests/scripts/test_rename_sharepoint_files.py b/tests/scripts/test_rename_sharepoint_files.py index c732383c7..cd27d8a29 100644 --- a/tests/scripts/test_rename_sharepoint_files.py +++ b/tests/scripts/test_rename_sharepoint_files.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock import pytest -from backend.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders +from domain.pashub_fetcher.sharepoint_subfolders import SharepointSubfolders from orchestration.sharepoint_renamer_orchestrator import ( ASSESSMENT_SUBFOLDER, BASE_PATH,